152 lines
6.3 KiB
YAML
152 lines
6.3 KiB
YAML
|
apiVersion: serving.kserve.io/v1beta1
|
||
|
|
kind: InferenceService
|
||
|
|
metadata:
|
||
|
|
annotations:
|
||
|
|
serving.kserve.io/deploymentMode: RawDeployment
|
||
|
|
# KServe injects the prometheus.io/{scrape,port,path} annotations into the
|
||
|
|
# generated pod only when this flag is on. Without it the predictor carries
|
||
|
|
# no scrape annotations at all, which is why no vLLM/SGLang metrics have
|
||
|
|
# ever reached Prometheus from this namespace.
|
||
|
|
serving.kserve.io/enable-prometheus-scraping: "true"
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: llm-reasoning
|
||
|
|
app.kubernetes.io/part-of: llm-serving
|
||
|
|
name: reasoning
|
||
|
|
namespace: llm-serving
|
||
|
|
spec:
|
||
|
|
predictor:
|
||
|
|
containers:
|
||
|
|
- args:
|
||
|
|
# Replaces the dense Qwen3-32B. Same serving name (`reasoning`), so every
|
||
|
|
# consumer keeps its model string and no client changes.
|
||
|
|
#
|
||
|
|
# Why a MoE: Qwen3-30B-A3B has 30.5B total but only 3.3B active per token.
|
||
|
|
# On sm70 the decode path is bandwidth- and kernel-launch-bound, not
|
||
|
|
# FLOP-bound, so activating 3.3B instead of 32B is the single biggest
|
||
|
|
# throughput win available on this hardware without changing GPUs.
|
||
|
|
#
|
||
|
|
# Quantization: GPTQ-Int4, pinned with --quantization=gptq. This repeats
|
||
|
|
# the finding recorded on the previous checkpoint: bnb-4bit works poorly
|
||
|
|
# on sm70 because there are no int4 tensor cores, and vLLM auto-upgrades
|
||
|
|
# AWQ/GPTQ to the Marlin kernel which requires sm80+. Pinning `gptq`
|
||
|
|
# forces the plain kernel, which is verified Volta-compatible here.
|
||
|
|
# Do not remove this flag.
|
||
|
|
#
|
||
|
|
# Context: 131072 via YaRN. The base model card documents 32,768 native
|
||
|
|
# and 131,072 with YaRN (factor 4.0, original_max_position_embeddings
|
||
|
|
# 32768). That is Qwen's own validated ceiling, so unlike a stretched
|
||
|
|
# window this is a documented capability.
|
||
|
|
#
|
||
|
|
# KV cache math, from config.json: 48 layers, 4 KV heads, head_dim 128,
|
||
|
|
# fp16 -> 2 * 4 * 128 * 2 = 2048 B/token/layer -> 96 KB/token.
|
||
|
|
# At 131072 tokens that is 12 GiB total. With TP=2 the KV heads split
|
||
|
|
# 2 per GPU, so each GPU carries ~6 GiB of KV. Weights are ~17 GB total,
|
||
|
|
# so ~8.5 GB per GPU. At gpu-memory-utilization=0.95 (about 30.4 GB
|
||
|
|
# usable per V100-32GB) that leaves roughly 21 GB per GPU for KV plus
|
||
|
|
# activations: comfortable for one full 128K request and a few shorter
|
||
|
|
# ones. Re-read vllm:cache_config_info after rollout for the real
|
||
|
|
# num_gpu_blocks value rather than trusting this projection.
|
||
|
|
#
|
||
|
|
# TP=2, not PP=2: the model has 4 KV heads, so TP must divide 4. TP=2
|
||
|
|
# splits cleanly; TP=3 would not. TP also shortens prefill at long
|
||
|
|
# context, which is the actual bottleneck at 128K on V100.
|
||
|
|
- --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
|
||
|
|
- --served-model-name=reasoning
|
||
|
|
- --quantization=gptq
|
||
|
|
- --dtype=float16
|
||
|
|
- --kv-cache-dtype=auto
|
||
|
|
- --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
|
||
|
|
- --tensor-parallel-size=2
|
||
|
|
- --max-model-len=131072
|
||
|
|
- --gpu-memory-utilization=0.95
|
||
|
|
- --max-num-seqs=4
|
||
|
|
# Prefill scheduling. The same reasoning as the previous checkpoint
|
||
|
|
# applies: vLLM V1 has no separate prefill phase, so a single long
|
||
|
|
# prompt can consume the whole step budget and starve every other
|
||
|
|
# request. The threshold must be strictly below the budget or it never
|
||
|
|
# binds, hence 8192 budget with a 2048 threshold.
|
||
|
|
- --max-num-batched-tokens=8192
|
||
|
|
- --long-prefill-token-threshold=2048
|
||
|
|
- --enable-chunked-prefill
|
||
|
|
- --enable-prefix-caching
|
||
|
|
# qwen3 is vLLM's reasoning parser for this family's <think> blocks.
|
||
|
|
- --reasoning-parser=qwen3
|
||
|
|
# hermes is the documented tool-call parser for general Qwen3 models.
|
||
|
|
- --enable-auto-tool-choice
|
||
|
|
- --tool-call-parser=hermes
|
||
|
|
- --host=0.0.0.0
|
||
|
|
- --port=8080
|
||
|
|
env:
|
||
|
|
# FlashInfer is sm80+. TRITON_ATTN is the Volta-compatible backend and is
|
||
|
|
# already the proven setting on this node. Clef-flash runs on SGLang
|
||
|
|
# instead, but Qwen stays on vLLM here because this exact combination is
|
||
|
|
# tested on this hardware; moving it to SGLang would add risk for no
|
||
|
|
# measured gain.
|
||
|
|
- name: VLLM_USE_FLASHINFER_SAMPLER
|
||
|
|
value: '0'
|
||
|
|
- name: VLLM_ATTENTION_BACKEND
|
||
|
|
value: TRITON_ATTN
|
||
|
|
- name: HF_HOME
|
||
|
|
value: /mnt/models
|
||
|
|
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
||
|
|
name: kserve-container
|
||
|
|
ports:
|
||
|
|
- containerPort: 8080
|
||
|
|
protocol: TCP
|
||
|
|
readinessProbe:
|
||
|
|
httpGet:
|
||
|
|
path: /health
|
||
|
|
port: 8080
|
||
|
|
periodSeconds: 10
|
||
|
|
resources:
|
||
|
|
limits:
|
||
|
|
cpu: '16'
|
||
|
|
memory: 36Gi
|
||
|
|
nvidia.com/gpu: '2'
|
||
|
|
requests:
|
||
|
|
cpu: '8'
|
||
|
|
memory: 12Gi
|
||
|
|
nvidia.com/gpu: '2'
|
||
|
|
startupProbe:
|
||
|
|
# 30.5B of int4 weights plus a 128K rope config: allow a generous
|
||
|
|
# first-boot window. failureThreshold 120 * 15s = 30 min ceiling.
|
||
|
|
failureThreshold: 120
|
||
|
|
httpGet:
|
||
|
|
path: /health
|
||
|
|
port: 8080
|
||
|
|
periodSeconds: 15
|
||
|
|
volumeMounts:
|
||
|
|
- mountPath: /mnt/models
|
||
|
|
name: models
|
||
|
|
- mountPath: /dev/shm
|
||
|
|
name: shm
|
||
|
|
# podMetadata is NOT a valid field on this KServe version. The CRD exposes
|
||
|
|
# annotations and labels directly under predictor. The older manifests in
|
||
|
|
# this directory still carry podMetadata and ArgoCD silently drops it on
|
||
|
|
# server-side apply; do not copy that pattern.
|
||
|
|
annotations:
|
||
|
|
prometheus.io/scrape: "true"
|
||
|
|
prometheus.io/port: "8080"
|
||
|
|
prometheus.io/path: "/metrics"
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: llm-reasoning
|
||
|
|
app.kubernetes.io/part-of: llm-serving
|
||
|
|
deploymentStrategy:
|
||
|
|
# Recreate, not RollingUpdate: GPU memory is a hard resource. A second
|
||
|
|
# pod cannot schedule alongside the first on a full 4-GPU node, so a
|
||
|
|
# rolling update would wedge with the new pod Pending forever.
|
||
|
|
type: Recreate
|
||
|
|
maxReplicas: 1
|
||
|
|
minReplicas: 1
|
||
|
|
nodeSelector:
|
||
|
|
kubernetes.io/hostname: worker-1
|
||
|
|
runtimeClassName: nvidia
|
||
|
|
volumes:
|
||
|
|
- name: models
|
||
|
|
persistentVolumeClaim:
|
||
|
|
claimName: llm-models
|
||
|
|
- emptyDir:
|
||
|
|
medium: Memory
|
||
|
|
sizeLimit: 2Gi
|
||
|
|
name: shm
|