rock
3eb6a91311
Replaces the Qwen3-32B dense deployment, which needed two V100s for 131k context. The MoE carries 96 KiB of KV per token against the dense model's 256 KiB, so the same 64k window fits on one card. Measured on this cluster before the switch: 15.55 GiB of weights, 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request, correct answer over a 5921-token prompt, and clean reasoning_content separation through the gateway. Three settings are load-bearing on sm_70 rather than tuning: - VLLM_USE_FLASHINFER_SAMPLER=0 flashinfer's check_cuda_arch raises on CC 7.0 - VLLM_ATTENTION_BACKEND=TRITON_ATTN FlashInfer and FA2 both need sm75+ - deploymentStrategy Recreate a rolling update deadlocks once every card is allocated The manifest is committed as well as applied: reasoning.yaml had never been in git.
103 lines
3.5 KiB
YAML
103 lines
3.5 KiB
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
name: reasoning
|
|
namespace: llm-serving
|
|
labels:
|
|
app.kubernetes.io/name: llm-reasoning
|
|
app.kubernetes.io/part-of: llm-serving
|
|
annotations:
|
|
serving.kserve.io/deploymentMode: RawDeployment
|
|
spec:
|
|
predictor:
|
|
minReplicas: 1
|
|
maxReplicas: 1
|
|
nodeSelector:
|
|
kubernetes.io/hostname: worker-1
|
|
# Every GPU card is allocated, so a rolling update deadlocks: the new pod needs a
|
|
# card the old pod still holds and sits Pending forever. Recreate frees the card
|
|
# first. The cost is a short gap in service for this model only.
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
containers:
|
|
- name: kserve-container
|
|
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
|
args:
|
|
# Qwen3-30B-A3B is a 30.5B MoE with 3.3B active per token. On a V100 that matters:
|
|
# the 32B dense model it replaces needed two cards for the same context, this needs
|
|
# one, because its KV cache is 96 KiB/token against the dense model's 256 KiB.
|
|
#
|
|
# Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
|
|
# 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
|
|
- --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
|
|
- --served-model-name=reasoning
|
|
- --quantization=gptq
|
|
- --dtype=float16
|
|
- --kv-cache-dtype=auto
|
|
# Native window is 40960. YaRN factor 2.0 reaches 81920, so 64k is inside it.
|
|
- --rope-scaling={"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}
|
|
- --tensor-parallel-size=1
|
|
- --pipeline-parallel-size=1
|
|
- --max-model-len=65536
|
|
- --gpu-memory-utilization=0.90
|
|
- --max-num-seqs=8
|
|
- --max-num-batched-tokens=8192
|
|
- --long-prefill-token-threshold=2048
|
|
- --enable-chunked-prefill
|
|
- --enable-prefix-caching
|
|
- --reasoning-parser=qwen3
|
|
- --enable-auto-tool-choice
|
|
- --tool-call-parser=hermes
|
|
- --host=0.0.0.0
|
|
- --port=8080
|
|
env:
|
|
# Both are required on this hardware, not tuning:
|
|
# - flashinfer's check_cuda_arch raises AttributeError on compute capability 7.0
|
|
# (flashinfer/jit/core.py:55, minor.isdigit() on an int), so the sampler must stay off
|
|
# - FlashInfer and FlashAttention-2 both need sm75+, so the Triton backend serves attention
|
|
- name: VLLM_USE_FLASHINFER_SAMPLER
|
|
value: "0"
|
|
- name: VLLM_ATTENTION_BACKEND
|
|
value: TRITON_ATTN
|
|
- name: HF_HOME
|
|
value: /mnt/models
|
|
ports:
|
|
- containerPort: 8080
|
|
protocol: TCP
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
periodSeconds: 10
|
|
startupProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
# Weight load took 18s from cache and 148s on a cold download, and the first
|
|
# torch.compile adds about 70s. 80 x 15s leaves ample room for a cold start.
|
|
failureThreshold: 80
|
|
periodSeconds: 15
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 12Gi
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "8"
|
|
memory: 36Gi
|
|
nvidia.com/gpu: "1"
|
|
volumeMounts:
|
|
- mountPath: /mnt/models
|
|
name: models
|
|
- mountPath: /dev/shm
|
|
name: shm
|
|
runtimeClassName: nvidia
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|
|
- name: shm
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 2Gi
|