diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index e500955..859929e 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -40,7 +40,13 @@ spec: - --tensor-parallel-size=1 - --max-model-len=16384 - --gpu-memory-utilization=0.90 - - --max-num-seqs=4 + # Halved from 4 -- weights alone (35B total, MoE, all experts loaded + # regardless of ~3B active) at on-the-fly bnb 4bit are ~19-21GB on a + # 32GB card at 0.90 util (~29.5GB budget), leaving only ~8-10GB for KV + # cache + activations. Old DeepSeek-32B was in the same weight + # ballpark but MoE bnb overhead here is untested. Starting at 2 to + # confirm real headroom via nvidia-smi/vLLM logs before raising back. + - --max-num-seqs=2 - --enable-chunked-prefill - --enable-prefix-caching # qwen3 parser handles this family's reasoning blocks (best