diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 19aae96..ebc72a9 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -12,25 +12,40 @@ spec: predictor: containers: - args: - # DeepSeek-R1-Distill-32B retired: tool_choice="auto" (what pi sends) - # hit a documented vLLM/R1-family conflict -- the model narrated fake - # tool_calls in its block instead of emitting real ones, - # regardless of parser. Tried swapping to a Kimi-distilled Qwen3.6 - # MoE checkpoint and an AWQ-quantized Qwen3-30B-A3B first -- both - # failed on real, separate blockers (unrecognized model_type; then - # marlin INT4 kernels needing compute capability 80+, but worker-1's - # GPU is sm70/V100). Landed on dense Qwen3-32B instead: native Qwen3 - # tool-call format (no narration bug), bnb-4bit works fine on sm70 - # (proven by the old DeepSeek config already), and no MoE - # arch/quantization risk this time. - - --model=unsloth/Qwen3-32B-bnb-4bit + # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then- + # matmul is two slow kernel launches instead of one fused int4 GEMM, + # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to + # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same + # hermes/qwen3 parser stack (no narration-bug risk, same as before), + # only the quant format changes. Plain (non-Marlin) GPTQ kernel is + # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try + # to auto-upgrade to it, so --quantization is pinned explicitly to + # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB + # (summed from the real safetensors index, not bits-per-param math). + # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN: + # factor=4.0, original_max_position_embeddings=32768) -- 200k was + # asked for but exceeds this architecturally regardless of VRAM. + # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128 + # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so + # each GPU carries ~9.67GB weights + ~128KB/token KV. At + # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves + # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably + # above the 131072 target with room to spare -- the old + # OffloadingConnector CPU-DRAM spillover (tuned for the previous + # model's much smaller 16384 context) is no longer needed and is + # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed, + # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no + # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth + # trying later, but not risking a parallelism-strategy change in the + # same rollout as the quant+context-length change. + - --model=JunHowie/Qwen3-32B-GPTQ-Int4 - --served-model-name=reasoning - - --quantization=bitsandbytes + - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto - --tensor-parallel-size=1 - --pipeline-parallel-size=2 - - --max-model-len=16384 + - --max-model-len=131072 - --gpu-memory-utilization=0.90 - --max-num-seqs=4 - --enable-chunked-prefill @@ -42,27 +57,6 @@ spec: # Qwen3 models -- native chat template support, not narrated text. - --enable-auto-tool-choice - --tool-call-parser=hermes - # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks - # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4 - # + max-model-len=16384 means concurrent long sequences compete for - # the same GPU KV space). No extra dependency, built into vLLM core. - # num_cpu_blocks=2000 hung the pod at startup on the old model (2000 x - # ~32MB/block blew well past the pod's memory limit). num_cpu_blocks=32 - # was the safe-recovery value after that -- only ~1GB of real DRAM - # (32 blocks x 128 tokens x 256KB/token-across-all-64-layers, fp16), - # basically a token-count safety valve, not real offload capacity. - # This model: 64 layers, 8 KV heads x 128 head_dim, fp16 -> ~256KB of - # KV per token across all layers -> ~32MB per 128-token block. - # num_cpu_blocks=256 -> ~8GB of actual DRAM offload (32,768 tokens), - # comfortably under the pod's 36Gi limit alongside the ~20GB bnb-4bit - # weights. Watch real host memory on boot before raising further -- - # block_size=128 tokens matches vLLM's own example. - # Note: 0.11.0 ships the original (fragmented, small-transfer-block) - # version of this connector -- 0.12.0 consolidates KV data into one - # contiguous block per request and is reported an order of magnitude - # faster for this specific feature, so this is a real but not yet - # optimal implementation until the image gets bumped. - - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":256,"block_size":128}} - --host=0.0.0.0 - --port=8080 env: