diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 2ddad69..19aae96 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -12,65 +12,57 @@ spec: predictor: containers: - args: - # dense Qwen3-32B-bnb-4bit retired: sm70/V100 bnb dequant kernel is - # slow (no int4 tensor cores pre-Turing, dequant-then-fp16-matmul is - # two kernel launches not one fused int4 GEMM), decode crawled at - # ~2.5-10 tok/s and blew the gateway's request timeout regardless of - # TP/PP config. bnb also outright rejects tensor-parallel-size>1 on - # prequant checkpoints ("Please try with pipeline parallelism"), - # which is why this went through a PP=2 detour first. - # Switched to Qwen3.5-35B-A3B (MoE, GDN hybrid attention) GPTQ-Int4. - # Requires vLLM >=0.17.0 -- v0.11.0 errors with "Model architectures - # ['Qwen3_5MoeForConditionalGeneration'] are not supported for now." - # moe_wna16 is the checkpoint's documented quantization kernel; - # confirmed working on sm70 -- weights loaded fine (10.6GiB/rank at - # TP=2, no crash). What actually broke TP=2 was a shm_broadcast - # deadlock between the two worker processes post-load (matches a - # known vLLM bug class, e.g. vllm-ascend#9405 -- logic bug in the - # broadcast ring buffer, unaffected by shm size/timeouts/eager-mode, - # all of which were tried there too). Dropped to tensor-parallel-size=1 - # to sidestep the cross-process sync entirely -- full unsharded - # weights (~21GB, roughly 2x the per-rank figure above) fit on one - # 32GB V100 with room for KV cache. gpu-memory-utilization raised - # accordingly (0.5 was sized for the TP=2 split, too low for - # unsharded weights on a single GPU). Second V100 sits idle for now; - # pipeline-parallel-size=2 is the next thing to try if it's needed - # back, since PP unblocked the previous model instead of TP too. - # --kv-cache-dtype stays auto, not - # fp8_e5m2: V100 has no FP8 tensor cores at all (Hopper/Ada only), - # hardware-blocked regardless of vLLM version. tool-call-parser - # changed hermes -> qwen3_coder per the checkpoint's own README - # example, not cosmetic. gpu-memory-utilization starts low (0.5) - # since real VRAM footprint for this arch+quant combo is unknown; - # raise once stable. Old OffloadingConnector kv-transfer-config - # dropped -- its block-size math was hand-tuned for the previous - # model's dense 64-layer/8-head attention and doesn't carry over to - # GDN's hybrid KV structure. Re-add once real numbers are known. - - --model=Qwen/Qwen3.5-35B-A3B-GPTQ-Int4 + # DeepSeek-R1-Distill-32B retired: tool_choice="auto" (what pi sends) + # hit a documented vLLM/R1-family conflict -- the model narrated fake + # tool_calls in its block instead of emitting real ones, + # regardless of parser. Tried swapping to a Kimi-distilled Qwen3.6 + # MoE checkpoint and an AWQ-quantized Qwen3-30B-A3B first -- both + # failed on real, separate blockers (unrecognized model_type; then + # marlin INT4 kernels needing compute capability 80+, but worker-1's + # GPU is sm70/V100). Landed on dense Qwen3-32B instead: native Qwen3 + # tool-call format (no narration bug), bnb-4bit works fine on sm70 + # (proven by the old DeepSeek config already), and no MoE + # arch/quantization risk this time. + - --model=unsloth/Qwen3-32B-bnb-4bit - --served-model-name=reasoning - - --quantization=moe_wna16 + - --quantization=bitsandbytes - --dtype=float16 - --kv-cache-dtype=auto - --tensor-parallel-size=1 + - --pipeline-parallel-size=2 - --max-model-len=16384 - - --gpu-memory-utilization=0.85 + - --gpu-memory-utilization=0.90 - --max-num-seqs=4 - --enable-chunked-prefill - --enable-prefix-caching # qwen3 is vLLM's dedicated reasoning parser for this family's - # blocks, also documented for Qwen3.5. + # blocks. - --reasoning-parser=qwen3 - # qwen3_coder is the checkpoint README's documented tool-call parser - # for this specific GPTQ-Int4 release -- not hermes. + # hermes is the documented tool-call parser for general (non-Coder) + # Qwen3 models -- native chat template support, not narrated text. - --enable-auto-tool-choice - - --tool-call-parser=qwen3_coder - # skip CUDA graph capture / torch.compile -- this arch's custom ops - # (mamba_mixer2, gdn_attention_core) are compiling for the first time - # ever on this hardware with no cache, and startupProbe kept killing - # the pod mid-compile every ~20min before it could finish. Trade some - # runtime throughput for a startup that actually completes; revisit - # once this is confirmed working end to end. - - --enforce-eager + - --tool-call-parser=hermes + # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks + # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4 + # + max-model-len=16384 means concurrent long sequences compete for + # the same GPU KV space). No extra dependency, built into vLLM core. + # num_cpu_blocks=2000 hung the pod at startup on the old model (2000 x + # ~32MB/block blew well past the pod's memory limit). num_cpu_blocks=32 + # was the safe-recovery value after that -- only ~1GB of real DRAM + # (32 blocks x 128 tokens x 256KB/token-across-all-64-layers, fp16), + # basically a token-count safety valve, not real offload capacity. + # This model: 64 layers, 8 KV heads x 128 head_dim, fp16 -> ~256KB of + # KV per token across all layers -> ~32MB per 128-token block. + # num_cpu_blocks=256 -> ~8GB of actual DRAM offload (32,768 tokens), + # comfortably under the pod's 36Gi limit alongside the ~20GB bnb-4bit + # weights. Watch real host memory on boot before raising further -- + # block_size=128 tokens matches vLLM's own example. + # Note: 0.11.0 ships the original (fragmented, small-transfer-block) + # version of this connector -- 0.12.0 consolidates KV data into one + # contiguous block per request and is reported an order of magnitude + # faster for this specific feature, so this is a real but not yet + # optimal implementation until the image gets bumped. + - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":256,"block_size":128}} - --host=0.0.0.0 - --port=8080 env: @@ -80,7 +72,7 @@ spec: value: TRITON_ATTN - name: HF_HOME value: /mnt/models - image: vllm/vllm-openai:v0.17.0@sha256:2296a2a7e1ce1dc59c6577ba5900f4e9910b76c4a0cb134833a8137f92404dfa + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c name: kserve-container ports: - containerPort: 8080 @@ -94,13 +86,13 @@ spec: limits: cpu: '16' memory: 36Gi - nvidia.com/gpu: '1' + nvidia.com/gpu: '2' requests: cpu: '8' memory: 12Gi - nvidia.com/gpu: '1' + nvidia.com/gpu: '2' startupProbe: - failureThreshold: 240 + failureThreshold: 80 httpGet: path: /health port: 8080