From 9ae0b90d4c06b7f71dc51ef372fc0a464223a49f Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:34:15 -0700 Subject: [PATCH] reasoning: revert to DeepSeek-R1-Distill-32B, retire Kimi/Qwen3 swap attempt Three straight failures on worker-1: Kimi-K2.6-distilled Qwen3.6-35B-A3B had an unrecognized model type (qwen3_5_moe); the AWQ-4bit fallback needed compute capability 80+ (marlin INT4 kernels) but this node's GPU is sm70 (V100); on-the-fly bitsandbytes against the full-precision Qwen3-30B-A3B kept crash-looping. Reverting to the last known-good config (596b5cb) -- tool-call narration bug on judge remains open, to revisit separately. --- k8s/apps/llm-serving/reasoning.yaml | 80 +++++++++-------------------- 1 file changed, 25 insertions(+), 55 deletions(-) diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index d6a3b1d..c14febf 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -12,70 +12,45 @@ spec: predictor: containers: - args: - # Swapped off DeepSeek-R1-Distill-Qwen-32B: tool_choice="auto" (what pi - # sends) hit a documented vLLM architecture conflict for R1-family - # models -- the model narrated fake tool-call completions in its - # block instead of emitting real tool_calls, regardless of - # parser combo tried (deepseek_v3 400s outright, hermes parsed but the - # model itself never called out to the real tool-call path). Root - # cause is upstream in the R1 distillation, not this config -- Qwen3's - # own tool-call format is natively supported instead. - # - # First attempt was a Kimi-K2.6-distilled Qwen3.6-35B-A3B checkpoint - # (lordx64/Qwen3.6-35B-A3B-Kimi-K2.6-Reasoning-Distilled) -- crashed on - # boot: `model type qwen3_5_moe` unrecognized by transformers/vLLM - # 0.11.0, not a config problem, a genuinely unsupported/obscure - # architecture. - # - # Second attempt was cpatonn's pre-quantized "AWQ-4bit" build of the - # official Qwen3-30B-A3B-Thinking-2507 -- also crashed, on two - # different real issues in sequence: (1) it's actually compressed-tensors - # format despite the repo name, and (2) once that was fixed, vLLM raised - # `RuntimeError: Quantization scheme is not supported for the current - # GPU. Min capability: 80. Current capability: 70.` -- worker-1's GPU is - # compute capability 7.0 (V100), and AWQ/GPTQ/compressed-tensors marlin - # INT4 kernels all require sm80+ (Ampere or newer). This node cannot run - # any of those quant formats, full stop -- only bitsandbytes or full - # precision work here (which is exactly why the old DeepSeek config used - # bitsandbytes to begin with). - # - # Using the official full-precision Qwen/Qwen3-30B-A3B-Thinking-2507 - # checkpoint with on-the-fly bitsandbytes quantization instead -- - # architecture itself already confirmed good (Qwen3MoeForCausalLM - # resolved cleanly in both prior attempts). - - --model=Qwen/Qwen3-30B-A3B-Thinking-2507 + - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit - --served-model-name=reasoning - --quantization=bitsandbytes - - --dtype=bfloat16 + - --dtype=float16 - --kv-cache-dtype=auto - --tensor-parallel-size=1 - --max-model-len=16384 - --gpu-memory-utilization=0.90 - # 30B total MoE at pre-quantized AWQ-4bit is ~15-16GB weights on a - # 32GB card at 0.90 util (~29.5GB budget) -- meaningfully more KV-cache - # headroom than the old DeepSeek-32B config had, so restoring the - # original max-num-seqs=4 rather than starting conservative again. - --max-num-seqs=4 - --enable-chunked-prefill - --enable-prefix-caching - # qwen3 is vLLM's dedicated reasoning parser for this family's - # blocks -- official pairing per Qwen3-Thinking's own deployment docs. - - --reasoning-parser=qwen3 - # hermes is the documented tool-call parser for general Qwen3 models - # (qwen3_coder/qwen3_xml are Coder-variant-only). + - --reasoning-parser=deepseek_r1 + # Without these, any client sending tool_choice="auto" (pi does, for + # Read/Bash/etc.) gets a 400: "auto" tool choice requires + # --enable-auto-tool-choice and --tool-call-parser to be set. + # deepseek_v3 (matching --reasoning-parser above) 400s here -- + # "DeepSeek-V3 Tool parser could not locate tool call start/end + # tokens in the tokenizer" -- this checkpoint is a Qwen2.5-32B base + # distilled on R1 traces, so its tokenizer never got DeepSeek-V3's + # own special tool-call tokens registered even though it kept R1's + # reasoning format. hermes parses tool calls from plain text + # patterns instead of special tokens, so it works against the + # underlying Qwen tokenizer regardless. Verified live: deepseek_v3 + # 400s, hermes returns a real tool_calls response. - --enable-auto-tool-choice - --tool-call-parser=hermes # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4 # + max-model-len=16384 means concurrent long sequences compete for # the same GPU KV space). No extra dependency, built into vLLM core. - # num_cpu_blocks=2000 hung the pod at startup on the old model (one - # pinned host tensor per layer, sized per CPU block -- oversized enough - # to stall pinning that much host memory). 32 is a small, known-safe - # starting point independent of this model's own layer count -- - # confirm it comes up healthy, then watch real host memory usage and - # raise it deliberately rather than guessing a round number again. - # block_size=128 tokens matches vLLM's own example. + # num_cpu_blocks=2000 hung the pod at startup ("Allocating 64 CPU + # tensors..." then nothing -- 64 is this model's layer count, one + # pinned host tensor per layer, each sized for every CPU block; 2000 + # was oversized enough to stall pinning that much host memory, likely + # blowing well past the pod's memory limit). Dropped to a small, + # known-safe starting point -- confirm it actually comes up healthy, + # then watch real host memory usage and raise it deliberately rather + # than guessing a round number again. block_size=128 tokens matches + # vLLM's own example. # Note: 0.11.0 ships the original (fragmented, small-transfer-block) # version of this connector -- 0.12.0 consolidates KV data into one # contiguous block per request and is reported an order of magnitude @@ -104,12 +79,7 @@ spec: resources: limits: cpu: '16' - # Bumped from 36Gi -- on-the-fly bnb quantization stages full - # bf16 shards (~60GB total model) transiently in host RAM during - # load before discarding them, unlike loading an already-quantized - # checkpoint. Request left unchanged (node is already at 84% memory - # request allocation) -- only the OOM ceiling moves. - memory: 48Gi + memory: 36Gi nvidia.com/gpu: '1' requests: cpu: '8'