From f21679f6017b6e8f1e49f84ad9d1b79134903c9c Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Wed, 19 Aug 2026 18:11:45 -0700
Subject: [PATCH] reasoning: swap DeepSeek-R1-Distill-32B for
 Kimi-K2.6-distilled Qwen3.6-35B-A3B

R1-family tool_choice=auto is a documented vLLM architecture conflict --
the model narrates fake tool_calls in <think> instead of emitting real
ones, regardless of parser (deepseek_v3 400s, hermes parses but the model
still doesn't call out). Qwen3's native tool-call format sidesteps this.

No pre-quantized AWQ/GPTQ/bnb checkpoint exists for this specific distill
(only GGUF, llama.cpp/Ollama-only) -- using on-the-fly bitsandbytes
quantization against the full bf16 checkpoint instead.
---
 k8s/apps/llm-serving/reasoning.yaml | 46 +++++++++++++++++++----------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml
index c14febf..e500955 100644
--- a/k8s/apps/llm-serving/reasoning.yaml
+++ b/k8s/apps/llm-serving/reasoning.yaml
@@ -12,10 +12,30 @@ spec:
   predictor:
     containers:
     - args:
-      - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
+      # Swapped off DeepSeek-R1-Distill-Qwen-32B: tool_choice="auto" (what pi
+      # sends) hit a documented vLLM architecture conflict for R1-family
+      # models -- the model narrated fake tool-call completions in its
+      # <think> block instead of emitting real tool_calls, regardless of
+      # parser combo tried (deepseek_v3 400s outright, hermes parsed but the
+      # model itself never called out to the real tool-call path). Root
+      # cause is upstream in the R1 distillation, not this config -- moving
+      # to a Qwen3-family model with a Kimi-K2.6 reasoning distillation
+      # instead, since Qwen3's own tool-call format is natively supported.
+      #
+      # No pre-quantized AWQ/GPTQ/bnb checkpoint exists for this specific
+      # distilled model (only a GGUF, which is llama.cpp/Ollama-only and not
+      # usable here) -- pointing --quantization=bitsandbytes at the full
+      # bf16 checkpoint directly, which makes vLLM quantize on load instead
+      # of requiring a pre-quantized repo. This on-the-fly bnb path is
+      # well-trodden for dense models but less battle-tested for MoE
+      # (this model is 35B total / ~3B active) -- watch first boot closely;
+      # if it OOMs or errors on the MoE expert weights, that's the likely
+      # cause.
+      - --model=lordx64/Qwen3.6-35B-A3B-Kimi-K2.6-Reasoning-Distilled
       - --served-model-name=reasoning
       - --quantization=bitsandbytes
-      - --dtype=float16
+      - --trust-remote-code
+      - --dtype=bfloat16
       - --kv-cache-dtype=auto
       - --tensor-parallel-size=1
       - --max-model-len=16384
@@ -23,19 +43,15 @@ spec:
       - --max-num-seqs=4
       - --enable-chunked-prefill
       - --enable-prefix-caching
-      - --reasoning-parser=deepseek_r1
-      # Without these, any client sending tool_choice="auto" (pi does, for
-      # Read/Bash/etc.) gets a 400: "auto" tool choice requires
-      # --enable-auto-tool-choice and --tool-call-parser to be set.
-      # deepseek_v3 (matching --reasoning-parser above) 400s here --
-      # "DeepSeek-V3 Tool parser could not locate tool call start/end
-      # tokens in the tokenizer" -- this checkpoint is a Qwen2.5-32B base
-      # distilled on R1 traces, so its tokenizer never got DeepSeek-V3's
-      # own special tool-call tokens registered even though it kept R1's
-      # <think> reasoning format. hermes parses tool calls from plain text
-      # patterns instead of special tokens, so it works against the
-      # underlying Qwen tokenizer regardless. Verified live: deepseek_v3
-      # 400s, hermes returns a real tool_calls response.
+      # qwen3 parser handles this family's <think> reasoning blocks (best
+      # match for this architecture; unverified against this exact
+      # checkpoint -- if it 400s or fails to strip <think> tags, that's the
+      # first thing to check).
+      - --reasoning-parser=qwen3
+      # hermes previously verified (on the old model) to work against a
+      # Qwen tokenizer's plain-text tool-call patterns without needing
+      # special tokens; Qwen3's native tool-call format is also
+      # hermes-style, so kept as-is.
       - --enable-auto-tool-choice
       - --tool-call-parser=hermes
       # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks