From 5a8fc1885a16285bdc0b79a63b876b5defe39199 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:56:40 -0700 Subject: [PATCH] fix(llm-serving): use hermes tool-call parser, not deepseek_v3 deepseek_v3 400s on this checkpoint: "could not locate tool call start/end tokens in the tokenizer". unsloth/DeepSeek-R1-Distill-Qwen-32B is a Qwen2.5 base distilled on R1 reasoning traces -- it kept R1's format but never got DeepSeek-V3's own special tool-call tokens registered in its tokenizer. hermes parses from text patterns instead of special tokens, so it works against the underlying Qwen tokenizer. --- k8s/apps/llm-serving/reasoning.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 66dfb0e..0ae0f8c 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -26,14 +26,18 @@ spec: - --reasoning-parser=deepseek_r1 # Without these, any client sending tool_choice="auto" (pi does, for # Read/Bash/etc.) gets a 400: "auto" tool choice requires - # --enable-auto-tool-choice and --tool-call-parser to be set. Verified - # against this exact deployed vLLM v0.11.0's registered parsers - # (ToolParserManager.tool_parsers) -- deepseek_v3 matches, same family - # as the deepseek_r1 reasoning-parser already set above (this model - # emits DeepSeek's own special-token format, not Qwen's Hermes-style, - # despite being a Qwen-base distillation). + # --enable-auto-tool-choice and --tool-call-parser to be set. + # deepseek_v3 (matching --reasoning-parser above) 400s here -- + # "DeepSeek-V3 Tool parser could not locate tool call start/end + # tokens in the tokenizer" -- this checkpoint is a Qwen2.5-32B base + # distilled on R1 traces, so its tokenizer never got DeepSeek-V3's + # own special tool-call tokens registered even though it kept R1's + # reasoning format. hermes parses tool calls from plain text + # patterns instead of special tokens, so it works against the + # underlying Qwen tokenizer regardless. Verified live: deepseek_v3 + # 400s, hermes returns a real tool_calls response. - --enable-auto-tool-choice - - --tool-call-parser=deepseek_v3 + - --tool-call-parser=hermes - --host=0.0.0.0 - --port=8080 env: