From 3aed5ea9486f928a340d9daa3467213638b0c6ff Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Wed, 19 Aug 2026 18:30:24 -0700
Subject: [PATCH] reasoning: switch to on-the-fly bnb quant, worker-1 GPU is
 sm70 (V100)

cpatonn's pre-quantized build failed with a real hardware constraint:
"Quantization scheme not supported for current GPU. Min capability: 80.
Current capability: 70." AWQ/GPTQ/compressed-tensors marlin INT4 kernels
all need sm80+ -- this node's GPU can't run any of them. Only bitsandbytes
or full precision work here. Switching to the official full-precision
Qwen/Qwen3-30B-A3B-Thinking-2507 with --quantization=bitsandbytes
on-the-fly, and bumping the memory limit (36Gi->48Gi, request unchanged)
for the transient bf16-shard staging during load.
---
 k8s/apps/llm-serving/reasoning.yaml | 40 +++++++++++++++++++----------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml
index 4b45795..d6a3b1d 100644
--- a/k8s/apps/llm-serving/reasoning.yaml
+++ b/k8s/apps/llm-serving/reasoning.yaml
@@ -25,19 +25,28 @@ spec:
       # (lordx64/Qwen3.6-35B-A3B-Kimi-K2.6-Reasoning-Distilled) -- crashed on
       # boot: `model type qwen3_5_moe` unrecognized by transformers/vLLM
       # 0.11.0, not a config problem, a genuinely unsupported/obscure
-      # architecture. Falling back to the official Qwen3-30B-A3B-Thinking-2507
-      # (no Kimi distillation, but native vLLM support confirmed) using
-      # cpatonn's pre-quantized AWQ-4bit build -- avoids repeating the
-      # on-the-fly bnb gamble a second time.
-      - --model=cpatonn/Qwen3-30B-A3B-Thinking-2507-AWQ-4bit
+      # architecture.
+      #
+      # Second attempt was cpatonn's pre-quantized "AWQ-4bit" build of the
+      # official Qwen3-30B-A3B-Thinking-2507 -- also crashed, on two
+      # different real issues in sequence: (1) it's actually compressed-tensors
+      # format despite the repo name, and (2) once that was fixed, vLLM raised
+      # `RuntimeError: Quantization scheme is not supported for the current
+      # GPU. Min capability: 80. Current capability: 70.` -- worker-1's GPU is
+      # compute capability 7.0 (V100), and AWQ/GPTQ/compressed-tensors marlin
+      # INT4 kernels all require sm80+ (Ampere or newer). This node cannot run
+      # any of those quant formats, full stop -- only bitsandbytes or full
+      # precision work here (which is exactly why the old DeepSeek config used
+      # bitsandbytes to begin with).
+      #
+      # Using the official full-precision Qwen/Qwen3-30B-A3B-Thinking-2507
+      # checkpoint with on-the-fly bitsandbytes quantization instead --
+      # architecture itself already confirmed good (Qwen3MoeForCausalLM
+      # resolved cleanly in both prior attempts).
+      - --model=Qwen/Qwen3-30B-A3B-Thinking-2507
       - --served-model-name=reasoning
-      # Named "AWQ" but actually quantized via llm-compressor -- config.json
-      # declares compressed-tensors, not classic AWQ. vLLM auto-detects this
-      # from the checkpoint; passing awq_marlin explicitly mismatches and
-      # 400s at config-validation time. Letting vLLM read it from the
-      # checkpoint instead of asserting the wrong format.
-      - --quantization=compressed-tensors
-      - --dtype=float16
+      - --quantization=bitsandbytes
+      - --dtype=bfloat16
       - --kv-cache-dtype=auto
       - --tensor-parallel-size=1
       - --max-model-len=16384
@@ -95,7 +104,12 @@ spec:
       resources:
         limits:
           cpu: '16'
-          memory: 36Gi
+          # Bumped from 36Gi -- on-the-fly bnb quantization stages full
+          # bf16 shards (~60GB total model) transiently in host RAM during
+          # load before discarding them, unlike loading an already-quantized
+          # checkpoint. Request left unchanged (node is already at 84% memory
+          # request allocation) -- only the OOM ceiling moves.
+          memory: 48Gi
           nvidia.com/gpu: '1'
         requests:
           cpu: '8'