diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 8aab953..4b45795 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -31,7 +31,12 @@ spec: # on-the-fly bnb gamble a second time. - --model=cpatonn/Qwen3-30B-A3B-Thinking-2507-AWQ-4bit - --served-model-name=reasoning - - --quantization=awq_marlin + # Named "AWQ" but actually quantized via llm-compressor -- config.json + # declares compressed-tensors, not classic AWQ. vLLM auto-detects this + # from the checkpoint; passing awq_marlin explicitly mismatches and + # 400s at config-validation time. Letting vLLM read it from the + # checkpoint instead of asserting the wrong format. + - --quantization=compressed-tensors - --dtype=float16 - --kv-cache-dtype=auto - --tensor-parallel-size=1