diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index ebc72a9..d9ff660 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -38,11 +38,19 @@ spec: # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth # trying later, but not risking a parallelism-strategy change in the # same rollout as the quant+context-length change. + # This GPTQ requant's own config.json ships max_position_embeddings= + # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw + # config.json), the base Qwen3-32B repo's YaRN block did NOT carry + # over during quantization. Re-applying it explicitly here restores + # the same math the base model documents (32768 * 4.0 = 131072); + # without this, --max-model-len=131072 fails ModelConfig validation + # against the checkpoint's own (unscaled) 40960 ceiling. - --model=JunHowie/Qwen3-32B-GPTQ-Int4 - --served-model-name=reasoning - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto + - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768} - --tensor-parallel-size=1 - --pipeline-parallel-size=2 - --max-model-len=131072