diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 77f5203..de87548 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -30,7 +30,28 @@ spec: # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights, # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request. - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4 - - --served-model-name=reasoning + # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map + # uses for this model, and the name qwen-finetune wiring targets. It is NOT a + # separate InferenceService: this model already occupies the only free card, and a + # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2, + # this 1, clef-serve 1). Serving both names from one process costs nothing and + # keeps the two callers working without a second weights copy. + # + # `reasoning` is deliberately still served here. poimen-memory pins + # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest + # immediately. coder-predictor also serves `reasoning`; that ambiguity is known + # and documented in the PR. Resolve it by moving the client host, not by + # removing the name here. + # + # The two names are separate list entries, NOT one comma-joined value. This + # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a + # single model whose name contains a comma, and api_server.py then iterates + # args.served_model_name to build the model list. Verified against vLLM 0.11.0: + # FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"]) + # -> ['a', 'b'] + - --served-model-name + - reasoning + - reasoning-long - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto