From 03e3e8393a07c64a2dc81ae1353f7057a1cb70ee Mon Sep 17 00:00:00 2001 From: rock Date: Sat, 31 Oct 2026 15:26:08 -0700 Subject: [PATCH] feat(llm-serving): add coder, serving Qwen3-Coder-30B-A3B on one V100 The name now matches the model. `coder` serves Qwen3-Coder-30B-A3B-Instruct, a 30.5B MoE with 3.3B active, quantized to 4-bit GPTQ. It is measured working on this node, not assumed. Measured, on worker-1, driver 580.159.04, compute capability 7.0: GET /v1/models -> id=coder, root=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A POST /v1/chat/completions model=coder -> 200 in 3.18s, "fn main() { println!(\"hi\"); }" POST /v1/chat/completions model=reasoning -> 200 in 0.21s Weights loaded in 15.61 GiB. torch.compile took 78s. The pod reached 1/1 Ready. `kubectl diff -f coder.yaml` returns empty, so the manifest matches live state. Two names are served on purpose: `coder` and `reasoning`. The rename therefore needs no change to poimen-memory's LLM_MODEL, only its host. That keeps the client working through the cutover. Why not Qwen3-Coder-Next, the model in the original plan. It cannot run here, and that is measured too. A v0.15.0 image on this free V100 gave: TORCH 2.9.1+cu129 ARCH_LIST [] <- empty RuntimeError: cudaGetDeviceCount() ... Error 803 fla: ModuleNotFoundError <- no GatedDeltaNet kernels vLLM 0.15.0 requires compute capability 7.5 or higher and fails here before it loads a model. Qwen3-Next's hybrid attention needs the `fla` module, which the image does not ship. So this stays on the pinned v0.11.0 image, which reports sm_70..sm_120 and works. Qwen publishes this model as BF16 (61.1 GB) and FP8 only. V100 has no FP8, and 61.1 GB does not fit 32 GB, so 4-bit GPTQ is the only form that fits one card. The checkpoint is a community quant; the manifest says so and names the reason. reasoning.yaml stays listed until poimen-memory's LLM_CHAT_BASE moves from reasoning-predictor to coder-predictor. Two GPUs are free, so coder rolled out beside reasoning with no downtime. --- k8s/apps/llm-serving/coder.yaml | 118 ++++++++++++++++++++++++ k8s/apps/llm-serving/kustomization.yaml | 4 + 2 files changed, 122 insertions(+) create mode 100644 k8s/apps/llm-serving/coder.yaml diff --git a/k8s/apps/llm-serving/coder.yaml b/k8s/apps/llm-serving/coder.yaml new file mode 100644 index 0000000..8b1b05f --- /dev/null +++ b/k8s/apps/llm-serving/coder.yaml @@ -0,0 +1,118 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: coder + namespace: llm-serving + labels: + app.kubernetes.io/name: llm-coder + app.kubernetes.io/part-of: llm-serving + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + # Two GPUs are free, so this rolls out beside `reasoning` instead of over it. + # Recreate still matters for the later swap: a rolling update needs a card the + # old pod holds and would sit Pending forever. + deploymentStrategy: + type: Recreate + containers: + - name: kserve-container + # Pinned to v0.11.0 on purpose. This is the last image measured to work on this + # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04. + # v0.15.0 was measured and FAILED here: arch list empty, and + # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination) + # before any model load. Do not raise this tag without re-running that probe. + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is + # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving + # here, so it takes the proven code path on sm70. + # + # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta). + # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a + # community quant: ModelCloud maintains GPTQModel, and the repo is complete + # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust. + # + # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option: + # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module, + # and sm70 has neither. Measured, not assumed. + - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A + # `reasoning` stays a served name so an existing client that sends + # model=reasoning keeps working after the DNS rename. vLLM takes the names + # as a space-separated list, so they go in as separate argv items. + - --served-model-name + - coder + - reasoning + - --quantization=gptq + - --dtype=float16 + - --kv-cache-dtype=auto + # Native window is 262144. 64k keeps the KV cache comfortable on one card; + # raise only after measuring. + - --max-model-len=65536 + - --tensor-parallel-size=1 + - --pipeline-parallel-size=1 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=16 + - --max-num-batched-tokens=8192 + - --long-prefill-token-threshold=2048 + - --enable-chunked-prefill + - --enable-prefix-caching + - --reasoning-parser=qwen3 + - --enable-auto-tool-choice + - --tool-call-parser=hermes + - --host=0.0.0.0 + - --port=8080 + env: + # Required on this hardware, not tuning: + # - flashinfer's check_cuda_arch raises on compute capability 7.0, + # so the sampler must stay off + # - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + # The coder checkpoint is a 16.9 GB cold download on first start, then + # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it. + failureThreshold: 80 + periodSeconds: 15 + resources: + requests: + cpu: "4" + memory: 12Gi + nvidia.com/gpu: "1" + limits: + cpu: "8" + memory: 36Gi + nvidia.com/gpu: "1" + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: 2Gi diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index e7da0a8..45baa58 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -10,8 +10,12 @@ kind: Kustomization # way after any edit here: a GPU predictor restart is a weights reload measured # in tens of seconds, not a rolling update. resources: + - coder.yaml - embeddings.yaml - ornith.yaml + # reasoning stays listed only until poimen-memory's LLM_CHAT_BASE moves to + # coder-predictor. coder already serves the name `reasoning`, so the client + # needs no model-name change, only the host change. Drop this line after that. - reasoning.yaml - reranker.yaml - qwen-cpu.yaml