diff --git a/k8s/apps/llm-serving/coder.yaml b/k8s/apps/llm-serving/coder.yaml new file mode 100644 index 0000000..8b1b05f --- /dev/null +++ b/k8s/apps/llm-serving/coder.yaml @@ -0,0 +1,118 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: coder + namespace: llm-serving + labels: + app.kubernetes.io/name: llm-coder + app.kubernetes.io/part-of: llm-serving + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + # Two GPUs are free, so this rolls out beside `reasoning` instead of over it. + # Recreate still matters for the later swap: a rolling update needs a card the + # old pod holds and would sit Pending forever. + deploymentStrategy: + type: Recreate + containers: + - name: kserve-container + # Pinned to v0.11.0 on purpose. This is the last image measured to work on this + # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04. + # v0.15.0 was measured and FAILED here: arch list empty, and + # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination) + # before any model load. Do not raise this tag without re-running that probe. + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is + # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving + # here, so it takes the proven code path on sm70. + # + # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta). + # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a + # community quant: ModelCloud maintains GPTQModel, and the repo is complete + # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust. + # + # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option: + # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module, + # and sm70 has neither. Measured, not assumed. + - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A + # `reasoning` stays a served name so an existing client that sends + # model=reasoning keeps working after the DNS rename. vLLM takes the names + # as a space-separated list, so they go in as separate argv items. + - --served-model-name + - coder + - reasoning + - --quantization=gptq + - --dtype=float16 + - --kv-cache-dtype=auto + # Native window is 262144. 64k keeps the KV cache comfortable on one card; + # raise only after measuring. + - --max-model-len=65536 + - --tensor-parallel-size=1 + - --pipeline-parallel-size=1 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=16 + - --max-num-batched-tokens=8192 + - --long-prefill-token-threshold=2048 + - --enable-chunked-prefill + - --enable-prefix-caching + - --reasoning-parser=qwen3 + - --enable-auto-tool-choice + - --tool-call-parser=hermes + - --host=0.0.0.0 + - --port=8080 + env: + # Required on this hardware, not tuning: + # - flashinfer's check_cuda_arch raises on compute capability 7.0, + # so the sampler must stay off + # - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + # The coder checkpoint is a 16.9 GB cold download on first start, then + # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it. + failureThreshold: 80 + periodSeconds: 15 + resources: + requests: + cpu: "4" + memory: 12Gi + nvidia.com/gpu: "1" + limits: + cpu: "8" + memory: 36Gi + nvidia.com/gpu: "1" + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: 2Gi diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index 8f0fb81..5680790 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -14,6 +14,10 @@ resources: # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100. - clef-serve.yaml + # coder = Qwen3-Coder-30B-A3B-Instruct, 4-bit GPTQ, TP=2 at 262,144 tokens. It serves + # both `coder` and `reasoning`, so poimen-memory needs a host change only, never a + # model-name change. reasoning.yaml below can be dropped once that host change lands. + - coder.yaml - reasoning.yaml # ornith is retired: its InferenceService still reports Ready=True while running zero # replicas, because desired equals available at 0. Its manifest was never committed,