homelab/k8s/apps/llm-serving/coder.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: coder
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-coder
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
    # Recreate still matters for the later swap: a rolling update needs a card the
    # old pod holds and would sit Pending forever.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
      # v0.15.0 was measured and FAILED here: arch list empty, and
      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
      # before any model load. Do not raise this tag without re-running that probe.
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
      # here, so it takes the proven code path on sm70.
      #
      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
      #
      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
      # and sm70 has neither. Measured, not assumed.
      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
      # `reasoning` stays a served name so an existing client that sends
      # model=reasoning keeps working after the DNS rename. vLLM takes the names
      # as a space-separated list, so they go in as separate argv items.
      - --served-model-name
      - coder
      - reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 262144. 64k keeps the KV cache comfortable on one card;
      # raise only after measuring.
      - --max-model-len=65536
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=1
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=16
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
      #    so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # The coder checkpoint is a 16.9 GB cold download on first start, then
        # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it.
        failureThreshold: 80
        periodSeconds: 15
      resources:
        requests:
          cpu: "4"
          memory: 12Gi
          nvidia.com/gpu: "1"
        limits:
          cpu: "8"
          memory: 36Gi
          nvidia.com/gpu: "1"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        sizeLimit: 2Gi