homelab/k8s/apps/llm-serving/coder.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: coder
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-coder
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Two GPUs are free, so this rolls out beside `reasoning` instead of over it.
    # Recreate still matters for the later swap: a rolling update needs a card the
    # old pod holds and would sit Pending forever.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      # Pinned to v0.11.0 on purpose. This is the last image measured to work on this
      # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04.
      # v0.15.0 was measured and FAILED here: arch list empty, and
      # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination)
      # before any model load. Do not raise this tag without re-running that probe.
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is
      # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving
      # here, so it takes the proven code path on sm70.
      #
      # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta).
      # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a
      # community quant: ModelCloud maintains GPTQModel, and the repo is complete
      # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust.
      #
      # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option:
      # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module,
      # and sm70 has neither. Measured, not assumed.
      - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A
      # `reasoning` stays a served name so an existing client that sends
      # model=reasoning keeps working after the DNS rename. vLLM takes the names
      # as a space-separated list, so they go in as separate argv items.
      - --served-model-name
      - coder
      - reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
      # One card cannot hold it: measured KV pool on a single V100 was 132,672
      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
      # for the larger pool. Measured after boot: see the KV line in the log.
      - --max-model-len=262144
      - --tensor-parallel-size=2
      - --pipeline-parallel-size=1
      - --gpu-memory-utilization=0.90
      # A single 262,144-token request consumes the whole pool, so keep few slots.
      - --max-num-seqs=4
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
      # "No available shared memory broadcast block found in 60 seconds". The front-end
      # never received the ready message. vLLM's custom all-reduce registers its own
      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
      # custom kernel anyway.
      - --disable-custom-all-reduce
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises on compute capability 7.0,
      #    so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
        failureThreshold: 120
        periodSeconds: 15
      resources:
        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
        # never schedules. Two vLLM workers keep the weights on the GPU and need
        # little host RAM, so the request stays small and the limit allows burst.
        requests:
          cpu: "8"
          memory: 12Gi
          nvidia.com/gpu: "2"
        limits:
          cpu: "16"
          memory: 40Gi
          nvidia.com/gpu: "2"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
        # too small and shows up as an init hang or a bus error.
        sizeLimit: 8Gi