homelab/k8s/apps/llm-serving/qwen3-moe.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # KServe injects the prometheus.io/{scrape,port,path} annotations into the
    # generated pod only when this flag is on. Without it the predictor carries
    # no scrape annotations at all, which is why no vLLM/SGLang metrics have
    # ever reached Prometheus from this namespace.
    serving.kserve.io/enable-prometheus-scraping: "true"
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
spec:
  predictor:
    containers:
    - args:
      # Replaces the dense Qwen3-32B. Same serving name (`reasoning`), so every
      # consumer keeps its model string and no client changes.
      #
      # Why a MoE: Qwen3-30B-A3B has 30.5B total but only 3.3B active per token.
      # On sm70 the decode path is bandwidth- and kernel-launch-bound, not
      # FLOP-bound, so activating 3.3B instead of 32B is the single biggest
      # throughput win available on this hardware without changing GPUs.
      #
      # Quantization: GPTQ-Int4, pinned with --quantization=gptq. This repeats
      # the finding recorded on the previous checkpoint: bnb-4bit works poorly
      # on sm70 because there are no int4 tensor cores, and vLLM auto-upgrades
      # AWQ/GPTQ to the Marlin kernel which requires sm80+. Pinning `gptq`
      # forces the plain kernel, which is verified Volta-compatible here.
      # Do not remove this flag.
      #
      # Context: 131072 via YaRN. The base model card documents 32,768 native
      # and 131,072 with YaRN (factor 4.0, original_max_position_embeddings
      # 32768). That is Qwen's own validated ceiling, so unlike a stretched
      # window this is a documented capability.
      #
      # KV cache math, from config.json: 48 layers, 4 KV heads, head_dim 128,
      # fp16 -> 2 * 4 * 128 * 2 = 2048 B/token/layer -> 96 KB/token.
      # At 131072 tokens that is 12 GiB total. With TP=2 the KV heads split
      # 2 per GPU, so each GPU carries ~6 GiB of KV. Weights are ~17 GB total,
      # so ~8.5 GB per GPU. At gpu-memory-utilization=0.95 (about 30.4 GB
      # usable per V100-32GB) that leaves roughly 21 GB per GPU for KV plus
      # activations: comfortable for one full 128K request and a few shorter
      # ones. Re-read vllm:cache_config_info after rollout for the real
      # num_gpu_blocks value rather than trusting this projection.
      #
      # TP=2, not PP=2: the model has 4 KV heads, so TP must divide 4. TP=2
      # splits cleanly; TP=3 would not. TP also shortens prefill at long
      # context, which is the actual bottleneck at 128K on V100.
      - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
      - --tensor-parallel-size=2
      - --max-model-len=131072
      - --gpu-memory-utilization=0.95
      - --max-num-seqs=4
      # Prefill scheduling. The same reasoning as the previous checkpoint
      # applies: vLLM V1 has no separate prefill phase, so a single long
      # prompt can consume the whole step budget and starve every other
      # request. The threshold must be strictly below the budget or it never
      # binds, hence 8192 budget with a 2048 threshold.
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # qwen3 is vLLM's reasoning parser for this family's <think> blocks.
      - --reasoning-parser=qwen3
      # hermes is the documented tool-call parser for general Qwen3 models.
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # FlashInfer is sm80+. TRITON_ATTN is the Volta-compatible backend and is
      # already the proven setting on this node. Clef-flash runs on SGLang
      # instead, but Qwen stays on vLLM here because this exact combination is
      # tested on this hardware; moving it to SGLang would add risk for no
      # measured gain.
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 36Gi
          nvidia.com/gpu: '2'
        requests:
          cpu: '8'
          memory: 12Gi
          nvidia.com/gpu: '2'
      startupProbe:
        # 30.5B of int4 weights plus a 128K rope config: allow a generous
        # first-boot window. failureThreshold 120 * 15s = 30 min ceiling.
        failureThreshold: 120
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    # podMetadata is NOT a valid field on this KServe version. The CRD exposes
    # annotations and labels directly under predictor. The older manifests in
    # this directory still carry podMetadata and ArgoCD silently drops it on
    # server-side apply; do not copy that pattern.
    annotations:
      prometheus.io/scrape: "true"
      prometheus.io/port: "8080"
      prometheus.io/path: "/metrics"
    labels:
      app.kubernetes.io/name: llm-reasoning
      app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      # Recreate, not RollingUpdate: GPU memory is a hard resource. A second
      # pod cannot schedule alongside the first on a full 4-GPU node, so a
      # rolling update would wedge with the new pod Pending forever.
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm