homelab/k8s/apps/llm-serving/reasoning.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
spec:
  predictor:
    containers:
    - args:
      # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then-
      # matmul is two slow kernel launches instead of one fused int4 GEMM,
      # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to
      # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same
      # hermes/qwen3 parser stack (no narration-bug risk, same as before),
      # only the quant format changes. Plain (non-Marlin) GPTQ kernel is
      # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try
      # to auto-upgrade to it, so --quantization is pinned explicitly to
      # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB
      # (summed from the real safetensors index, not bits-per-param math).
      # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN:
      # factor=4.0, original_max_position_embeddings=32768) -- 200k was
      # asked for but exceeds this architecturally regardless of VRAM.
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
      # metric after the rollout -- max_num_batched_tokens=8192 also raises
      # activation memory out of the same budget, so the post-boot
      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
      # only valid if it still covers 2x65k + 3x8k. The old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
      # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no
      # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth
      # trying later, but not risking a parallelism-strategy change in the
      # same rollout as the quant+context-length change.
      # This GPTQ requant's own config.json ships max_position_embeddings=
      # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw
      # config.json), the base Qwen3-32B repo's YaRN block did NOT carry
      # over during quantization. Re-applying it explicitly here restores
      # the same math the base model documents (32768 * 4.0 = 131072);
      # without this, --max-model-len=131072 fails ModelConfig validation
      # against the checkpoint's own (unscaled) 40960 ceiling.
      - --model=JunHowie/Qwen3-32B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
      - --gpu-memory-utilization=0.95
      - --max-num-seqs=5
      # Prefill scheduling -- see the measured starvation this fixes.
      # V1's scheduler has no prefill/decode phase: it walks self.running and
      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
      # per request is applied BEFORE the budget clamp:
      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
      #     take = min(take, token_budget)
      # The loop is `while ... and token_budget > 0`. With the threshold at its
      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
      # budget, token_budget hits 0, the loop exits and every other request is
      # skipped for that step -- measured as 262 tok/s contended prefill and
      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
      # Hence threshold must be strictly BELOW the budget or it never binds:
      # at 8192/8192 a single request is served per step (still starved), while
      # 8192/2048 serves four. Verified by replaying the loop.
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      #
      # NOT set, deliberately: --max-num-partial-prefills and
      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
      # rejects any value other than the default for either one, calling
      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
      # -> AssertionError, because there is no V0 engine to fall back to.
      # The Deployment uses strategy Recreate, so setting them is an outage.
      # V1 does concurrent chunked prefill natively; it needs no flag, only the
      # budget/threshold ratio above.
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
      # hermes is the documented tool-call parser for general (non-Coder)
      # Qwen3 models -- native chat template support, not narrated text.
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 36Gi
          nvidia.com/gpu: '2'
        requests:
          cpu: '8'
          memory: 12Gi
          nvidia.com/gpu: '2'
      startupProbe:
        failureThreshold: 80
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    podMetadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
      labels:
        app.kubernetes.io/name: llm-reasoning
        app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm