apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
spec:
  predictor:
    containers:
    - args:
      # dense Qwen3-32B-bnb-4bit retired: sm70/V100 bnb dequant kernel is
      # slow (no int4 tensor cores pre-Turing, dequant-then-fp16-matmul is
      # two kernel launches not one fused int4 GEMM), decode crawled at
      # ~2.5-10 tok/s and blew the gateway's request timeout regardless of
      # TP/PP config. bnb also outright rejects tensor-parallel-size>1 on
      # prequant checkpoints ("Please try with pipeline parallelism"),
      # which is why this went through a PP=2 detour first.
      # Switched to Qwen3.5-35B-A3B (MoE, GDN hybrid attention) GPTQ-Int4.
      # Requires vLLM >=0.17.0 -- v0.11.0 errors with "Model architectures
      # ['Qwen3_5MoeForConditionalGeneration'] are not supported for now."
      # moe_wna16 is the checkpoint's documented quantization kernel;
      # compute-capability requirement on sm70 is UNVERIFIED going in --
      # this rollout is the real test. --kv-cache-dtype stays auto, not
      # fp8_e5m2: V100 has no FP8 tensor cores at all (Hopper/Ada only),
      # hardware-blocked regardless of vLLM version. tool-call-parser
      # changed hermes -> qwen3_coder per the checkpoint's own README
      # example, not cosmetic. gpu-memory-utilization starts low (0.5)
      # since real VRAM footprint for this arch+quant combo is unknown;
      # raise once stable. Old OffloadingConnector kv-transfer-config
      # dropped -- its block-size math was hand-tuned for the previous
      # model's dense 64-layer/8-head attention and doesn't carry over to
      # GDN's hybrid KV structure. Re-add once real numbers are known.
      - --model=Qwen/Qwen3.5-35B-A3B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=moe_wna16
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --tensor-parallel-size=2
      - --max-model-len=16384
      - --gpu-memory-utilization=0.5
      - --max-num-seqs=4
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks, also documented for Qwen3.5.
      - --reasoning-parser=qwen3
      # qwen3_coder is the checkpoint README's documented tool-call parser
      # for this specific GPTQ-Int4 release -- not hermes.
      - --enable-auto-tool-choice
      - --tool-call-parser=qwen3_coder
      # skip CUDA graph capture / torch.compile -- this arch's custom ops
      # (mamba_mixer2, gdn_attention_core) are compiling for the first time
      # ever on this hardware with no cache, and startupProbe kept killing
      # the pod mid-compile every ~20min before it could finish. Trade some
      # runtime throughput for a startup that actually completes; revisit
      # once this is confirmed working end to end.
      - --enforce-eager
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.17.0@sha256:2296a2a7e1ce1dc59c6577ba5900f4e9910b76c4a0cb134833a8137f92404dfa
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 36Gi
          nvidia.com/gpu: '2'
        requests:
          cpu: '8'
          memory: 12Gi
          nvidia.com/gpu: '2'
      startupProbe:
        failureThreshold: 240
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm