apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: reasoning
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Every GPU card is allocated, so a rolling update deadlocks: the new pod needs a
    # card the old pod still holds and sits Pending forever. Recreate frees the card
    # first. The cost is a short gap in service for this model only.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-30B-A3B is a 30.5B MoE with 3.3B active per token. On a V100 that matters:
      # the 32B dense model it replaces needed two cards for the same context, this needs
      # one, because its KV cache is 96 KiB/token against the dense model's 256 KiB.
      #
      # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
      # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
      - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 40960. YaRN factor 2.0 reaches 81920, so 64k is inside it.
      - --rope-scaling={"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=1
      - --max-model-len=65536
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=8
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Both are required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises AttributeError on compute capability 7.0
      #    (flashinfer/jit/core.py:55, minor.isdigit() on an int), so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 both need sm75+, so the Triton backend serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # Weight load took 18s from cache and 148s on a cold download, and the first
        # torch.compile adds about 70s. 80 x 15s leaves ample room for a cold start.
        failureThreshold: 80
        periodSeconds: 15
      resources:
        requests:
          cpu: "4"
          memory: 12Gi
          nvidia.com/gpu: "1"
        limits:
          cpu: "8"
          memory: 36Gi
          nvidia.com/gpu: "1"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        sizeLimit: 2Gi