homelab/k8s/apps/llm-serving/inferenceservice-reasoning.yaml

# Reasoning engine — DeepSeek-R1-Distill-Qwen-32B, GPTQ INT4, vLLM.
#
# TP=1 with 2 data-parallel replicas (GPU0 + GPU1) rather than one TP=2 engine:
# worker-1 has NO NVLink, so tensor-parallel's per-token all-reduce would cross
# PCIe on every decode step. Two independent replicas need zero inter-GPU
# communication and KServe load-balances them behind one Service.
#
# vLLM is pinned to v0.11.0 — the LAST release that compiles sm_70 (Volta)
# kernels. v0.11.1 dropped 7.0 from CUDA_SUPPORTED_ARCHS. Do not bump this
# without re-checking CMakeLists.txt, or every pod dies with "no kernel image".
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: reasoning
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
spec:
  predictor:
    minReplicas: 2
    # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4,
    # so a surge pod has no card to claim and sits Pending while the old pod is
    # never torn down — a deadlock. Recreate tears down first, accepting a brief
    # gap during updates.
    deploymentStrategy:
      type: Recreate
    maxReplicas: 2
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    containers:
      - name: kserve-container
        image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
        args:
          # GPTQ, NOT AWQ. vLLM hard-refuses AWQ below compute capability 75:
          #   "The quantization method awq is not supported for the current GPU.
          #    Minimum capability: 75. Current capability: 70."
          # V100 is sm_70. GPTQ's min capability is 60, so it runs. (gptq_marlin
          # needs 80 and fp8 needs 80 — both also out.) Same 19.3GB footprint.
          # desc_act=False in this build: no activation reordering, faster.
          - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
          - --served-model-name=reasoning
          # bitsandbytes nf4. GPTQ passed vLLM's min_capability=60 check but was
          # numerically WRONG on sm_70 (garbage logits) — proven by the fp16
          # control run producing correct text with the identical backend. bnb
          # declares min_capability=70, but treat that as unverified until the
          # output itself is checked.
          # NOTE the repo sets bnb_4bit_compute_dtype=bfloat16, which Volta does
          # not have; --dtype=float16 must override it.
          - --quantization=bitsandbytes
          # Volta has no bf16 — must be explicit, the repo's weights are bf16.
          - --dtype=float16
          # No FP8 KV on Volta; stays fp16.
          - --kv-cache-dtype=auto
          - --tensor-parallel-size=1
          - --max-model-len=16384
          # VRAM budget on a 32GiB V100: 0.92 => ~29.4GiB, minus ~18GiB of GPTQ
          # weights leaves ~11GiB for KV + activations. One full 32K sequence
          # costs 32768 x 256KB = 8GiB of KV, so 8 concurrent full-length
          # sequences is not physically possible here — 4 is honest, and a
          # serial single-user harness never needs more.
          - --gpu-memory-utilization=0.90
          - --max-num-seqs=4
          # Smooths Volta's slow prefill (no FlashAttention2 on sm_70).
          - --enable-chunked-prefill
          - --enable-prefix-caching
          # Splits <think>…</think> into its own channel.
          - --reasoning-parser=deepseek_r1
          - --host=0.0.0.0
          - --port=8080
        env:
          # FlashAttention2 requires sm_80; Volta must fall back to xformers.
          # flashinfer's check_cuda_arch() has an upstream bug that crashes on
          # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit()
          # on an int, so instead of reporting "unsupported" it raises
          #   AttributeError: 'int' object has no attribute 'isdigit'
          # and engine init dies. Default is None (auto-detect), which walks
          # straight into that path. 0 disables the flashinfer sampler outright.
          # Only affects the generate runner — the verifier (pooling) never hits
          # the sampler, which is why it started fine and this did not.
          - name: VLLM_USE_FLASHINFER_SAMPLER
            value: "0"
          # TRITON_ATTN, not XFORMERS. On sm_70 every xformers kernel is
          # rejected for V1's paged-attention bias type:
          #   fa2F / triton_splitKF -> require sm_80
          #   cutlassF              -> supports sm_70 but not
          #                            PagedBlockDiagonalCausalWithOffsetPaddedKeysMask
          # -> NotImplementedError kills EngineCore on the FIRST request, which
          # takes the whole pod down (vLLM treats engine death as fatal).
          # V0, whose hand-written paged kernels did support sm_70, was REMOVED
          # in v0.11.0, so VLLM_USE_V1=0 has nothing to fall back to.
          # Triton JIT-compiles for the local arch, so it is the last option.
          - name: VLLM_ATTENTION_BACKEND
            value: TRITON_ATTN
          - name: HF_HOME
            value: /mnt/models
        ports:
          - containerPort: 8080
            protocol: TCP
        resources:
          requests:
            cpu: "8"
            memory: 8Gi
            nvidia.com/gpu: "1"
          limits:
            cpu: "16"
            memory: 16Gi
            nvidia.com/gpu: "1"
        volumeMounts:
          - name: models
            mountPath: /mnt/models
          - name: shm
            mountPath: /dev/shm
        startupProbe:
          httpGet:
            path: /health
            port: 8080
          # Cold start pulls ~18Gi of weights over Longhorn, then loads to VRAM.
          periodSeconds: 15
          failureThreshold: 80
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 10
    volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llm-models
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 2Gi