homelab/k8s/apps/llm-serving/reasoning.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: reasoning
  namespace: llm-serving
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
spec:
  predictor:
    minReplicas: 1
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    # Every GPU card is allocated, so a rolling update deadlocks: the new pod needs a
    # card the old pod still holds and sits Pending forever. Recreate frees the card
    # first. The cost is a short gap in service for this model only.
    deploymentStrategy:
      type: Recreate
    containers:
    - name: kserve-container
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      args:
      # Qwen3-30B-A3B is a 30.5B MoE with 3.3B active per token. On a V100 that matters:
      # the 32B dense model it replaces needed two cards for the same context, this needs
      # one, because its KV cache is 96 KiB/token against the dense model's 256 KiB.
      #
      # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights,
      # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request.
      - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4
      # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map
      # uses for this model, and the name qwen-finetune wiring targets. It is NOT a
      # separate InferenceService: this model already occupies the only free card, and a
      # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2,
      # this 1, clef-serve 1). Serving both names from one process costs nothing and
      # keeps the two callers working without a second weights copy.
      #
      # `reasoning` is deliberately still served here. poimen-memory pins
      # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest
      # immediately. coder-predictor also serves `reasoning`; that ambiguity is known
      # and documented in the PR. Resolve it by moving the client host, not by
      # removing the name here.
      #
      # The two names are separate list entries, NOT one comma-joined value. This
      # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a
      # single model whose name contains a comma, and api_server.py then iterates
      # args.served_model_name to build the model list. Verified against vLLM 0.11.0:
      #   FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"])
      #     -> ['a', 'b']
      - --served-model-name
      - reasoning
      - reasoning-long
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      # Native window is 40960. YaRN factor 2.0 reaches 81920, so 64k is inside it.
      - --rope-scaling={"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960}
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=1
      - --max-model-len=65536
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=8
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --reasoning-parser=qwen3
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      # Both are required on this hardware, not tuning:
      #  - flashinfer's check_cuda_arch raises AttributeError on compute capability 7.0
      #    (flashinfer/jit/core.py:55, minor.isdigit() on an int), so the sampler must stay off
      #  - FlashInfer and FlashAttention-2 both need sm75+, so the Triton backend serves attention
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: "0"
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      startupProbe:
        httpGet:
          path: /health
          port: 8080
        # Weight load took 18s from cache and 148s on a cold download, and the first
        # torch.compile adds about 70s. 80 x 15s leaves ample room for a cold start.
        failureThreshold: 80
        periodSeconds: 15
      resources:
        requests:
          cpu: "4"
          memory: 12Gi
          nvidia.com/gpu: "1"
        limits:
          cpu: "8"
          memory: 36Gi
          nvidia.com/gpu: "1"
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - name: shm
      emptyDir:
        medium: Memory
        sizeLimit: 2Gi