# Verifier — Qwen2.5-Math-PRM-7B, step-level process reward model, vLLM.
#
# Model choice was constrained by vLLM v0.11.0's registry: its arch
# (Qwen2ForProcessRewardModel) is natively registered, whereas the smaller
# community PRMs are Qwen2ForTokenClassification / Qwen2ForPrmModel, neither of
# which v0.11.0 can load (ForTokenClassification is absent from
# _SUFFIX_TO_DEFAULTS, so it won't even auto-convert).
#
# --runner pooling, NOT --task reward: --task is [DEPRECATED] in v0.11.0.
# Scoring goes to /pooling, not /v1/completions — this is a reward model, it
# returns scores, not tokens.
#
# Gets a whole dedicated GPU despite only needing ~15Gi: it sits on the
# harness's critical path (every reasoning->action->verify iteration waits on
# it), so isolation from the generation engines' decode loops matters more than
# the idle VRAM.
apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: verifier
  labels:
    app.kubernetes.io/name: llm-verifier
    app.kubernetes.io/part-of: llm-serving
spec:
  predictor:
    minReplicas: 1
    # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4,
    # so a surge pod has no card to claim and sits Pending while the old pod is
    # never torn down — a deadlock. Recreate tears down first, accepting a brief
    # gap during updates.
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    containers:
      - name: kserve-container
        image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
        args:
          - --model=Qwen/Qwen2.5-Math-PRM-7B
          - --served-model-name=verifier
          # Pooling runner => reward scoring. Weights are bf16; Volta needs fp16.
          - --runner=pooling
          - --dtype=float16
          - --tensor-parallel-size=1
          - --max-model-len=4096
          - --max-num-seqs=8
          - --host=0.0.0.0
          - --port=8080
        env:
          # flashinfer's check_cuda_arch() has an upstream bug that crashes on
          # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit()
          # on an int, so instead of reporting "unsupported" it raises
          #   AttributeError: 'int' object has no attribute 'isdigit'
          # and engine init dies. Default is None (auto-detect), which walks
          # straight into that path. 0 disables the flashinfer sampler outright.
          # Only affects the generate runner — the verifier (pooling) never hits
          # the sampler, which is why it started fine and this did not.
          - name: VLLM_USE_FLASHINFER_SAMPLER
            value: "0"
          - name: VLLM_ATTENTION_BACKEND
            value: XFORMERS
          - name: HF_HOME
            value: /mnt/models
        ports:
          - containerPort: 8080
            protocol: TCP
        resources:
          requests:
            cpu: "4"
            memory: 8Gi
            nvidia.com/gpu: "1"
          limits:
            cpu: "16"
            memory: 16Gi
            nvidia.com/gpu: "1"
        volumeMounts:
          - name: models
            mountPath: /mnt/models
          - name: shm
            mountPath: /dev/shm
        startupProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 15
          failureThreshold: 60
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 10
    volumes:
      - name: models
        persistentVolumeClaim:
          claimName: llm-models
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 1Gi