feat(llm-serving): retire verifier-predictor, add grm (qwen2.5:3b)

Frees verifier's GPU from an underused vLLM PRM deployment. qwen2.5:3b-instruct moves off ornith-predictor's shared pod onto its own dedicated GPU (grm.yaml), so verification/judge traffic stops contending with ornith:35b's agent traffic. /v1/qwen/chat/completions now points at grm-predictor; path unchanged.
2026-08-18 18:18:31 -07:00
parent e6ada95b39
commit 50d00ae350
5 changed files with 115 additions and 126 deletions
@@ -9,10 +9,14 @@
 # ── Model -> upstream map (verified live) ───────────────────────────────────
 #   reasoning              -> reasoning-predictor   vLLM, DeepSeek-R1-Distill-32B
 #   ornith:35b             -> ornith-predictor      Ollama
-#   qwen2.5:3b-instruct    -> ornith-predictor      Ollama  (same pod!)
+#   qwen2.5:3b-instruct    -> grm-predictor         Ollama  (dedicated GPU --
+#                                                    retired verifier-predictor's
+#                                                    vLLM PRM slot; verification/
+#                                                    judge traffic no longer
+#                                                    contends with ornith:35b's
+#                                                    planner/implementer traffic)
 #   nomic-embed-text-v2    -> embeddings-predictor  TEI
 #   bge-reranker-base      -> reranker-predictor    TEI
-#   Qwen2.5-Math-PRM-7B    -> verifier-predictor    vLLM pooling
 #
 # ── Why path-per-model, and why the body is rewritten ───────────────────────
 # Kong matches routes on host, path, method and headers — never on the request
@@ -20,12 +24,10 @@
 # `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
 # multi-target model routing, is Enterprise-only).
 #
-# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct`
-# share ONE Ollama pod, and Ollama still reads which model to load from the
-# body's `model` field. If only the path selected the route, a client calling
-# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the
-# 35B model. So each chat route force-overwrites `model` in the body, making the
-# path the single source of truth. Callers may omit `model` entirely.
+# Hence the model is in the path, and each chat route force-overwrites `model`
+# in the body regardless, so a client calling /v1/qwen/... with some other
+# `model` value in the body can't silently get routed to the wrong weights.
+# Callers may omit `model` entirely.
 #
 # ── Timeouts ───────────────────────────────────────────────────────────────
 # Kong's upstream timeouts default to 60000ms. A 32B model generating a long
@@ -56,8 +58,7 @@ config:
      {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
      {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
      {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
-      {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0},
-      {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0}
+      {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}
    ]}
 ---
 apiVersion: networking.k8s.io/v1
@@ -213,7 +214,7 @@ spec:
            pathType: Prefix
            backend:
              service:
-                name: ornith-predictor
+                name: grm-predictor
                port:
                  number: 80
 ---
@@ -286,32 +287,3 @@ spec:
                name: reranker-predictor
                port:
                  number: 80
---
-# ── POST /v1/score ──────────────────────────────────────────────────────────
-# The process reward model. Returns scores, not tokens, so it is deliberately
-# not under /chat/completions. vLLM serves /v1/score natively (verified), so no
-# rewrite is needed.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-score
-  namespace: llm-serving
-  annotations:
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "600000"
-    konghq.com/write-timeout: "600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/score
-            pathType: Prefix
-            backend:
-              service:
-                name: verifier-predictor
-                port:
-                  number: 80
@@ -0,0 +1,96 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    # Same Kong-timeout rationale as ornith.yaml: these configure the Service
+    # KServe generates, not the Ingress, and matter once OLLAMA_KEEP_ALIVE=-1
+    # stops covering a cold load after a pod restart.
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
+  labels:
+    app.kubernetes.io/name: llm-grm
+    app.kubernetes.io/part-of: llm-serving
+  name: grm
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - command:
+      - /bin/sh
+      - -c
+      - 'set -e
+
+        ollama serve &
+
+        SERVE_PID=$!
+
+        until ollama list >/dev/null 2>&1; do sleep 2; done
+
+        ollama pull qwen2.5:3b-instruct
+
+        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+
+        wait $SERVE_PID
+
+        '
+      env:
+      - name: OLLAMA_HOST
+        value: 0.0.0.0:8080
+      - name: OLLAMA_MODELS
+        value: /mnt/models/ollama
+      - name: OLLAMA_CONTEXT_LENGTH
+        value: '32768'
+      - name: OLLAMA_KEEP_ALIVE
+        value: '-1'
+      - name: OLLAMA_NUM_PARALLEL
+        value: '1'
+      # Room for a second verification/reward model alongside qwen2.5:3b
+      # without a redeploy -- matches ornith.yaml's pattern, one dedicated
+      # GPU now free for it instead of contending with ornith:35b's.
+      - name: OLLAMA_MAX_LOADED_MODELS
+        value: '2'
+      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q qwen2.5
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q qwen2.5
+        failureThreshold: 120
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
@@ -11,9 +11,9 @@ kind: Kustomization
 # in tens of seconds, not a rolling update.
 resources:
  - embeddings.yaml
+  - grm.yaml
  - ornith.yaml
  - reasoning.yaml
  - reranker.yaml
-  - verifier.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.
@@ -38,12 +38,8 @@ spec:

        ollama pull ornith:35b

-        ollama pull qwen2.5:3b-instruct
-
        ollama run ornith:35b "ok" >/dev/null 2>&1 || true

-        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
-
        wait $SERVE_PID

        '
@@ -58,8 +54,11 @@ spec:
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
+      # qwen2.5:3b-instruct moved to its own dedicated GPU (llm-serving/grm.yaml)
+      # so verification/judge traffic no longer contends with ornith:35b's
+      # planner/implementer traffic on this one -- single model here now.
      - name: OLLAMA_MAX_LOADED_MODELS
-        value: '2'
+        value: '1'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
@@ -70,8 +69,7 @@ spec:
          command:
          - /bin/sh
          - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
-            grep -q qwen2.5
+          - ollama ps 2>/dev/null | grep -q ornith
        periodSeconds: 10
      resources:
        limits:
@@ -87,8 +85,7 @@ spec:
          command:
          - /bin/sh
          - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
-            grep -q qwen2.5
+          - ollama ps 2>/dev/null | grep -q ornith
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
@@ -1,76 +0,0 @@
-apiVersion: serving.kserve.io/v1beta1
-kind: InferenceService
-metadata:
-  annotations:
-    serving.kserve.io/deploymentMode: RawDeployment
-  labels:
-    app.kubernetes.io/name: llm-verifier
-    app.kubernetes.io/part-of: llm-serving
-  name: verifier
-  namespace: llm-serving
-spec:
-  predictor:
-    containers:
-    - args:
-      - --model=Qwen/Qwen2.5-Math-PRM-7B
-      - --served-model-name=verifier
-      - --runner=pooling
-      - --dtype=float16
-      - --tensor-parallel-size=1
-      - --max-model-len=4096
-      - --max-num-seqs=8
-      - --host=0.0.0.0
-      - --port=8080
-      env:
-      - name: VLLM_USE_FLASHINFER_SAMPLER
-        value: '0'
-      - name: VLLM_ATTENTION_BACKEND
-        value: XFORMERS
-      - name: HF_HOME
-        value: /mnt/models
-      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
-      name: kserve-container
-      ports:
-      - containerPort: 8080
-        protocol: TCP
-      readinessProbe:
-        httpGet:
-          path: /health
-          port: 8080
-        periodSeconds: 10
-      resources:
-        limits:
-          cpu: '16'
-          memory: 16Gi
-          nvidia.com/gpu: '1'
-        requests:
-          cpu: '4'
-          memory: 8Gi
-          nvidia.com/gpu: '1'
-      startupProbe:
-        failureThreshold: 60
-        httpGet:
-          path: /health
-          port: 8080
-        periodSeconds: 15
-      volumeMounts:
-      - mountPath: /mnt/models
-        name: models
-      - mountPath: /dev/shm
-        name: shm
-    deploymentStrategy:
-      type: Recreate
-    maxReplicas: 1
-    minReplicas: 1
-    nodeSelector:
-      kubernetes.io/hostname: worker-1
-    runtimeClassName: nvidia
-    volumes:
-    - name: models
-      persistentVolumeClaim:
-        claimName: llm-models
-    - emptyDir:
-        medium: Memory
-        sizeLimit: 1Gi
-      name: shm
-