feat(llm-serving): retire verifier-predictor, add grm (qwen2.5:3b)

Frees verifier's GPU from an underused vLLM PRM deployment. qwen2.5:3b-instruct moves off ornith-predictor's shared pod onto its own dedicated GPU (grm.yaml), so verification/judge traffic stops contending with ornith:35b's agent traffic. /v1/qwen/chat/completions now points at grm-predictor; path unchanged.
This commit is contained in:
Story Crater Bot
2026-08-18 18:18:31 -07:00
parent e6ada95b39
commit 50d00ae350
5 changed files with 115 additions and 126 deletions
+6 -9
View File
@@ -38,12 +38,8 @@ spec:
ollama pull ornith:35b
ollama pull qwen2.5:3b-instruct
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
wait $SERVE_PID
'
@@ -58,8 +54,11 @@ spec:
value: '-1'
- name: OLLAMA_NUM_PARALLEL
value: '1'
# qwen2.5:3b-instruct moved to its own dedicated GPU (llm-serving/grm.yaml)
# so verification/judge traffic no longer contends with ornith:35b's
# planner/implementer traffic on this one -- single model here now.
- name: OLLAMA_MAX_LOADED_MODELS
value: '2'
value: '1'
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
name: kserve-container
ports:
@@ -70,8 +69,7 @@ spec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
- ollama ps 2>/dev/null | grep -q ornith
periodSeconds: 10
resources:
limits:
@@ -87,8 +85,7 @@ spec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
- ollama ps 2>/dev/null | grep -q ornith
failureThreshold: 120
periodSeconds: 15
volumeMounts: