feat(llm-serving): scale ornith to 2 replicas instead of a dedicated grm GPU

reasoning keeps its 2 GPUs untouched. verifier's freed GPU goes to a second ornith replica instead of a standalone qwen-only pod -- both replicas load ornith:35b + qwen2.5:3b-instruct, k8s Service load-balances across them, so 2 concurrent implementer-style calls get independent instances.
This commit is contained in:
Story Crater Bot
2026-08-18 18:25:12 -07:00
parent 50d00ae350
commit 5d7b9acd43
4 changed files with 24 additions and 114 deletions
+16 -8
View File
@@ -38,8 +38,12 @@ spec:
ollama pull ornith:35b
ollama pull qwen2.5:3b-instruct
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
wait $SERVE_PID
'
@@ -54,11 +58,8 @@ spec:
value: '-1'
- name: OLLAMA_NUM_PARALLEL
value: '1'
# qwen2.5:3b-instruct moved to its own dedicated GPU (llm-serving/grm.yaml)
# so verification/judge traffic no longer contends with ornith:35b's
# planner/implementer traffic on this one -- single model here now.
- name: OLLAMA_MAX_LOADED_MODELS
value: '1'
value: '2'
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
name: kserve-container
ports:
@@ -69,7 +70,8 @@ spec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
periodSeconds: 10
resources:
limits:
@@ -85,7 +87,8 @@ spec:
command:
- /bin/sh
- -c
- ollama ps 2>/dev/null | grep -q ornith
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
grep -q qwen2.5
failureThreshold: 120
periodSeconds: 15
volumeMounts:
@@ -93,8 +96,13 @@ spec:
name: models
deploymentStrategy:
type: Recreate
maxReplicas: 1
minReplicas: 1
# 2 replicas -- each its own GPU, each loading both ornith:35b and
# qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
# get an independent instance instead of contending on one, at the
# cost of judge/qwen traffic still sharing whichever replica an
# implementer call also lands on.
maxReplicas: 2
minReplicas: 2
nodeSelector:
kubernetes.io/hostname: worker-1
runtimeClassName: nvidia