feat: add ComfyUI, rebalance GPU allocation
GPU rebalance (4× V100 32GB on worker-1): - reasoning: 2 GPU (unchanged, PP=2 for Qwen3-32B) - ornith: 2→1 GPU (scale to 1 replica, ornith:35b only) - comfyui: 0→1 GPU (new) - embeddings/reranker: 0 GPU (CPU, unchanged) qwen2.5:3b-instruct moved to CPU on talos-cp-2 (144GB RAM). Separate Ollama deployment + 5Gi PVC, pulls model on first start. Gateway config updated in homelab-frontend (separate commit). Co-authored-by: poimen <[email protected]>
This commit is contained in:
@@ -33,12 +33,8 @@ spec:
|
||||
|
||||
ollama pull ornith:35b
|
||||
|
||||
ollama pull qwen2.5:3b-instruct
|
||||
|
||||
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
|
||||
|
||||
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
|
||||
|
||||
wait $SERVE_PID
|
||||
|
||||
'
|
||||
@@ -54,7 +50,7 @@ spec:
|
||||
- name: OLLAMA_NUM_PARALLEL
|
||||
value: '1'
|
||||
- name: OLLAMA_MAX_LOADED_MODELS
|
||||
value: '2'
|
||||
value: '1'
|
||||
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
|
||||
name: kserve-container
|
||||
ports:
|
||||
@@ -65,8 +61,7 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
||||
grep -q qwen2.5
|
||||
- ollama ps 2>/dev/null | grep -q ornith
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
@@ -82,8 +77,7 @@ spec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
||||
grep -q qwen2.5
|
||||
- ollama ps 2>/dev/null | grep -q ornith
|
||||
failureThreshold: 120
|
||||
periodSeconds: 15
|
||||
volumeMounts:
|
||||
@@ -91,13 +85,10 @@ spec:
|
||||
name: models
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
# 2 replicas -- each its own GPU, each loading both ornith:35b and
|
||||
# qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
|
||||
# get an independent instance instead of contending on one, at the
|
||||
# cost of judge/qwen traffic still sharing whichever replica an
|
||||
# implementer call also lands on.
|
||||
maxReplicas: 2
|
||||
minReplicas: 2
|
||||
# 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
|
||||
# Frees 1 GPU for ComfyUI.
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
|
||||
Reference in New Issue
Block a user