rock
8790de6038
## GPU Rebalance (4× V100 32GB) | Pod | Before | After | |-----|--------|-------| | reasoning (PP=2) | 2 GPU | 2 GPU | | ornith | 2 GPU (2 replicas) | 1 GPU (1 replica) | | comfyui | — | 1 GPU (**new**) | | qwen-cpu | — | CPU on cp-2 (**new**) | | embeddings/reranker | CPU | CPU | ## Changes - `ornith.yaml`: scale 2→1, remove qwen2.5 co-loading, MAX_LOADED_MODELS=1 - `qwen-cpu.yaml`: new Ollama deployment on talos-cp-2 (144GB RAM), 5Gi PVC - `k8s/apps/comfyui/`: new ComfyUI deployment (1 GPU, 50Gi model PVC, ingress) - `58-comfyui.yaml`: ArgoCD Application (wave 8) Gateway route update in separate PR (homelab-frontend).Reviewed-on: #17 Co-authored-by: rock <[email protected]>
116 lines
3.1 KiB
YAML
116 lines
3.1 KiB
YAML
# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
|
|
# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
|
|
# than GPU but sufficient for lightweight tasks (summarization,
|
|
# classification, quick answers).
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: qwen-cpu
|
|
namespace: llm-serving
|
|
labels:
|
|
app: qwen-cpu
|
|
app.kubernetes.io/name: qwen-cpu
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: qwen-cpu
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: qwen-cpu
|
|
app.kubernetes.io/name: qwen-cpu
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/hostname: talos-cp-2
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: ollama
|
|
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
ollama serve &
|
|
SERVE_PID=$!
|
|
until ollama list >/dev/null 2>&1; do sleep 2; done
|
|
ollama pull qwen2.5:3b-instruct
|
|
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
|
|
wait $SERVE_PID
|
|
env:
|
|
- name: OLLAMA_HOST
|
|
value: "0.0.0.0:8080"
|
|
- name: OLLAMA_MODELS
|
|
value: /root/.ollama/models
|
|
- name: OLLAMA_CONTEXT_LENGTH
|
|
value: "32768"
|
|
- name: OLLAMA_KEEP_ALIVE
|
|
value: "-1"
|
|
- name: OLLAMA_MAX_LOADED_MODELS
|
|
value: "1"
|
|
- name: OLLAMA_NUM_PARALLEL
|
|
value: "2"
|
|
ports:
|
|
- containerPort: 8080
|
|
protocol: TCP
|
|
readinessProbe:
|
|
exec:
|
|
command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
|
|
periodSeconds: 10
|
|
startupProbe:
|
|
exec:
|
|
command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
|
|
failureThreshold: 60
|
|
periodSeconds: 10
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 4Gi
|
|
limits:
|
|
cpu: "8"
|
|
memory: 8Gi
|
|
volumeMounts:
|
|
- mountPath: /root/.ollama
|
|
name: ollama-data
|
|
volumes:
|
|
- name: ollama-data
|
|
persistentVolumeClaim:
|
|
claimName: qwen-cpu-data
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: qwen-cpu
|
|
namespace: llm-serving
|
|
labels:
|
|
app: qwen-cpu
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
selector:
|
|
app: qwen-cpu
|
|
ports:
|
|
- port: 80
|
|
targetPort: 8080
|
|
protocol: TCP
|
|
---
|
|
# Small PVC for qwen2.5:3b model weights (~1.9GB).
|
|
# Separate from llm-models PVC which is pinned to worker-1.
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: qwen-cpu-data
|
|
namespace: llm-serving
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|