Story Crater Bot
7e40a4baf6
Ingress api/api now backs onto api-gateway:8080; the kong Application, its Helm values, plugins and llm-routes are removed. Gateway image v0.0.0 is in the Forgejo registry and the pull secret is in the api namespace.
109 lines
3.1 KiB
YAML
109 lines
3.1 KiB
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/deploymentMode: RawDeployment
|
|
# The konghq.com/{connect,read,write}-timeout annotations that used to live
|
|
# here went with Kong (retired 2026-08-19). They existed because Kong read
|
|
# its upstream timeouts off the Kubernetes Service, and its 60s default cut
|
|
# off the first request after any pod restart — a restart flushes VRAM and
|
|
# reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
|
|
# problem in steady state.
|
|
#
|
|
# The equivalent budget now belongs to the Go gateway's per-route timeout
|
|
# config in rock/homelab-frontend, not to an annotation on this object.
|
|
labels:
|
|
app.kubernetes.io/name: llm-ornith
|
|
app.kubernetes.io/part-of: llm-serving
|
|
name: ornith
|
|
namespace: llm-serving
|
|
spec:
|
|
predictor:
|
|
containers:
|
|
- command:
|
|
- /bin/sh
|
|
- -c
|
|
- 'set -e
|
|
|
|
ollama serve &
|
|
|
|
SERVE_PID=$!
|
|
|
|
until ollama list >/dev/null 2>&1; do sleep 2; done
|
|
|
|
ollama pull ornith:35b
|
|
|
|
ollama pull qwen2.5:3b-instruct
|
|
|
|
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
|
|
|
|
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
|
|
|
|
wait $SERVE_PID
|
|
|
|
'
|
|
env:
|
|
- name: OLLAMA_HOST
|
|
value: 0.0.0.0:8080
|
|
- name: OLLAMA_MODELS
|
|
value: /mnt/models/ollama
|
|
- name: OLLAMA_CONTEXT_LENGTH
|
|
value: '32768'
|
|
- name: OLLAMA_KEEP_ALIVE
|
|
value: '-1'
|
|
- name: OLLAMA_NUM_PARALLEL
|
|
value: '1'
|
|
- name: OLLAMA_MAX_LOADED_MODELS
|
|
value: '2'
|
|
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
|
|
name: kserve-container
|
|
ports:
|
|
- containerPort: 8080
|
|
protocol: TCP
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
|
grep -q qwen2.5
|
|
periodSeconds: 10
|
|
resources:
|
|
limits:
|
|
cpu: '16'
|
|
memory: 16Gi
|
|
nvidia.com/gpu: '1'
|
|
requests:
|
|
cpu: '8'
|
|
memory: 8Gi
|
|
nvidia.com/gpu: '1'
|
|
startupProbe:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
|
grep -q qwen2.5
|
|
failureThreshold: 120
|
|
periodSeconds: 15
|
|
volumeMounts:
|
|
- mountPath: /mnt/models
|
|
name: models
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
# 2 replicas -- each its own GPU, each loading both ornith:35b and
|
|
# qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
|
|
# get an independent instance instead of contending on one, at the
|
|
# cost of judge/qwen traffic still sharing whichever replica an
|
|
# implementer call also lands on.
|
|
maxReplicas: 2
|
|
minReplicas: 2
|
|
nodeSelector:
|
|
kubernetes.io/hostname: worker-1
|
|
runtimeClassName: nvidia
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|
|
|