Story Crater Bot
5d7b9acd43
reasoning keeps its 2 GPUs untouched. verifier's freed GPU goes to a second ornith replica instead of a standalone qwen-only pod -- both replicas load ornith:35b + qwen2.5:3b-instruct, k8s Service load-balances across them, so 2 concurrent implementer-style calls get independent instances.
114 lines
3.3 KiB
YAML
114 lines
3.3 KiB
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/deploymentMode: RawDeployment
|
|
# Kong reads its timeouts from the Kubernetes Service, not the Ingress —
|
|
# Ingress annotations configure Route entities (strip-path, methods,
|
|
# plugins), these configure the Service entity. They were on
|
|
# llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
|
|
# default in force. KServe propagates InferenceService annotations to the
|
|
# Service it generates, which is how they reach Kong from here.
|
|
#
|
|
# This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
|
|
# request ever waited on a cold load. A pod restart flushes VRAM, and
|
|
# loading ornith:35b takes longer than 60s, so the first request after any
|
|
# restart returned 504.
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
labels:
|
|
app.kubernetes.io/name: llm-ornith
|
|
app.kubernetes.io/part-of: llm-serving
|
|
name: ornith
|
|
namespace: llm-serving
|
|
spec:
|
|
predictor:
|
|
containers:
|
|
- command:
|
|
- /bin/sh
|
|
- -c
|
|
- 'set -e
|
|
|
|
ollama serve &
|
|
|
|
SERVE_PID=$!
|
|
|
|
until ollama list >/dev/null 2>&1; do sleep 2; done
|
|
|
|
ollama pull ornith:35b
|
|
|
|
ollama pull qwen2.5:3b-instruct
|
|
|
|
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
|
|
|
|
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
|
|
|
|
wait $SERVE_PID
|
|
|
|
'
|
|
env:
|
|
- name: OLLAMA_HOST
|
|
value: 0.0.0.0:8080
|
|
- name: OLLAMA_MODELS
|
|
value: /mnt/models/ollama
|
|
- name: OLLAMA_CONTEXT_LENGTH
|
|
value: '32768'
|
|
- name: OLLAMA_KEEP_ALIVE
|
|
value: '-1'
|
|
- name: OLLAMA_NUM_PARALLEL
|
|
value: '1'
|
|
- name: OLLAMA_MAX_LOADED_MODELS
|
|
value: '2'
|
|
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
|
|
name: kserve-container
|
|
ports:
|
|
- containerPort: 8080
|
|
protocol: TCP
|
|
readinessProbe:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
|
grep -q qwen2.5
|
|
periodSeconds: 10
|
|
resources:
|
|
limits:
|
|
cpu: '16'
|
|
memory: 16Gi
|
|
nvidia.com/gpu: '1'
|
|
requests:
|
|
cpu: '8'
|
|
memory: 8Gi
|
|
nvidia.com/gpu: '1'
|
|
startupProbe:
|
|
exec:
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
|
grep -q qwen2.5
|
|
failureThreshold: 120
|
|
periodSeconds: 15
|
|
volumeMounts:
|
|
- mountPath: /mnt/models
|
|
name: models
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
# 2 replicas -- each its own GPU, each loading both ornith:35b and
|
|
# qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
|
|
# get an independent instance instead of contending on one, at the
|
|
# cost of judge/qwen traffic still sharing whichever replica an
|
|
# implementer call also lands on.
|
|
maxReplicas: 2
|
|
minReplicas: 2
|
|
nodeSelector:
|
|
kubernetes.io/hostname: worker-1
|
|
runtimeClassName: nvidia
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|
|
|