diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml index ec8754e..fb0c4bc 100644 --- a/k8s/apps/llm-serving/ornith.yaml +++ b/k8s/apps/llm-serving/ornith.yaml @@ -3,6 +3,20 @@ kind: InferenceService metadata: annotations: serving.kserve.io/deploymentMode: RawDeployment + # Kong reads its timeouts from the Kubernetes Service, not the Ingress — + # Ingress annotations configure Route entities (strip-path, methods, + # plugins), these configure the Service entity. They were on + # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s + # default in force. KServe propagates InferenceService annotations to the + # Service it generates, which is how they reach Kong from here. + # + # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no + # request ever waited on a cold load. A pod restart flushes VRAM, and + # loading ornith:35b takes longer than 60s, so the first request after any + # restart returned 504. + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" labels: app.kubernetes.io/name: llm-ornith app.kubernetes.io/part-of: llm-serving @@ -26,6 +40,10 @@ spec: ollama pull qwen2.5:3b-instruct + ollama run ornith:35b "ok" >/dev/null 2>&1 || true + + ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true + wait $SERVE_PID ' @@ -52,7 +70,7 @@ spec: command: - /bin/sh - -c - - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | + - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | grep -q qwen2.5 periodSeconds: 10 resources: @@ -69,7 +87,7 @@ spec: command: - /bin/sh - -c - - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | + - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | grep -q qwen2.5 failureThreshold: 120 periodSeconds: 15