From a005a076486dd3223c2363c0847a236a0202532f Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Sat, 15 Aug 2026 22:58:24 -0700
Subject: [PATCH] fix(llm-serving): ornith 504 on cold start
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two bugs. Kong timeouts were on the Ingress; it reads them from the Service, so
its 60s default applied. Moved to the isvc, which KServe propagates.

Probes ran 'ollama list' — models on disk, not in VRAM — so the pod went Ready
before it could serve. Now 'ollama ps', and both models are warmed at startup.
---
 k8s/apps/llm-serving/ornith.yaml | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml
index ec8754e..fb0c4bc 100644
--- a/k8s/apps/llm-serving/ornith.yaml
+++ b/k8s/apps/llm-serving/ornith.yaml
@@ -3,6 +3,20 @@ kind: InferenceService
 metadata:
   annotations:
     serving.kserve.io/deploymentMode: RawDeployment
+    # Kong reads its timeouts from the Kubernetes Service, not the Ingress —
+    # Ingress annotations configure Route entities (strip-path, methods,
+    # plugins), these configure the Service entity. They were on
+    # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
+    # default in force. KServe propagates InferenceService annotations to the
+    # Service it generates, which is how they reach Kong from here.
+    #
+    # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
+    # request ever waited on a cold load. A pod restart flushes VRAM, and
+    # loading ornith:35b takes longer than 60s, so the first request after any
+    # restart returned 504.
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
   labels:
     app.kubernetes.io/name: llm-ornith
     app.kubernetes.io/part-of: llm-serving
@@ -26,6 +40,10 @@ spec:
 
         ollama pull qwen2.5:3b-instruct
 
+        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
+
+        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+
         wait $SERVE_PID
 
         '
@@ -52,7 +70,7 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null |
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
             grep -q qwen2.5
         periodSeconds: 10
       resources:
@@ -69,7 +87,7 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null |
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
             grep -q qwen2.5
         failureThreshold: 120
         periodSeconds: 15