fix: grant queue-operator create/delete RBAC on temporalworkers

ClusterRole only had get/list/watch/update/patch on temporalworkers
(inherited from a combined queues+temporalworkers rule), missing create
needed by reconcileTemporalWorker's cross-namespace (sqs -> temporal)
CreateOrUpdate call. Split into separate rule blocks per resource so each
carries its own correct verb set.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

k8s/apps/agent-pod/configmap.yaml

@@ -1,8 +1,4 @@
 apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: pi-config
-  namespace: agent-pod
 data:
   config.json: |
     {
@@ -39,3 +35,7 @@ data:
       "defaultThinkingLevel": "medium",
       "theme": "light"
     }
+kind: ConfigMap
+metadata:
+  name: pi-config
+  namespace: agent-pod

k8s/apps/agent-pod/console-ingress.yaml

@@ -0,0 +1,36 @@
+# Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
+# (trigger a new session) -- both are routes on the same hub.js service.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: console
+  namespace: agent-pod
+  annotations:
+    konghq.com/strip-path: "false"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /console
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /run
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /sessions
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090

k8s/apps/agent-pod/deployment.yaml

@@ -13,13 +13,28 @@ spec:
       labels:
         app: agent-pod
     spec:
+      # api.riotpiao.com has no in-cluster DNS record (only resolves from the
+      # home network's own resolver) -- pin it to ingress-nginx-controller's
+      # ClusterIP so pi's models.json baseUrl works unchanged. TLS still
+      # terminates correctly since SNI/Host still say api.riotpiao.com.
+      hostAliases:
+        - ip: "10.101.128.185"
+          hostnames:
+            - "api.riotpiao.com"
       containers:
+        # hub.js runs in the same container as pi (not a sidecar) so it can
+        # spawn `pi -p --mode json` directly via child_process -- a separate
+        # container can't exec into another container's filesystem/PATH.
+        # It IS the container's long-running process now; no more `sleep
+        # infinity` placeholder.
         - name: pi
           image: node:22-slim
           command:
             - sh
             - -c
-            - "npm install -g @earendil-works/[email protected] && sleep infinity"
+            - "npm install -g @earendil-works/[email protected] && npm install --prefix /root ws && node /root/hub.js"
+          ports:
+            - containerPort: 9090
           resources:
             requests:
               cpu: "2"
@@ -37,6 +52,9 @@ spec:
             - name: pi-models
               mountPath: /root/.pi/agent/models.json
               subPath: models.json
+            - name: hub-src
+              mountPath: /root/hub.js
+              subPath: hub.js
       volumes:
         - name: pi-config
           configMap:
@@ -44,3 +62,6 @@ spec:
         - name: pi-models
           secret:
             secretName: pi-models
+        - name: hub-src
+          configMap:
+            name: hub-src

k8s/apps/agent-pod/hub-configmap.yaml

@@ -0,0 +1,119 @@
+apiVersion: v1
+data:
+  hub.js: |
+    #!/usr/bin/env node
+    // agent-hub: lives inside the pi container (not a sidecar) so it can spawn
+    // `pi -p --mode json` directly. One persistent in-cluster service --
+    // POST /run to trigger an agent, GET /console (WebSocket) to watch every
+    // concurrent run live as it happens, relaying pi's own session protocol
+    // verbatim (same event shape Claude Code sessions use).
+    const http = require("node:http");
+    const crypto = require("node:crypto");
+    const { spawn } = require("node:child_process");
+    const readline = require("node:readline");
+    const { WebSocketServer } = require("ws");
+
+    const PORT = process.env.HUB_PORT || 9090;
+
+    const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt}
+    const viewers = new Set(); // WebSocket connections watching /console
+
+    function broadcast(type, session) {
+      const msg = JSON.stringify({ type, session });
+      for (const ws of viewers) {
+        if (ws.readyState === ws.OPEN) ws.send(msg);
+      }
+    }
+
+    function startSession(agent) {
+      const id = crypto.randomUUID();
+      const session = { id, agent, status: "running", events: [], startedAt: new Date().toISOString() };
+      sessions.set(id, session);
+      broadcast("start", session);
+      return session;
+    }
+
+    function addEvent(session, rawLine) {
+      session.events.push(JSON.parse(rawLine));
+      broadcast("event", session);
+    }
+
+    function endSession(session, status) {
+      session.status = status;
+      session.endedAt = new Date().toISOString();
+      broadcast("end", session);
+    }
+
+    // Spawns `pi -p --mode json <prompt>` and relays it live. Same shape as the
+    // old agent-run.js wrapper, just in-process instead of a separate exec.
+    function runAgent(agent, prompt, extraArgs = []) {
+      const session = startSession(agent);
+      const child = spawn("pi", ["-p", "--mode", "json", ...extraArgs, prompt], {
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      const rl = readline.createInterface({ input: child.stdout });
+      rl.on("line", (line) => {
+        if (!line.trim()) return;
+        try {
+          addEvent(session, line);
+        } catch {
+          // non-JSON stdout noise, ignore
+        }
+      });
+      child.stderr.on("data", (chunk) => process.stderr.write(chunk));
+      child.on("close", (code) => endSession(session, code === 0 ? "done" : "error"));
+      return session;
+    }
+
+    const server = http.createServer((req, res) => {
+      const url = new URL(req.url, "http://localhost");
+
+      if (url.pathname === "/healthz") {
+        res.writeHead(200).end();
+        return;
+      }
+
+      if (url.pathname === "/sessions" && req.method === "GET") {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify([...sessions.values()]));
+        return;
+      }
+
+      if (url.pathname === "/run" && req.method === "POST") {
+        let body = "";
+        req.on("data", (chunk) => (body += chunk));
+        req.on("end", () => {
+          try {
+            const { agent, prompt, provider, model } = JSON.parse(body);
+            if (!agent || !prompt) throw new Error("agent and prompt are required");
+            const extraArgs = [];
+            if (provider) extraArgs.push("--provider", provider);
+            if (model) extraArgs.push("--model", model);
+            const session = runAgent(agent, prompt, extraArgs);
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ id: session.id }));
+          } catch (err) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      res.writeHead(404).end();
+    });
+
+    const wss = new WebSocketServer({ server, path: "/console" });
+    wss.on("connection", (ws) => {
+      for (const session of sessions.values()) {
+        ws.send(JSON.stringify({ type: "snapshot", session }));
+      }
+      viewers.add(ws);
+      ws.on("close", () => viewers.delete(ws));
+    });
+
+    server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
+kind: ConfigMap
+metadata:
+  name: hub-src
+  namespace: agent-pod

k8s/apps/agent-pod/hub-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-hub
+  namespace: agent-pod
+spec:
+  selector:
+    app: agent-pod
+  ports:
+    - port: 9090
+      targetPort: 9090

k8s/apps/agent-pod/kustomization.yaml

@@ -4,3 +4,6 @@ namespace: agent-pod
 resources:
   - deployment.yaml
   - configmap.yaml
+  - hub-configmap.yaml
+  - hub-service.yaml
+  - console-ingress.yaml

k8s/apps/api/kong-metrics.yaml

@@ -0,0 +1,19 @@
+# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the
+# ingress controller apply it to every route on this Kong instance, so all
+# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics
+# without touching llm-routes.yaml. Scraped via kong-values.yaml's
+# serviceMonitor (status listener, already on by chart default at :8100).
+apiVersion: configuration.konghq.com/v1
+kind: KongClusterPlugin
+metadata:
+  name: prometheus
+  annotations:
+    kubernetes.io/ingress.class: kong
+  labels:
+    global: "true"
+plugin: prometheus
+config:
+  status_code_metrics: true
+  latency_metrics: true
+  bandwidth_metrics: true
+  upstream_health_metrics: true

k8s/apps/api/kong-values.yaml

@@ -118,6 +118,16 @@ podDisruptionBudget:
   enabled: true
   minAvailable: 1
 
+# Status listener (metrics/health) is on by default at :8100 (chart default,
+# verified via `helm show values`). This just wires the ServiceMonitor the
+# chart already knows how to generate for it, so kong_http_requests_total /
+# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the
+# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml.
+serviceMonitor:
+  enabled: true
+  labels:
+    release: kube-prometheus-stack
+
 # Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
 # situation degrades to co-location instead of leaving a pod Pending.
 topologySpreadConstraints:

k8s/apps/api/kustomization.yaml

@@ -6,6 +6,7 @@ kind: Kustomization
 # or it is silently dropped with no error and no drift shown.
 resources:
   - ingress.yaml
+  - kong-metrics.yaml
   - llm-routes.yaml
   - model-auth.yaml
 # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own

k8s/apps/messaging/queue-crd/templates/rbac.yaml

@@ -10,13 +10,22 @@ metadata:
   name: queue-operator
 rules:
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues", "temporalworkers"]
+    resources: ["queues"]
     verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues/status", "temporalworkers/status"]
+    resources: ["queues/status"]
     verbs: ["get", "update", "patch"]
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues/finalizers", "temporalworkers/finalizers"]
+    resources: ["queues/finalizers"]
+    verbs: ["update"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/finalizers"]
     verbs: ["update"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]

k8s/argocd/secrets/agent-pod-models.enc.yaml

@@ -1,23 +1,23 @@
-apiVersion: ENC[AES256_GCM,data:14E=,iv:Yn6YBsUfovWOVtf1hnEs1Ik51lZMp6/xxqnvzoxMgKo=,tag:cK5OnF87og+gqtg0Z34hLQ==,type:str]
-kind: ENC[AES256_GCM,data:0jXc1dAD,iv:LvMgwGEQ0BRXElkr6jZVrr7j+WsFUKwB6tlHu/pnc1g=,tag:FOpJOzJobFaZuj4h2LsQqQ==,type:str]
+apiVersion: ENC[AES256_GCM,data:YW0=,iv:O7vbb38oX3ADRID4OTXB9mg5+3PbNxXpiwQ1AQYF8ik=,tag:FW5yRAA60uho0wsgYDUBSw==,type:str]
+kind: ENC[AES256_GCM,data:TkhPw8oX,iv:QXfb5UW8ys9o5FRpmimvVuxrYAPZEJ4jdQV68MqVCj8=,tag:rJ7ffXik9lo5EUWgfvoMHA==,type:str]
 metadata:
-    name: ENC[AES256_GCM,data:1qwIvq1m5NDH,iv:qRno92eJrWQ9m7gdudfNw/lUGQiDVI7tKXfIA2ePW5Q=,tag:l4IjZ5lzY7pYg9NuMRTOQw==,type:str]
-    namespace: ENC[AES256_GCM,data:Jxb8Rc91TtRA,iv:Fb1NzwxL4g/T5IsnFpPeAgG6VowM5m5jIwtyc9KfwMk=,tag:T0dg51rZ+0TRREo7J/NK9A==,type:str]
-type: ENC[AES256_GCM,data:utgCSGXC,iv:GukJ0oAORZS24IxS0TnK9kuFbK8IEoC2NIPQV0OGpYc=,tag:BjjIAzYio1HHrgbiSXETug==,type:str]
+    name: ENC[AES256_GCM,data:XohVV8OM9UQZ,iv:O/Rrpb/T8bIuSt969zDSnZ1OYRxvjdAGH/F887P+fPs=,tag:T7zlbUuQClpsDpDaB1b3Sg==,type:str]
+    namespace: ENC[AES256_GCM,data:p6zXMIJ/xGhp,iv:WSIURjth4nf2nu5vLbgHkPXHeuRMoXGW5Z+LfHxigF8=,tag:KDT9atV3/o8ycVcujFKQmA==,type:str]
+type: ENC[AES256_GCM,data:6qGRnFhJ,iv:koA1exWU7EaKRMqjXomD+u0dfcszCmPuV9U9iwKIK7Q=,tag:7Bt+cmcZrE70NH057ho3Dw==,type:str]
 stringData:
-    models.json: ENC[AES256_GCM,data:ywnjstl9mQ7xxi+XJjOPmrthKtobi1NeukcF7Cp3nsFfvRnYErKE7vtW6pWkq0cLclQLjKXWP1KRTqwAUlQOf07ET8or6Q/gL2fPwoC7PE9x2p05Fz25tv5YOczLfA+lj+4FjHPuUodjSjJHWws8aiawd/dlkJkyYv1F0gn4o4UYO2qHFKjhgaYEaTpvK2jRXg0CY3fLa4M0X32XT6ZwrVenr9oEzRSo0QQzCF8tTuabtAHr0vE1DoTsfhR+WTloKwxO+IUYKFzlUnok3JB0PpUpQeoYM7Ji7gYK/auBD9KMzqjOlHDBrjCZpsoVCw3IiXFGmbYzWt8Y5RhIxLy+VYBaYjMAN3twr8bPgQEsO2UklgAANsylIESbbJYiw9TrmErunYsQ40MSzlwIJ5zSfYC6zxnFWVcjaGu3y418A82Tr9CqrSAR8eOpnZ+abWhg/JAWUedlOhy+AMHBu8UCiyOEYDTiBBFg+yWm5fZwe3LXpkdy12G8rgqLV1c4W1Q+EQKo9qTl30j94ZOVEwY76emXhLzrkVcA2pjCck8Yu3PnCD4qZys7hqV8x8riRDlvqM1UGG1X608YbspBvsZimiA+CIpuSvtH70h36l9EbzvfZJ7OZYC3ZBuKYiMcnu29mrMWA/ffwq8Gayncrrb7aUici0K3n1JuYMFWP0dc4IeiQ80ZZA6KK+zfzPPkUblUhkEGnVguwRXrHvlDJAH4J6QqmhIB+qVt+AKusO7McnrYOCf4cw0b41G53Lf+64RgkX7z1qRDDLIswrw/Jcv2278b9AWNb5SyXIf6YzZcEf3kouc03EcR+yOkHX4X9xgMQNhSrJLMYCqOMAGYoJAskwab0sYFK2WHMlLx1ZTa27cqqIdwDnO/WH1msS0mjqJGHQA5j3XdewEWGSor6ETpbv3SRhdS9ul8r50bVUS3wRbaagSuoWqg1jR7vQsc2CVdcX04A8IuDoyK1dHPBMyvTDCm4TmGdRuPgn9JDTjkc+TMFhk6XryDSHbcrMHzEszPzsR3PXbzqO+gDqgddB7LEyUoSoxP8dV+gqffr2Zq6uvCgm6U6Ro8VWqNx4ecQbl3l3hrkArKXMWhoTybWSXF4uThm+uFaJp1AsDKJql0uhyXoZNELarkqBgtwTLO47G/pbZzf9IuBZzKW4UOaxmNmR6eWrIIQSw7nmbKElnTE8u6p9I+HDLRHpMxYmFqeVmHsyGevwMzcvStDQPNzaULVrFb5sfzVcyoFnJBMFBAEYzEDvcHBtcWpxqjh1iAHDH7qPBJ4xeDS9WZpguLJiMGuP1sRj6j+Yg/lbsxcnCocmmdsacP3hXv60p1ZfpkPRnPj+jfHeToT5o7vKpamL/Qp5E1cU7MKmmfLtOdKKXk/89xpe9Y+uhEMv7eW9q6k+odiw4ctOw8jNF82pfz6cteQZ3JRXG29OcX+Tg1fsD9yzDWagCP8bn1P/QGA1EostnVF84NW6zQLJgt9pfByDq6PP1CMGv1evV4Ocm7mLgvaq1sEo/3Vws4gLAtcXKJBo0LZLlf3hvGoVazLyls4vBazVl86nYHip+Fkg7vi4CgapX42nd6YRSvPyJBBr/dDoQkyA5dsSfZ81F5rzrlRbbufVSWUdWHW6w7h/g+ScR9E/9s4/xW9ZqLUJ7r4DtuQv/CVK/oYD0UjjysxvRdW0b4X8jr5xfwKFdMzfJ43ObWYlIvERT//tXS9jBjYsR7slo+a6D2qVpCtBnfQ+vMeTvQOQ1yJfKp6KGMEtkwTQvzIUiXy5+kxvJcBNLFlYgRPWFYqPkhRfZ8d7s/yYJApCo5wCaR/E1IE/XHf3HT1HyG8g8bBTk4w5gw/dLGZDUFRtDT7yPVG7gVIlVByCAYA1iM3HgYcsvenaJA0sZ4BTljBnl3M2CiByQiKhhXeYT3HVKMj+R1cv1T+As+4CWJQ55Y6RzUrn9tVvg0DNZnM+mfyRtADnhTEYqLAmajanTAYaWgGeOdsZue+LE/Amc9NeeyPTatA8q5usWBGb2Bje9+jFvVrohit9sBxhHNjsv0YNKDf9urqQF50nziS8mOOJ8DbEz5FLGnsDnNwBlQ4IBhHBxrHT6igazKp0ZHWAFM21g2AkBQuO/vQGkCGA5ZUDoO93ltxZlnulnrw3cjIZStVDjPDoq+1EbpGQzjBlGcgyW1M2UiLKjyY8ITmdDrE9n5022SwrRvRc3OyRpO0GhwQAggD1H4pDx7hXkJ6h5Q+tIDLeEdmnvUZ21engZHrB+Ucr3e8esgMoPi+KP6hHyNMFmqaEJlrttUB58L5qWlrIaVd6jcS8gXpUdLsUAwZLgR8cHP5XyBw0BkXvvtbFgN9WfOrHICA8/nvH5eRX96D1zycf1Ki98yOM1TSDvWiEGea0rVn4U/em4pOMdX6HpuMgz9kjmGw/sHXcspv9AglVVsXVTCjqadprC2aOnSLjwvwu2vLp5V7QhxS6SufjbzPLUboc1f6LRP/+pqtHw9vmpG02KZ0WTIPzG1Z+7kODbZ3FV1R+lgWJtLJDAhAYSuCmKI9T9aytsFbaoex3OZ3JIUMGOCnbMNNAFY4pNoKn/FwoKgtK2p79Yh52LhdAFGL12047vskgf4pBMQpeJrK2BS5ooJFAKlgGMXquF3cGCDJwBgUNvfhmq/EU30f7R5diYWPfsdu0PP0UFLelV19343AAjM2bzVTFjU9PTAzqDOBZw3ZpBcOPdjy1T2CGszcVcLk354maIyLCa7nLf/HgPv8YfylH8UkFceJ5NvpFKeVCe9SYQi,iv:wPOMkbODxIVYHyzQ61EGujK3/gvjPkBSaZ80plmY8Ug=,tag:a1yxmGj7pnwoXq2HjXBlFg==,type:str]
+    models.json: ENC[AES256_GCM,data:zvrx7vM5tqUg0cZaYEbGX/MHOr6S+yJiJd1rPDWIrXTrDc6SeqfCeTmT129tt1FAfn6htczOFT50N2wFzZXBhXGEhjAqKLHoYRziH+qIDLKaRPF1XvkNLCTjDS7dlsPLLR2/SvUshnmYeyCFGNj+N4n8gXZo8BJzMJTxPWLVVV4MCfwE8xjIihwAA5pOxT8lk4Ffo1mOrnobZGFybINvj5yn6i8pMeoIYgQ6ToCWHLKnC8rzaR1Ve5G1km4nIn5zODKwGaQeOr2ooKsvekwJRY/quJU7aKCjLF6CmJxmU9g+Kcuzm/gKPgaeePl5TXmclWnUJmjH3nkvHIvJQ70W+vaFpWDskHz6RoLWo/qtVLcirqPVHAyK2PjcTBa6wcyQQWFgD74hbaHnXXvHIWYARUEPvXvN01wRySVp7cthTDUtn1rtFmmEnmHrS+uwadsl03mm9Lo6jfXIWur0avUXQ8Qb0+pK3BavD8hI/O9omTPjXc1VBxHmg58KEYHIgerZ08YKB2C1/RsvdFFcJam4EN00G7XxBzHdtijOCqPuGPdOIYkqKOeJ9Ht5/hJY3By6SV4Q22gFBqMidvCMLtrwR4OfHmYJZVn29VhYx3CL8omUEXbuewOc4srK2/zA05bEBm7HkxOIAkb5wBmFYq0IKLpbgvRljI/T6O6kCnk/nd3spdJFUh6C+ee+x+YB/1sPrMby5XBd+vlFhgIkrF6xKT5iBZQllYa+QYbP/ytITkzMCFZJZAXx4+ZxGMxlAIfAkoj7VYRRLXgb0pz225IIZrKP4w5PSgcd+JXStZsD5yIT3//zZnzqXf9UfpMegsZ23GWCNuFpJ2ZEBNPUZVBqVvIsCxcFgSshSwPUXfBKaOmvOrV0+8Vwx1tkC+TmRPdH+sCxHH9C7UCH6EITWGOgsM64blFTd4D5C8z/SBITxIH7iJ4U0Wcg6RIBq5yRkNV5UyddLPuBZcNPfWY8aPwR1JL8eseMtg4/Q7tIsTHNqIz+80ixwvEHDMZrxgiWXPKO5cWgdVHyImuOvTkuCusTzF37A9duB9ylch8Bn14wAK/cEZu019Y3mbkYhcGHB2ex+ddSEc4VxpXd1xXLg09Y8XC21cudq61sCUZ9oQB6NCqhEE2B0xB56ZvP+SpcRmoTmn8IxM1vM2GhcneEhNLSYg/ZoRKwJyX2kb1raCxe4Q16NvEiQYTkRRy5CfeTEx/M39RZIhxtagIRh0R2kxr4olwA1VCGjmBvHRPOpzjwZHdwL+qptcbRZNQN04gGRGDPn6nICh/jOGzdxTQBaD2aVGutoHqtMD0q7aDw52Byb6mU7u1714p5RMM5VsbA0oMxBVwH0XMfp7coKA0VoksiTfGnX75mu8AURiXypkMQOXTVwV6fRmMPhIJOE7MjQF6m/vyplsa/ggJIWRy5prIXeYQFDXwUkQ9SkZT8lDITK8I9zwSwdxEm3RgZnvBaroKtEnK+X+KrOKN/Aa3ApXHyKmRgQ9fhxSsKcDOsT/+L88YP02mL8Hoj+8dwjMZ7NQyh7WlwEF6SuZeSLPLgfVWP5/ph+vLFOb15f7CsTp0NlJyCgFsf49WdI5NQyHMWkk+mE856N4hxCFhSm/vw9p3nypnHtzhmzJ6zvT01C4WsHyIWb9/jnVi7Df5hpxk71Nrd0I1HsA5dUwZJMKUw6IhuKP0+Z38euFjmJEyKiLRMA3weFnSkDOgPwFb9KH1lIYuc0K48nBwYtRu6IBB7k38Xe8L7ResZDgMgUYbDniUBWLXEXiAFJ9dTv2wh90TjQn97RQ5lVXALOR/RZPSz/LybWxEHVTTAfIk0oXIJBryquPqaolzB3MSiQZIn1wxv3GOyvDPsPK7nkf/Or5mS14DZ0YMfEaAoiu7CigEnhvc79QHi8dVeNWIhB6WGGa8NivSwq9jiqQoCHiaw3ClTQ/j3ZHv2cAAoioU/bPR2hZ/FtU95mGhzHw8wF2QnwSnb7pHYQO9TevJkUB8v+0+F/bJRJ/iOKWu/4l9d078zczuCfvF6JFoBK/GqmOMff6i8XdtF9o9t0s61qbDMGh0vezmx4/kqRzdZVwzmjd9PQtmNIlYrH5JIPlczJksRbJc3JIQ0OJePJXT1NxXQvemsbLcacF3ju9OR4e1lQ33qy36MT0pNfKnDaMDHaF9COY0/uV/zBYeOi3dfOJ8rsEKhdoqsQa7c8TrBOedAUh1y6Amaec6Rrw1Uqollyror65jSGMtDNu/4gfsEMHfA0hK6Rwh5DFMbQc8TCL1Vzn2HpsHZX2A7fesmlfNanFXId4EeAPsBQtZhyTuJNzubkRSruoV9MPATEuF/1DEGwhMEDVSG8sWIvPn1c5Z1MgMpxyDLkxNO6cRpsPgJSqgKDYkQefKOXdXfsU5B4FNtGxU/MR6fgC5PnWzqCx/x6Wd0dfP76fC40eOmKMNlb1y+cgLHeWMkUexEFYTZM8wFSITfiOf/1HAhUW51LmMAvWeTjZfNRi+Dtj94h7F/yEmkmuIPzgV5nlnicFAQMat7PjedxGXRtobaBuRWvGAG3EOBlSoD+m9vRqfoGXZSpUL9rTA3iLcTqSQgE3BK4zZ/RaxxtSOMJg5Ax5NnFAhi3C8i+0RKha047+/OXDFOdeDvOX8sRQTlPA5tf26fxS48lgS0BzxfI1rpGv8vVO/bKt0j5iUP,iv:UAn2rUgOgXT9OW8qwpQIljVVttNQ06Q8hOSJllvPWE0=,tag:2cl0oo/At+kMtbuAbV6XNw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKdUZQcXoyUFFUOGQ5TUw4
-            WTVyTWZoWUQ4YkYwV2FFMENpbko4QUlzN0YwCnpZZm9TYnB0TjdJRzV6OVREM0Qy
-            OGJXdUJJejdJQjEyQWpkZEJMKzF3VlEKLS0tIGxoSndLdkZPTWNUZ3IyKzlrTnZu
-            aG0yS1o1VmFmcjdBVlc3eHNqazU3SG8KjOkqI4Qmaoyr0RTV8bFbuaaGoYKmbWCY
-            ZZz3kcax8PuoS8clzYFIgBTnV/jCspODsuWLVDQXs8iKysnI6w+GPg==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3Mjk5czNMYzE5ME5sam8y
+            WmZpQTgrZ1MyTXBUVGVwMUl4cCtzQzdzUkVzCjRmQnVyN2RlU1dITDViZWF2OVNB
+            dkwrN0lPYTdncFFZRUZJK0lFNlhTMVUKLS0tIEJSanBnci81Kzl5dWtqZDJxYzZY
+            YXJCNGdiUHpRcmEwSC9UMzBnVk4zYjgK5faS8kzQ49gGI1pOVKO7h4AEBj/ZbeYE
+            AU227F5JEsF03Ob6ZUkUFs0Pde9S+KnASPGEpyQQa6r2zvJMRQdytQ==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-16T07:20:03Z"
-    mac: ENC[AES256_GCM,data:LucvLtHREgLD5k4f50Zy6Isix2GbuMhz0Li4JRMY9Q/4vEKikX6ArtgJmNR2hySit7tj7tsptNTMyheaeRfVjQ3EtkQDokbYw7CdSWyD6J3+rrIo9Pe8WdilNjz1u+TDjhIQWQjgjJskewjp95ofnVJ2WtwFnENRWNdL8aJe/hI=,iv:vDbhjRyRU3QvOTUfxlp2C/2p5mmehqDixc8yyr0Sk2A=,tag:ABnWz2Dl2d4rQNdfIfN7/g==,type:str]
+    lastmodified: "2026-08-16T13:47:18Z"
+    mac: ENC[AES256_GCM,data:XXhC4+jTt0cE8l5WvoMmkShnuhfF4HLWXou3Y3Ui21K8v6Ni1T3kdUCL8W1R+duvQrJpV+Q2L7HIniG/kRvk+R82qpfzqWSexOaXhP+qkL3UK6GNrNyLz+NyGylK7W16pH4T4iMVr2WQHd2OfPVCPyQA4pZcPHMyv+Eo7gZg02E=,iv:D65Zra216I+KsDLo19wJGupikQtHi5wycCbhXxHIjPE=,tag:O5BmhN8Vs4OhC5hSZ6Wa+w==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

k8s/infra/monitoring/dashboards/llm-frontend.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llm-frontend-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "LLM"
+    # Request rate/error/latency/bandwidth now come from Kong's prometheus
+    # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
+    # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
+    # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
+    # embeddings/rerank uniformly without per-backend instrumentation.
+    # Token-count metrics are still not available: that needs response-body
+    # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
+    # Predictor-level metrics (native Ollama/vLLM stats) also still need a
+    # dedicated exporter -- not added here.
+data:
+  llm-frontend.json: |
+    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}

k8s/infra/monitoring/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - dashboards/control-plane-logs.yaml
   - dashboards/hardware-overview.yaml
   - dashboards/kube-controller-health.yaml
+  - dashboards/llm-frontend.yaml
   - dashboards/service-availability.yaml
   - dashboards/service-golden-signals.yaml
   - dashboards/service-internals.yaml