fix(agent-pod): remote tui session for multi-agent

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -7,6 +7,22 @@ metadata:
   namespace: longhorn-system
   annotations:
     storageclass.kubernetes.io/is-default-class: "false"
+    # StorageClass.parameters is immutable. mkfsParams below was added after
+    # this class already existed, so every sync failed with
+    #   parameters: Invalid value: {...}: field is immutable
+    # and the Application sat OutOfSync permanently — ArgoCD retrying a change
+    # the API server can never accept.
+    #
+    # Replace=true makes ArgoCD delete and recreate instead of patching. Safe
+    # for a StorageClass: it is consulted only at provisioning time, so bound
+    # PVs and their data are untouched. New PVCs briefly fail if one is created
+    # during the window.
+    #
+    # Note what this does NOT do: the nine CNPG volumes already provisioned were
+    # formatted without mkfsParams and keep that format. Only volumes created
+    # after this recreate get it. Existing volumes that need to expand still
+    # need handling separately.
+    argocd.argoproj.io/sync-options: Replace=true,Force=true
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 parameters:

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,6 +1,16 @@
 # Longhorn Node CRDs for cp-2 and cp-3.
 # These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
 # Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
+#
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
+#
+# What these objects are actually for is `allowScheduling: true` on tainted
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
@@ -10,12 +20,6 @@ metadata:
 spec:
   name: talos-cp-2
   allowScheduling: true
-  disks:
-    default-disk:
-      allowScheduling: true
-      path: /var/lib/longhorn
-      storageReserved: 0
-      tags: []
   tags: []
 ---
 apiVersion: longhorn.io/v1beta2
@@ -26,10 +30,4 @@ metadata:
 spec:
   name: talos-cp-3
   allowScheduling: true
-  disks:
-    default-disk:
-      allowScheduling: true
-      path: /var/lib/longhorn
-      storageReserved: 0
-      tags: []
   tags: []

k8s/infra/monitoring/dashboards/llm-frontend.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llm-frontend-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "LLM"
+    # Request rate/error/latency/bandwidth now come from Kong's prometheus
+    # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
+    # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
+    # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
+    # embeddings/rerank uniformly without per-backend instrumentation.
+    # Token-count metrics are still not available: that needs response-body
+    # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
+    # Predictor-level metrics (native Ollama/vLLM stats) also still need a
+    # dedicated exporter -- not added here.
+data:
+  llm-frontend.json: |
+    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}

k8s/infra/monitoring/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - dashboards/control-plane-logs.yaml
   - dashboards/hardware-overview.yaml
   - dashboards/kube-controller-health.yaml
+  - dashboards/llm-frontend.yaml
   - dashboards/service-availability.yaml
   - dashboards/service-golden-signals.yaml
   - dashboards/service-internals.yaml