docs(monitoring): correct the gateway latency-metric claim

Previous comments described llm_ttft_seconds / llm_itl_seconds /
llm_tokens_total as "streaming-only". That was wrong, and optimistic.

They are dead code. internal/proxy/llm_metrics.go defines
LLMMetricsCapture, which is the only thing that calls RecordTTFT /
RecordITL / RecordTokens — but NewLLMMetricsCapture() has zero call sites
anywhere in the repo, verified by grep. The exporter skips keys with no
samples, so those three series render as HELP-only stubs and can never be
graphed, streaming or not.

No behaviour change here; only the comments and the dashboard generator's
header are corrected so the next person does not build on a false
assumption. The gateway latency panels are left in place so the gap stays
visible — they populate unchanged once the capture wrapper is wired into
the streaming proxy path.

Series that DO work from the gateway, confirmed live with 3 series (one
per replica): queue_full_requests_total, warmup_requests_delayed_total,
vllm_prefix_cache_hit_tokens_total.

k8s/apps/gotify/deployment.yaml

@@ -23,7 +23,7 @@ spec:
       containers:
         # --- Gotify server ---
         - name: gotify
-          image: ghcr.io/gotify/server:2.6.1
+          image: ghcr.io/gotify/server:3.1.0
           command: ["/bin/sh", "-c"]
           args:
             - |
@@ -84,7 +84,7 @@ spec:
         # Watches Gotify WebSocket stream, forwards messages as email.
         # https://github.com/eternal-flame-AD/gotify-broadcast
         - name: smtp-emailer
-          image: ghcr.io/gotify/server:2.6.1
+          image: ghcr.io/gotify/server:3.1.0
           command:
             - /bin/sh
             - -c

k8s/apps/gotify/igotify-deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: igotify
+  namespace: notifications
+  labels:
+    app: igotify
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: igotify
+  template:
+    metadata:
+      labels:
+        app: igotify
+    spec:
+      containers:
+        - name: igotify
+          image: ghcr.io/androidseb25/igotify-notification-assist:v1.6.0.1
+          ports:
+            - containerPort: 8080
+              protocol: TCP
+          env:
+            - name: ENABLE_CONSOLE_LOG
+              value: "true"
+            - name: ENABLE_SCALAR_UI
+              value: "true"
+            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
+            # left unset — not required when Gotify is reachable via public domain
+            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
+          volumeMounts:
+            - name: data
+              mountPath: /app/data
+          livenessProbe:
+            httpGet:
+              path: /Version
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /Version
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: igotify-data

k8s/apps/gotify/igotify-ingress.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: igotify
+  namespace: notifications
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: igotify.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: igotify
+                port:
+                  number: 80

k8s/apps/gotify/igotify-pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: igotify-data
+  namespace: notifications
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

k8s/apps/gotify/igotify-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: igotify
+  namespace: notifications
+  labels:
+    app: igotify
+spec:
+  selector:
+    app: igotify
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP

k8s/apps/gotify/ingress.yaml

@@ -6,11 +6,9 @@ metadata:
   annotations:
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    # WebSocket support for Gotify client connections
+    # WebSocket support for Gotify client connections (/stream endpoint)
+    # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "upgrade";
 spec:
   ingressClassName: nginx
   rules:

k8s/apps/gotify/kustomization.yaml

@@ -6,3 +6,7 @@ resources:
   - deployment.yaml
   - service.yaml
   - ingress.yaml
+  - igotify-pvc.yaml
+  - igotify-deployment.yaml
+  - igotify-service.yaml
+  - igotify-ingress.yaml

k8s/apps/llm-serving/ornith.yaml

@@ -44,7 +44,7 @@ spec:
       - name: OLLAMA_MODELS
         value: /mnt/models/ollama
       - name: OLLAMA_CONTEXT_LENGTH
-        value: '32768'
+        value: '65536'
       - name: OLLAMA_KEEP_ALIVE
         value: '-1'
       - name: OLLAMA_NUM_PARALLEL

k8s/apps/paperless/configmap.yaml

@@ -20,3 +20,7 @@ data:
   # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
   # on 'account_confirm_email' without this.
   PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
+  # Auto-connect social accounts to existing users with matching email
+  PAPERLESS_SOCIAL_AUTO_SIGNUP: "true"
+  # Allow automatic linking of social accounts to existing users
+  PAPERLESS_ACCOUNT_ALLOW_SIGNUPS: "true"

k8s/apps/paperless/kustomization.yaml

@@ -4,6 +4,7 @@ namespace: paperless
 resources:
   - pvc.yaml
   - configmap.yaml
+  - paperless-ai-config.yaml
   - redis.yaml
   - deployment.yaml
   - service.yaml

k8s/apps/paperless/paperless-ai-config.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: paperless-ai-env
+  namespace: paperless
+data:
+  paperless-ai-config.env: |
+    # Paperless-NGX API configuration
+    PAPERLESS_URL=http://paperless.paperless.svc.cluster.local:8000
+    PAPERLESS_API_TOKEN=7b89463e04c141f4172fbcddf78d623547d327ec
+
+    # AI Processing settings
+    ENABLE_AI_PROCESSING=yes
+    AI_MODEL=reasoning
+    LLM_API_URL=https://api.riotpiao.com/v1
+    LLM_API_TOKEN_FILE=/app/data/llm_token.txt
+
+    # Auto-tagging configuration
+    ENABLE_AUTO_TAGGING=yes
+    ENABLE_CORRESPONDENT_EXTRACTION=yes
+    ENABLE_DOCUMENT_TYPE_EXTRACTION=yes
+    ENABLE_TITLE_GENERATION=yes
+    
+    # Advanced tagging with AI
+    ENABLE_INTELLIGENT_TAGGING=yes
+    ENABLE_DATE_EXTRACTION=yes
+    ENABLE_TIMELINE_TAGGING=yes
+    AUTO_CREATE_TAGS=yes
+    
+    # Tagging prompts for custom extraction
+    CUSTOM_EXTRACTION_PROMPT=Extract document date, time period, financial year, and temporal context. Create tags like YYYY, Q1-YYYY, Last-Month, Current-Year, Historic
+    TAG_EXTRACTION_MODEL=reasoning
+
+    # Scanning behavior
+    SCAN_INTERVAL_SECONDS=60
+    PROCESS_EXISTING_DOCUMENTS=no
+    ADD_AI_TAG=yes
+    AI_TAG_NAME=ai-processed
+
+    # Date/Timeline tagging configuration
+    DATE_EXTRACTION_FORMAT=iso8601
+    TIMELINE_TAG_GRANULARITY=year,quarter,month
+    AUTO_TAG_HISTORICAL=yes
+    AUTO_TAG_RECENT=yes
+    
+    # Performance
+    MAX_CONCURRENT_REQUESTS=2
+    REQUEST_TIMEOUT=60
+    DATE_EXTRACTION_TIMEOUT=30

k8s/apps/paperless/paperless-ai-new.yaml

@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-ai-data
+  namespace: paperless
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperless-ai
+  namespace: paperless
+  labels:
+    app.kubernetes.io/name: paperless-ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: paperless-ai
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: paperless-ai
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      
+      volumes:
+        - name: paperless-ai-data
+          persistentVolumeClaim:
+            claimName: paperless-ai-data
+      
+      initContainers:
+        - name: fetch-llm-token
+          image: curlimages/curl:8.12.0
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
+              
+              # Get JWT token via client_credentials grant
+              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+                -d "grant_type=client_credentials" \
+                -d "client_id=paperless-ai-agent" \
+                -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
+                -d "scope=openid llm:inference" 2>/dev/null)
+              
+              # Extract token
+              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
+              
+              if [ -z "$TOKEN" ]; then
+                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
+                exit 1
+              fi
+              
+              # Store token in file for main container to read
+              mkdir -p /data
+              echo "$TOKEN" > /data/llm_token.txt
+              echo "[init] Token fetched and stored successfully"
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /data
+      
+      containers:
+        - name: paperless-ai
+          image: clusterzx/paperless-ai:latest
+          imagePullPolicy: Always
+          env:
+            # Paperless-ngx connection
+            - name: PAPERLESS_API_URL
+              value: "http://paperless.paperless.svc.cluster.local:8000"
+            - name: PAPERLESS_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-ai-config
+                  key: PAPERLESS_API_TOKEN
+            - name: PAPERLESS_USERNAME
+              value: "admin"
+            
+            # LLM API configuration
+            - name: AI_PROVIDER
+              value: "custom"
+            - name: CUSTOM_BASE_URL
+              value: "https://api.riotpiao.com/v1"
+            # Token file created by init container with JWT from Authentik
+            - name: CUSTOM_API_KEY_FILE
+              value: "/app/data/llm_token.txt"
+            - name: CUSTOM_MODEL
+              value: "qwen2.5:3b-instruct"
+            
+            # Scanning behavior
+            - name: SCAN_INTERVAL
+              value: "60"
+            - name: PROCESS_PREDEFINED_DOCUMENTS
+              value: "no"
+            
+            # Tagging configuration
+            - name: ADD_AI_TAG
+              value: "yes"
+            - name: AI_TAG_NAME
+              value: "ai-processed"
+            - name: ADD_AI_DESCRIPTIONS
+              value: "yes"
+            - name: USE_PROMPT_TAGS
+              value: "yes"
+          
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /app/data
+          
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi

k8s/apps/paperless/paperless-ai.yaml

@@ -25,9 +25,15 @@ spec:
         - name: paperless-ai-data
           persistentVolumeClaim:
             claimName: paperless-ai-data
+        - name: paperless-ai-env-config
+          configMap:
+            name: paperless-ai-env
       initContainers:
         - name: fetch-llm-token
           image: curlimages/curl:8.12.0
+          securityContext:
+            runAsUser: 0
+            fsGroup: 0
           command:
             - sh
             - -c
@@ -43,8 +49,8 @@ spec:
                 -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
                 -d "scope=openid llm:inference" 2>/dev/null)
               
-              # Extract token
+              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
+              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
               
               if [ -z "$TOKEN" ]; then
                 echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
@@ -52,9 +58,13 @@ spec:
               fi
               
               # Store token in file for main container to read
-              mkdir -p /data
+              # Write to both locations for compatibility
-              echo "$TOKEN" > /data/llm_token.txt
+              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "[init] Token fetched and stored successfully"
+              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
+              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
+              echo "[init] Token fetched and stored"
+              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
+              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
           env:
             - name: LLM_AUTH_CLIENT_SECRET
               valueFrom:
@@ -63,7 +73,7 @@ spec:
                   key: LLM_AUTH_CLIENT_SECRET
           volumeMounts:
             - name: paperless-ai-data
-              mountPath: /data
+              mountPath: /app/data
       containers:
         - name: paperless-ai
           image: clusterzx/paperless-ai:latest
@@ -105,6 +115,9 @@ spec:
           volumeMounts:
             - name: paperless-ai-data
               mountPath: /app/data
+            - name: paperless-ai-env-config
+              mountPath: /app/data/.env
+              subPath: paperless-ai-config.env
           resources:
             requests:
               cpu: 100m

k8s/apps/poimen-frontend/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: poimen-frontend
+  namespace: poimen
+  annotations:
+    # Next.js SSE + LLM streaming responses need long timeouts + no buffering.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: poimen.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: poimen-frontend
+                port:
+                  number: 3000

k8s/apps/poimen-frontend/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Edge Ingress for poimen.riotpiao.com.
+# Same pattern as k8s/apps/api/ (api-gw ingress).
+# App manifests (Deployment, Service, NetworkPolicy) live in
+# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
+resources:
+  - ingress.yaml

k8s/argocd/apps/01-image-updater.yaml

@@ -1,33 +0,0 @@
-# ArgoCD Image Updater - auto-updates Application images from registry
-# Watches forgejo.riotpiao.com for new image tags and updates Applications
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: argocd-image-updater
-  namespace: argocd
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  sources:
-    - repoURL: https://argoproj.github.io/argo-helm
-      chart: argocd-image-updater
-      targetRevision: "0.11.2"
-      helm:
-        valueFiles:
-          - $values/k8s/infra/argocd-image-updater/values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: argocd
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=false

k8s/argocd/apps/30-security.yaml

@@ -152,13 +152,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -180,13 +175,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -211,13 +201,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:

k8s/argocd/apps/55-api-gateway.yaml

@@ -36,11 +36,9 @@ metadata:
     app.kubernetes.io/component: gateway
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # ArgoCD Image Updater - auto-update on new image push
+    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
-    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
+    # → patches this Application's kustomize.images via k8s API.
-    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   revisionHistoryLimit: 3

k8s/argocd/apps/60-applications.yaml

@@ -246,11 +246,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "8"
-    # ArgoCD Image Updater - auto-update on new image push
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/app.update-strategy: digest
-    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   source:

k8s/argocd/apps/70-poimen.yaml

@@ -1,37 +1,81 @@
+# Poimen — unified ArgoCD Application (wave 7).
+#
+# Consolidates what was previously five separate apps:
+#   poimen            → source 1 (poimen-workflows workers)
+#   poimen-workflows  → duplicate of source 1 (deleted)
+#   poimen-memory     → source 2 (memory service + redis)
+#   poimen-memory-app → duplicate of source 2 (deleted)
+#   poimen-memory-database → source 3 (CNPG cluster, moved to k8s/infra/poimen)
+#
+# poimen-toolkit is intentionally kept as a separate Application because it
+# manages cluster-scoped CRDs (ToolDefinition). A bad CRD can affect the
+# whole cluster; keeping it separate means it has its own sync history,
+# independent rollback, and can be paused without touching app workloads.
+#
+# Sync ordering via argocd sync-wave annotations:
+#   wave -1 → DB (CNPG Cluster) — must be Ready before app pods start
+#   wave  0 → all application Deployments (default)
+#
+# Image Updater tracks all four images. Write-back via argocd (in-cluster
+# annotation patch — no git commit per image update).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
   name: poimen
   namespace: argocd
+  labels:
+    app.kubernetes.io/part-of: poimen
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # Image Updater: auto-update on new image push (SHA tag filter)
+    # ── Image promotion ──────────────────────────────────────────────────────
-    argocd-image-updater.argoproj.io/image-list: |
+    # Forgejo package webhook → api-gw patches kustomize.images per source.
-      memory=forgejo.riotpiao.com/rock/poimen-memory
+    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
-      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-      frontend=forgejo.riotpiao.com/rock/poimen-frontend
-    argocd-image-updater.argoproj.io/memory.update-strategy: digest
-    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
-    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
-    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
-  source:
+  # Multi-source: each repo owns its own k8s manifests.
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+  # ArgoCD syncs all sources in parallel; sync-wave annotations on individual
+  # resources control ordering within the sync operation.
+  sources:
+    # 1. Workers — Temporal activity + workflow workers, pi-worker pool
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
       targetRevision: main
       path: k8s
+
+    # 2. Memory service — poimen-memory API, Redis cache, NetworkPolicies
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
+      targetRevision: main
+      path: k8s/app
+
+    # 3. Database infrastructure — CNPG Postgres cluster (wave -1)
+    #    Moved from k8s/infra/databases to isolate poimen DB from other tenants.
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      path: k8s/infra/poimen
+
+    # 4. Frontend app manifests — Deployment, Service, NetworkPolicy.
+    #    Each repo owns its own k8s/ (same pattern as homelab-frontend for api-gw).
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
+      targetRevision: main
+      path: k8s
+
+    # 5. Frontend ingress — poimen.riotpiao.com edge route.
+    #    Lives in homelab (same pattern as k8s/apps/api/ for api-gw).
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      path: k8s/apps/poimen-frontend
+
   destination:
     server: https://kubernetes.default.svc
     namespace: poimen
+
   syncPolicy:
     automated:
       prune: true
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+      - SkipDryRunOnMissingResource=true
     retry:
       limit: 5
       backoff:

k8s/argocd/apps/71-poimen-toolkit.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: poimen-toolkit
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Tracks: poimen-toolkit-controller, pi-worker.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
+    targetRevision: main
+    path: k8s
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: poimen
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/projects/homelab-project.yaml

@@ -24,6 +24,7 @@ spec:
     - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
     - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
     - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
     - https://forgejo.riotpiao.com/rock/riotpiao.com.git
     # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
     - https://cloudnative-pg.github.io/charts

k8s/argocd/secrets/forgejo-registry-pat-api.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: forgejo-registry-pat
+    namespace: api
+type: Opaque
+stringData:
+    pat: ENC[AES256_GCM,data:alYPvHBVqnf13NZu6jtgA6WdFM4x46s5mIY8uEnfWdq0Vp+7oKrY1A==,iv:JbUjes5ZB9eKQ1nkV02++7F/BaWIROfT6gjShzajXro=,tag:ApjawJold0QApnX1GUXGhw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6bFZhU3JLSUZEVWhYdEdZ
+            MURoSWNlZjY0RkxyaWlwU2ZTVjJ5V2FnMVVZCis5YlBwQnZrWnVJNFh4UnNldmx6
+            Y2JjdnpkRTVGSjRSZ0U0THdDZHIrNHcKLS0tIHZDd1BDVGRDSHJEQThuTExORWs2
+            b3BxenY4WHhpRkRTZHJGMkNYMm14RnMKH//Xpqbf/ZBNFQ43D0eFUWwRcQWqA3CC
+            RG8U9wyxC6YrlruU2hY6eB2x0OjsVic34ZGaR752Bwyp0Pz49C/k3g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-09-28T03:10:55Z"
+    mac: ENC[AES256_GCM,data:nLZueHygckw+Gddlea90tRU8h8nEGpe21JUi9nCdpc0ctkSlTDqcR3Q7JFPwbEGHGFU5UJ2xnkBZEDIVmExJryqPBMXT7hQd3ySeQgPwAPGt6zEhlRc5jXhqmmrpJiiN5XUL52AZVDwcWACg7kVzBXIClzihFRW+UqjJZhdtELM=,iv:C0H4AFfdfpsBvP1z2chVvW/VTejxYxwNn/KuA50LeH0=,tag:ZEkTCTKvoZH7DjrcwfL3dw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/paperless-ai-secrets.enc.yaml

@@ -1,23 +1,24 @@
-apiVersion: ENC[AES256_GCM,data:l7I=,iv:NZY7r3JVW3zVwxeiScvWKpAQUDa9+nckHd0qWVrGU88=,tag:qpowp5OILdYKtTux/nlWpg==,type:str]
+apiVersion: v1
-kind: ENC[AES256_GCM,data:6oeEbuIk,iv:5flI9TtcYQ961wOYPBPhvQpitHFYAf8yMdNBXqytbJs=,tag:WNhlbKmSeZEqZ9ZgiB/BYg==,type:str]
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:fvJMsgy+wPZqMCdxm9hoV3n/+Q==,iv:I3rVtHIJBOME+bxhPws58Zjhj5i+KT5UtB9o7Vus5EU=,tag:6h4FnUu7PGxlQPIQxPYCRg==,type:str]
+    name: paperless-ai-config
-    namespace: ENC[AES256_GCM,data:CDriLoLovROW,iv:rHXcN1xm5+t2D/Tq/2sx9lQFF8anD5jH24ZntPuBA8U=,tag:XstJAz6S8IM1c/pp9Cg0Ww==,type:str]
+    namespace: paperless
-type: ENC[AES256_GCM,data:JdTwBbag,iv:9Ys15Ketl0ghNK0u0N8IOpUt7+KoloutiG5M9zHPXxw=,tag:teau/udf41Ty34A5wLAm6Q==,type:str]
+type: Opaque
 stringData:
-    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:TuQeDx8po3h4loTRABlItVYQJ7gFjnmIn3zQQGtUcoNFMKpkqLK/GQ==,iv:TDg0stpca5pDtatqu8DFU7R0Bm/S/BI9ZoiG4K8mCT4=,tag:BfjX25V5gL7AeIsscClRAg==,type:str]
+    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
+    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKVDN3aFVqVmFXY1VQVXZP
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
-            OUsrNHdNdlRvRFgvTDQrNE5uL2xaazVENTBjCkNPbGR5Vk16RXNDOW15OGNTRmFR
+            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
-            U0JOeTllaWU1dzNVY3lBbEVyVG5tOEkKLS0tIEZvajlvcUtJdFNNUkkzV3FKOFRj
+            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
-            UUE2TDBzT0xVc2E1NlUvQXAyZytMZEUKBv+ChaoQCstA742L3Bq5mBJlW/UC4Pyw
+            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
-            ZvFAyYbs1NaEqhjtHq+4T62jTWcH/St/vKgUuFQ9LCUQhYd8DUzAow==
+            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T21:04:26Z"
+    encrypted_regex: ^(data|stringData)$
-    mac: ENC[AES256_GCM,data:0ks96xQzOa8ygNDDYfKV8EJ8dWLBaC0PCnLG73NinMByF/KoWkCdwMDPC34W9xxVoTD2NiF1i/3pgCG4QFezTE7tPHPPKdYcQfwda9fkooiXW4Nh2M/sgbq/xgsy9N3qpHD/O5iILgpehb9y0DuErOyxcn2AIYizynU7m8e63pc=,iv:fvPWBsfE6YHskKzdlxJidp14dUgRR0XdMkEu6IxMMSo=,tag:5FiuIrFOg/GXvlQVy7drJQ==,type:str]
+    lastmodified: "2026-09-14T14:19:39Z"
-    unencrypted_suffix: _unencrypted
+    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
     version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -12,6 +12,7 @@ files:
   - authentik-secrets.enc.yaml
   - cloudflare-secrets.enc.yaml
   - forgejo-registry-pat.enc.yaml
+  - forgejo-registry-pat-api.enc.yaml
   - forgejo-registry-pull.enc.yaml
   - forgejo-runner-token.enc.yaml
   - forgejo-secrets.enc.yaml

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -46,6 +46,12 @@ gitea:
       SSH_PORT: 2222
       SSH_LISTEN_PORT: 2222
 
+    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
+    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
+    # the admin only; admin create-user via UI/API is unaffected by this flag.
+    service:
+      DISABLE_REGISTRATION: true
+
     database:
       DB_TYPE: postgres
       HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -166,3 +172,8 @@ deployment:
         secretKeyRef:
           name: forgejo-smtp
           key: password
+    # Allow webhooks to reach api-gw through the cluster ingress (resolves to
+    # internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
+    # api.riotpiao.com is our own gateway — safe to allowlist.
+    - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
+      value: "external,api.riotpiao.com"

k8s/infra/argocd-image-updater/values.yaml

@@ -1,70 +0,0 @@
-# ArgoCD Image Updater configuration
-# Watches Forgejo registry and updates ArgoCD Applications with new image tags
-
-config:
-  # Registry configuration - Forgejo allows anonymous pulls
-  registries:
-    - name: forgejo
-      api_url: https://forgejo.riotpiao.com
-      prefix: forgejo.riotpiao.com
-      default: true
-      insecure: false
-
-  # Log level
-  logLevel: debug
-
-  # ArgoCD API server
-  argocd:
-    grpcWeb: true
-    serverAddress: argocd-server.argocd.svc.cluster.local
-    insecure: true
-    plaintext: true
-
-  # Git write-back configuration (for multi-source Applications)
-  git:
-    # Commit author for image updates
-    user:
-      name: "ArgoCD Image Updater"
-      email: "[email protected]"
-    # Use SSH keys from ArgoCD's known hosts + credentials
-    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
-
-# Mount ArgoCD's git credentials for write-back
-extraVolumes:
-  - name: argocd-ssh-known-hosts-cm
-    configMap:
-      name: argocd-ssh-known-hosts-cm
-      defaultMode: 0644
-  - name: argocd-gpg-keys-cm
-    configMap:
-      name: argocd-gpg-keys-cm
-      optional: true
-      defaultMode: 0644
-  - name: argocd-gpg-pubring
-    configMap:
-      name: argocd-gpg-pubring-cm
-      optional: true
-      defaultMode: 0644
-
-extraVolumeMounts:
-  - name: argocd-ssh-known-hosts-cm
-    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
-    subPath: ssh_known_hosts
-  - name: argocd-gpg-keys-cm
-    mountPath: /etc/gpg/source
-  - name: argocd-gpg-pubring
-    mountPath: /etc/gpg/pubring
-
-# Extra environment variables
-extraEnv:
-  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
-    value: "true"
-
-# Resources
-resources:
-  requests:
-    cpu: 50m
-    memory: 64Mi
-  limits:
-    cpu: 200m
-    memory: 128Mi

k8s/infra/databases/kustomization.yaml

@@ -8,5 +8,4 @@ resources:
   - namespaces.yaml
   - authentik-db.yaml
   - temporal-db.yaml
-  - memory-db.yaml
   - paperless-db.yaml

k8s/infra/logging/grafana-values.yaml

@@ -73,7 +73,7 @@ grafana.ini:
   auth.generic_oauth:
     enabled: true
     name: Authentik
-    allow_sign_up: true
+    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
     client_id: grafana
     scopes: openid email profile groups
     auth_url: https://authentik.riotpiao.com/application/o/authorize/
@@ -84,13 +84,13 @@ grafana.ini:
     # Authentik doesn't serve — request 404s with "Error getting email address"
     # and the whole OAuth login fails.
     email_attribute_path: email
-    login_attribute_path: preferred_username
+    login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
     name_attribute_path: name
     role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
     allow_assign_grafana_admin: true
     use_pkce: false
     use_refresh_token: false
-    skip_org_role_sync: false
+    skip_org_role_sync: false  # Sync org roles from OAuth groups
     tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
 
 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -117,6 +117,22 @@ spec:
             echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
           fi
 
+          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
+          # clear eviction flag that was set when scheduling was disabled.
+          echo "Configuring worker-1 disk..."
+          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
+            "spec": {
+              "disks": {
+                "default-disk-1030400000000": {
+                  "allowScheduling": true,
+                  "evictionRequested": false,
+                  "storageReserved": 429496729600
+                }
+              }
+            }
+          }'
+          echo "  ✓ worker-1 disk configured (400Gi reserved)"
+
           echo
           echo "Waiting for disks to be ready..."
           sleep 10

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,33 +1,25 @@
-# Longhorn Node CRDs for cp-2 and cp-3.
+# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
-# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
-# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
-# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
-# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
-# `default-disk` key here never matched the live one, so the Application sat
+# for general replica scheduling).
-# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
-# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# What these objects are actually for is `allowScheduling: true` on tainted
+# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
-# control-plane nodes. That is all they need to declare.
+# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
+# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
+metadata: { name: talos-cp-2, namespace: longhorn-system }
-  name: talos-cp-2
+spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
-  namespace: longhorn-system
-spec:
-  name: talos-cp-2
-  allowScheduling: true
-  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
+metadata: { name: talos-cp-3, namespace: longhorn-system }
-  name: talos-cp-3
+spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
-  namespace: longhorn-system
+---
-spec:
+apiVersion: longhorn.io/v1beta2
-  name: talos-cp-3
+kind: Node
-  allowScheduling: true
+metadata: { name: worker-1, namespace: longhorn-system }
-  tags: []
+spec: { name: worker-1, allowScheduling: true, tags: [] }

k8s/infra/minio/minio-tenant.yaml

@@ -112,10 +112,16 @@ spec:
           name: minio-oidc
           key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
-      value: "policy"
+      value: "groups"
+    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
+      value: ""
     - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
       value: "https://minio.riotpiao.com/oauth_callback"
     - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
       value: "Authentik"
     - name: MINIO_IDENTITY_OPENID_SCOPES
-      value: "openid,profile,email,minio"
+      value: "openid,profile,email,groups"
+    - name: MINIO_BROWSER_LOGIN_ANIMATION
+      value: "off"
+    - name: MINIO_BROWSER_REDIRECT_URL
+      value: "https://minio.riotpiao.com"

k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

@@ -0,0 +1,76 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cag-kv-cache-rules
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
+    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
+    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
+    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
+    # prefix_cache_queries_total — so this rule produced no data and
+    # VLLMPrefixCacheHitRateLow could never fire.
+    - name: vllm.recording
+      rules:
+        - record: vllm:prefix_cache_hit_rate
+          expr: |
+            rate(vllm:prefix_cache_hits_total[5m])
+            /
+            clamp_min(
+              rate(vllm:prefix_cache_queries_total[5m]),
+              0.001
+            )
+          labels:
+            job: "reasoning-predictor"
+
+    # Alert rules: CAG pipeline health
+    - name: cag.alerts
+      rules:
+        # KV warmup consistently failing — cache never gets warm
+        - alert: KVWarmupFailureHigh
+          expr: |
+            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
+            description: |
+              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
+              for 5 consecutive minutes. The prefix KV cache is not being warmed;
+              agents will see full prefill latency on every call.
+              Check reasoning-predictor health and poimen-memory NetworkPolicy.
+
+        # Warmup misses: vLLM reachable but not caching
+        - alert: KVWarmupMissHigh
+          expr: |
+            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
+            description: |
+              vLLM is reachable but cached_tokens=0 on warmup responses.
+              Possible causes: prefix caching disabled in vLLM, max-num-seqs
+              saturated during warmup, or context format changed.
+
+        # Prefix cache hit rate low after warmup should be established
+        - alert: VLLMPrefixCacheHitRateLow
+          expr: |
+            vllm:prefix_cache_hit_rate < 0.70
+          for: 10m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
+            description: |
+              vLLM prefix cache hit rate below 70% for 10 minutes.
+              Expected >= 85% in steady state with CAG warming active.
+              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.

k8s/infra/monitoring/dashboards/generate-dashboards.py

@@ -265,6 +265,189 @@ def build_api_gateway():
     }
 
 
+# ============================================================================
+# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
+# ============================================================================
+#
+# Two independent metric sources, deliberately both shown:
+#
+#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
+#                               that expose /metrics. Ollama (ornith:35b,
+#                               qwen2.5:3b-instruct) exposes nothing at all.
+#   api-gateway (/metrics)    — intended to cover EVERY model including the
+#                               Ollama ones, but see the dead-metrics note
+#                               below: its latency series are not recorded.
+#
+# NOTE — llm_ttft_seconds, llm_itl_seconds and llm_tokens_total are DEAD as of
+# 2026-09-29. The only recorder is LLMMetricsCapture
+# (internal/proxy/llm_metrics.go), and NewLLMMetricsCapture() has ZERO call
+# sites in the whole repo, so the exporter emits HELP-only stubs and any panel
+# querying them stays empty forever. Verified by grep, not assumed.
+#
+# The gateway latency panels are kept deliberately so the gap is visible — they
+# light up unchanged once the capture wrapper is wired into the streaming proxy
+# path. Until then vLLM's own histograms are the latency source for `reasoning`,
+# and the two Ollama models have no latency visibility from anywhere.
+#
+# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
+# vllm_prefix_cache_hit_tokens_total DO work today (3 series, one per replica).
+#
+# All metric names below were verified against a live scrape, not assumed:
+#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
+#   NOT cache_query_hit_total / cache_query_miss_total.
+
+def build_llm_observability():
+    def hq(metric, q, by="le"):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
+            f"p{int(q * 100)}",
+        )
+
+    def hq_gw(metric, q):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
+            f"p{int(q * 100)} · {{{{model}}}}",
+        )
+
+    def avg_of(metric):
+        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
+        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
+
+    panels = []
+
+    # ── TTFT ────────────────────────────────────────────────────────────────
+    panels.append(row(100, "Latency — Time To First Token", 0, [
+        stat_panel(101, "TTFT p95 (gateway)",
+                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
+                   0, 1, w=4, h=4, unit="s"),
+        stat_panel(102, "TTFT p95 (vLLM)",
+                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
+                   4, 1, w=4, h=4, unit="s"),
+        stat_panel(103, "KV cache in use",
+                   "max(vllm:kv_cache_usage_perc)",
+                   8, 1, w=4, h=4, unit="percentunit"),
+        stat_panel(104, "Requests running",
+                   "sum(vllm:num_requests_running)",
+                   12, 1, w=4, h=4, unit="short"),
+        stat_panel(105, "Requests waiting",
+                   "sum(vllm:num_requests_waiting)",
+                   16, 1, w=4, h=4, unit="short"),
+        stat_panel(106, "Preemptions (5m)",
+                   "increase(vllm:num_preemptions_total[5m])",
+                   20, 1, w=4, h=4, unit="short"),
+        ts_panel(110, "TTFT percentile by model (gateway)",
+                 [hq_gw("llm_ttft_seconds", 0.50),
+                  hq_gw("llm_ttft_seconds", 0.95),
+                  hq_gw("llm_ttft_seconds", 0.99)],
+                 0, 2, w=12, h=8, unit="s"),
+        ts_panel(111, "TTFT percentile — vLLM reasoning",
+                 [hq("vllm:time_to_first_token_seconds", 0.50),
+                  hq("vllm:time_to_first_token_seconds", 0.95),
+                  hq("vllm:time_to_first_token_seconds", 0.99)],
+                 12, 2, w=12, h=8, unit="s"),
+        ts_panel(112, "Prefill vs decode (mean)",
+                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
+                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
+                 0, 3, w=12, h=8, unit="s"),
+        ts_panel(113, "Queue wait before scheduling (mean)",
+                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
+                 12, 3, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Inter-token latency ─────────────────────────────────────────────────
+    panels.append(row(120, "Latency — Inter-Token", 4, [
+        ts_panel(121, "Inter-token latency by model (gateway)",
+                 [hq_gw("llm_itl_seconds", 0.50),
+                  hq_gw("llm_itl_seconds", 0.95),
+                  hq_gw("llm_itl_seconds", 0.99)],
+                 0, 5, w=12, h=8, unit="s"),
+        ts_panel(122, "Inter-token latency — vLLM reasoning",
+                 [hq("vllm:inter_token_latency_seconds", 0.50),
+                  hq("vllm:inter_token_latency_seconds", 0.95),
+                  hq("vllm:inter_token_latency_seconds", 0.99)],
+                 12, 5, w=12, h=8, unit="s"),
+        ts_panel(123, "Decode throughput (tokens/s per request)",
+                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
+                 0, 6, w=12, h=8, unit="short"),
+        ts_panel(124, "End-to-end request latency p50/p95/p99",
+                 [hq("vllm:e2e_request_latency_seconds", 0.50),
+                  hq("vllm:e2e_request_latency_seconds", 0.95),
+                  hq("vllm:e2e_request_latency_seconds", 0.99)],
+                 12, 6, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Batching / scheduling ───────────────────────────────────────────────
+    panels.append(row(130, "Batching & Scheduling", 7, [
+        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
+                 [("sum(vllm:num_requests_running)", "running (in batch)"),
+                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
+                 0, 8, w=12, h=8, unit="short"),
+        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
+                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
+                 12, 8, w=12, h=8, unit="short"),
+        ts_panel(133, "TEI batch size (embeddings / reranker)",
+                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
+                   "{{app}}")],
+                 0, 9, w=12, h=8, unit="short"),
+        ts_panel(134, "TEI queue depth",
+                 [("max(te_queue_size) by (app)", "{{app}}")],
+                 12, 9, w=12, h=8, unit="short"),
+    ]))
+
+    # ── KV cache ────────────────────────────────────────────────────────────
+    panels.append(row(140, "KV Cache", 10, [
+        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
+                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
+                 0, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
+                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
+                 12, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(143, "Prefix cache hits vs queries",
+                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
+                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
+                 0, 12, w=12, h=8, unit="short"),
+        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
+                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
+                 12, 12, w=12, h=8, unit="short"),
+    ]))
+
+    # ── Throughput & health ─────────────────────────────────────────────────
+    panels.append(row(150, "Throughput & Health", 13, [
+        ts_panel(151, "Token throughput",
+                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
+                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
+                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
+                 0, 14, w=12, h=8, unit="short"),
+        ts_panel(152, "Successful requests/s",
+                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
+                  ("sum(rate(te_request_count[5m]))", "TEI")],
+                 12, 14, w=12, h=8, unit="short"),
+        ts_panel(153, "Backpressure — queue-full rejects",
+                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
+                 0, 15, w=8, h=8, unit="short"),
+        ts_panel(154, "KV warmup requests delayed",
+                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
+                 8, 15, w=8, h=8, unit="short"),
+        ts_panel(155, "Embedding / rerank latency p95",
+                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
+                   "{{app}} embed"),
+                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
+                   "{{app}} queue")],
+                 16, 15, w=8, h=8, unit="s"),
+    ]))
+
+    return {
+        "title": "LLM Observability",
+        "uid": "llm-observability",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
+        "panels": panels,
+    }
+
+
 # ============================================================================
 # Generate
 # ============================================================================
@@ -277,4 +460,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
 
+# Dashboard 4
+write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
+
 print("Done.")

k8s/infra/monitoring/dashboards/llm-observability.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+data:
+  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
+    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
+    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
+    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
+    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
+    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
+    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
+    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
+    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
+    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
+    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
+    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
+    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
+    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
+    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
+    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
+    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
+    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
+    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
+    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
+    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
+    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
+    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
+    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
+    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
+    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
+    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
+    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
+    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
+    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
+    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
+    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
+    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
+    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
+    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
+    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
+    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
+    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
+    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
+    by (le, app))","legendFormat":"{{app}} queue"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: LLM
+  labels:
+    grafana_dashboard: '1'
+  name: llm-observability-dashboard
+  namespace: logging

k8s/infra/monitoring/kustomization.yaml

@@ -14,10 +14,14 @@ resources:
   - alerts/svc-vault-rules.yaml
   - alerts/cluster-alerts.yaml
   - alerts/api-gateway-alerts.yaml
+  - alerts/cag-kv-cache-rules.yaml
   - servicemonitors/argocd.yaml
   - servicemonitors/authentik.yaml
   - servicemonitors/forgejo.yaml
   - servicemonitors/minio.yaml
   - servicemonitors/ingress-nginx.yaml
+  - servicemonitors/vllm.yaml
+  - servicemonitors/api-gateway.yaml
   - dashboards/cluster-infrastructure.yaml
   - dashboards/api-gateway.yaml
+  - dashboards/llm-observability.yaml

k8s/infra/monitoring/prometheus-values.yaml

@@ -54,8 +54,8 @@ alertmanager:
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:
   prometheusSpec:
-    retention: 15d
+    retention: 30d
-    retentionSize: "18GB"
+    retentionSize: "95GB"
 
     # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
     # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -64,10 +64,9 @@ prometheus:
     scrapeTimeout: 60s
     evaluationInterval: 30s
 
-    # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
+    # Pin to worker-1 for more storage capacity
-    # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
     nodeSelector:
-      topology.kubernetes.io/zone: az-a
+      kubernetes.io/hostname: worker-1
 
     # Persistent storage — metrics survive node reboots and pod restarts.
     # Uses the default `longhorn` StorageClass; nodeSelector above already pins
@@ -79,7 +78,7 @@ prometheus:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: 20Gi
+              storage: 25Gi
 
     resources:
       requests:

k8s/infra/monitoring/servicemonitors/api-gateway.yaml

@@ -0,0 +1,51 @@
+# WARNING — three of the series documented below are DEAD CODE as of
+# 2026-09-29 and will never produce data:
+#
+#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
+#
+# They are declared and exported in internal/observability/prometheus.go, but
+# the only thing that would ever record samples is LLMMetricsCapture
+# (internal/proxy/llm_metrics.go) — and NewLLMMetricsCapture() has ZERO call
+# sites. Verified: `grep -rn NewLLMMetricsCapture` returns only the definition,
+# and RecordTTFT/RecordITL/RecordTokens are called from nowhere else. The
+# exporter skips keys with no samples, so these render as HELP-only stubs.
+#
+# The gateway latency panels on the llm-observability dashboard depend on that
+# wiring. Until it lands, use vLLM's own histograms as the latency source for
+# `reasoning`; the two Ollama models (ornith:35b, qwen2.5:3b-instruct) have no
+# latency visibility at all, because Ollama exposes no metrics endpoint either.
+#
+# These three ARE recorded and do work (3 series, one per replica):
+#   queue_full_requests_total           counter    backpressure rejects
+#   warmup_requests_delayed_total       counter    vLLM slot contention
+#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
+#   request_duration_seconds            histogram  full request
+#
+# Also note the Deployment already carried prometheus.io/scrape annotations,
+# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
+# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
+# collected before this ServiceMonitor existed.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: api-gateway
+    app.kubernetes.io/part-of: api
+spec:
+  namespaceSelector:
+    matchNames:
+      - api
+  selector:
+    matchLabels:
+      app: api-gateway
+  endpoints:
+    # api-gateway Service port IS named `http` (unlike the KServe ones), so
+    # the name form works here.
+    - port: http
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -11,20 +11,39 @@ spec:
     matchNames:
       - llm-serving
   selector:
-    matchLabels:
+    # Only the three engines that actually serve /metrics.
-      app.kubernetes.io/part-of: llm-serving
+    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
+    # endpoint (/metrics returns 404) — selecting them would add permanently
+    # DOWN targets. Ollama-model visibility comes from the api-gateway
+    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
+    # measured at the proxy, so they cover every model including Ollama).
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
+          - llm-embeddings  # TEI 1.8.2 — CPU
+          - llm-reranker    # TEI 1.8.2 — CPU
   endpoints:
-    - port: http
+    # targetPort, not port: the KServe-generated Services on llm-serving have
+    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
+    # editing the InferenceServices (predictor restart = full weights reload,
+    # tens of seconds on the vLLM slot), so scrape the pod port directly.
+    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
+    - targetPort: 8080
       interval: 30s
       scrapeTimeout: 10s
       path: /metrics
       scheme: http
       relabelings:
-        - sourceLabels: [__meta_kubernetes_namespace]
+        # Prometheus's Kubernetes service discovery already supplies the
-          targetLabel: namespace
+        # namespace, pod and service labels; the only one that needs deriving
-        - sourceLabels: [__meta_kubernetes_pod_name]
+        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
-          targetLabel: pod
+        # not promote arbitrary pod labels.
-        - sourceLabels: [__meta_kubernetes_service_name]
+        #
-          targetLabel: service
+        # `action: replace` is written explicitly. It is the API server's
-        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        # default for relabeling rules, so omitting it leaves the live object
+        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
           targetLabel: app

k8s/infra/poimen/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Poimen infrastructure — CNPG Postgres for memory service.
+# Part of the `poimen` ArgoCD Application (multi-source, source 3).
+# Sync-wave -1 ensures the DB cluster is healthy before application
+# deployments (wave 0) are applied.
+resources:
+  - memory-db.yaml
+  - poimen-sandbox-namespace.yaml
+  - poimen-sandbox-quota.yaml
+  - pi-session-netpol.yaml

k8s/infra/poimen/memory-db.yaml

@@ -0,0 +1,37 @@
+# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
+# Uses default longhorn storage class (3 replicas, dataLocality disabled).
+# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: memory-db
+  namespace: poimen
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: memory
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+      postInitApplicationSQL:
+        - "CREATE EXTENSION vector;"
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 20Gi
+    storageClass: longhorn
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/infra/poimen/pi-session-netpol.yaml

@@ -0,0 +1,12 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pi-session-deny-kube-api
+  namespace: poimen-sandbox
+spec:
+  endpointSelector:
+    matchLabels:
+      app: pi-session
+  egressDeny:
+    - toEntities:
+        - kube-apiserver

k8s/infra/poimen/poimen-sandbox-namespace.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: poimen-sandbox
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

k8s/infra/poimen/poimen-sandbox-quota.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: sandbox-quota
+  namespace: poimen-sandbox
+spec:
+  hard:
+    pods: "20"
+    requests.cpu: "10"
+    requests.memory: 20Gi
+    limits.cpu: "20"
+    limits.memory: 40Gi

k8s/infra/rbac/kustomization.yaml

@@ -16,6 +16,7 @@ resources:
   - temporal-operator-role.yaml
   - llm-serving-operator-role.yaml
   - poimen-memory-operator-role.yaml
+  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until

k8s/infra/rbac/sandbox-observer-role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sandbox-observer
+  namespace: poimen-sandbox
+rules:
+  - apiGroups: [""]
+    resources: [pods, pods/log, services]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: poimen-devs-sandbox
+  namespace: poimen-sandbox
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: "oidc:poimen-devs"
+roleRef:
+  kind: Role
+  name: sandbox-observer
+  apiGroup: rbac.authorization.k8s.io

k8s/infra/tracing/api-latency-dashboard.yaml

@@ -123,6 +123,156 @@ data:
               "custom": {"fillOpacity": 10}
             }
           }
+        },
+        {
+          "id": 7,
+          "title": "vLLM Prefix Cache Hit Rate",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "vllm:prefix_cache_hit_rate",
+              "legendFormat": "prefix cache hit rate"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percentunit",
+              "min": 0,
+              "max": 1,
+              "custom": {"fillOpacity": 15},
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {"color": "red", "value": 0},
+                  {"color": "yellow", "value": 0.7},
+                  {"color": "green", "value": 0.85}
+                ]
+              }
+            }
+          }
+        },
+        {
+          "id": 8,
+          "title": "KV Warmup Hit vs Miss",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
+              "legendFormat": "warmup hit"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
+              "legendFormat": "warmup miss"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_error_total[5m])",
+              "legendFormat": "warmup error"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 9,
+          "title": "KV Warmup Latency p50/p95 (ms)",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
+              "legendFormat": "p50 warmup latency"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
+              "legendFormat": "p95 warmup latency"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ms",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 10,
+          "title": "Standing Context Cache Tier Hits",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
+              "legendFormat": "Redis L1 hit"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_hit_db_total[5m])",
+              "legendFormat": "DB L2 hit"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_recompute_total[5m])",
+              "legendFormat": "recompute (L3)"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_304_total[5m])",
+              "legendFormat": "304 Not Modified"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 11,
+          "title": "Standing Context Alert: Stale > 10min post-ingest",
+          "type": "timeseries",
+          "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
+          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
+              "legendFormat": "recompute rate (non-zero = fresh)"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
+              "legendFormat": "warmup miss rate (alert if > 3/5min)"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10},
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {"color": "green", "value": 0},
+                  {"color": "red", "value": 0.01}
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
+                "properties": [
+                  {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
+                  {"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
+                ]
+              }
+            ]
+          }
         }
       ]
     }

scripts/iam/provision-rbac.py

@@ -145,6 +145,7 @@ SERVICE_ACCOUNTS: Dict[str, Dict[str, Any]] = {
 SCOPE_MAPPINGS: Dict[str, Dict[str, str]] = {
     "roles": {"expression": 'return user.attributes.get("roles", [])'},
     "permissions": {"expression": 'return ["*"] if any(user.groups.filter(is_superuser=True)) else list(user.groups.values_list("name", flat=True))'},
+    "groups": {"expression": 'return {"groups": list(user.groups.values_list("name", flat=True))}'},
     "minio_buckets": {"expression": 'return user.attributes.get("minio_buckets", [])'},
     "paperless_doctypes": {"expression": 'return user.attributes.get("paperless_doctypes", [])'},
     "memory_projects": {"expression": 'return user.attributes.get("memory_projects", [])'},
@@ -246,6 +247,25 @@ for scope_name, scope_spec in SCOPE_MAPPINGS.items():
         else:
             print(f"  {scope_name}: FAILED {status} {res}")
 
+# Fix built-in 'openid' scope to include preferred_username (needed by Grafana, Forgejo, etc.)
+# Note: sub MUST be a string, not int, for JSON compatibility with Go struct unmarshaling
+if "openid" in existing_scopes:
+    openid_scope = existing_scopes["openid"]
+    openid_pk = openid_scope["pk"]
+    openid_expression = 'return {"sub": str(user.pk), "preferred_username": user.username}'
+    if openid_expression not in openid_scope.get("expression", ""):
+        status, res = api("PUT", f"/api/v3/propertymappings/provider/scope/{openid_pk}/", {
+            "name": openid_scope["name"],
+            "scope_name": "openid",
+            "expression": openid_expression,
+        })
+        if status in (200, 201):
+            print(f"  openid: updated (sub as string + preferred_username)")
+        else:
+            print(f"  openid: UPDATE FAILED {status} {res}")
+    else:
+        print(f"  openid: already correct (sub as string)")
+
 # ===========================================================================
 # Phase 4: Get flow UUIDs (needed for providers)
 # ===========================================================================

terraform/files/coredns/Corefile

@@ -38,6 +38,8 @@
     rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
     kubernetes cluster.local in-addr.arpa ip6.arpa {