fix: vendor Tekton release.yaml for proper ArgoCD management

ROOT CAUSE: - tektoncd/pipeline config/ dir uses ko:// image refs (build-time placeholders) - ArgoCD synced the raw dev manifests → InvalidImageName on all pods - tektoncd/operator requires its own CRDs and controller (too heavy) - Tekton has no official Helm chart FIX: - Vendor the pre-built release.yaml (v0.68.0) into k8s/infra/tekton/ - Point ArgoCD Application at our own repo (forgejo) - Release contains real container images (ghcr.io/tektoncd/pipeline/*) - Remove external tektoncd repo from AppProject (not needed) TO UPGRADE TEKTON: 1. Download new release from github.com/tektoncd/pipeline/releases 2. Replace k8s/infra/tekton/release.yaml 3. Commit — ArgoCD syncs automatically
fix: use tektoncd/operator for proper K8s-native Tekton installation
2026-09-13 15:05:26 +09:00 · 2026-09-13 14:51:54 +09:00 · 2026-09-13 14:46:00 +09:00
58 changed files with 256 additions and 1639 deletions
@@ -23,7 +23,7 @@ spec:
      containers:
        # --- Gotify server ---
        - name: gotify
-          image: ghcr.io/gotify/server:3.1.0
+          image: ghcr.io/gotify/server:2.6.1
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -84,7 +84,7 @@ spec:
        # Watches Gotify WebSocket stream, forwards messages as email.
        # https://github.com/eternal-flame-AD/gotify-broadcast
        - name: smtp-emailer
-          image: ghcr.io/gotify/server:3.1.0
+          image: ghcr.io/gotify/server:2.6.1
          command:
            - /bin/sh
            - -c
@@ -1,59 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: igotify
-  template:
-    metadata:
-      labels:
-        app: igotify
-    spec:
-      containers:
-        - name: igotify
-          image: ghcr.io/androidseb25/igotify-notification-assist:v1.6.0.1
-          ports:
-            - containerPort: 8080
-              protocol: TCP
-          env:
-            - name: ENABLE_CONSOLE_LOG
-              value: "true"
-            - name: ENABLE_SCALAR_UI
-              value: "true"
-            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
-            # left unset — not required when Gotify is reachable via public domain
-            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
-          volumeMounts:
-            - name: data
-              mountPath: /app/data
-          livenessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 10
-            periodSeconds: 30
-          readinessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 200m
-              memory: 128Mi
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: igotify-data
@@ -1,22 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: igotify
-  namespace: notifications
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: igotify.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: igotify
-                port:
-                  number: 80
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: igotify-data
-  namespace: notifications
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  selector:
-    app: igotify
-  ports:
-    - port: 80
-      targetPort: 8080
-      protocol: TCP
@@ -6,9 +6,11 @@ metadata:
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    # WebSocket support for Gotify client connections (/stream endpoint)
-    # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
+    # WebSocket support for Gotify client connections
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
 spec:
  ingressClassName: nginx
  rules:
@@ -6,7 +6,3 @@ resources:
  - deployment.yaml
  - service.yaml
  - ingress.yaml
-  - igotify-pvc.yaml
-  - igotify-deployment.yaml
-  - igotify-service.yaml
-  - igotify-ingress.yaml
@@ -52,20 +52,6 @@ spec:
          port: 8080
        - protocol: TCP
          port: 9000
-    # Allow from paperless namespace (paperless-ai auto-tagging)
-    # Bypasses gateway until service-account JWT token exchange is implemented.
-    # paperless-ai-agent has llm:inference role in Authentik.
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: paperless
-      ports:
-        - protocol: TCP
-          port: 8080
-        - protocol: TCP
-          port: 80
-        - protocol: TCP
-          port: 8000
    # Allow intra-namespace (pod-to-pod within llm-serving)
    - from:
        - podSelector:
@@ -44,7 +44,7 @@ spec:
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
-        value: '65536'
+        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
@@ -28,14 +28,9 @@ spec:
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
-      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
-      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
-      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
-      # metric after the rollout -- max_num_batched_tokens=8192 also raises
-      # activation memory out of the same budget, so the post-boot
-      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
-      # only valid if it still covers 2x65k + 3x8k. The old
+      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
+      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
+      # above the 131072 target with room to spare -- the old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -59,37 +54,10 @@ spec:
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
-      - --gpu-memory-utilization=0.95
-      - --max-num-seqs=5
-      # Prefill scheduling -- see the measured starvation this fixes.
-      # V1's scheduler has no prefill/decode phase: it walks self.running and
-      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
-      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
-      # per request is applied BEFORE the budget clamp:
-      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
-      #     take = min(take, token_budget)
-      # The loop is `while ... and token_budget > 0`. With the threshold at its
-      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
-      # budget, token_budget hits 0, the loop exits and every other request is
-      # skipped for that step -- measured as 262 tok/s contended prefill and
-      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
-      # Hence threshold must be strictly BELOW the budget or it never binds:
-      # at 8192/8192 a single request is served per step (still starved), while
-      # 8192/2048 serves four. Verified by replaying the loop.
-      - --max-num-batched-tokens=8192
-      - --long-prefill-token-threshold=2048
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
      - --enable-chunked-prefill
      - --enable-prefix-caching
-      #
-      # NOT set, deliberately: --max-num-partial-prefills and
-      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
-      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
-      # rejects any value other than the default for either one, calling
-      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
-      # -> AssertionError, because there is no V0 engine to fall back to.
-      # The Deployment uses strategy Recreate, so setting them is an outage.
-      # V1 does concurrent chunked prefill natively; it needs no flag, only the
-      # budget/threshold ratio above.
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
@@ -48,8 +48,7 @@ spec:
                  mountPath: /backup
          containers:
            - name: mc-mirror
-              # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
-              image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3
+              image: quay.io/minio/mc:latest
              env:
                - name: ACCESS_KEY
                  valueFrom:
@@ -20,7 +20,3 @@ data:
  # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
  # on 'account_confirm_email' without this.
  PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
-  # Auto-connect social accounts to existing users with matching email
-  PAPERLESS_SOCIAL_AUTO_SIGNUP: "true"
-  # Allow automatic linking of social accounts to existing users
-  PAPERLESS_ACCOUNT_ALLOW_SIGNUPS: "true"
@@ -4,7 +4,6 @@ namespace: paperless
 resources:
  - pvc.yaml
  - configmap.yaml
-  - paperless-ai-config.yaml
  - redis.yaml
  - deployment.yaml
  - service.yaml
@@ -1,49 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-ai-env
-  namespace: paperless
-data:
-  paperless-ai-config.env: |
-    # Paperless-NGX API configuration
-    PAPERLESS_URL=http://paperless.paperless.svc.cluster.local:8000
-    PAPERLESS_API_TOKEN=7b89463e04c141f4172fbcddf78d623547d327ec
-
-    # AI Processing settings
-    ENABLE_AI_PROCESSING=yes
-    AI_MODEL=reasoning
-    LLM_API_URL=https://api.riotpiao.com/v1
-    LLM_API_TOKEN_FILE=/app/data/llm_token.txt
-
-    # Auto-tagging configuration
-    ENABLE_AUTO_TAGGING=yes
-    ENABLE_CORRESPONDENT_EXTRACTION=yes
-    ENABLE_DOCUMENT_TYPE_EXTRACTION=yes
-    ENABLE_TITLE_GENERATION=yes
-    
-    # Advanced tagging with AI
-    ENABLE_INTELLIGENT_TAGGING=yes
-    ENABLE_DATE_EXTRACTION=yes
-    ENABLE_TIMELINE_TAGGING=yes
-    AUTO_CREATE_TAGS=yes
-    
-    # Tagging prompts for custom extraction
-    CUSTOM_EXTRACTION_PROMPT=Extract document date, time period, financial year, and temporal context. Create tags like YYYY, Q1-YYYY, Last-Month, Current-Year, Historic
-    TAG_EXTRACTION_MODEL=reasoning
-
-    # Scanning behavior
-    SCAN_INTERVAL_SECONDS=60
-    PROCESS_EXISTING_DOCUMENTS=no
-    ADD_AI_TAG=yes
-    AI_TAG_NAME=ai-processed
-
-    # Date/Timeline tagging configuration
-    DATE_EXTRACTION_FORMAT=iso8601
-    TIMELINE_TAG_GRANULARITY=year,quarter,month
-    AUTO_TAG_HISTORICAL=yes
-    AUTO_TAG_RECENT=yes
-    
-    # Performance
-    MAX_CONCURRENT_REQUESTS=2
-    REQUEST_TIMEOUT=60
-    DATE_EXTRACTION_TIMEOUT=30
@@ -1,127 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-ai-data
-  namespace: paperless
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 2Gi
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-ai
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-ai
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-ai
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-ai
-    spec:
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-      
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token
-              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              mkdir -p /data
-              echo "$TOKEN" > /data/llm_token.txt
-              echo "[init] Token fetched and stored successfully"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /data
-      
-      containers:
-        - name: paperless-ai
-          image: clusterzx/paperless-ai:latest
-          imagePullPolicy: Always
-          env:
-            # Paperless-ngx connection
-            - name: PAPERLESS_API_URL
-              value: "http://paperless.paperless.svc.cluster.local:8000"
-            - name: PAPERLESS_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: PAPERLESS_API_TOKEN
-            - name: PAPERLESS_USERNAME
-              value: "admin"
-            
-            # LLM API configuration
-            - name: AI_PROVIDER
-              value: "custom"
-            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token file created by init container with JWT from Authentik
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
-            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            
-            # Scanning behavior
-            - name: SCAN_INTERVAL
-              value: "60"
-            - name: PROCESS_PREDEFINED_DOCUMENTS
-              value: "no"
-            
-            # Tagging configuration
-            - name: ADD_AI_TAG
-              value: "yes"
-            - name: AI_TAG_NAME
-              value: "ai-processed"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-            - name: USE_PROMPT_TAGS
-              value: "yes"
-          
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-          
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi
@@ -21,59 +21,6 @@ spec:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-        - name: paperless-ai-env-config
-          configMap:
-            name: paperless-ai-env
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          securityContext:
-            runAsUser: 0
-            fsGroup: 0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              # Write to both locations for compatibility
-              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
-              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
-              echo "[init] Token fetched and stored"
-              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
-              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
-          env:
-            - name: LLM_AUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: LLM_AUTH_CLIENT_SECRET
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
      containers:
        - name: paperless-ai
          image: clusterzx/paperless-ai:latest
@@ -88,20 +35,18 @@ spec:
                  key: PAPERLESS_API_TOKEN
            - name: PAPERLESS_USERNAME
              value: "admin"
-            # LLM API — via public gateway with JWT auth
+            # LLM API — local gateway, no auth required (phase 3 not built yet)
            - name: AI_PROVIDER
              value: "custom"
            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token will be read from file at runtime by the application
-            # The init container fetches it and stores in /app/data/llm_token.txt
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
+              value: "http://api-gateway.api.svc.cluster.local:8080/v1"
+            - name: CUSTOM_API_KEY
+              value: "not-required"
            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            # Behavior - scan for new documents and tag them
+              value: "reasoning"
+            # Behavior
            - name: SCAN_INTERVAL
-              value: "60"
+              value: "300"
            - name: PROCESS_PREDEFINED_DOCUMENTS
              value: "no"
            - name: ADD_AI_TAG
@@ -110,14 +55,6 @@ spec:
              value: "ai-processed"
            - name: USE_PROMPT_TAGS
              value: "yes"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-            - name: paperless-ai-env-config
-              mountPath: /app/data/.env
-              subPath: paperless-ai-config.env
          resources:
            requests:
              cpu: 100m
@@ -1,24 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: poimen-frontend
-  namespace: poimen
-  annotations:
-    # Next.js SSE + LLM streaming responses need long timeouts + no buffering.
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-buffering: "off"
-    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: poimen.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: poimen-frontend
-                port:
-                  number: 3000
@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Edge Ingress for poimen.riotpiao.com.
-# Same pattern as k8s/apps/api/ (api-gw ingress).
-# App manifests (Deployment, Service, NetworkPolicy) live in
-# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
-resources:
-  - ingress.yaml
@@ -0,0 +1,33 @@
+# ArgoCD Image Updater - auto-updates Application images from registry
+# Watches forgejo.riotpiao.com for new image tags and updates Applications
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-image-updater
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://argoproj.github.io/argo-helm
+      chart: argocd-image-updater
+      targetRevision: "0.11.2"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/argocd-image-updater/values.yaml
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false
@@ -152,8 +152,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -175,8 +180,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -201,8 +211,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -36,9 +36,11 @@ metadata:
    app.kubernetes.io/component: gateway
  annotations:
    argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
-    # → patches this Application's kustomize.images via k8s API.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
+    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
  revisionHistoryLimit: 3
@@ -246,8 +246,11 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "8"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
+    argocd-image-updater.argoproj.io/app.update-strategy: digest
+    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
  source:
@@ -1,81 +1,37 @@
-# Poimen — unified ArgoCD Application (wave 7).
-#
-# Consolidates what was previously five separate apps:
-#   poimen            → source 1 (poimen-workflows workers)
-#   poimen-workflows  → duplicate of source 1 (deleted)
-#   poimen-memory     → source 2 (memory service + redis)
-#   poimen-memory-app → duplicate of source 2 (deleted)
-#   poimen-memory-database → source 3 (CNPG cluster, moved to k8s/infra/poimen)
-#
-# poimen-toolkit is intentionally kept as a separate Application because it
-# manages cluster-scoped CRDs (ToolDefinition). A bad CRD can affect the
-# whole cluster; keeping it separate means it has its own sync history,
-# independent rollback, and can be paused without touching app workloads.
-#
-# Sync ordering via argocd sync-wave annotations:
-#   wave -1 → DB (CNPG Cluster) — must be Ready before app pods start
-#   wave  0 → all application Deployments (default)
-#
-# Image Updater tracks all four images. Write-back via argocd (in-cluster
-# annotation patch — no git commit per image update).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: poimen
  namespace: argocd
-  labels:
-    app.kubernetes.io/part-of: poimen
  annotations:
    argocd.argoproj.io/sync-wave: "7"
-    # ── Image promotion ──────────────────────────────────────────────────────
-    # Forgejo package webhook → api-gw patches kustomize.images per source.
-    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: digest
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
-  # Multi-source: each repo owns its own k8s manifests.
-  # ArgoCD syncs all sources in parallel; sync-wave annotations on individual
-  # resources control ordering within the sync operation.
-  sources:
-    # 1. Workers — Temporal activity + workflow workers, pi-worker pool
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
-      targetRevision: main
-      path: k8s
-
-    # 2. Memory service — poimen-memory API, Redis cache, NetworkPolicies
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
-      targetRevision: main
-      path: k8s/app
-
-    # 3. Database infrastructure — CNPG Postgres cluster (wave -1)
-    #    Moved from k8s/infra/databases to isolate poimen DB from other tenants.
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      path: k8s/infra/poimen
-
-    # 4. Frontend app manifests — Deployment, Service, NetworkPolicy.
-    #    Each repo owns its own k8s/ (same pattern as homelab-frontend for api-gw).
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
-      targetRevision: main
-      path: k8s
-
-    # 5. Frontend ingress — poimen.riotpiao.com edge route.
-    #    Lives in homelab (same pattern as k8s/apps/api/ for api-gw).
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      path: k8s/apps/poimen-frontend
-
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+    targetRevision: main
+    path: k8s
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
-
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
-      - SkipDryRunOnMissingResource=true
    retry:
      limit: 5
      backoff:
@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: poimen-toolkit
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Tracks: poimen-toolkit-controller, pi-worker.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
-    targetRevision: main
-    path: k8s
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: poimen
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m
@@ -24,7 +24,6 @@ spec:
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
    - https://forgejo.riotpiao.com/rock/riotpiao.com.git
    # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
    - https://cloudnative-pg.github.io/charts
@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: forgejo-registry-pat
-    namespace: api
-type: Opaque
-stringData:
-    pat: ENC[AES256_GCM,data:alYPvHBVqnf13NZu6jtgA6WdFM4x46s5mIY8uEnfWdq0Vp+7oKrY1A==,iv:JbUjes5ZB9eKQ1nkV02++7F/BaWIROfT6gjShzajXro=,tag:ApjawJold0QApnX1GUXGhw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6bFZhU3JLSUZEVWhYdEdZ
-            MURoSWNlZjY0RkxyaWlwU2ZTVjJ5V2FnMVVZCis5YlBwQnZrWnVJNFh4UnNldmx6
-            Y2JjdnpkRTVGSjRSZ0U0THdDZHIrNHcKLS0tIHZDd1BDVGRDSHJEQThuTExORWs2
-            b3BxenY4WHhpRkRTZHJGMkNYMm14RnMKH//Xpqbf/ZBNFQ43D0eFUWwRcQWqA3CC
-            RG8U9wyxC6YrlruU2hY6eB2x0OjsVic34ZGaR752Bwyp0Pz49C/k3g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-28T03:10:55Z"
-    mac: ENC[AES256_GCM,data:nLZueHygckw+Gddlea90tRU8h8nEGpe21JUi9nCdpc0ctkSlTDqcR3Q7JFPwbEGHGFU5UJ2xnkBZEDIVmExJryqPBMXT7hQd3ySeQgPwAPGt6zEhlRc5jXhqmmrpJiiN5XUL52AZVDwcWACg7kVzBXIClzihFRW+UqjJZhdtELM=,iv:C0H4AFfdfpsBvP1z2chVvW/VTejxYxwNn/KuA50LeH0=,tag:ZEkTCTKvoZH7DjrcwfL3dw==,type:str]
-    version: 3.13.2
@@ -1,24 +1,23 @@
-apiVersion: v1
-kind: Secret
+apiVersion: ENC[AES256_GCM,data:l7I=,iv:NZY7r3JVW3zVwxeiScvWKpAQUDa9+nckHd0qWVrGU88=,tag:qpowp5OILdYKtTux/nlWpg==,type:str]
+kind: ENC[AES256_GCM,data:6oeEbuIk,iv:5flI9TtcYQ961wOYPBPhvQpitHFYAf8yMdNBXqytbJs=,tag:WNhlbKmSeZEqZ9ZgiB/BYg==,type:str]
 metadata:
-    name: paperless-ai-config
-    namespace: paperless
-type: Opaque
+    name: ENC[AES256_GCM,data:fvJMsgy+wPZqMCdxm9hoV3n/+Q==,iv:I3rVtHIJBOME+bxhPws58Zjhj5i+KT5UtB9o7Vus5EU=,tag:6h4FnUu7PGxlQPIQxPYCRg==,type:str]
+    namespace: ENC[AES256_GCM,data:CDriLoLovROW,iv:rHXcN1xm5+t2D/Tq/2sx9lQFF8anD5jH24ZntPuBA8U=,tag:XstJAz6S8IM1c/pp9Cg0Ww==,type:str]
+type: ENC[AES256_GCM,data:JdTwBbag,iv:9Ys15Ketl0ghNK0u0N8IOpUt7+KoloutiG5M9zHPXxw=,tag:teau/udf41Ty34A5wLAm6Q==,type:str]
 stringData:
-    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
-    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
+    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:TuQeDx8po3h4loTRABlItVYQJ7gFjnmIn3zQQGtUcoNFMKpkqLK/GQ==,iv:TDg0stpca5pDtatqu8DFU7R0Bm/S/BI9ZoiG4K8mCT4=,tag:BfjX25V5gL7AeIsscClRAg==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
-            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
-            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
-            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
-            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKVDN3aFVqVmFXY1VQVXZP
+            OUsrNHdNdlRvRFgvTDQrNE5uL2xaazVENTBjCkNPbGR5Vk16RXNDOW15OGNTRmFR
+            U0JOeTllaWU1dzNVY3lBbEVyVG5tOEkKLS0tIEZvajlvcUtJdFNNUkkzV3FKOFRj
+            UUE2TDBzT0xVc2E1NlUvQXAyZytMZEUKBv+ChaoQCstA742L3Bq5mBJlW/UC4Pyw
+            ZvFAyYbs1NaEqhjtHq+4T62jTWcH/St/vKgUuFQ9LCUQhYd8DUzAow==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-14T14:19:39Z"
-    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
+    lastmodified: "2026-09-12T21:04:26Z"
+    mac: ENC[AES256_GCM,data:0ks96xQzOa8ygNDDYfKV8EJ8dWLBaC0PCnLG73NinMByF/KoWkCdwMDPC34W9xxVoTD2NiF1i/3pgCG4QFezTE7tPHPPKdYcQfwda9fkooiXW4Nh2M/sgbq/xgsy9N3qpHD/O5iILgpehb9y0DuErOyxcn2AIYizynU7m8e63pc=,iv:fvPWBsfE6YHskKzdlxJidp14dUgRR0XdMkEu6IxMMSo=,tag:5FiuIrFOg/GXvlQVy7drJQ==,type:str]
+    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: pi-code-agent-api-keys
-    namespace: poimen-sandbox
-stringData:
-    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
-    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
-            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
-            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
-            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
-            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T01:19:49Z"
-    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
-    version: 3.13.2
@@ -12,7 +12,6 @@ files:
  - authentik-secrets.enc.yaml
  - cloudflare-secrets.enc.yaml
  - forgejo-registry-pat.enc.yaml
-  - forgejo-registry-pat-api.enc.yaml
  - forgejo-registry-pull.enc.yaml
  - forgejo-runner-token.enc.yaml
  - forgejo-secrets.enc.yaml
@@ -33,4 +32,3 @@ files:
  - gotify-smtp-secrets.enc.yaml
  - forgejo-smtp-secrets.enc.yaml
  - paperless-ai-secrets.enc.yaml
-    - pi-sandbox-secrets.enc.yaml
@@ -45,13 +45,7 @@ gitea:
      SSH_DOMAIN: git.riotpiao.com
      SSH_PORT: 2222
      SSH_LISTEN_PORT: 2222
-
-    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
-    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
-    # the admin only; admin create-user via UI/API is unaffected by this flag.
-    service:
-      DISABLE_REGISTRATION: true
-
+      
    database:
      DB_TYPE: postgres
      HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -172,8 +166,3 @@ deployment:
        secretKeyRef:
          name: forgejo-smtp
          key: password
-    # Allow webhooks to reach api-gw through the cluster ingress (resolves to
-    # internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
-    # api.riotpiao.com is our own gateway — safe to allowlist.
-    - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
-      value: "external,api.riotpiao.com"
@@ -0,0 +1,70 @@
+# ArgoCD Image Updater configuration
+# Watches Forgejo registry and updates ArgoCD Applications with new image tags
+
+config:
+  # Registry configuration - Forgejo allows anonymous pulls
+  registries:
+    - name: forgejo
+      api_url: https://forgejo.riotpiao.com
+      prefix: forgejo.riotpiao.com
+      default: true
+      insecure: false
+
+  # Log level
+  logLevel: debug
+
+  # ArgoCD API server
+  argocd:
+    grpcWeb: true
+    serverAddress: argocd-server.argocd.svc.cluster.local
+    insecure: true
+    plaintext: true
+
+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
+# Extra environment variables
+extraEnv:
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
+    value: "true"
+
+# Resources
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs every 2 minutes. Deletes:
-#   - Failed Jobs older than 2m (any namespace)
-#   - Completed standalone Jobs older than 2m with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
+# Runs daily at 04:00 UTC. Deletes:
+#   - Failed Jobs older than 24h (any namespace)
+#   - Completed Jobs older than 72h with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted state older than 1h
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
-# and linger forever. Aggressive schedule keeps the cluster clean.
+# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
+# and linger forever.

 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
  labels:
    app: stale-job-cleanup
 spec:
-  schedule: "*/2 * * * *"
+  schedule: "0 4 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
@@ -74,9 +74,7 @@ spec:
                  set -e
                  NOW=$(date +%s)

-                  TTL=120  # 2 minutes in seconds
-
-                  echo "=== Cleaning failed Jobs older than 2m ==="
+                  echo "=== Cleaning failed Jobs older than 24h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.conditions[]?.type == "Failed") |
@@ -84,15 +82,15 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 24 ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
+                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.succeeded >= 1) |
@@ -100,20 +98,20 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 72 ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
+                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
+                  # Evicted pods show as Failed with reason Evicted
                  kubectl get pods --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(
                        .status.phase == "Failed" or
-                        .status.phase == "Succeeded" or
                        (.status.reason // "") == "Evicted" or
                        (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                      ) |
@@ -121,9 +119,9 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - POD_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
+                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 1 ]; then
+                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
                      kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                    fi
                  done
@@ -8,4 +8,5 @@ resources:
  - namespaces.yaml
  - authentik-db.yaml
  - temporal-db.yaml
+  - memory-db.yaml
  - paperless-db.yaml
@@ -1,25 +0,0 @@
-# CiliumNetworkPolicy for kube-apiserver access.
-#
-# Standard K8s NetworkPolicy ipBlock rules don't work for the API server
-# under Cilium — the except clause on 192.168.1.0/24 blocks the post-DNAT
-# destination even when a separate rule re-allows a /32 or subnet.
-#
-# Cilium's native `kube-apiserver` entity resolves this correctly: it
-# tracks the API server endpoints regardless of ClusterIP vs node-IP
-# routing, so the policy stays valid across node changes and NAT paths.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: {{ .Release.Name }}-apiserver
-  namespace: {{ .Release.Namespace }}
-spec:
-  endpointSelector:
-    matchLabels:
-      app: {{ .Release.Name }}
-  egress:
-  - toEntities:
-    - kube-apiserver
-    toPorts:
-    - ports:
-      - port: "6443"
-        protocol: TCP
@@ -73,7 +73,7 @@ grafana.ini:
  auth.generic_oauth:
    enabled: true
    name: Authentik
-    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
+    allow_sign_up: true
    client_id: grafana
    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
@@ -84,13 +84,13 @@ grafana.ini:
    # Authentik doesn't serve — request 404s with "Error getting email address"
    # and the whole OAuth login fails.
    email_attribute_path: email
-    login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
+    login_attribute_path: preferred_username
    name_attribute_path: name
    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
-    skip_org_role_sync: false  # Sync org roles from OAuth groups
+    skip_org_role_sync: false
    tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod

 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
@@ -117,22 +117,6 @@ spec:
            echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
          fi

-          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
-          # clear eviction flag that was set when scheduling was disabled.
-          echo "Configuring worker-1 disk..."
-          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
-            "spec": {
-              "disks": {
-                "default-disk-1030400000000": {
-                  "allowScheduling": true,
-                  "evictionRequested": false,
-                  "storageReserved": 429496729600
-                }
-              }
-            }
-          }'
-          echo "  ✓ worker-1 disk configured (400Gi reserved)"
-
          echo
          echo "Waiting for disks to be ready..."
          sleep 10
@@ -1,25 +1,33 @@
-# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
+# Longhorn Node CRDs for cp-2 and cp-3.
+# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
-# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
-# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
-# for general replica scheduling).
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
-# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
-# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
+# What these objects are actually for is `allowScheduling: true` on tainted
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-2, namespace: longhorn-system }
-spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-2
+  namespace: longhorn-system
+spec:
+  name: talos-cp-2
+  allowScheduling: true
+  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-3, namespace: longhorn-system }
-spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
---
-apiVersion: longhorn.io/v1beta2
-kind: Node
-metadata: { name: worker-1, namespace: longhorn-system }
-spec: { name: worker-1, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-3
+  namespace: longhorn-system
+spec:
+  name: talos-cp-3
+  allowScheduling: true
+  tags: []
@@ -59,8 +59,7 @@ spec:
              mountPath: /shared
      containers:
        - name: provision
-          # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
-          image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3
+          image: quay.io/minio/mc:latest
          volumeMounts:
            - name: shared
              mountPath: /shared
@@ -11,12 +11,7 @@ metadata:
    prometheus.io/port: "9000"
    prometheus.io/scrape: "true"
 spec:
-  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
-  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
-  # backend could not resolve minio-cluster-hl. Official images now need registry auth
-  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
-  # release; swap it for the official image once MinIO registry credentials exist.
-  image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z
+  image: minio/minio:RELEASE.2025-07-23T15-54-02Z

  # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
  # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -117,16 +112,10 @@ spec:
          name: minio-oidc
          key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
-      value: "groups"
-    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
-      value: ""
+      value: "policy"
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: "https://minio.riotpiao.com/oauth_callback"
    - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
      value: "Authentik"
    - name: MINIO_IDENTITY_OPENID_SCOPES
-      value: "openid,profile,email,groups"
-    - name: MINIO_BROWSER_LOGIN_ANIMATION
-      value: "off"
-    - name: MINIO_BROWSER_REDIRECT_URL
-      value: "https://minio.riotpiao.com"
+      value: "openid,profile,email,minio"
@@ -158,66 +158,3 @@ spec:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
-
-    - name: api-gateway.admission
-      # The admission gate fails OPEN when its view of the engine gauges goes
-      # stale, so a broken observer degrades silently to the pre-#64 behaviour
-      # of admitting everything. These two series are the only detection —
-      # the gate has no automated tests.
-      rules:
-        # The gate fails OPEN when it cannot read the engine, so "no data" must
-        # alert: otherwise a missing emitter, a renamed metric, or a broken
-        # scrape is indistinguishable from a healthy observer. Both expressions
-        # therefore OR in an absent() guard on the metric itself.
-        - alert: APIGatewayObserverDegraded
-          expr: observer_degraded == 1 or absent(observer_degraded)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
-            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
-
-        # Engine gauges went stale, or are not being reported at all
-        - alert: GatewayEngineStateStale
-          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
-            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
-
-        # The acceptance metric for reasoning.yaml's prefill change. Preemption
-        # of a 65k prompt costs a full re-prefill (~250s at the measured
-        # 262 tok/s), so any sustained rate is a regression, not noise.
-        - alert: LLMPreemptions
-          # Guarded absent() is deliberately NOT used bare here: the engine
-          # restarts (strategy Recreate) and the series legitimately vanishes
-          # for the length of a boot, so a bare absent() would false-fire on
-          # every rollout. It only fires when the metric is missing WHILE the
-          # predictor is up, which is the real "renamed or unscraped" case.
-          expr: |
-            increase(vllm:num_preemptions_total[15m]) > 0
-            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
-            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
-
-        # This change trades single-request latency for fairness: capping a long
-        # prompt at 2048 tokens/step means it takes more steps to finish, so a
-        # lone large prompt can now be slower even while preemptions stay 0.
-        # Without this alert the change has no way to show a regression.
-        - alert: LLMPrefillStarvation
-          expr: |
-            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
-            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
-            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."
@@ -1,76 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: cag-kv-cache-rules
-  namespace: monitoring
-  labels:
-    release: prometheus
-spec:
-  groups:
-    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
-    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
-    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
-    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
-    # prefix_cache_queries_total — so this rule produced no data and
-    # VLLMPrefixCacheHitRateLow could never fire.
-    - name: vllm.recording
-      rules:
-        - record: vllm:prefix_cache_hit_rate
-          expr: |
-            rate(vllm:prefix_cache_hits_total[5m])
-            /
-            clamp_min(
-              rate(vllm:prefix_cache_queries_total[5m]),
-              0.001
-            )
-          labels:
-            job: "reasoning-predictor"
-
-    # Alert rules: CAG pipeline health
-    - name: cag.alerts
-      rules:
-        # KV warmup consistently failing — cache never gets warm
-        - alert: KVWarmupFailureHigh
-          expr: |
-            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
-            description: |
-              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
-              for 5 consecutive minutes. The prefix KV cache is not being warmed;
-              agents will see full prefill latency on every call.
-              Check reasoning-predictor health and poimen-memory NetworkPolicy.
-
-        # Warmup misses: vLLM reachable but not caching
-        - alert: KVWarmupMissHigh
-          expr: |
-            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
-            description: |
-              vLLM is reachable but cached_tokens=0 on warmup responses.
-              Possible causes: prefix caching disabled in vLLM, max-num-seqs
-              saturated during warmup, or context format changed.
-
-        # Prefix cache hit rate low after warmup should be established
-        - alert: VLLMPrefixCacheHitRateLow
-          expr: |
-            vllm:prefix_cache_hit_rate < 0.70
-          for: 10m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
-            description: |
-              vLLM prefix cache hit rate below 70% for 10 minutes.
-              Expected >= 85% in steady state with CAG warming active.
-              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.
@@ -26,23 +26,20 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
    return p


-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
    targets = []
    for e in exprs:
        if isinstance(e, tuple):
            targets.append({"expr": e[0], "legendFormat": e[1]})
        else:
            targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    p = {
+    return {
        "id": id, "title": title, "type": "timeseries",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "fieldConfig": {"defaults": {"unit": unit}},
        "targets": targets,
    }
-    if description:
-        p["description"] = description
-    return p


 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -268,237 +265,6 @@ def build_api_gateway():
    }


-# ============================================================================
-# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
-# ============================================================================
-#
-# Two independent metric sources, deliberately both shown:
-#
-#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
-#                               that expose /metrics. Ollama (ornith:35b,
-#                               qwen2.5:3b-instruct) exposes nothing at all.
-#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
-#                               ones, derived from the SSE byte stream: TTFT is
-#                               request-start to first byte, ITL is the gap
-#                               between SSE writes carrying "data: ", so it
-#                               counts SSE chunks rather than true tokens.
-#                               vLLM's own histograms remain authoritative for
-#                               `reasoning`.
-#
-# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
-# verified against the running gateway and Prometheus:
-#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
-#
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
-# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
-# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
-# homelab-frontend#63 fixed exactly that — the wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
-# The "dead code" note that used to live in this header is superseded.
-#
-# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
-# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
-#
-#
-# All metric names below were verified against a live scrape, not assumed:
-#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
-#   NOT cache_query_hit_total / cache_query_miss_total.
-
-def build_llm_observability():
-    def hq(metric, q, by="le"):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
-            f"p{int(q * 100)}",
-        )
-
-    def hq_gw(metric, q):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
-            f"p{int(q * 100)} · {{{{model}}}}",
-        )
-
-    def avg_of(metric):
-        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
-        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
-
-    panels = []
-
-    # ── TTFT ────────────────────────────────────────────────────────────────
-    panels.append(row(100, "Latency — Time To First Token", 0, [
-        stat_panel(101, "TTFT p95 (gateway)",
-                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
-                   0, 1, w=4, h=4, unit="s"),
-        stat_panel(102, "TTFT p95 (vLLM)",
-                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
-                   4, 1, w=4, h=4, unit="s"),
-        stat_panel(103, "KV cache in use",
-                   "max(vllm:kv_cache_usage_perc)",
-                   8, 1, w=4, h=4, unit="percentunit"),
-        stat_panel(104, "Requests running",
-                   "sum(vllm:num_requests_running)",
-                   12, 1, w=4, h=4, unit="short"),
-        stat_panel(105, "Requests waiting",
-                   "sum(vllm:num_requests_waiting)",
-                   16, 1, w=4, h=4, unit="short"),
-        stat_panel(106, "Preemptions (5m)",
-                   "increase(vllm:num_preemptions_total[5m])",
-                   20, 1, w=4, h=4, unit="short"),
-        ts_panel(110, "TTFT percentile by model (gateway)",
-                 [hq_gw("llm_ttft_seconds", 0.50),
-                  hq_gw("llm_ttft_seconds", 0.95),
-                  hq_gw("llm_ttft_seconds", 0.99)],
-                 0, 2, w=12, h=8, unit="s"),
-        ts_panel(111, "TTFT percentile — vLLM reasoning",
-                 [hq("vllm:time_to_first_token_seconds", 0.50),
-                  hq("vllm:time_to_first_token_seconds", 0.95),
-                  hq("vllm:time_to_first_token_seconds", 0.99)],
-                 12, 2, w=12, h=8, unit="s"),
-        ts_panel(112, "Prefill vs decode (mean)",
-                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
-                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
-                 0, 3, w=12, h=8, unit="s"),
-        ts_panel(113, "Queue wait before scheduling (mean)",
-                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
-                 12, 3, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Inter-token latency ─────────────────────────────────────────────────
-    panels.append(row(120, "Latency — Inter-Token", 4, [
-        ts_panel(121, "Inter-token latency by model (gateway)",
-                 [hq_gw("llm_itl_seconds", 0.50),
-                  hq_gw("llm_itl_seconds", 0.95),
-                  hq_gw("llm_itl_seconds", 0.99)],
-                 0, 5, w=12, h=8, unit="s"),
-        ts_panel(122, "Inter-token latency — vLLM reasoning",
-                 [hq("vllm:inter_token_latency_seconds", 0.50),
-                  hq("vllm:inter_token_latency_seconds", 0.95),
-                  hq("vllm:inter_token_latency_seconds", 0.99)],
-                 12, 5, w=12, h=8, unit="s"),
-        ts_panel(123, "Decode throughput (tokens/s per request)",
-                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
-                 0, 6, w=12, h=8, unit="short"),
-        ts_panel(124, "End-to-end request latency p50/p95/p99",
-                 [hq("vllm:e2e_request_latency_seconds", 0.50),
-                  hq("vllm:e2e_request_latency_seconds", 0.95),
-                  hq("vllm:e2e_request_latency_seconds", 0.99)],
-                 12, 6, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Batching / scheduling ───────────────────────────────────────────────
-    panels.append(row(130, "Batching & Scheduling", 7, [
-        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
-                 [("sum(vllm:num_requests_running)", "running (in batch)"),
-                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
-                 0, 8, w=12, h=8, unit="short"),
-        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
-                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
-                 12, 8, w=12, h=8, unit="short"),
-        ts_panel(133, "TEI batch size (embeddings / reranker)",
-                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
-                   "{{app}}")],
-                 0, 9, w=12, h=8, unit="short"),
-        ts_panel(134, "TEI queue depth",
-                 [("max(te_queue_size) by (app)", "{{app}}")],
-                 12, 9, w=12, h=8, unit="short"),
-    ]))
-
-    # ── KV cache ────────────────────────────────────────────────────────────
-    panels.append(row(140, "KV Cache", 10, [
-        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
-                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
-                 0, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
-                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
-                 12, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(143, "Prefix cache hits vs queries",
-                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
-                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
-                 0, 12, w=12, h=8, unit="short"),
-        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
-                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
-                 12, 12, w=12, h=8, unit="short"),
-    ]))
-
-    # ── Throughput & health ─────────────────────────────────────────────────
-    panels.append(row(150, "Throughput & Health", 13, [
-        ts_panel(151, "Token throughput",
-                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
-                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
-                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
-                 0, 14, w=12, h=8, unit="short"),
-        ts_panel(152, "Successful requests/s",
-                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
-                  ("sum(rate(te_request_count[5m]))", "TEI")],
-                 12, 14, w=12, h=8, unit="short"),
-        ts_panel(153, "Backpressure — queue-full rejects",
-                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
-                 0, 15, w=8, h=8, unit="short"),
-        ts_panel(154, "KV warmup requests delayed",
-                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
-                 8, 15, w=8, h=8, unit="short"),
-        ts_panel(155, "Embedding / rerank latency p95",
-                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
-                   "{{app}} embed"),
-                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
-                   "{{app}} queue")],
-                 16, 15, w=8, h=8, unit="s"),
-    ]))
-
-    # ── Admission control ───────────────────────────────────────────────────
-    # These four panels read the gateway's admission gate, which is the
-    # companion change in homelab-frontend#64. That emitter is NOT deployed
-    # yet (verified against the live /metrics endpoint: 0 series matching
-    # ^(admission_|observer_|engine_state_)), so they render "No data" until
-    # #64 ships. That is expected, and each panel carries a description
-    # saying so — an unexplained blank panel would read as a broken
-    # dashboard, which is exactly the failure mode this row exists to catch.
-    panels.append(row(160, "Admission Control", 16, [
-        ts_panel(161, "Admission parked (5m)",
-                 [("increase(admission_parked_total[5m])", "parked")],
-                 0, 17, w=8, h=8, unit="short",
-                 description="Requests the gateway held back rather than admitting. "
-                             "Emitted by the admission gate (homelab-frontend#64); "
-                             "empty until that change is deployed."),
-        ts_panel(162, "Admission rejected by reason",
-                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
-                 8, 17, w=8, h=8, unit="short",
-                 description="Admission rejections split by the gate's own reason label. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(163, "Engine state stale",
-                 [("increase(engine_state_stale_total[5m])", "stale")],
-                 16, 17, w=8, h=8, unit="short",
-                 description="Times the gateway's view of the engine gauges went stale. "
-                             "The gate fails OPEN when this happens, so any non-zero rate "
-                             "means admission is silently degraded to admit-everything. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(164, "Gateway view vs engine truth",
-                 [("max(vllm_requests_running)", "gateway: running"),
-                  ("sum(vllm:num_requests_running)", "engine: running"),
-                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
-                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
-                 0, 18, w=24, h=8, unit="short",
-                 description="The gateway's own reading of the engine against the engine's "
-                             "self-reported value. A disagreement between a pair is how a "
-                             "stale observer or a mistuned admission target shows itself, "
-                             "and nothing else in this dashboard reveals it. The two "
-                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
-                             "ship with homelab-frontend#64 and are absent until it is "
-                             "deployed — until then this panel shows the engine's pair only."),
-    ]))
-
-    return {
-        "title": "LLM Observability",
-        "uid": "llm-observability",
-        "schemaVersion": 39,
-        "timezone": "browser",
-        "time": {"from": "now-6h", "to": "now"},
-        "refresh": "30s",
-        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
-        "panels": panels,
-    }
-
-
 # ============================================================================
 # Generate
 # ============================================================================
@@ -511,7 +277,4 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")

-# Dashboard 4
-write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
-
 print("Done.")
@@ -1,95 +0,0 @@
-apiVersion: v1
-data:
-  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
-    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
-    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
-    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
-    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
-    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
-    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
-    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
-    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
-    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
-    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
-    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
-    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
-    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
-    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
-    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
-    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
-    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
-    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
-    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
-    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
-    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
-    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
-    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
-    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
-    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
-    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
-    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
-    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
-    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
-    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
-    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
-    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
-    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
-    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
-    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
-    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
-    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
-    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
-    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
-    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
-    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
-    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
-    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
-    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
-    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
-    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
-    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
-    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
-    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
-    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
-    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
-    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
-    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
-    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
-    the engine''s self-reported value. A disagreement between a pair is how a stale
-    observer or a mistuned admission target shows itself, and nothing else in this
-    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
-    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
-    then this panel shows the engine''s pair only."}]}]}'
-kind: ConfigMap
-metadata:
-  annotations:
-    grafana_folder: LLM
-  labels:
-    grafana_dashboard: '1'
-  name: llm-observability-dashboard
-  namespace: logging
@@ -14,14 +14,10 @@ resources:
  - alerts/svc-vault-rules.yaml
  - alerts/cluster-alerts.yaml
  - alerts/api-gateway-alerts.yaml
-  - alerts/cag-kv-cache-rules.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
  - servicemonitors/ingress-nginx.yaml
-  - servicemonitors/vllm.yaml
-  - servicemonitors/api-gateway.yaml
  - dashboards/cluster-infrastructure.yaml
  - dashboards/api-gateway.yaml
-  - dashboards/llm-observability.yaml
@@ -54,8 +54,8 @@ alertmanager:
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:
  prometheusSpec:
-    retention: 30d
-    retentionSize: "95GB"
+    retention: 15d
+    retentionSize: "18GB"

    # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
    # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -64,9 +64,10 @@ prometheus:
    scrapeTimeout: 60s
    evaluationInterval: 30s

-    # Pin to worker-1 for more storage capacity
+    # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
+    # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
    nodeSelector:
-      kubernetes.io/hostname: worker-1
+      topology.kubernetes.io/zone: az-a

    # Persistent storage — metrics survive node reboots and pod restarts.
    # Uses the default `longhorn` StorageClass; nodeSelector above already pins
@@ -78,7 +79,7 @@ prometheus:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
-              storage: 25Gi
+              storage: 20Gi

    resources:
      requests:
@@ -1,62 +0,0 @@
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
-# three latency series below recorded nothing. internal/proxy/llm_metrics.go
-# declared a capture wrapper (LLMMetricsCapture) that was never constructed
-# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
-# stayed empty.
-#
-# homelab-frontend#63 fixed exactly that. The wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
-# request path. Verified live on 2026-09-29:
-#   llm_ttft_seconds  count=7  sum=598.212s
-#   llm_itl_seconds   count=16116
-#   llm_tokens_total  3 series
-#
-# These derived-from-SSE series cover EVERY model including the two Ollama ones
-# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
-# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
-# between SSE writes, so both are granular to SSE write boundaries rather than
-# to the engine's own token cadence. llm_tokens_total counts `data:` events,
-# and the documented assumption in internal/proxy/llm_metrics.go is that these
-# upstreams emit one such event per token — so it is a token count only as far
-# as that assumption holds. vLLM's own histograms remain authoritative.
-#
-# Also recorded and working (3 series, one per replica):
-#   queue_full_requests_total           counter    backpressure rejects
-#   warmup_requests_delayed_total       counter    vLLM slot contention
-#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
-#
-# NOT working, do not build on it: request_duration_seconds is declared and
-# exported but has 0 samples, and its only writer Metrics.RecordRequest has
-# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
-# the same dead-code shape this branch already corrected for the latency
-# series, so it is listed here rather than left to be rediscovered.
-#
-# Also note the Deployment already carried prometheus.io/scrape annotations,
-# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
-# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
-# collected before this ServiceMonitor existed.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: api-gateway
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: api-gateway
-    app.kubernetes.io/part-of: api
-spec:
-  namespaceSelector:
-    matchNames:
-      - api
-  selector:
-    matchLabels:
-      app: api-gateway
-  endpoints:
-    # api-gateway Service port IS named `http` (unlike the KServe ones), so
-    # the name form works here.
-    - port: http
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-
@@ -11,39 +11,20 @@ spec:
    matchNames:
      - llm-serving
  selector:
-    # Only the three engines that actually serve /metrics.
-    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
-    # endpoint (/metrics returns 404) — selecting them would add permanently
-    # DOWN targets. Ollama-model visibility comes from the api-gateway
-    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
-    # measured at the proxy, so they cover every model including Ollama).
-    matchExpressions:
-      - key: app.kubernetes.io/name
-        operator: In
-        values:
-          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
-          - llm-embeddings  # TEI 1.8.2 — CPU
-          - llm-reranker    # TEI 1.8.2 — CPU
+    matchLabels:
+      app.kubernetes.io/part-of: llm-serving
  endpoints:
-    # targetPort, not port: the KServe-generated Services on llm-serving have
-    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
-    # editing the InferenceServices (predictor restart = full weights reload,
-    # tens of seconds on the vLLM slot), so scrape the pod port directly.
-    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
-    - targetPort: 8080
+    - port: http
      interval: 30s
      scrapeTimeout: 10s
      path: /metrics
      scheme: http
      relabelings:
-        # Prometheus's Kubernetes service discovery already supplies the
-        # namespace, pod and service labels; the only one that needs deriving
-        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
-        # not promote arbitrary pod labels.
-        #
-        # `action: replace` is written explicitly. It is the API server's
-        # default for relabeling rules, so omitting it leaves the live object
-        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
-        - action: replace
-          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        - sourceLabels: [__meta_kubernetes_namespace]
+          targetLabel: namespace
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          targetLabel: pod
+        - sourceLabels: [__meta_kubernetes_service_name]
+          targetLabel: service
+        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
          targetLabel: app
@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Poimen infrastructure — CNPG Postgres for memory service.
-# Part of the `poimen` ArgoCD Application (multi-source, source 3).
-# Sync-wave -1 ensures the DB cluster is healthy before application
-# deployments (wave 0) are applied.
-resources:
-  - memory-db.yaml
-  - poimen-sandbox-namespace.yaml
-  - poimen-sandbox-quota.yaml
-  - pi-session-netpol.yaml
-  - pi-agent-workspace-pv.yaml
@@ -1,37 +0,0 @@
-# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
-# Uses default longhorn storage class (3 replicas, dataLocality disabled).
-# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: memory-db
-  namespace: poimen
-  annotations:
-    argocd.argoproj.io/sync-wave: "-1"
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-spec:
-  instances: 3
-  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
-  bootstrap:
-    initdb:
-      database: memory
-      owner: app
-      encoding: UTF8
-      localeCollate: C
-      localeCType: C
-      postInitApplicationSQL:
-        - "CREATE EXTENSION vector;"
-  enableSuperuserAccess: false
-  resources:
-    requests: { memory: "512Mi", cpu: "250m" }
-    limits: { memory: "2Gi", cpu: "1" }
-  storage:
-    size: 20Gi
-    storageClass: longhorn
-  affinity:
-    podAntiAffinityType: preferred
-    topologyKey: kubernetes.io/hostname
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  finalizers:
-  - kubernetes.io/pv-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace-pv
-  annotations:
-    argocd.argoproj.io/sync-wave: '-1'
-spec:
-  accessModes:
-  - ReadWriteOnce
-  capacity:
-    storage: 10Gi
-  claimRef:
-    apiVersion: v1
-    kind: PersistentVolumeClaim
-    name: pi-agent-workspace
-    namespace: poimen-sandbox
-  hostPath:
-    path: /var/lib/pi-agent-sandbox
-    type: DirectoryOrCreate
-  nodeAffinity:
-    required:
-      nodeSelectorTerms:
-      - matchExpressions:
-        - key: kubernetes.io/hostname
-          operator: In
-          values:
-          - talos-cp-2
-  persistentVolumeReclaimPolicy: Retain
-  volumeMode: Filesystem
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  annotations:
-    pv.kubernetes.io/bind-completed: 'yes'
-    argocd.argoproj.io/sync-wave: '-1'
-  finalizers:
-  - kubernetes.io/pvc-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace
-  namespace: poimen-sandbox
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: ''
-  volumeMode: Filesystem
-  volumeName: pi-agent-workspace-pv
@@ -1,12 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: pi-session-deny-kube-api
-  namespace: poimen-sandbox
-spec:
-  endpointSelector:
-    matchLabels:
-      app: pi-session
-  egressDeny:
-    - toEntities:
-        - kube-apiserver
@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: poimen-sandbox
-  labels:
-    pod-security.kubernetes.io/enforce: restricted
-    pod-security.kubernetes.io/enforce-version: latest
-    pod-security.kubernetes.io/audit: restricted
-    pod-security.kubernetes.io/warn: restricted
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ResourceQuota
-metadata:
-  name: sandbox-quota
-  namespace: poimen-sandbox
-spec:
-  hard:
-    pods: "20"
-    requests.cpu: "10"
-    requests.memory: 20Gi
-    limits.cpu: "20"
-    limits.memory: 40Gi
@@ -16,7 +16,6 @@ resources:
  - temporal-operator-role.yaml
  - llm-serving-operator-role.yaml
  - poimen-memory-operator-role.yaml
-  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until
@@ -1,23 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: sandbox-observer
-  namespace: poimen-sandbox
-rules:
-  - apiGroups: [""]
-    resources: [pods, pods/log, services]
-    verbs: [get, list, watch]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: poimen-devs-sandbox
-  namespace: poimen-sandbox
-subjects:
-  - kind: Group
-    apiGroup: rbac.authorization.k8s.io
-    name: "oidc:poimen-devs"
-roleRef:
-  kind: Role
-  name: sandbox-observer
-  apiGroup: rbac.authorization.k8s.io
@@ -123,156 +123,6 @@ data:
              "custom": {"fillOpacity": 10}
            }
          }
-        },
-        {
-          "id": 7,
-          "title": "vLLM Prefix Cache Hit Rate",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "vllm:prefix_cache_hit_rate",
-              "legendFormat": "prefix cache hit rate"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "percentunit",
-              "min": 0,
-              "max": 1,
-              "custom": {"fillOpacity": 15},
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {"color": "red", "value": 0},
-                  {"color": "yellow", "value": 0.7},
-                  {"color": "green", "value": 0.85}
-                ]
-              }
-            }
-          }
-        },
-        {
-          "id": 8,
-          "title": "KV Warmup Hit vs Miss",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
-              "legendFormat": "warmup hit"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
-              "legendFormat": "warmup miss"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_error_total[5m])",
-              "legendFormat": "warmup error"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 9,
-          "title": "KV Warmup Latency p50/p95 (ms)",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
-              "legendFormat": "p50 warmup latency"
-            },
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
-              "legendFormat": "p95 warmup latency"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ms",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 10,
-          "title": "Standing Context Cache Tier Hits",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
-              "legendFormat": "Redis L1 hit"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_hit_db_total[5m])",
-              "legendFormat": "DB L2 hit"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_recompute_total[5m])",
-              "legendFormat": "recompute (L3)"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_304_total[5m])",
-              "legendFormat": "304 Not Modified"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 11,
-          "title": "Standing Context Alert: Stale > 10min post-ingest",
-          "type": "timeseries",
-          "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
-          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
-              "legendFormat": "recompute rate (non-zero = fresh)"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
-              "legendFormat": "warmup miss rate (alert if > 3/5min)"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10},
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {"color": "green", "value": 0},
-                  {"color": "red", "value": 0.01}
-                ]
-              }
-            },
-            "overrides": [
-              {
-                "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
-                "properties": [
-                  {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
-                  {"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
-                ]
-              }
-            ]
-          }
        }
      ]
    }
@@ -145,7 +145,6 @@ SERVICE_ACCOUNTS: Dict[str, Dict[str, Any]] = {
 SCOPE_MAPPINGS: Dict[str, Dict[str, str]] = {
    "roles": {"expression": 'return user.attributes.get("roles", [])'},
    "permissions": {"expression": 'return ["*"] if any(user.groups.filter(is_superuser=True)) else list(user.groups.values_list("name", flat=True))'},
-    "groups": {"expression": 'return {"groups": list(user.groups.values_list("name", flat=True))}'},
    "minio_buckets": {"expression": 'return user.attributes.get("minio_buckets", [])'},
    "paperless_doctypes": {"expression": 'return user.attributes.get("paperless_doctypes", [])'},
    "memory_projects": {"expression": 'return user.attributes.get("memory_projects", [])'},
@@ -247,25 +246,6 @@ for scope_name, scope_spec in SCOPE_MAPPINGS.items():
        else:
            print(f"  {scope_name}: FAILED {status} {res}")

-# Fix built-in 'openid' scope to include preferred_username (needed by Grafana, Forgejo, etc.)
-# Note: sub MUST be a string, not int, for JSON compatibility with Go struct unmarshaling
-if "openid" in existing_scopes:
-    openid_scope = existing_scopes["openid"]
-    openid_pk = openid_scope["pk"]
-    openid_expression = 'return {"sub": str(user.pk), "preferred_username": user.username}'
-    if openid_expression not in openid_scope.get("expression", ""):
-        status, res = api("PUT", f"/api/v3/propertymappings/provider/scope/{openid_pk}/", {
-            "name": openid_scope["name"],
-            "scope_name": "openid",
-            "expression": openid_expression,
-        })
-        if status in (200, 201):
-            print(f"  openid: updated (sub as string + preferred_username)")
-        else:
-            print(f"  openid: UPDATE FAILED {status} {res}")
-    else:
-        print(f"  openid: already correct (sub as string)")
-
 # ===========================================================================
 # Phase 4: Get flow UUIDs (needed for providers)
 # ===========================================================================
@@ -38,9 +38,6 @@
    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local

    kubernetes cluster.local in-addr.arpa ip6.arpa {