fix(cnpg): scale paperless-db, immich-db, gotify-db to 3 instances

3 CP nodes available — run one CNPG instance per node for proper HA. 2 instances loses quorum on single node failure. memory-db stays at 2 (acceptable for that workload).
2026-09-11 07:56:21 +09:00
79 changed files with 481 additions and 6594 deletions
@@ -2,5 +2,4 @@ creation_rules:
  # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
  # and was committed in plaintext to a public remote.
  - path_regex: k8s/.*secrets?.*\.ya?ml
-    encrypted_regex: ^(data|stringData)$
    age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
@@ -0,0 +1,206 @@
+# Authentik Auth Integration for NextJS
+
+## Current State
+
+### Gateway Auth Status
+
+| Endpoint | Auth Status | Notes |
+|----------|-------------|-------|
+| `/v1/chat/completions` | ❌ **OFF** | LLM routes have no auth middleware |
+| `/v1/embeddings` | ❌ **OFF** | Same - no auth |
+| `/v1/rerank` | ❌ **OFF** | Same - no auth |
+| `X-Service: sqs` | ✅ **ON** | JWT validated via `internal/auth/jwt.go` |
+| `/workflow` | ❌ **OFF** | Pass-through to Temporal |
+
+**Auth module exists** at `homelab-frontend/internal/auth/jwt.go` but only wired for SQS.
+LLM routes in `internal/proxy/proxy.go` have no auth middleware.
+
+### Authentik App
+
+Authentik app `local-llm` exists for LLM API auth:
+- **Client ID**: `local-llm`
+- **Client Secret**: `kubectl -n llm-serving get secret local-llm-jwt -o jsonpath='{.data.client-secret}' | base64 -d`
+- **Token endpoint**: `https://authentik.riotpiao.com/application/o/token/`
+- **Userinfo endpoint**: `https://authentik.riotpiao.com/application/o/userinfo/`
+- **OIDC discovery**: `https://authentik.riotpiao.com/application/o/local-llm/.well-known/openid-configuration`
+
+## Sign-in Methods
+
+### 1. Resource Owner Password Credentials (ROPC)
+
+Direct username/password login. Server-side only (needs client_secret).
+
+```typescript
+// API Route: app/api/auth/login/route.ts
+const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+  method: 'POST',
+  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+  body: new URLSearchParams({
+    grant_type: 'password',
+    client_id: 'local-llm',
+    client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+    username: '[email protected]',
+    password: 'userpassword',
+    scope: 'openid email profile groups',
+  }),
+});
+
+const tokens = await response.json();
+// { access_token, refresh_token, expires_in, token_type }
+```
+
+### 2. Authorization Code Flow (Browser Redirect)
+
+Requires adding redirect URIs to `local-llm` Authentik app:
+
+```python
+# In k8s/infra/iam/scripts/authentik-provision.py, update:
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",  # dev
+        "https://your-nextjs-app.com/api/auth/callback",  # prod
+    ],
+}
+```
+
+Then standard OIDC flow:
+1. Redirect to `https://authentik.riotpiao.com/application/o/authorize/?client_id=local-llm&redirect_uri=...&response_type=code&scope=openid email profile groups`
+2. User logs in via Authentik UI
+3. Callback receives `code`, exchange for tokens
+
+## JWT Token Persistence
+
+### Browser (localStorage)
+
+```typescript
+const TOKEN_KEY = 'llm_auth_token';
+
+// Save
+localStorage.setItem(TOKEN_KEY, JSON.stringify({
+  access_token: tokens.access_token,
+  refresh_token: tokens.refresh_token,
+  expires_at: Date.now() + tokens.expires_in * 1000,
+}));
+
+// Load
+const stored = JSON.parse(localStorage.getItem(TOKEN_KEY) || 'null');
+if (stored && stored.expires_at > Date.now()) {
+  // Token valid
+}
+
+// Clear (logout)
+localStorage.removeItem(TOKEN_KEY);
+```
+
+### Server-side (HTTP-only cookies)
+
+```typescript
+// app/api/auth/login/route.ts
+import { cookies } from 'next/headers';
+
+// After successful login
+cookies().set('llm_auth_token', JSON.stringify(tokens), {
+  httpOnly: true,
+  secure: process.env.NODE_ENV === 'production',
+  sameSite: 'lax',
+  maxAge: tokens.expires_in,
+  path: '/',
+});
+
+// Read in middleware or API routes
+const tokenCookie = cookies().get('llm_auth_token');
+const tokens = JSON.parse(tokenCookie?.value || 'null');
+```
+
+## Token Refresh
+
+```typescript
+async function refreshAccessToken(refresh_token: string) {
+  const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'refresh_token',
+      client_id: 'local-llm',
+      client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+      refresh_token,
+    }),
+  });
+  return response.json();
+}
+```
+
+## Environment Variables
+
+```bash
+# .env.local
+AUTHENTIK_URL=https://authentik.riotpiao.com
+AUTHENTIK_CLIENT_ID=local-llm
+AUTHENTIK_CLIENT_SECRET=<from-secret>
+
+# For client-side (public)
+NEXT_PUBLIC_AUTHENTIK_URL=https://authentik.riotpiao.com
+NEXT_PUBLIC_AUTHENTIK_CLIENT_ID=local-llm
+```
+
+## Using Token with LLM API
+
+```typescript
+const token = await getValidToken(); // from localStorage or cookie
+
+const response = await fetch('https://api.riotpiao.com/v1/chat/completions', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${token}`,  // JWT from Authentik
+  },
+  body: JSON.stringify({
+    model: 'reasoning',
+    messages: [{ role: 'user', content: 'Hello' }],
+  }),
+});
+```
+
+## TODO
+
+### Gateway-side (homelab-frontend)
+
+- [ ] Wire `internal/auth/jwt.go` into LLM proxy handler (`internal/proxy/proxy.go`)
+- [ ] Add `authRequired: true` to model config or create LLM-specific middleware
+- [ ] Example pattern from SQS (in `internal/serviceadapter/router.go`):
+
+```go
+// In proxy.go ServeHTTP, before dispatching to LLM upstream:
+if strings.HasPrefix(r.URL.Path, "/v1/") {
+    authHeader := r.Header.Get("Authorization")
+    claims, err := llmJWTAuth.ValidateBearerToken(authHeader)
+    if err != nil {
+        // Return 401/403
+    }
+    if !llmJWTAuth.CheckPermissions(claims, "llm:inference", "*") {
+        // Return 403 insufficient permissions
+    }
+}
+```
+
+### Authentik-side
+
+- [ ] Enable ROPC grant in Authentik provider settings (if not already)
+- [ ] Add redirect URIs to `local-llm` app if browser OAuth flow needed:
+
+```python
+# k8s/infra/iam/scripts/authentik-provision.py
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",
+        "https://your-app.com/api/auth/callback",
+    ],
+}
+```
+
+### NextJS-side
+
+- [ ] Until gateway auth is wired, LLM API works without token
+- [ ] Once wired, add `Authorization: Bearer <token>` to all LLM requests
@@ -22,10 +22,6 @@ spec:
      nodeSelector:
        kubernetes.io/hostname: worker-1
      runtimeClassName: nvidia
-      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
-      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
-      # Disable service link injection to avoid the collision.
-      enableServiceLinks: false
      containers:
        - name: comfyui
          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
@@ -35,6 +31,8 @@ spec:
          env:
            - name: NVIDIA_VISIBLE_DEVICES
              value: "all"
+            - name: COMFYUI_FLAGS
+              value: "--listen 0.0.0.0 --port 8188"
          resources:
            requests:
              cpu: "4"
@@ -59,7 +57,7 @@ spec:
            httpGet:
              path: /
              port: 8188
-            failureThreshold: 120
+            failureThreshold: 60
            periodSeconds: 10
      volumes:
        - name: models
@@ -4,24 +4,16 @@ metadata:
  name: comfyui
  namespace: comfyui
  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    # WebSocket support for ComfyUI's live preview
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "upgrade";
+    nginx.ingress.kubernetes.io/proxy-set-headers: "Upgrade"
 spec:
  ingressClassName: nginx
-  tls:
-    - secretName: comfyui-tls
-      hosts:
-        - comfyui.riotpiao.com
  rules:
-    - host: comfyui.riotpiao.com
+    - host: comfy.riotpiao.com
      http:
        paths:
          - path: /
@@ -23,7 +23,7 @@ spec:
      containers:
        # --- Gotify server ---
        - name: gotify
-          image: ghcr.io/gotify/server:3.1.0
+          image: ghcr.io/gotify/server:2.6.1
          command: ["/bin/sh", "-c"]
          args:
            - |
@@ -84,7 +84,7 @@ spec:
        # Watches Gotify WebSocket stream, forwards messages as email.
        # https://github.com/eternal-flame-AD/gotify-broadcast
        - name: smtp-emailer
-          image: ghcr.io/gotify/server:3.1.0
+          image: ghcr.io/gotify/server:2.6.1
          command:
            - /bin/sh
            - -c
@@ -1,59 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: igotify
-  template:
-    metadata:
-      labels:
-        app: igotify
-    spec:
-      containers:
-        - name: igotify
-          image: ghcr.io/androidseb25/igotify-notification-assist:v1.6.0.1
-          ports:
-            - containerPort: 8080
-              protocol: TCP
-          env:
-            - name: ENABLE_CONSOLE_LOG
-              value: "true"
-            - name: ENABLE_SCALAR_UI
-              value: "true"
-            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
-            # left unset — not required when Gotify is reachable via public domain
-            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
-          volumeMounts:
-            - name: data
-              mountPath: /app/data
-          livenessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 10
-            periodSeconds: 30
-          readinessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 200m
-              memory: 128Mi
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: igotify-data
@@ -1,22 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: igotify
-  namespace: notifications
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: igotify.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: igotify
-                port:
-                  number: 80
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: igotify-data
-  namespace: notifications
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  selector:
-    app: igotify
-  ports:
-    - port: 80
-      targetPort: 8080
-      protocol: TCP
@@ -6,9 +6,11 @@ metadata:
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    # WebSocket support for Gotify client connections (/stream endpoint)
-    # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
+    # WebSocket support for Gotify client connections
    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
 spec:
  ingressClassName: nginx
  rules:
@@ -6,7 +6,3 @@ resources:
  - deployment.yaml
  - service.yaml
  - ingress.yaml
-  - igotify-pvc.yaml
-  - igotify-deployment.yaml
-  - igotify-service.yaml
-  - igotify-ingress.yaml
@@ -45,14 +45,6 @@ spec:
      volumeMounts:
      - mountPath: /mnt/models
        name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-embeddings
-        app.kubernetes.io/part-of: llm-serving
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
@@ -52,20 +52,6 @@ spec:
          port: 8080
        - protocol: TCP
          port: 9000
-    # Allow from paperless namespace (paperless-ai auto-tagging)
-    # Bypasses gateway until service-account JWT token exchange is implemented.
-    # paperless-ai-agent has llm:inference role in Authentik.
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: paperless
-      ports:
-        - protocol: TCP
-          port: 8080
-        - protocol: TCP
-          port: 80
-        - protocol: TCP
-          port: 8000
    # Allow intra-namespace (pod-to-pod within llm-serving)
    - from:
        - podSelector:
@@ -44,7 +44,7 @@ spec:
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
-        value: '65536'
+        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
@@ -83,14 +83,6 @@ spec:
      volumeMounts:
      - mountPath: /mnt/models
        name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-ornith
-        app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
@@ -28,14 +28,9 @@ spec:
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
-      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
-      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
-      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
-      # metric after the rollout -- max_num_batched_tokens=8192 also raises
-      # activation memory out of the same budget, so the post-boot
-      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
-      # only valid if it still covers 2x65k + 3x8k. The old
+      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
+      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
+      # above the 131072 target with room to spare -- the old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -59,37 +54,10 @@ spec:
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
-      - --gpu-memory-utilization=0.95
-      - --max-num-seqs=5
-      # Prefill scheduling -- see the measured starvation this fixes.
-      # V1's scheduler has no prefill/decode phase: it walks self.running and
-      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
-      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
-      # per request is applied BEFORE the budget clamp:
-      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
-      #     take = min(take, token_budget)
-      # The loop is `while ... and token_budget > 0`. With the threshold at its
-      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
-      # budget, token_budget hits 0, the loop exits and every other request is
-      # skipped for that step -- measured as 262 tok/s contended prefill and
-      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
-      # Hence threshold must be strictly BELOW the budget or it never binds:
-      # at 8192/8192 a single request is served per step (still starved), while
-      # 8192/2048 serves four. Verified by replaying the loop.
-      - --max-num-batched-tokens=8192
-      - --long-prefill-token-threshold=2048
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
      - --enable-chunked-prefill
      - --enable-prefix-caching
-      #
-      # NOT set, deliberately: --max-num-partial-prefills and
-      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
-      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
-      # rejects any value other than the default for either one, calling
-      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
-      # -> AssertionError, because there is no V0 engine to fall back to.
-      # The Deployment uses strategy Recreate, so setting them is an outage.
-      # V1 does concurrent chunked prefill natively; it needs no flag, only the
-      # budget/threshold ratio above.
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
@@ -136,14 +104,6 @@ spec:
        name: models
      - mountPath: /dev/shm
        name: shm
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reasoning
-        app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
@@ -45,14 +45,6 @@ spec:
      volumeMounts:
      - mountPath: /mnt/models
        name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reranker
-        app.kubernetes.io/part-of: llm-serving
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
@@ -48,8 +48,7 @@ spec:
                  mountPath: /backup
          containers:
            - name: mc-mirror
-              # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
-              image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3
+              image: minio/mc:latest
              env:
                - name: ACCESS_KEY
                  valueFrom:
@@ -20,7 +20,3 @@ data:
  # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
  # on 'account_confirm_email' without this.
  PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
-  # Auto-connect social accounts to existing users with matching email
-  PAPERLESS_SOCIAL_AUTO_SIGNUP: "true"
-  # Allow automatic linking of social accounts to existing users
-  PAPERLESS_ACCOUNT_ALLOW_SIGNUPS: "true"
@@ -4,7 +4,6 @@ namespace: paperless
 resources:
  - pvc.yaml
  - configmap.yaml
-  - paperless-ai-config.yaml
  - redis.yaml
  - deployment.yaml
  - service.yaml
@@ -12,7 +11,6 @@ resources:
  - backup-cronjob.yaml
  - adapter-configmap.yaml
  - rbac.yaml
-  - paperless-ai.yaml
 # postgres: paperless-db CNPG Cluster, deployed by k8s/infra/databases (wave 2,
 # before this app at wave 8) - not duplicated here. Same for the paperless-oidc
 # and paperless-minio-creds Secrets, written by PostSync provisioning Jobs in
@@ -1,49 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-ai-env
-  namespace: paperless
-data:
-  paperless-ai-config.env: |
-    # Paperless-NGX API configuration
-    PAPERLESS_URL=http://paperless.paperless.svc.cluster.local:8000
-    PAPERLESS_API_TOKEN=7b89463e04c141f4172fbcddf78d623547d327ec
-
-    # AI Processing settings
-    ENABLE_AI_PROCESSING=yes
-    AI_MODEL=reasoning
-    LLM_API_URL=https://api.riotpiao.com/v1
-    LLM_API_TOKEN_FILE=/app/data/llm_token.txt
-
-    # Auto-tagging configuration
-    ENABLE_AUTO_TAGGING=yes
-    ENABLE_CORRESPONDENT_EXTRACTION=yes
-    ENABLE_DOCUMENT_TYPE_EXTRACTION=yes
-    ENABLE_TITLE_GENERATION=yes
-    
-    # Advanced tagging with AI
-    ENABLE_INTELLIGENT_TAGGING=yes
-    ENABLE_DATE_EXTRACTION=yes
-    ENABLE_TIMELINE_TAGGING=yes
-    AUTO_CREATE_TAGS=yes
-    
-    # Tagging prompts for custom extraction
-    CUSTOM_EXTRACTION_PROMPT=Extract document date, time period, financial year, and temporal context. Create tags like YYYY, Q1-YYYY, Last-Month, Current-Year, Historic
-    TAG_EXTRACTION_MODEL=reasoning
-
-    # Scanning behavior
-    SCAN_INTERVAL_SECONDS=60
-    PROCESS_EXISTING_DOCUMENTS=no
-    ADD_AI_TAG=yes
-    AI_TAG_NAME=ai-processed
-
-    # Date/Timeline tagging configuration
-    DATE_EXTRACTION_FORMAT=iso8601
-    TIMELINE_TAG_GRANULARITY=year,quarter,month
-    AUTO_TAG_HISTORICAL=yes
-    AUTO_TAG_RECENT=yes
-    
-    # Performance
-    MAX_CONCURRENT_REQUESTS=2
-    REQUEST_TIMEOUT=60
-    DATE_EXTRACTION_TIMEOUT=30
@@ -1,127 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-ai-data
-  namespace: paperless
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 2Gi
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-ai
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-ai
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-ai
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-ai
-    spec:
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-      
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token
-              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              mkdir -p /data
-              echo "$TOKEN" > /data/llm_token.txt
-              echo "[init] Token fetched and stored successfully"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /data
-      
-      containers:
-        - name: paperless-ai
-          image: clusterzx/paperless-ai:latest
-          imagePullPolicy: Always
-          env:
-            # Paperless-ngx connection
-            - name: PAPERLESS_API_URL
-              value: "http://paperless.paperless.svc.cluster.local:8000"
-            - name: PAPERLESS_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: PAPERLESS_API_TOKEN
-            - name: PAPERLESS_USERNAME
-              value: "admin"
-            
-            # LLM API configuration
-            - name: AI_PROVIDER
-              value: "custom"
-            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token file created by init container with JWT from Authentik
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
-            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            
-            # Scanning behavior
-            - name: SCAN_INTERVAL
-              value: "60"
-            - name: PROCESS_PREDEFINED_DOCUMENTS
-              value: "no"
-            
-            # Tagging configuration
-            - name: ADD_AI_TAG
-              value: "yes"
-            - name: AI_TAG_NAME
-              value: "ai-processed"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-            - name: USE_PROMPT_TAGS
-              value: "yes"
-          
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-          
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi
@@ -1,127 +0,0 @@
-# Secret paperless-ai-config managed via SOPS (argocd/secrets/paperless-ai-secrets.enc.yaml)
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-ai
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-ai
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-ai
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-ai
-    spec:
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-        - name: paperless-ai-env-config
-          configMap:
-            name: paperless-ai-env
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          securityContext:
-            runAsUser: 0
-            fsGroup: 0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              # Write to both locations for compatibility
-              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
-              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
-              echo "[init] Token fetched and stored"
-              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
-              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
-          env:
-            - name: LLM_AUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: LLM_AUTH_CLIENT_SECRET
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-      containers:
-        - name: paperless-ai
-          image: clusterzx/paperless-ai:latest
-          env:
-            # Paperless-ngx connection
-            - name: PAPERLESS_API_URL
-              value: "http://paperless.paperless.svc.cluster.local:8000"
-            - name: PAPERLESS_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: PAPERLESS_API_TOKEN
-            - name: PAPERLESS_USERNAME
-              value: "admin"
-            # LLM API — via public gateway with JWT auth
-            - name: AI_PROVIDER
-              value: "custom"
-            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token will be read from file at runtime by the application
-            # The init container fetches it and stores in /app/data/llm_token.txt
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
-            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            # Behavior - scan for new documents and tag them
-            - name: SCAN_INTERVAL
-              value: "60"
-            - name: PROCESS_PREDEFINED_DOCUMENTS
-              value: "no"
-            - name: ADD_AI_TAG
-              value: "yes"
-            - name: AI_TAG_NAME
-              value: "ai-processed"
-            - name: USE_PROMPT_TAGS
-              value: "yes"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-            - name: paperless-ai-env-config
-              mountPath: /app/data/.env
-              subPath: paperless-ai-config.env
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi
@@ -1,24 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: poimen-frontend
-  namespace: poimen
-  annotations:
-    # Next.js SSE + LLM streaming responses need long timeouts + no buffering.
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-buffering: "off"
-    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: poimen.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: poimen-frontend
-                port:
-                  number: 3000
@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Edge Ingress for poimen.riotpiao.com.
-# Same pattern as k8s/apps/api/ (api-gw ingress).
-# App manifests (Deployment, Service, NetworkPolicy) live in
-# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
-resources:
-  - ingress.yaml
@@ -0,0 +1,33 @@
+# ArgoCD Image Updater - auto-updates Application images from registry
+# Watches forgejo.riotpiao.com for new image tags and updates Applications
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-image-updater
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://argoproj.github.io/argo-helm
+      chart: argocd-image-updater
+      targetRevision: "0.11.2"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/argocd-image-updater/values.yaml
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false
@@ -1,55 +0,0 @@
-# Tekton Pipelines v0.68.0
-#
-# Install method: vendored release.yaml in k8s/infra/tekton/
-# downloaded from https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.68.0/release.yaml
-#
-# To upgrade:
-#   1. Download new release.yaml from https://github.com/tektoncd/pipeline/releases
-#   2. Replace k8s/infra/tekton/release.yaml
-#   3. Commit and push — ArgoCD syncs automatically
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: tekton-pipelines
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: tekton-pipelines
-    app.kubernetes.io/part-of: homelab-infra
-    wave: "06"
-spec:
-  project: homelab
-
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/infra/tekton
-
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: tekton-pipelines
-
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m
-
-  ignoreDifferences:
-  - group: admissionregistration.k8s.io
-    kind: ValidatingWebhookConfiguration
-    jsonPointers:
-    - /webhooks/0/clientConfig/caBundle
-    - /webhooks
-  - group: admissionregistration.k8s.io
-    kind: MutatingWebhookConfiguration
-    jsonPointers:
-    - /webhooks/0/clientConfig/caBundle
-    - /webhooks
@@ -152,8 +152,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -175,8 +180,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -201,8 +211,13 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
@@ -36,9 +36,11 @@ metadata:
    app.kubernetes.io/component: gateway
  annotations:
    argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
-    # → patches this Application's kustomize.images via k8s API.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
+    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
  revisionHistoryLimit: 3
@@ -246,8 +246,11 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "8"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
+    argocd-image-updater.argoproj.io/app.update-strategy: digest
+    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
  source:
@@ -1,81 +1,43 @@
-# Poimen — unified ArgoCD Application (wave 7).
-#
-# Consolidates what was previously five separate apps:
-#   poimen            → source 1 (poimen-workflows workers)
-#   poimen-workflows  → duplicate of source 1 (deleted)
-#   poimen-memory     → source 2 (memory service + redis)
-#   poimen-memory-app → duplicate of source 2 (deleted)
-#   poimen-memory-database → source 3 (CNPG cluster, moved to k8s/infra/poimen)
-#
-# poimen-toolkit is intentionally kept as a separate Application because it
-# manages cluster-scoped CRDs (ToolDefinition). A bad CRD can affect the
-# whole cluster; keeping it separate means it has its own sync history,
-# independent rollback, and can be paused without touching app workloads.
-#
-# Sync ordering via argocd sync-wave annotations:
-#   wave -1 → DB (CNPG Cluster) — must be Ready before app pods start
-#   wave  0 → all application Deployments (default)
-#
-# Image Updater tracks all four images. Write-back via argocd (in-cluster
-# annotation patch — no git commit per image update).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: poimen
  namespace: argocd
-  labels:
-    app.kubernetes.io/part-of: poimen
  annotations:
    argocd.argoproj.io/sync-wave: "7"
-    # ── Image promotion ──────────────────────────────────────────────────────
-    # Forgejo package webhook → api-gw patches kustomize.images per source.
-    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: digest
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
-  # Multi-source: each repo owns its own k8s manifests.
-  # ArgoCD syncs all sources in parallel; sync-wave annotations on individual
-  # resources control ordering within the sync operation.
  sources:
-    # 1. Workers — Temporal activity + workflow workers, pi-worker pool
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
-      targetRevision: main
-      path: k8s
-
-    # 2. Memory service — poimen-memory API, Redis cache, NetworkPolicies
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
      targetRevision: main
-      path: k8s/app
-
-    # 3. Database infrastructure — CNPG Postgres cluster (wave -1)
-    #    Moved from k8s/infra/databases to isolate poimen DB from other tenants.
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
      targetRevision: main
-      path: k8s/infra/poimen
-
-    # 4. Frontend app manifests — Deployment, Service, NetworkPolicy.
-    #    Each repo owns its own k8s/ (same pattern as homelab-frontend for api-gw).
+      path: k8s/argocd
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
      targetRevision: main
-      path: k8s
-
-    # 5. Frontend ingress — poimen.riotpiao.com edge route.
-    #    Lives in homelab (same pattern as k8s/apps/api/ for api-gw).
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      path: k8s/apps/poimen-frontend
-
+      path: k8s/argocd
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
-
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
-      - SkipDryRunOnMissingResource=true
    retry:
      limit: 5
      backoff:
@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: poimen-toolkit
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Tracks: poimen-toolkit-controller, pi-worker.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
-    targetRevision: main
-    path: k8s
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: poimen
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m
@@ -24,7 +24,6 @@ spec:
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
    - https://forgejo.riotpiao.com/rock/riotpiao.com.git
    # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
    - https://cloudnative-pg.github.io/charts
@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: forgejo-registry-pat
-    namespace: api
-type: Opaque
-stringData:
-    pat: ENC[AES256_GCM,data:alYPvHBVqnf13NZu6jtgA6WdFM4x46s5mIY8uEnfWdq0Vp+7oKrY1A==,iv:JbUjes5ZB9eKQ1nkV02++7F/BaWIROfT6gjShzajXro=,tag:ApjawJold0QApnX1GUXGhw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6bFZhU3JLSUZEVWhYdEdZ
-            MURoSWNlZjY0RkxyaWlwU2ZTVjJ5V2FnMVVZCis5YlBwQnZrWnVJNFh4UnNldmx6
-            Y2JjdnpkRTVGSjRSZ0U0THdDZHIrNHcKLS0tIHZDd1BDVGRDSHJEQThuTExORWs2
-            b3BxenY4WHhpRkRTZHJGMkNYMm14RnMKH//Xpqbf/ZBNFQ43D0eFUWwRcQWqA3CC
-            RG8U9wyxC6YrlruU2hY6eB2x0OjsVic34ZGaR752Bwyp0Pz49C/k3g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-28T03:10:55Z"
-    mac: ENC[AES256_GCM,data:nLZueHygckw+Gddlea90tRU8h8nEGpe21JUi9nCdpc0ctkSlTDqcR3Q7JFPwbEGHGFU5UJ2xnkBZEDIVmExJryqPBMXT7hQd3ySeQgPwAPGt6zEhlRc5jXhqmmrpJiiN5XUL52AZVDwcWACg7kVzBXIClzihFRW+UqjJZhdtELM=,iv:C0H4AFfdfpsBvP1z2chVvW/VTejxYxwNn/KuA50LeH0=,tag:ZEkTCTKvoZH7DjrcwfL3dw==,type:str]
-    version: 3.13.2
@@ -1,24 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:uS8=,iv:EEoo9U+C244eAJMSTOQVkf5AE6BeHrc5DWPjtWnPRdk=,tag:H+YmBSGvcON3wh0p22LXDQ==,type:str]
-kind: ENC[AES256_GCM,data:j1/PFkTS,iv:ja4q8X+nzE/ZczwcY+Qe2DnnsxG64W1fkRPQvOaoqvA=,tag:WiilMGagudEKUlc2JdbGyg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:J9iAIHboMyHu6xn/,iv:p3z3uzlkM7VDzOT3WDCelCdZ2t+QqSPmFtqYfvXPQMs=,tag:rKRtpUexVkuZKOJ34a0Xjw==,type:str]
-    namespace: ENC[AES256_GCM,data:su0FvA==,iv:6SPvwxZ/4aoL0Z07zdPC9r6L7HOHuKv3RodXpVcii04=,tag:njwkj+yvSgN8sliJP8T/Kw==,type:str]
-type: ENC[AES256_GCM,data:Qd6WJN1c,iv:M3fc78LvemcEbWzesuYeQ/GZyIOl7Eg/0EmUe3p0qAk=,tag:Tnzwewn0vwdowxy/EyuPdA==,type:str]
-stringData:
-    user: ENC[AES256_GCM,data:/Z7gPrsTUZOLzfoZ8cZGD10wrZE=,iv:0ydBSeq+yHB2b1J4W7FwIv55Eh+N3qfQ6Q7PwoUAvYo=,tag:CO19F8nElaYZyFiFR6N/Tg==,type:str]
-    password: ENC[AES256_GCM,data:AOl4StpeQIHSLX+oEFnkfg==,iv:mYsZ+5sum6YqyUPndtPCniTraxldKc803ULlKs8Gsaw=,tag:hK3w51ifZ/omAL7XDf8seg==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaMlhKVEM2bFdFOFAwV1lp
-            WUd6SEN4ZTF1TEpGYXhOYmJLK0tpcWZyc2w4ClVzZmI2Qk1KUnV1OXpyTEV2WWFa
-            VlJ2eHRSaEhqUnA2dUJVbWJUcEgxcFkKLS0tIG9IaFVnenNpSFRzdStiWjJvakVL
-            aDk2bTZGR3Zya3ROUS9vd1hEQVRFaG8KbeXA6IebHEaB79N6u795336aHesHOgzO
-            uZvvBUzSBy3t3jfFk8bJP4aH79I33Ha2eK5rsvdsiv/orwCMXUINKg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T21:00:46Z"
-    mac: ENC[AES256_GCM,data:DjzPtR+Ueihh166Bvd3jCLtZFQdrvrxOoF87cv6lxKGWPEptT5vbw/EfuIFJBb6lvEJFDWKRC/r5ASdGwComYsPn0DZs4BPCzeKBtLfP9K2OGCXnAC7eGqt4mzMrrW0CQd1QSGcuXw8CY7uJGkMPGPKe3/0mQWiis2OSpHxTM18=,iv:/QqSEFU9RuX9z5Z6aSaD7KlP/cgGTOYvTH+VJOnDTYM=,tag:yFbFXn+iWqQZx7wW4dKppg==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2
@@ -1,28 +1,30 @@
-apiVersion: ENC[AES256_GCM,data:FOg=,iv:15mfPeWXV5LQEYahaYvNf1z2bHbxk4j5i8DXT6mdG/0=,tag:1f9/jnnu+wGeeiXGNFgKzA==,type:str]
-kind: ENC[AES256_GCM,data:dOFfNQiM,iv:HVLosbRpcCgu4iiYKV8MDLiQ0uhj8DXBfVpRBW2NFNo=,tag:mCbRIzD1OHRhBLc+7xcVug==,type:str]
+apiVersion: ENC[AES256_GCM,data:xUA=,iv:YH1WYnoXhxbeES3Dmo8ZrMDwFcwCykt6+Zn4YSgN4gc=,tag:t3bIWo/LetlQv4tJ0nJ2Sw==,type:str]
+kind: ENC[AES256_GCM,data:PJVzDTcT,iv:aRxvzLUuCzSDryOjA/v67csL6E2FcCpoK8aD7KNStno=,tag:cQgbowSTTV99nQ41yFOlsA==,type:str]
 metadata:
-    name: ENC[AES256_GCM,data:kHoKhGyiIQvCfng=,iv:dAhxjeLlXK3yueMKfrHxmh9YmNA7AYKFBquLikCNzcE=,tag:3Xoaw7YNBCU/GINlaQOpBw==,type:str]
-    namespace: ENC[AES256_GCM,data:fTJMZhbqSQWD3/cnWg==,iv:D7zWa91+Fgerfyrywf8VA5YNzGrThhLz7CvYXucfiq0=,tag:RcIegCl2UR1JsbfIQm928w==,type:str]
-type: ENC[AES256_GCM,data:Qh51bTsM,iv:2Htv0Xze7Hd1m6zkP6rtFXAlg8qm0AKylSjwJxRjpBk=,tag:NCu5iVUTNVfYgErZSvwHIg==,type:str]
+    name: ENC[AES256_GCM,data:atkSiP0rjyuWFjs=,iv:l/48X9W9lrkAyw6Q/p39V775L6xbr0tOIFI4vjYrg+E=,tag:LZrQxN5h6T5tehFG8L9KFw==,type:str]
+    namespace: ENC[AES256_GCM,data:fbklNyooBeqsogKbeg==,iv:z2wLgl81q8xfzGGVQ9A4Pkst/zLmuZpyBzHrXzmVfm4=,tag:QKJik4IWc4nrOBINjGCxvQ==,type:str]
+type: ENC[AES256_GCM,data:bfAFBd/7,iv:9/W3X1KefqDAsurIatT6kBuEIQvWVRTq4G9A+9ZfYKs=,tag:Optlb+gZ2udbqIOerqV2zQ==,type:str]
 stringData:
-    host: ENC[AES256_GCM,data:aZXSvQ8B7h78q1kHmh0=,iv:uUyL9X3ehIHqztGAtXtAQWgduvbSsSwZBZlTFMdOlBo=,tag:OM/vdO19VKDSa4W5WQAKNQ==,type:str]
-    port: ENC[AES256_GCM,data:5ZBB,iv:1/XAfq+PJKdBsufx+EPvvB/cm9nbEwYigc8eACXjR8g=,tag:WlNyz7gTPh0KMe5oKVd0BA==,type:str]
-    from: ENC[AES256_GCM,data:2/akr8cXgmgQGnqJaFcLJMDcWw==,iv:0rT/6aqyXxn1jIJ5umYCDZR4S8Pbh4vUgaXNrxd3JF8=,tag:k0pQrPOrqWTyE1Xu9oSzVA==,type:str]
-    user: ENC[AES256_GCM,data:d9BLaFYOYm++HZMzlf1gVauKkUQ=,iv:Wd48T5UPVn6z3bS0t02X9GbrnWal3QoIQv2EgM9z9Wg=,tag:cmg2KnLY4Atwco+j2wVdeg==,type:str]
-    password: ENC[AES256_GCM,data:pmqEj9623A6bThKrkCCjuA==,iv:M/QwJ0s//UvuOjOljl67EDhvZt/9Wp8JicvCcows51A=,tag:bfRyIdj1cgIZxU3WPbKteg==,type:str]
-    notify-email: ENC[AES256_GCM,data:+Xo22g8U2wDKsnPnFq/+GKxgYOI=,iv:PE+C1etlp6046ragiv1iMDQbhfo5IULd/5akD0+Sc6A=,tag:cCZmXByA85fuZL9yozVLjw==,type:str]
+    #ENC[AES256_GCM,data:RZaw/FNQZOf0F3+XfFmGW+4gjKXNdjrAbMRjBt4hptwk,iv:IEd/+Bsv/h08nKak+bcovNiohC7nNjEQ+QdtIFMUGng=,tag:zsF5ZQ3+5JijF4MU+RhnyA==,type:comment]
+    #ENC[AES256_GCM,data:ux3ONHnuI0uFEihElbLI1rKlrN91Pn0iLLZfmez0+t5d8yhH6OD3L/3oi6yHbdgD8bV90kRK/lw=,iv:7Ed+W/TeOY9Af8bKO0Fu3XOuaK8b0Hw70rwobfmosxM=,tag:Z0RbIOjdDroK0drC6P+Y8A==,type:comment]
+    host: ENC[AES256_GCM,data:YKgas/8hT5ZOGIW7O0c=,iv:WtBz1MFrwd6EgDDS5fMqtcVGIZGNLgbXnmkqlNWoXy8=,tag:+H9ARGPi5XocgZYp4oA3wA==,type:str]
+    port: ENC[AES256_GCM,data:f0hx,iv:HkSnJcCyzyfBTA39BNw+muhHTyrNLzzGxZ7M3bcLVu4=,tag:4+xL1xNyYPaV2b0mwnbQVQ==,type:str]
+    from: ENC[AES256_GCM,data:lqYqXxJuI4CAWPlX+5cltCdxqA==,iv:ZgZYySJXvjuTNIkMaDxynUJ6PFdp4fibqlgoA1vNDKE=,tag:vb/XKl2N1Hl5u7ndZXQvbg==,type:str]
+    user: ENC[AES256_GCM,data:ajljGVbRO//G,iv:E4Q0LGDCd5wnHc+Oq2A9ZhWGbCmkKlhllt922UmppIg=,tag:Cf4ft1JKFi9PaJEfPfq92Q==,type:str]
+    password: ENC[AES256_GCM,data:YNbtoiWLsQGl,iv:FJ3y1lWkqKkM8GVr7FZljAXJ331GkDgy4TenAbdIY3w=,tag:KIvhv+RL49ZMwWhYrU0SPA==,type:str]
+    notify-email: ENC[AES256_GCM,data:s5wRkDuaiflCyu4IyAnaocVPA3I=,iv:dK5PsweYa3JotYcDcz74DPNDXmhW5ngNEbY6JuG1ctg=,tag:/Dt2UoCtfau6qv0G7VzmrQ==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpVVdxQTZFTHcwMXFFZEFk
-            ckNsYk4rdWtkeXFMVnlyVmdaRDBuRWsvZkQ4CitSSTBXOTZhWVpUZFFlR0JITVV4
-            Uy9lSlNHNjFNaFhHNTN0WnRaRlNNVGcKLS0tIHRlMnZpQVlScGRYYm5nT3Z5TWd6
-            ak1UZ1RobkpQZHBXR3MyenBDcU53Mk0KvH9O6bgwrjay0+1/A6TGX8GhITiDjWoO
-            RNX4fDtqNwhzmCfXbVjK30vlBjOFe+Bb7Z2n+hWMmHHDgFdS0xIAzA==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQlJVekcwb01ESnhxSE9m
+            MzliVmlaYUttZVdiMDVtMlQ2L29LMXBWRzBZCkxRMi9RcE5LMzQ1MHFqUUxLaFZo
+            MUZ2UTg5eUNsMFVmenlhL2F3NDd0akkKLS0tIGFwbUUrSGFUTDFVOWlxM3V1bVdy
+            RUVVNDFMRHZCSWptdSt1bFVoN2JPWVkKXMlrJNPbm7Sj9vd/H3o6iqPRLy4LyjVe
+            xJyh9vn9RCwS9Jr6BN2Q9P2UZfTGwvIYc6zdmV+b2uRwvi4t4lMsBA==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T20:57:18Z"
-    mac: ENC[AES256_GCM,data:V/wgjnpUBvaV39BCTlecCwQy2/1h+0vixEXleJc/I9y+AOvuwVZNH7M86hdjoAdsKiIoNYySj4Flz+MJ9C8jQoAxBTDPbolP9UwDFWQ5KMJTKPPDoJGLNjy7bUq51WoyePUM6WWAYIiWHIB3OvAxHaSzECzL+ZoAhQy92cPKa1o=,iv:vLtIoD/C5yeXPEr+2Lim6XnWzAj42vr9qQgsxKpuhA8=,tag:PBq8qPHrx4RRgAKCWrxG3Q==,type:str]
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:g1YjOANlnvXErPqtDdDjRB9yTUiGEhX8QDkmf1aLfQBwoXbGUpoPIZibF8A4cAL1yUSZ2rD4l9TIe9nK8U3oF4WiKXf9u0bFK2g//bOv2A5JPewwmzoazkjIDXcJ/76nOQ5c+v8/r5tUG4c0SyzBH/gFP0D0+P5Yo3SEXjTxj2w=,iv:gW+AUwwAWkXZ+aLBXjjNDZw8VEraMpSE16RJGQc/L5g=,tag:DM/DUtil/7yLhfGYQXKFGA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: paperless-ai-config
-    namespace: paperless
-type: Opaque
-stringData:
-    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
-    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
-            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
-            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
-            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
-            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-14T14:19:39Z"
-    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
-    version: 3.13.2
@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: pi-code-agent-api-keys
-    namespace: poimen-sandbox
-stringData:
-    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
-    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
-            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
-            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
-            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
-            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T01:19:49Z"
-    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
-    version: 3.13.2
@@ -12,7 +12,6 @@ files:
  - authentik-secrets.enc.yaml
  - cloudflare-secrets.enc.yaml
  - forgejo-registry-pat.enc.yaml
-  - forgejo-registry-pat-api.enc.yaml
  - forgejo-registry-pull.enc.yaml
  - forgejo-runner-token.enc.yaml
  - forgejo-secrets.enc.yaml
@@ -31,6 +30,3 @@ files:
  - gotify-admin-secrets.enc.yaml
  - gotify-tokens-secrets.enc.yaml
  - gotify-smtp-secrets.enc.yaml
-  - forgejo-smtp-secrets.enc.yaml
-  - paperless-ai-secrets.enc.yaml
-    - pi-sandbox-secrets.enc.yaml
@@ -12,7 +12,6 @@ defaultSettings:
  replicaSoftAntiAffinity: false  # REQUIRED for true HA
  replicaAutoBalance: best-effort
  storageMinimalAvailablePercentage: 10
-  storageOverProvisioningPercentage: 200  # Actual usage is ~10% of scheduled; 200% unblocks all 3-replica scheduling on cp-1
  
  # Performance tuning
  defaultDataPath: /var/lib/longhorn
@@ -46,12 +46,6 @@ gitea:
      SSH_PORT: 2222
      SSH_LISTEN_PORT: 2222
      
-    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
-    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
-    # the admin only; admin create-user via UI/API is unaffected by this flag.
-    service:
-      DISABLE_REGISTRATION: true
-
    database:
      DB_TYPE: postgres
      HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -86,14 +80,6 @@ gitea:
    actions:
      ENABLED: true

-    mailer:
-      ENABLED: true
-      PROTOCOL: smtp+starttls
-      SMTP_ADDR: smtp.gmail.com
-      SMTP_PORT: 587
-      FROM: "Forgejo <[email protected]>"
-      # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
-
 # Persistence (shared storage for repos)
 persistence:
  enabled: true
@@ -162,18 +148,3 @@ deployment:
        secretKeyRef:
          name: forgejo-db-app
          key: password
-    - name: GITEA__MAILER__USER
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-smtp
-          key: user
-    - name: GITEA__MAILER__PASSWD
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-smtp
-          key: password
-    # Allow webhooks to reach api-gw through the cluster ingress (resolves to
-    # internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
-    # api.riotpiao.com is our own gateway — safe to allowlist.
-    - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
-      value: "external,api.riotpiao.com"
@@ -0,0 +1,72 @@
+# ArgoCD Image Updater configuration
+# Watches Forgejo registry and updates ArgoCD Applications with new image tags
+
+config:
+  # Registry configuration - Forgejo allows anonymous pulls
+  registries:
+    - name: forgejo
+      api_url: https://forgejo.riotpiao.com
+      prefix: forgejo.riotpiao.com
+      default: true
+      insecure: false
+
+  # Log level
+  logLevel: debug
+
+  # ArgoCD API server
+  argocd:
+    grpcWeb: true
+    serverAddress: argocd-server.argocd.svc.cluster.local
+    insecure: true
+    plaintext: true
+
+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
+# Extra environment variables
+extraEnv:
+  - name: ARGOCD_GRPC_WEB
+    value: "true"
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
+    value: "true"
+
+# Resources
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs every 2 minutes. Deletes:
-#   - Failed Jobs older than 2m (any namespace)
-#   - Completed standalone Jobs older than 2m with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
+# Runs daily at 04:00 UTC. Deletes:
+#   - Failed Jobs older than 24h (any namespace)
+#   - Completed Jobs older than 72h with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted state older than 1h
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
-# and linger forever. Aggressive schedule keeps the cluster clean.
+# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
+# and linger forever.

 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
  labels:
    app: stale-job-cleanup
 spec:
-  schedule: "*/2 * * * *"
+  schedule: "0 4 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
@@ -74,9 +74,7 @@ spec:
                  set -e
                  NOW=$(date +%s)

-                  TTL=120  # 2 minutes in seconds
-
-                  echo "=== Cleaning failed Jobs older than 2m ==="
+                  echo "=== Cleaning failed Jobs older than 24h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.conditions[]?.type == "Failed") |
@@ -84,15 +82,15 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 24 ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
+                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.succeeded >= 1) |
@@ -100,20 +98,20 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 72 ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
+                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
+                  # Evicted pods show as Failed with reason Evicted
                  kubectl get pods --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(
                        .status.phase == "Failed" or
-                        .status.phase == "Succeeded" or
                        (.status.reason // "") == "Evicted" or
                        (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                      ) |
@@ -121,9 +119,9 @@ spec:
                      "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - POD_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
+                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 1 ]; then
+                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
                      kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                    fi
                  done
@@ -8,4 +8,6 @@ resources:
  - namespaces.yaml
  - authentik-db.yaml
  - temporal-db.yaml
+  - memory-db.yaml
  - paperless-db.yaml
+  - obsidian-vault-pvc.yaml
@@ -9,7 +9,7 @@ metadata:
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  instances: 2
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
  bootstrap:
    initdb:
@@ -0,0 +1,18 @@
+---
+# Obsidian vault PVC — shared storage for REST API + UI pods
+# ReadWriteMany so both obsidian-server and obsidian-ui can mount simultaneously
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: obsidian-vault
+  namespace: poimen
+  labels:
+    app.kubernetes.io/name: obsidian-server
+    app.kubernetes.io/part-of: poimen-memory
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi
@@ -1,25 +0,0 @@
-# CiliumNetworkPolicy for kube-apiserver access.
-#
-# Standard K8s NetworkPolicy ipBlock rules don't work for the API server
-# under Cilium — the except clause on 192.168.1.0/24 blocks the post-DNAT
-# destination even when a separate rule re-allows a /32 or subnet.
-#
-# Cilium's native `kube-apiserver` entity resolves this correctly: it
-# tracks the API server endpoints regardless of ClusterIP vs node-IP
-# routing, so the policy stays valid across node changes and NAT paths.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: {{ .Release.Name }}-apiserver
-  namespace: {{ .Release.Namespace }}
-spec:
-  endpointSelector:
-    matchLabels:
-      app: {{ .Release.Name }}
-  egress:
-  - toEntities:
-    - kube-apiserver
-    toPorts:
-    - ports:
-      - port: "6443"
-        protocol: TCP
@@ -153,11 +153,9 @@ server:
  # checks aren't treated as failures. (Only these fields are overridden; the
  # chart deep-merges the rest of each probe, incl. the httpGet path.)
  livenessProbe:
-    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
    timeoutSeconds: 15
    failureThreshold: 6
  readinessProbe:
-    initialDelaySeconds: 300  # skip probe until migrations complete
    timeoutSeconds: 15
    failureThreshold: 6
  startupProbe:
@@ -217,13 +215,6 @@ worker:
  podAnnotations:
    configmap.reloader.stakater.com/reload: "homelab-ca"
    homelab.io/restart-at: "2026-06-21T13-40"
-  livenessProbe:
-    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
-  readinessProbe:
-    initialDelaySeconds: 300  # skip probe until migrations complete
-  startupProbe:
-    initialDelaySeconds: 30  # let server finish DB work first
-    failureThreshold: 120
  metrics:
    enabled: true
    serviceMonitor:
@@ -73,7 +73,7 @@ grafana.ini:
  auth.generic_oauth:
    enabled: true
    name: Authentik
-    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
+    allow_sign_up: true
    client_id: grafana
    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
@@ -84,13 +84,13 @@ grafana.ini:
    # Authentik doesn't serve — request 404s with "Error getting email address"
    # and the whole OAuth login fails.
    email_attribute_path: email
-    login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
+    login_attribute_path: preferred_username
    name_attribute_path: name
    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
-    skip_org_role_sync: false  # Sync org roles from OAuth groups
+    skip_org_role_sync: false
    tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod

 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
@@ -117,22 +117,6 @@ spec:
            echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
          fi

-          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
-          # clear eviction flag that was set when scheduling was disabled.
-          echo "Configuring worker-1 disk..."
-          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
-            "spec": {
-              "disks": {
-                "default-disk-1030400000000": {
-                  "allowScheduling": true,
-                  "evictionRequested": false,
-                  "storageReserved": 429496729600
-                }
-              }
-            }
-          }'
-          echo "  ✓ worker-1 disk configured (400Gi reserved)"
-
          echo
          echo "Waiting for disks to be ready..."
          sleep 10
@@ -35,5 +35,8 @@ parameters:
  mkfsParams: "-O ^64bit,^metadata_csum"
 mountOptions:
  - "noatime"
+  # Critical: mount with postgres UID/GID (26:26) to avoid permission issues
+  - "uid=26"
+  - "gid=26"
 reclaimPolicy: Delete
 volumeBindingMode: Immediate
@@ -1,25 +1,33 @@
-# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
+# Longhorn Node CRDs for cp-2 and cp-3.
+# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
-# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
-# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
-# for general replica scheduling).
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
-# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
-# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
+# What these objects are actually for is `allowScheduling: true` on tainted
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-2, namespace: longhorn-system }
-spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-2
+  namespace: longhorn-system
+spec:
+  name: talos-cp-2
+  allowScheduling: true
+  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-3, namespace: longhorn-system }
-spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
---
-apiVersion: longhorn.io/v1beta2
-kind: Node
-metadata: { name: worker-1, namespace: longhorn-system }
-spec: { name: worker-1, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-3
+  namespace: longhorn-system
+spec:
+  name: talos-cp-3
+  allowScheduling: true
+  tags: []
@@ -59,8 +59,7 @@ spec:
              mountPath: /shared
      containers:
        - name: provision
-          # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
-          image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3
+          image: minio/mc:latest
          volumeMounts:
            - name: shared
              mountPath: /shared
@@ -82,9 +81,6 @@ spec:
              mc alias set m http://minio-cluster-hl.storage.svc.cluster.local:9000 \
                "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"

-              echo "Ensuring paperless bucket exists..."
-              mc mb --ignore-existing m/paperless
-
              echo "Checking for existing paperless-minio-creds secret..."
              if kubectl -n paperless get secret paperless-minio-creds >/dev/null 2>&1; then
                ACCESS_KEY=$(kubectl -n paperless get secret paperless-minio-creds -o jsonpath='{.data.ACCESS_KEY}' | base64 -d)
@@ -11,12 +11,7 @@ metadata:
    prometheus.io/port: "9000"
    prometheus.io/scrape: "true"
 spec:
-  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
-  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
-  # backend could not resolve minio-cluster-hl. Official images now need registry auth
-  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
-  # release; swap it for the official image once MinIO registry credentials exist.
-  image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z
+  image: minio/minio:RELEASE.2025-07-23T15-54-02Z

  # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
  # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -117,16 +112,10 @@ spec:
          name: minio-oidc
          key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
-      value: "groups"
-    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
-      value: ""
+      value: "policy"
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: "https://minio.riotpiao.com/oauth_callback"
    - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
      value: "Authentik"
    - name: MINIO_IDENTITY_OPENID_SCOPES
-      value: "openid,profile,email,groups"
-    - name: MINIO_BROWSER_LOGIN_ANIMATION
-      value: "off"
-    - name: MINIO_BROWSER_REDIRECT_URL
-      value: "https://minio.riotpiao.com"
+      value: "openid,profile,email,minio"
@@ -158,66 +158,3 @@ spec:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
-
-    - name: api-gateway.admission
-      # The admission gate fails OPEN when its view of the engine gauges goes
-      # stale, so a broken observer degrades silently to the pre-#64 behaviour
-      # of admitting everything. These two series are the only detection —
-      # the gate has no automated tests.
-      rules:
-        # The gate fails OPEN when it cannot read the engine, so "no data" must
-        # alert: otherwise a missing emitter, a renamed metric, or a broken
-        # scrape is indistinguishable from a healthy observer. Both expressions
-        # therefore OR in an absent() guard on the metric itself.
-        - alert: APIGatewayObserverDegraded
-          expr: observer_degraded == 1 or absent(observer_degraded)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
-            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
-
-        # Engine gauges went stale, or are not being reported at all
-        - alert: GatewayEngineStateStale
-          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
-            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
-
-        # The acceptance metric for reasoning.yaml's prefill change. Preemption
-        # of a 65k prompt costs a full re-prefill (~250s at the measured
-        # 262 tok/s), so any sustained rate is a regression, not noise.
-        - alert: LLMPreemptions
-          # Guarded absent() is deliberately NOT used bare here: the engine
-          # restarts (strategy Recreate) and the series legitimately vanishes
-          # for the length of a boot, so a bare absent() would false-fire on
-          # every rollout. It only fires when the metric is missing WHILE the
-          # predictor is up, which is the real "renamed or unscraped" case.
-          expr: |
-            increase(vllm:num_preemptions_total[15m]) > 0
-            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
-            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
-
-        # This change trades single-request latency for fairness: capping a long
-        # prompt at 2048 tokens/step means it takes more steps to finish, so a
-        # lone large prompt can now be slower even while preemptions stay 0.
-        # Without this alert the change has no way to show a regression.
-        - alert: LLMPrefillStarvation
-          expr: |
-            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
-            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
-            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."
@@ -1,76 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: cag-kv-cache-rules
-  namespace: monitoring
-  labels:
-    release: prometheus
-spec:
-  groups:
-    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
-    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
-    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
-    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
-    # prefix_cache_queries_total — so this rule produced no data and
-    # VLLMPrefixCacheHitRateLow could never fire.
-    - name: vllm.recording
-      rules:
-        - record: vllm:prefix_cache_hit_rate
-          expr: |
-            rate(vllm:prefix_cache_hits_total[5m])
-            /
-            clamp_min(
-              rate(vllm:prefix_cache_queries_total[5m]),
-              0.001
-            )
-          labels:
-            job: "reasoning-predictor"
-
-    # Alert rules: CAG pipeline health
-    - name: cag.alerts
-      rules:
-        # KV warmup consistently failing — cache never gets warm
-        - alert: KVWarmupFailureHigh
-          expr: |
-            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
-            description: |
-              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
-              for 5 consecutive minutes. The prefix KV cache is not being warmed;
-              agents will see full prefill latency on every call.
-              Check reasoning-predictor health and poimen-memory NetworkPolicy.
-
-        # Warmup misses: vLLM reachable but not caching
-        - alert: KVWarmupMissHigh
-          expr: |
-            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
-            description: |
-              vLLM is reachable but cached_tokens=0 on warmup responses.
-              Possible causes: prefix caching disabled in vLLM, max-num-seqs
-              saturated during warmup, or context format changed.
-
-        # Prefix cache hit rate low after warmup should be established
-        - alert: VLLMPrefixCacheHitRateLow
-          expr: |
-            vllm:prefix_cache_hit_rate < 0.70
-          for: 10m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
-            description: |
-              vLLM prefix cache hit rate below 70% for 10 minutes.
-              Expected >= 85% in steady state with CAG warming active.
-              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.
@@ -26,23 +26,20 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
    return p


-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
    targets = []
    for e in exprs:
        if isinstance(e, tuple):
            targets.append({"expr": e[0], "legendFormat": e[1]})
        else:
            targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    p = {
+    return {
        "id": id, "title": title, "type": "timeseries",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "fieldConfig": {"defaults": {"unit": unit}},
        "targets": targets,
    }
-    if description:
-        p["description"] = description
-    return p


 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -268,237 +265,6 @@ def build_api_gateway():
    }


-# ============================================================================
-# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
-# ============================================================================
-#
-# Two independent metric sources, deliberately both shown:
-#
-#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
-#                               that expose /metrics. Ollama (ornith:35b,
-#                               qwen2.5:3b-instruct) exposes nothing at all.
-#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
-#                               ones, derived from the SSE byte stream: TTFT is
-#                               request-start to first byte, ITL is the gap
-#                               between SSE writes carrying "data: ", so it
-#                               counts SSE chunks rather than true tokens.
-#                               vLLM's own histograms remain authoritative for
-#                               `reasoning`.
-#
-# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
-# verified against the running gateway and Prometheus:
-#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
-#
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
-# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
-# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
-# homelab-frontend#63 fixed exactly that — the wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
-# The "dead code" note that used to live in this header is superseded.
-#
-# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
-# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
-#
-#
-# All metric names below were verified against a live scrape, not assumed:
-#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
-#   NOT cache_query_hit_total / cache_query_miss_total.
-
-def build_llm_observability():
-    def hq(metric, q, by="le"):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
-            f"p{int(q * 100)}",
-        )
-
-    def hq_gw(metric, q):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
-            f"p{int(q * 100)} · {{{{model}}}}",
-        )
-
-    def avg_of(metric):
-        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
-        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
-
-    panels = []
-
-    # ── TTFT ────────────────────────────────────────────────────────────────
-    panels.append(row(100, "Latency — Time To First Token", 0, [
-        stat_panel(101, "TTFT p95 (gateway)",
-                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
-                   0, 1, w=4, h=4, unit="s"),
-        stat_panel(102, "TTFT p95 (vLLM)",
-                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
-                   4, 1, w=4, h=4, unit="s"),
-        stat_panel(103, "KV cache in use",
-                   "max(vllm:kv_cache_usage_perc)",
-                   8, 1, w=4, h=4, unit="percentunit"),
-        stat_panel(104, "Requests running",
-                   "sum(vllm:num_requests_running)",
-                   12, 1, w=4, h=4, unit="short"),
-        stat_panel(105, "Requests waiting",
-                   "sum(vllm:num_requests_waiting)",
-                   16, 1, w=4, h=4, unit="short"),
-        stat_panel(106, "Preemptions (5m)",
-                   "increase(vllm:num_preemptions_total[5m])",
-                   20, 1, w=4, h=4, unit="short"),
-        ts_panel(110, "TTFT percentile by model (gateway)",
-                 [hq_gw("llm_ttft_seconds", 0.50),
-                  hq_gw("llm_ttft_seconds", 0.95),
-                  hq_gw("llm_ttft_seconds", 0.99)],
-                 0, 2, w=12, h=8, unit="s"),
-        ts_panel(111, "TTFT percentile — vLLM reasoning",
-                 [hq("vllm:time_to_first_token_seconds", 0.50),
-                  hq("vllm:time_to_first_token_seconds", 0.95),
-                  hq("vllm:time_to_first_token_seconds", 0.99)],
-                 12, 2, w=12, h=8, unit="s"),
-        ts_panel(112, "Prefill vs decode (mean)",
-                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
-                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
-                 0, 3, w=12, h=8, unit="s"),
-        ts_panel(113, "Queue wait before scheduling (mean)",
-                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
-                 12, 3, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Inter-token latency ─────────────────────────────────────────────────
-    panels.append(row(120, "Latency — Inter-Token", 4, [
-        ts_panel(121, "Inter-token latency by model (gateway)",
-                 [hq_gw("llm_itl_seconds", 0.50),
-                  hq_gw("llm_itl_seconds", 0.95),
-                  hq_gw("llm_itl_seconds", 0.99)],
-                 0, 5, w=12, h=8, unit="s"),
-        ts_panel(122, "Inter-token latency — vLLM reasoning",
-                 [hq("vllm:inter_token_latency_seconds", 0.50),
-                  hq("vllm:inter_token_latency_seconds", 0.95),
-                  hq("vllm:inter_token_latency_seconds", 0.99)],
-                 12, 5, w=12, h=8, unit="s"),
-        ts_panel(123, "Decode throughput (tokens/s per request)",
-                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
-                 0, 6, w=12, h=8, unit="short"),
-        ts_panel(124, "End-to-end request latency p50/p95/p99",
-                 [hq("vllm:e2e_request_latency_seconds", 0.50),
-                  hq("vllm:e2e_request_latency_seconds", 0.95),
-                  hq("vllm:e2e_request_latency_seconds", 0.99)],
-                 12, 6, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Batching / scheduling ───────────────────────────────────────────────
-    panels.append(row(130, "Batching & Scheduling", 7, [
-        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
-                 [("sum(vllm:num_requests_running)", "running (in batch)"),
-                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
-                 0, 8, w=12, h=8, unit="short"),
-        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
-                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
-                 12, 8, w=12, h=8, unit="short"),
-        ts_panel(133, "TEI batch size (embeddings / reranker)",
-                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
-                   "{{app}}")],
-                 0, 9, w=12, h=8, unit="short"),
-        ts_panel(134, "TEI queue depth",
-                 [("max(te_queue_size) by (app)", "{{app}}")],
-                 12, 9, w=12, h=8, unit="short"),
-    ]))
-
-    # ── KV cache ────────────────────────────────────────────────────────────
-    panels.append(row(140, "KV Cache", 10, [
-        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
-                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
-                 0, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
-                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
-                 12, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(143, "Prefix cache hits vs queries",
-                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
-                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
-                 0, 12, w=12, h=8, unit="short"),
-        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
-                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
-                 12, 12, w=12, h=8, unit="short"),
-    ]))
-
-    # ── Throughput & health ─────────────────────────────────────────────────
-    panels.append(row(150, "Throughput & Health", 13, [
-        ts_panel(151, "Token throughput",
-                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
-                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
-                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
-                 0, 14, w=12, h=8, unit="short"),
-        ts_panel(152, "Successful requests/s",
-                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
-                  ("sum(rate(te_request_count[5m]))", "TEI")],
-                 12, 14, w=12, h=8, unit="short"),
-        ts_panel(153, "Backpressure — queue-full rejects",
-                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
-                 0, 15, w=8, h=8, unit="short"),
-        ts_panel(154, "KV warmup requests delayed",
-                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
-                 8, 15, w=8, h=8, unit="short"),
-        ts_panel(155, "Embedding / rerank latency p95",
-                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
-                   "{{app}} embed"),
-                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
-                   "{{app}} queue")],
-                 16, 15, w=8, h=8, unit="s"),
-    ]))
-
-    # ── Admission control ───────────────────────────────────────────────────
-    # These four panels read the gateway's admission gate, which is the
-    # companion change in homelab-frontend#64. That emitter is NOT deployed
-    # yet (verified against the live /metrics endpoint: 0 series matching
-    # ^(admission_|observer_|engine_state_)), so they render "No data" until
-    # #64 ships. That is expected, and each panel carries a description
-    # saying so — an unexplained blank panel would read as a broken
-    # dashboard, which is exactly the failure mode this row exists to catch.
-    panels.append(row(160, "Admission Control", 16, [
-        ts_panel(161, "Admission parked (5m)",
-                 [("increase(admission_parked_total[5m])", "parked")],
-                 0, 17, w=8, h=8, unit="short",
-                 description="Requests the gateway held back rather than admitting. "
-                             "Emitted by the admission gate (homelab-frontend#64); "
-                             "empty until that change is deployed."),
-        ts_panel(162, "Admission rejected by reason",
-                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
-                 8, 17, w=8, h=8, unit="short",
-                 description="Admission rejections split by the gate's own reason label. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(163, "Engine state stale",
-                 [("increase(engine_state_stale_total[5m])", "stale")],
-                 16, 17, w=8, h=8, unit="short",
-                 description="Times the gateway's view of the engine gauges went stale. "
-                             "The gate fails OPEN when this happens, so any non-zero rate "
-                             "means admission is silently degraded to admit-everything. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(164, "Gateway view vs engine truth",
-                 [("max(vllm_requests_running)", "gateway: running"),
-                  ("sum(vllm:num_requests_running)", "engine: running"),
-                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
-                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
-                 0, 18, w=24, h=8, unit="short",
-                 description="The gateway's own reading of the engine against the engine's "
-                             "self-reported value. A disagreement between a pair is how a "
-                             "stale observer or a mistuned admission target shows itself, "
-                             "and nothing else in this dashboard reveals it. The two "
-                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
-                             "ship with homelab-frontend#64 and are absent until it is "
-                             "deployed — until then this panel shows the engine's pair only."),
-    ]))
-
-    return {
-        "title": "LLM Observability",
-        "uid": "llm-observability",
-        "schemaVersion": 39,
-        "timezone": "browser",
-        "time": {"from": "now-6h", "to": "now"},
-        "refresh": "30s",
-        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
-        "panels": panels,
-    }
-
-
 # ============================================================================
 # Generate
 # ============================================================================
@@ -511,7 +277,4 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")

-# Dashboard 4
-write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
-
 print("Done.")
@@ -1,95 +0,0 @@
-apiVersion: v1
-data:
-  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
-    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
-    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
-    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
-    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
-    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
-    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
-    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
-    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
-    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
-    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
-    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
-    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
-    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
-    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
-    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
-    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
-    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
-    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
-    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
-    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
-    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
-    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
-    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
-    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
-    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
-    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
-    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
-    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
-    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
-    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
-    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
-    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
-    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
-    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
-    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
-    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
-    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
-    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
-    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
-    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
-    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
-    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
-    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
-    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
-    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
-    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
-    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
-    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
-    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
-    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
-    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
-    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
-    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
-    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
-    the engine''s self-reported value. A disagreement between a pair is how a stale
-    observer or a mistuned admission target shows itself, and nothing else in this
-    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
-    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
-    then this panel shows the engine''s pair only."}]}]}'
-kind: ConfigMap
-metadata:
-  annotations:
-    grafana_folder: LLM
-  labels:
-    grafana_dashboard: '1'
-  name: llm-observability-dashboard
-  namespace: logging
@@ -14,14 +14,10 @@ resources:
  - alerts/svc-vault-rules.yaml
  - alerts/cluster-alerts.yaml
  - alerts/api-gateway-alerts.yaml
-  - alerts/cag-kv-cache-rules.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
  - servicemonitors/ingress-nginx.yaml
-  - servicemonitors/vllm.yaml
-  - servicemonitors/api-gateway.yaml
  - dashboards/cluster-infrastructure.yaml
  - dashboards/api-gateway.yaml
-  - dashboards/llm-observability.yaml
@@ -54,8 +54,8 @@ alertmanager:
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:
  prometheusSpec:
-    retention: 30d
-    retentionSize: "95GB"
+    retention: 15d
+    retentionSize: "18GB"

    # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
    # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -64,9 +64,10 @@ prometheus:
    scrapeTimeout: 60s
    evaluationInterval: 30s

-    # Pin to worker-1 for more storage capacity
+    # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
+    # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
    nodeSelector:
-      kubernetes.io/hostname: worker-1
+      topology.kubernetes.io/zone: az-a

    # Persistent storage — metrics survive node reboots and pod restarts.
    # Uses the default `longhorn` StorageClass; nodeSelector above already pins
@@ -78,7 +79,7 @@ prometheus:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
-              storage: 25Gi
+              storage: 20Gi

    resources:
      requests:
@@ -1,62 +0,0 @@
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
-# three latency series below recorded nothing. internal/proxy/llm_metrics.go
-# declared a capture wrapper (LLMMetricsCapture) that was never constructed
-# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
-# stayed empty.
-#
-# homelab-frontend#63 fixed exactly that. The wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
-# request path. Verified live on 2026-09-29:
-#   llm_ttft_seconds  count=7  sum=598.212s
-#   llm_itl_seconds   count=16116
-#   llm_tokens_total  3 series
-#
-# These derived-from-SSE series cover EVERY model including the two Ollama ones
-# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
-# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
-# between SSE writes, so both are granular to SSE write boundaries rather than
-# to the engine's own token cadence. llm_tokens_total counts `data:` events,
-# and the documented assumption in internal/proxy/llm_metrics.go is that these
-# upstreams emit one such event per token — so it is a token count only as far
-# as that assumption holds. vLLM's own histograms remain authoritative.
-#
-# Also recorded and working (3 series, one per replica):
-#   queue_full_requests_total           counter    backpressure rejects
-#   warmup_requests_delayed_total       counter    vLLM slot contention
-#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
-#
-# NOT working, do not build on it: request_duration_seconds is declared and
-# exported but has 0 samples, and its only writer Metrics.RecordRequest has
-# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
-# the same dead-code shape this branch already corrected for the latency
-# series, so it is listed here rather than left to be rediscovered.
-#
-# Also note the Deployment already carried prometheus.io/scrape annotations,
-# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
-# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
-# collected before this ServiceMonitor existed.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: api-gateway
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: api-gateway
-    app.kubernetes.io/part-of: api
-spec:
-  namespaceSelector:
-    matchNames:
-      - api
-  selector:
-    matchLabels:
-      app: api-gateway
-  endpoints:
-    # api-gateway Service port IS named `http` (unlike the KServe ones), so
-    # the name form works here.
-    - port: http
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-
@@ -1,49 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: vllm
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: vllm
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  namespaceSelector:
-    matchNames:
-      - llm-serving
-  selector:
-    # Only the three engines that actually serve /metrics.
-    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
-    # endpoint (/metrics returns 404) — selecting them would add permanently
-    # DOWN targets. Ollama-model visibility comes from the api-gateway
-    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
-    # measured at the proxy, so they cover every model including Ollama).
-    matchExpressions:
-      - key: app.kubernetes.io/name
-        operator: In
-        values:
-          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
-          - llm-embeddings  # TEI 1.8.2 — CPU
-          - llm-reranker    # TEI 1.8.2 — CPU
-  endpoints:
-    # targetPort, not port: the KServe-generated Services on llm-serving have
-    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
-    # editing the InferenceServices (predictor restart = full weights reload,
-    # tens of seconds on the vLLM slot), so scrape the pod port directly.
-    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
-    - targetPort: 8080
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-      relabelings:
-        # Prometheus's Kubernetes service discovery already supplies the
-        # namespace, pod and service labels; the only one that needs deriving
-        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
-        # not promote arbitrary pod labels.
-        #
-        # `action: replace` is written explicitly. It is the API server's
-        # default for relabeling rules, so omitting it leaves the live object
-        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
-        - action: replace
-          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
-          targetLabel: app
@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Poimen infrastructure — CNPG Postgres for memory service.
-# Part of the `poimen` ArgoCD Application (multi-source, source 3).
-# Sync-wave -1 ensures the DB cluster is healthy before application
-# deployments (wave 0) are applied.
-resources:
-  - memory-db.yaml
-  - poimen-sandbox-namespace.yaml
-  - poimen-sandbox-quota.yaml
-  - pi-session-netpol.yaml
-  - pi-agent-workspace-pv.yaml
@@ -1,37 +0,0 @@
-# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
-# Uses default longhorn storage class (3 replicas, dataLocality disabled).
-# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: memory-db
-  namespace: poimen
-  annotations:
-    argocd.argoproj.io/sync-wave: "-1"
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-spec:
-  instances: 3
-  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
-  bootstrap:
-    initdb:
-      database: memory
-      owner: app
-      encoding: UTF8
-      localeCollate: C
-      localeCType: C
-      postInitApplicationSQL:
-        - "CREATE EXTENSION vector;"
-  enableSuperuserAccess: false
-  resources:
-    requests: { memory: "512Mi", cpu: "250m" }
-    limits: { memory: "2Gi", cpu: "1" }
-  storage:
-    size: 20Gi
-    storageClass: longhorn
-  affinity:
-    podAntiAffinityType: preferred
-    topologyKey: kubernetes.io/hostname
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule
@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  finalizers:
-  - kubernetes.io/pv-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace-pv
-  annotations:
-    argocd.argoproj.io/sync-wave: '-1'
-spec:
-  accessModes:
-  - ReadWriteOnce
-  capacity:
-    storage: 10Gi
-  claimRef:
-    apiVersion: v1
-    kind: PersistentVolumeClaim
-    name: pi-agent-workspace
-    namespace: poimen-sandbox
-  hostPath:
-    path: /var/lib/pi-agent-sandbox
-    type: DirectoryOrCreate
-  nodeAffinity:
-    required:
-      nodeSelectorTerms:
-      - matchExpressions:
-        - key: kubernetes.io/hostname
-          operator: In
-          values:
-          - talos-cp-2
-  persistentVolumeReclaimPolicy: Retain
-  volumeMode: Filesystem
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  annotations:
-    pv.kubernetes.io/bind-completed: 'yes'
-    argocd.argoproj.io/sync-wave: '-1'
-  finalizers:
-  - kubernetes.io/pvc-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace
-  namespace: poimen-sandbox
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: ''
-  volumeMode: Filesystem
-  volumeName: pi-agent-workspace-pv
@@ -1,12 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: pi-session-deny-kube-api
-  namespace: poimen-sandbox
-spec:
-  endpointSelector:
-    matchLabels:
-      app: pi-session
-  egressDeny:
-    - toEntities:
-        - kube-apiserver
@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: poimen-sandbox
-  labels:
-    pod-security.kubernetes.io/enforce: restricted
-    pod-security.kubernetes.io/enforce-version: latest
-    pod-security.kubernetes.io/audit: restricted
-    pod-security.kubernetes.io/warn: restricted
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ResourceQuota
-metadata:
-  name: sandbox-quota
-  namespace: poimen-sandbox
-spec:
-  hard:
-    pods: "20"
-    requests.cpu: "10"
-    requests.memory: 20Gi
-    limits.cpu: "20"
-    limits.memory: 40Gi
@@ -16,7 +16,6 @@ resources:
  - temporal-operator-role.yaml
  - llm-serving-operator-role.yaml
  - poimen-memory-operator-role.yaml
-  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until
@@ -1,23 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: sandbox-observer
-  namespace: poimen-sandbox
-rules:
-  - apiGroups: [""]
-    resources: [pods, pods/log, services]
-    verbs: [get, list, watch]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: poimen-devs-sandbox
-  namespace: poimen-sandbox
-subjects:
-  - kind: Group
-    apiGroup: rbac.authorization.k8s.io
-    name: "oidc:poimen-devs"
-roleRef:
-  kind: Role
-  name: sandbox-observer
-  apiGroup: rbac.authorization.k8s.io
@@ -123,156 +123,6 @@ data:
              "custom": {"fillOpacity": 10}
            }
          }
-        },
-        {
-          "id": 7,
-          "title": "vLLM Prefix Cache Hit Rate",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "vllm:prefix_cache_hit_rate",
-              "legendFormat": "prefix cache hit rate"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "percentunit",
-              "min": 0,
-              "max": 1,
-              "custom": {"fillOpacity": 15},
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {"color": "red", "value": 0},
-                  {"color": "yellow", "value": 0.7},
-                  {"color": "green", "value": 0.85}
-                ]
-              }
-            }
-          }
-        },
-        {
-          "id": 8,
-          "title": "KV Warmup Hit vs Miss",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
-              "legendFormat": "warmup hit"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
-              "legendFormat": "warmup miss"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_error_total[5m])",
-              "legendFormat": "warmup error"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 9,
-          "title": "KV Warmup Latency p50/p95 (ms)",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
-              "legendFormat": "p50 warmup latency"
-            },
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
-              "legendFormat": "p95 warmup latency"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ms",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 10,
-          "title": "Standing Context Cache Tier Hits",
-          "type": "timeseries",
-          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
-              "legendFormat": "Redis L1 hit"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_hit_db_total[5m])",
-              "legendFormat": "DB L2 hit"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_recompute_total[5m])",
-              "legendFormat": "recompute (L3)"
-            },
-            {
-              "expr": "rate(memory_standing_ctx_304_total[5m])",
-              "legendFormat": "304 Not Modified"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10}
-            }
-          }
-        },
-        {
-          "id": 11,
-          "title": "Standing Context Alert: Stale > 10min post-ingest",
-          "type": "timeseries",
-          "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
-          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
-          "datasource": {"type": "prometheus", "uid": "prometheus"},
-          "targets": [
-            {
-              "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
-              "legendFormat": "recompute rate (non-zero = fresh)"
-            },
-            {
-              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
-              "legendFormat": "warmup miss rate (alert if > 3/5min)"
-            }
-          ],
-          "fieldConfig": {
-            "defaults": {
-              "unit": "ops",
-              "custom": {"fillOpacity": 10},
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {"color": "green", "value": 0},
-                  {"color": "red", "value": 0.01}
-                ]
-              }
-            },
-            "overrides": [
-              {
-                "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
-                "properties": [
-                  {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
-                  {"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
-                ]
-              }
-            ]
-          }
        }
      ]
    }
@@ -1,162 +0,0 @@
-# Gotify Notifications Setup
-
-## Overview
-
-Gotify is a self-hosted push notification server deployed in the `notifications` namespace. The homelab-frontend gateway provides a `sendMsg` endpoint that accepts email and SMS notification requests and sends them directly via SMTP (SMS provider TBD).
-
-## Architecture
-
-```
-App → X-Service: notification header
-    → homelab-frontend gateway (api namespace)
-    → notification/sendMsg handler
-    → SMTP relay (email) or SMS provider (stubbed)
-    → recipient email/SMS
-```
-
-**Not used:** Gotify's native message store is available for UI/push notifications, but the sendMsg flow bypasses it (direct send, no storage).
-
-## Usage
-
-### Send Email
-
-```bash
-curl -X POST https://api.riotpiao.com \
-  -H 'X-Service: notification' \
-  -H 'X-Resource: sendMsg' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "format": "smtp",
-    "title": "Alert",
-    "message": "System CPU high",
-    "priority": 5,
-    "extras": {
-      "to_email": "admin@example.com",
-      "cc": "ops@example.com"
-    }
-  }'
-```
-
-**Response (success):**
-```json
-{
-  "status": "success",
-  "messageId": "email-admin@example.com"
-}
-```
-
-**Response (error):**
-```json
-{
-  "status": "error",
-  "error": "failed to send email: connection refused"
-}
-```
-
-### Send SMS (Stubbed)
-
-SMS support is stubbed. Currently returns "not implemented" error. To enable:
-
-1. Choose SMS provider (Twilio, AWS SNS, Vonage, etc.)
-2. Set `SMS_API_URL` and `SMS_API_KEY` environment vars in gateway Deployment
-3. Implement provider integration in `internal/notification/handler.go` sendSMS() method
-
-```bash
-curl -X POST https://api.riotpiao.com \
-  -H 'X-Service: notification' \
-  -H 'X-Resource: sendMsg' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "format": "sms",
-    "message": "System CPU high",
-    "extras": {
-      "phone": "+12025551234"
-    }
-  }'
-```
-
-## Configuration
-
-### SMTP Settings
-
-Gateway reads SMTP config from environment variables (pulled from `smtp-credentials` Secret in `api` namespace):
- `SMTP_HOST` — SMTP server hostname
- `SMTP_PORT` — SMTP server port (587 TLS or 465 SSL)
- `SMTP_FROM` — Sender email address
- `SMTP_USER` — SMTP auth username
- `SMTP_PASS` — SMTP auth password
-
-Secret is SOPS-encrypted in git. Create via:
-```bash
-kubectl create secret generic smtp-credentials \
-  --from-literal=host=mail.riotpiao.com \
-  --from-literal=port=587 \
-  --from-literal=from=alerts@riotpiao.com \
-  --from-literal=user=smtp-user \
-  --from-literal=password=smtp-password \
-  -n api \
-  -o yaml | sops -e /dev/stdin > k8s/smtp-secrets.enc.yaml
-```
-
-Then add to `k8s/kustomization.yaml`:
-```yaml
-resources:
- smtp-secrets.enc.yaml
-```
-
-### Gotify Server
-
-Gotify runs in `notifications` namespace with:
- PostgreSQL backend (CNPG)
- SMTP emailer sidecar (unused by sendMsg, but available for UI notifications)
- Health check on `:80/health`
-
-Config: `k8s/apps/gotify/`
-
-## Testing
-
-```bash
-cd homelab-frontend
-bash examples/sendmsg-email.sh https://api.riotpiao.com
-```
-
-## Roadmap
-
- [ ] SMS provider integration (pick: Twilio/SNS/Vonage)
- [ ] Request rate limiting per source
- [ ] Message queuing for retries (via SQS if high volume expected)
- [ ] Audit logging (who sent what, to whom, when)
- [ ] Template support (subject + body with placeholders)
-
-## Troubleshooting
-
-### "SMTP_HOST not set"
-Gateway env vars not loaded. Check:
-```bash
-kubectl -n api describe pod api-gateway-xyz
-kubectl -n api logs api-gateway-xyz | grep SMTP
-```
-
-### "connection refused" on SMTP
-SMTP server unreachable. Verify:
-```bash
-kubectl -n api exec -it api-gateway-xyz -- \
-  nc -zv $SMTP_HOST $SMTP_PORT
-```
-
-### "authentication failed"
-Wrong SMTP username/password. Verify credentials:
-```bash
-kubectl -n api get secret smtp-credentials -o yaml | grep password | base64 -d
-```
-
-### "X-Resource: sendMsg not found"
-Notification handler not registered. Check `internal/server/router.go`:
- Verify `X-Service: notification` case exists
- Confirm `notification.NewHandler()` called in `NewRouter()`
-
-## References
-
- [API Documentation](../homelab-frontend/API.md#notification-services)
- [Gotify Server Docs](https://gotify.net)
- [SMTP Configuration Best Practices](https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol)
@@ -1,669 +0,0 @@
-#!/usr/bin/env python3
-"""
-Provision RBAC groups, service account roles, fine-grained claims, and auth flows.
-
-Idempotent — safe to re-run. Provisions:
-  1. Global admin groups (homelab-admins)
-  2. Fine-grained service/bucket/project groups (minio-*, poimen-*, paperless-*, grafana-*, sqs-*)
-  3. Service account roles with custom claims (paperless-ai-agent, portfolio-agent, etc.)
-  4. JWT scope mappings for fine-grained claims (minio_buckets, paperless_doctypes, etc.)
-  5. OAuth2 providers with scopes (api-gw, minio, poimen, paperless, grafana)
-  6. Auth flows (password grant on api-gw provider)
-
-Usage:
-    source ~/.env
-    export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-        -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
-    python3 scripts/iam/provision-rbac.py
-
-DO NOT commit this file to git — .gitignore covers scripts/iam/*.py.
-"""
-import json
-import os
-import sys
-import urllib.error
-import urllib.request
-from typing import Dict, List, Any
-from pathlib import Path
-
-# Load ~/.env for OAuth2 provider secrets
-env_file = Path.home() / ".env"
-if env_file.exists():
-    with open(env_file) as f:
-        for line in f:
-            line = line.strip()
-            if line.startswith("export ") and "=" in line:
-                key, _, value = line[7:].partition("=")
-                key = key.strip()
-                value = value.strip().strip('"').strip("'")
-                os.environ[key] = value
-
-AUTHENTIK_URL = "https://authentik.riotpiao.com"
-TOKEN = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN")
-if not TOKEN:
-    print("Error: AUTHENTIK_BOOTSTRAP_TOKEN not set")
-    print("  source ~/.env")
-    print("  export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \\")
-    print("      -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)")
-    sys.exit(1)
-
-
-def api(method, path, data=None):
-    url = f"{AUTHENTIK_URL}{path}"
-    body = json.dumps(data).encode() if data is not None else None
-    req = urllib.request.Request(url, data=body, method=method, headers={
-        "Authorization": f"Bearer {TOKEN}",
-        "Content-Type": "application/json",
-    })
-    try:
-        with urllib.request.urlopen(req, timeout=30) as resp:
-            raw = resp.read()
-            return resp.status, json.loads(raw) if raw else {}
-    except urllib.error.HTTPError as e:
-        raw = e.read()
-        try:
-            parsed = json.loads(raw) if raw else {}
-        except json.JSONDecodeError:
-            parsed = {"raw": raw.decode(errors="replace")}
-        return e.code, parsed
-
-
-def die(msg):
-    print(f"FATAL: {msg}", file=sys.stderr)
-    sys.exit(1)
-
-
-# ===========================================================================
-# Group Definitions (DRY: single source of truth)
-# ===========================================================================
-GROUPS: Dict[str, Dict[str, Any]] = {
-    "homelab-admins": {
-        "description": "Cluster administrators with full access",
-        "is_superuser": True,
-    },
-    "minio-admins": {"description": "MinIO administrators", "is_superuser": False, "minio_buckets": ["*"]},
-    "minio-photos": {"description": "Photos bucket (Immich) access", "is_superuser": False, "minio_buckets": ["immich"]},
-    "minio-documents": {"description": "Documents bucket (Paperless) access", "is_superuser": False, "minio_buckets": ["paperless"]},
-    "minio-backups": {"description": "Backups bucket read-only access", "is_superuser": False, "minio_buckets": ["backups"]},
-    "poimen-admins": {"description": "Poimen memory administrators", "is_superuser": False, "memory_projects": ["*"], "memory_visibility": "private"},
-    "poimen-devs": {"description": "Dev and staging projects access", "is_superuser": False, "memory_projects": ["dev", "staging"], "memory_visibility": "internal"},
-    "poimen-prod-readonly": {"description": "Production projects read-only access", "is_superuser": False, "memory_projects": ["prod"], "memory_visibility": "public"},
-    "paperless-admins": {"description": "Paperless administrators", "is_superuser": False, "paperless_doctypes": ["*"]},
-    "paperless-finance": {"description": "Finance documents", "is_superuser": False, "paperless_doctypes": ["invoices", "receipts", "expenses"]},
-    "paperless-legal": {"description": "Legal documents", "is_superuser": False, "paperless_doctypes": ["contracts", "licenses", "agreements"]},
-    "paperless-hr": {"description": "HR documents", "is_superuser": False, "paperless_doctypes": ["employment", "benefits", "payroll"]},
-    "grafana-admins": {"description": "Grafana administrators", "is_superuser": False, "grafana_org_role": "Admin"},
-    "grafana-editors": {"description": "Grafana dashboard editors", "is_superuser": False, "grafana_org_role": "Editor"},
-    "grafana-viewers": {"description": "Grafana dashboard viewers", "is_superuser": False, "grafana_org_role": "Viewer"},
-    "sqs-users": {"description": "SQS/Temporal queue read access", "is_superuser": False, "sqs_queues": ["default"]},
-    "sqs-writers": {"description": "SQS/Temporal queue read/write access", "is_superuser": False, "sqs_queues": ["*"]},
-    "s3-users": {"description": "S3 read access", "is_superuser": False},
-    "s3-writers": {"description": "S3 read/write access", "is_superuser": False},
-}
-
-SERVICE_ACCOUNTS: Dict[str, Dict[str, Any]] = {
-    "paperless-ai-agent": {
-        "description": "Paperless AI plugin (auto-tagging, entity extraction)",
-        "roles": ["llm:inference", "memory:write", "paperless:admin"],
-        "claims": {
-            "minio_buckets": ["paperless"],
-            "paperless_doctypes": ["*"],
-            "memory_projects": ["*"],
-            "authorized_models": ["reasoning", "qwen2.5:3b"],
-        },
-    },
-    "portfolio-agent": {
-        "description": "Portfolio service agent",
-        "roles": ["llm:inference", "memory:read"],
-        "claims": {
-            "memory_projects": ["homelab", "portfolio"],
-            "memory_visibility": "public",
-            "authorized_models": ["ornith:35b"],
-            "minio_buckets": ["backups"],
-        },
-    },
-    "memory-agent": {
-        "description": "Memory service agent",
-        "roles": ["llm:inference", "memory:read", "memory:write"],
-        "claims": {
-            "memory_projects": ["*"],
-            "memory_visibility": "private",
-            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
-        },
-    },
-    "temporal-worker-agent": {
-        "description": "Temporal workflow worker",
-        "roles": ["llm:inference", "workflow:execute", "memory:read", "memory:write", "queue:send"],
-        "claims": {
-            "memory_projects": ["*"],
-            "sqs_queues": ["*"],
-            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
-        },
-    },
-}
-
-SCOPE_MAPPINGS: Dict[str, Dict[str, str]] = {
-    "roles": {"expression": 'return user.attributes.get("roles", [])'},
-    "permissions": {"expression": 'return ["*"] if any(user.groups.filter(is_superuser=True)) else list(user.groups.values_list("name", flat=True))'},
-    "groups": {"expression": 'return {"groups": list(user.groups.values_list("name", flat=True))}'},
-    "minio_buckets": {"expression": 'return user.attributes.get("minio_buckets", [])'},
-    "paperless_doctypes": {"expression": 'return user.attributes.get("paperless_doctypes", [])'},
-    "memory_projects": {"expression": 'return user.attributes.get("memory_projects", [])'},
-    "memory_visibility": {"expression": 'return user.attributes.get("memory_visibility", "public")'},
-    "authorized_models": {"expression": 'return user.attributes.get("authorized_models", [])'},
-    "sqs_queues": {"expression": 'return user.attributes.get("sqs_queues", [])'},
-    "grafana_org_role": {"expression": 'return user.attributes.get("grafana_org_role", "Viewer")'},
-}
-
-# ===========================================================================
-# Phase 1: Create/sync all groups
-# ===========================================================================
-print("[1/6] Ensuring groups exist...")
-
-status, res = api("GET", "/api/v3/core/groups/?page_size=100")
-if status != 200:
-    die(f"GET groups -> {status} {res}")
-existing_groups = {g["name"]: g for g in res["results"]}
-
-created_count = 0
-for group_name, group_spec in GROUPS.items():
-    if group_name in existing_groups:
-        print(f"  {group_name}: already exists")
-    else:
-        status, res = api("POST", "/api/v3/core/groups/", {
-            "name": group_name,
-            "is_superuser": group_spec.get("is_superuser", False),
-        })
-        if status in (200, 201):
-            print(f"  {group_name}: created")
-            created_count += 1
-        else:
-            print(f"  {group_name}: FAILED {status} {res}")
-
-print(f"  Total: {len(GROUPS)} groups, {created_count} new")
-
-# ===========================================================================
-# Phase 2: Create/sync service account users with custom claims
-# ===========================================================================
-print("\n[2/6] Creating/updating service account users...")
-
-status, res = api("GET", "/api/v3/core/users/?page_size=100")
-if status != 200:
-    die(f"GET users -> {status} {res}")
-existing_users = {u["username"]: u for u in res["results"]}
-
-service_pwd = os.environ.get("AUTHENTIK_SERVICE_ACCOUNT_PASSWORD", "DefaultPassword123!")
-
-for agent_name, agent_spec in SERVICE_ACCOUNTS.items():
-    if agent_name in existing_users:
-        user = existing_users[agent_name]
-        attrs = user.get("attributes", {})
-        attrs.update(agent_spec.get("claims", {}))
-        attrs["roles"] = agent_spec.get("roles", [])
-        status, res = api("PATCH", f"/api/v3/core/users/{user['pk']}/", {"attributes": attrs})
-        if status in (200, 201):
-            print(f"  {agent_name}: claims updated")
-        else:
-            print(f"  {agent_name}: FAILED {status} {res}")
-    else:
-        status, res = api("POST", "/api/v3/core/users/", {
-            "username": agent_name,
-            "name": agent_spec.get("description", agent_name),
-            "email": f"{agent_name}@homelab.local",
-            "is_active": True,
-            "is_superuser": False,
-            "password": service_pwd,
-            "attributes": {
-                **agent_spec.get("claims", {}),
-                "roles": agent_spec.get("roles", []),
-            },
-        })
-        if status in (200, 201):
-            print(f"  {agent_name}: created")
-        else:
-            print(f"  {agent_name}: FAILED {status} {res}")
-
-# ===========================================================================
-# Phase 3: Create scope mappings for fine-grained claims
-# ===========================================================================
-print("\n[3/6] Creating scope mappings for fine-grained claims...")
-
-status, res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
-if status != 200:
-    die(f"GET scope mappings -> {status} {res}")
-existing_scopes = {m["scope_name"]: m for m in res["results"]}
-
-for scope_name, scope_spec in SCOPE_MAPPINGS.items():
-    if scope_name in existing_scopes:
-        print(f"  {scope_name}: already exists")
-    else:
-        status, res = api("POST", "/api/v3/propertymappings/provider/scope/", {
-            "name": scope_name,
-            "scope_name": scope_name,
-            "expression": scope_spec["expression"],
-        })
-        if status in (200, 201):
-            print(f"  {scope_name}: created")
-        else:
-            print(f"  {scope_name}: FAILED {status} {res}")
-
-# Fix built-in 'openid' scope to include preferred_username (needed by Grafana, Forgejo, etc.)
-# Note: sub MUST be a string, not int, for JSON compatibility with Go struct unmarshaling
-if "openid" in existing_scopes:
-    openid_scope = existing_scopes["openid"]
-    openid_pk = openid_scope["pk"]
-    openid_expression = 'return {"sub": str(user.pk), "preferred_username": user.username}'
-    if openid_expression not in openid_scope.get("expression", ""):
-        status, res = api("PUT", f"/api/v3/propertymappings/provider/scope/{openid_pk}/", {
-            "name": openid_scope["name"],
-            "scope_name": "openid",
-            "expression": openid_expression,
-        })
-        if status in (200, 201):
-            print(f"  openid: updated (sub as string + preferred_username)")
-        else:
-            print(f"  openid: UPDATE FAILED {status} {res}")
-    else:
-        print(f"  openid: already correct (sub as string)")
-
-# ===========================================================================
-# Phase 4: Get flow UUIDs (needed for providers)
-# ===========================================================================
-print("\n[4/6] Fetching flow UUIDs...")
-
-status, res = api("GET", "/api/v3/flows/instances/?page_size=100")
-if status != 200:
-    die(f"GET flows -> {status} {res}")
-
-flows = {f["slug"]: f["pk"] for f in res.get("results", [])}
-auth_flow = flows.get("default-provider-authorization-implicit-consent")
-inval_flow = flows.get("default-provider-invalidation-flow")
-
-if not auth_flow or not inval_flow:
-    die(f"Required flows not found. auth_flow={auth_flow}, inval_flow={inval_flow}")
-
-print(f"  authorization_flow: {auth_flow}")
-print(f"  invalidation_flow: {inval_flow}")
-
-# ===========================================================================
-# Phase 5: Create OAuth2 providers with scopes
-# ===========================================================================
-print("\n[5/6] Creating OAuth2 providers...")
-
-status, res = api("GET", "/api/v3/providers/oauth2/?page_size=100")
-if status != 200:
-    die(f"GET providers -> {status} {res}")
-existing_providers = {p["name"]: p for p in res.get("results", [])}
-
-# Fetch scope mapping PKs
-status, scopes_res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
-if status != 200:
-    print("  WARNING: could not fetch scope mappings")
-    scope_pks = {}
-else:
-    scope_pks = {m["scope_name"]: m["pk"] for m in scopes_res.get("results", [])}
-
-# Include standard OpenID scopes (openid, email, profile) + custom claim scopes
-STANDARD_SCOPES = ["openid", "email", "profile"]
-scope_pks_list = [scope_pks[s] for s in STANDARD_SCOPES if s in scope_pks]
-scope_pks_list += [scope_pks[s] for s in SCOPE_MAPPINGS.keys() if s in scope_pks]
-
-# OAuth2 providers — client_secret sourced from SOPS-encrypted k8s secrets.
-# These are the real secrets the services use. Authentik must match.
-OAuth2_PROVIDERS = {
-    "api-gw": {
-        "client_id": "api-gw",
-        "client_secret_env": "AUTHENTIK_PROVIDER_API_GW_SECRET",
-        "redirect_uris": ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"],
-    },
-    "minio": {
-        "client_id": "minio",
-        "client_secret": "9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219",
-        "redirect_uris": ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"],
-    },
-    "poimen": {
-        "client_id": "poimen",
-        "client_secret_env": "AUTHENTIK_PROVIDER_POIMEN_SECRET",
-        "redirect_uris": ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"],
-    },
-    "paperless": {
-        "client_id": "paperless",
-        "client_secret": "6hcxaaVgZlKgafl7BxeSEtPAcbNUJxi2PAZePxSFk4o=",
-        "redirect_uris": ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"],
-    },
-    "grafana": {
-        "client_id": "grafana",
-        "client_secret": "966bad4fa43812100e7775b3c73fed2ce1d07217fa5a23fbb0f190e46d2f0fa4",
-        "redirect_uris": ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"],
-    },
-    "queue": {
-        "client_id": "queue-sqs",
-        "client_secret_env": "AUTHENTIK_PROVIDER_QUEUE_SECRET",
-        "redirect_uris": ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"],
-    },
-    "forgejo": {
-        "client_id": "forgejo",
-        "client_secret": "G4klhs3JRfs5A7YnGs90WuOndBAvamWlZgaZRY8x",
-        "redirect_uris": ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"],
-    },
-    "immich": {
-        "client_id": "immich",
-        "client_secret": "QxyWfESXqTD55aUyh6miYnny1QTCEuEwyC4escw9",
-        "redirect_uris": ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"],
-    },
-    "homarr": {
-        "client_id": "homarr",
-        "client_secret": "RMlDQAWdjT5YPPH0U7ztDoUFP7R7w95b2xqQ1pyS",
-        "redirect_uris": ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"],
-    },
-    "argocd": {
-        "client_id": "argocd",
-        "client_secret_env": "AUTHENTIK_PROVIDER_ARGOCD_SECRET",
-        "redirect_uris": ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"],
-    },
-    "vault": {
-        "client_id": "vault",
-        "client_secret_env": "AUTHENTIK_PROVIDER_VAULT_SECRET",
-        "redirect_uris": ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"],
-    },
-}
-
-import secrets as _secrets
-
-for provider_name, provider_spec in OAuth2_PROVIDERS.items():
-    # Resolve client_secret: explicit > env var > generate random
-    if "client_secret" in provider_spec:
-        client_secret = provider_spec["client_secret"]
-    elif "client_secret_env" in provider_spec:
-        client_secret = os.environ.get(provider_spec["client_secret_env"], _secrets.token_urlsafe(32))
-    else:
-        client_secret = _secrets.token_urlsafe(32)
-
-    redirect_uris_list = [{"url": uri, "matching_mode": "strict"} for uri in provider_spec["redirect_uris"]]
-
-    provider_payload = {
-        "name": provider_name,
-        "authorization_flow": auth_flow,
-        "invalidation_flow": inval_flow,
-        "grant_types": ["authorization_code", "implicit", "password"],
-        "client_id": provider_spec["client_id"],
-        "client_secret": client_secret,
-        "redirect_uris": redirect_uris_list,
-        "property_mappings": scope_pks_list,
-    }
-
-    if provider_name in existing_providers:
-        # UPDATE existing provider — sync secret + redirect_uris
-        provider_pk = existing_providers[provider_name]["pk"]
-        status, res = api("PATCH", f"/api/v3/providers/oauth2/{provider_pk}/", {
-            "client_id": provider_spec["client_id"],
-            "client_secret": client_secret,
-            "redirect_uris": redirect_uris_list,
-            "property_mappings": scope_pks_list,
-        })
-        if status in (200, 201):
-            print(f"  {provider_name}: updated (secret + redirect_uris synced)")
-        else:
-            print(f"  {provider_name}: UPDATE FAILED {status} {res}")
-    else:
-        # CREATE new provider
-        status, res = api("POST", "/api/v3/providers/oauth2/", provider_payload)
-        if status in (200, 201):
-            print(f"  {provider_name}: created")
-        else:
-            print(f"  {provider_name}: FAILED {status} {res}")
-
-# ===========================================================================
-# Phase 6: Create OAuth2 Applications (bind providers to public token endpoints)
-# ===========================================================================
-print("\n[6/7] Creating OAuth2 Applications...")
-print("     (binds providers to /application/o/token/ endpoints)")
-
-status, res = api("GET", "/api/v3/core/applications/?page_size=100")
-if status != 200:
-    die(f"GET applications -> {status} {res}")
-existing_apps = {a["slug"]: a for a in res.get("results", [])}
-
-for provider_name in OAuth2_PROVIDERS.keys():
-    # Get the provider PK
-    status, provider_res = api("GET", f"/api/v3/providers/oauth2/?name={provider_name}")
-    if status != 200 or not provider_res.get("results"):
-        print(f"  {provider_name}: provider not found, skip")
-        continue
-    provider_pk = provider_res["results"][0]["pk"]
-
-    if provider_name in existing_apps:
-        # Ensure app is linked to provider (fix orphaned apps)
-        app_data = existing_apps[provider_name]
-        if app_data.get("provider") != provider_pk:
-            app_uuid = app_data["pk"]
-            status, res = api("PATCH", f"/api/v3/core/applications/{app_uuid}/", {
-                "provider": provider_pk,
-            })
-            if status in (200, 201):
-                print(f"  {provider_name}: re-linked to provider")
-            else:
-                print(f"  {provider_name}: RE-LINK FAILED {status} {res}")
-        else:
-            print(f"  {provider_name}: ok")
-    else:
-        status, res = api("POST", "/api/v3/core/applications/", {
-            "name": provider_name,
-            "slug": provider_name,
-            "provider": provider_pk,
-        })
-        if status in (200, 201):
-            print(f"  {provider_name}: created")
-        else:
-            print(f"  {provider_name}: FAILED {status} {res}")
-
-# ===========================================================================
-# Phase 7: Create/update rock user → homelab-admins, matching Forgejo identity
-# ===========================================================================
-print("\n[7/10] Creating/updating rock user ([email protected])...")
-
-ROCK_EMAIL = "[email protected]"
-ROCK_PASSWORD = os.environ.get("ROCK_PASSWORD", "")
-
-status, res = api("GET", "/api/v3/core/users/?username=rock")
-if status == 200 and res.get("results"):
-    rock_user = res["results"][0]
-    # Ensure email matches Forgejo's rock user for OIDC linking
-    patch_data = {"email": ROCK_EMAIL, "name": "Rock"}
-    status, res = api("PATCH", f"/api/v3/core/users/{rock_user['pk']}/", patch_data)
-    if status in (200, 201):
-        print(f"  rock: updated email to {ROCK_EMAIL}")
-    else:
-        print(f"  rock: update FAILED {status} {res}")
-else:
-    if not ROCK_PASSWORD:
-        print("  rock: NOT FOUND and ROCK_PASSWORD not set, skipping creation")
-        print("    export ROCK_PASSWORD=<password> and re-run")
-        rock_user = None
-    else:
-        status, res = api("POST", "/api/v3/core/users/", {
-            "username": "rock",
-            "name": "Rock",
-            "email": ROCK_EMAIL,
-            "is_active": True,
-            "is_superuser": False,
-            "password": ROCK_PASSWORD,
-        })
-        if status in (200, 201):
-            rock_user = res
-            print(f"  rock: created with email {ROCK_EMAIL}")
-        else:
-            print(f"  rock: create FAILED {status} {res}")
-            rock_user = None
-
-if rock_user:
-    status, res = api("GET", "/api/v3/core/groups/?name=homelab-admins")
-    if status == 200 and res.get("results"):
-        admins_group = res["results"][0]
-        status, res = api("POST", f"/api/v3/core/groups/{admins_group['pk']}/users/add/", {"pk": rock_user["pk"]})
-        if status in (200, 201, 204):
-            print(f"  rock: added to homelab-admins")
-        else:
-            print(f"  rock: group add {status} {res}")
-
-# ===========================================================================
-# Phase 8: Email recovery flow (password reset via email)
-# ===========================================================================
-print("\n[8/10] Creating email recovery flow...")
-
-# Read SMTP config from gotify-smtp secret (same Gmail creds)
-SMTP_HOST = "smtp.gmail.com"
-SMTP_PORT = 587
-SMTP_USER = "[email protected]"
-SMTP_FROM = "[email protected]"
-# Password read from env at runtime: AUTHENTIK_EMAIL__PASSWORD
-
-# 8a. Create email stage for recovery
-status, res = api("GET", "/api/v3/stages/email/?name=email-recovery")
-if status == 200 and res.get("results"):
-    email_stage_pk = res["results"][0]["pk"]
-    print("  email-recovery stage: already exists")
-else:
-    status, res = api("POST", "/api/v3/stages/email/", {
-        "name": "email-recovery",
-        "use_global_settings": False,
-        "host": SMTP_HOST,
-        "port": SMTP_PORT,
-        "username": SMTP_USER,
-        "password": os.environ.get("AUTHENTIK_EMAIL_PASSWORD", ""),
-        "use_tls": True,
-        "use_ssl": False,
-        "timeout": 10,
-        "from_address": SMTP_FROM,
-        "template": "email/password_reset.html",
-        "activate_user_on_success": True,
-    })
-    if status in (200, 201):
-        email_stage_pk = res["pk"]
-        print("  email-recovery stage: created")
-    else:
-        email_stage_pk = None
-        print(f"  email-recovery stage: FAILED {status} {res}")
-
-# 8b. Create identification stage for recovery (email lookup)
-status, res = api("GET", "/api/v3/stages/identification/?name=recovery-identification")
-if status == 200 and res.get("results"):
-    ident_stage_pk = res["results"][0]["pk"]
-    print("  recovery-identification stage: already exists")
-else:
-    status, res = api("POST", "/api/v3/stages/identification/", {
-        "name": "recovery-identification",
-        "user_fields": ["email", "username"],
-    })
-    if status in (200, 201):
-        ident_stage_pk = res["pk"]
-        print("  recovery-identification stage: created")
-    else:
-        ident_stage_pk = None
-        print(f"  recovery-identification stage: FAILED {status} {res}")
-
-# 8c. Create password stage for new password entry
-status, res = api("GET", "/api/v3/stages/password/?name=recovery-password-change")
-if status == 200 and res.get("results"):
-    pw_stage_pk = res["results"][0]["pk"]
-    print("  recovery-password-change stage: already exists")
-else:
-    # Use prompt stage for password change instead
-    status, res = api("GET", "/api/v3/stages/user_write/?name=recovery-user-write")
-    if status == 200 and res.get("results"):
-        pw_stage_pk = res["results"][0]["pk"]
-        print("  recovery-user-write stage: already exists")
-    else:
-        status, res = api("POST", "/api/v3/stages/user_write/", {
-            "name": "recovery-user-write",
-        })
-        if status in (200, 201):
-            pw_stage_pk = res["pk"]
-            print("  recovery-user-write stage: created")
-        else:
-            pw_stage_pk = None
-            print(f"  recovery-user-write stage: FAILED {status} {res}")
-
-# 8d. Create recovery flow
-status, res = api("GET", "/api/v3/flows/instances/?slug=password-recovery")
-if status == 200 and res.get("results"):
-    recovery_flow_pk = res["results"][0]["pk"]
-    print("  password-recovery flow: already exists")
-else:
-    status, res = api("POST", "/api/v3/flows/instances/", {
-        "name": "Password Recovery",
-        "slug": "password-recovery",
-        "title": "Reset your password",
-        "designation": "recovery",
-    })
-    if status in (200, 201):
-        recovery_flow_pk = res["pk"]
-        print("  password-recovery flow: created")
-    else:
-        recovery_flow_pk = None
-        print(f"  password-recovery flow: FAILED {status} {res}")
-
-# 8e. Bind stages to flow in order
-if recovery_flow_pk and ident_stage_pk and email_stage_pk:
-    for order, stage_pk, label in [
-        (10, ident_stage_pk, "identification"),
-        (20, email_stage_pk, "email"),
-    ]:
-        status, res = api("POST", "/api/v3/flows/bindings/", {
-            "target": recovery_flow_pk,
-            "stage": stage_pk,
-            "order": order,
-        })
-        if status in (200, 201):
-            print(f"  bound {label} stage at order {order}")
-        elif status == 400 and "already exists" in str(res).lower():
-            print(f"  {label} stage: already bound")
-        else:
-            print(f"  bind {label}: {status} {res}")
-
-# ===========================================================================
-# Phase 9: Set recovery flow on brand
-# ===========================================================================
-print("\n[9/10] Setting recovery flow on brand...")
-
-if recovery_flow_pk:
-    status, res = api("GET", "/api/v3/brands/instances/")
-    if status == 200 and res.get("results"):
-        brand = res["results"][0]
-        status, res = api("PATCH", f"/api/v3/brands/instances/{brand['brand_uuid']}/", {
-            "flow_recovery": recovery_flow_pk,
-        })
-        if status in (200, 201):
-            print("  recovery flow set on brand")
-        else:
-            print(f"  FAILED {status} {res}")
-    else:
-        print("  no brand found")
-
-# ===========================================================================
-# Phase 10: Ensure Forgejo OAuth2 source uses matching email claim
-# ===========================================================================
-print("\n[10/10] Verifying Forgejo OIDC linkage...")
-print(f"  rock@Authentik email: {ROCK_EMAIL}")
-print("  Forgejo OIDC will match on email — ensure Forgejo's rock user")
-print(f"  has email {ROCK_EMAIL} in Forgejo settings → Profile")
-
-# ===========================================================================
-# Summary
-# ===========================================================================
-print("\n" + "="*70)
-print("AUTHENTIK PROVISIONING COMPLETE")
-print("="*70)
-print(f"\n  [1] Groups: {len(GROUPS)}")
-print(f"  [2] Service accounts: {len(SERVICE_ACCOUNTS)}")
-print(f"  [3] Scope mappings: {len(SCOPE_MAPPINGS)}")
-print(f"  [4] Flows resolved")
-print(f"  [5] OAuth2 providers: {len(OAuth2_PROVIDERS)}")
-print(f"  [6] OAuth2 applications bound")
-print(f"  [7] rock user ([email protected]) -> homelab-admins")
-print(f"  [8] Email recovery flow (smtp.gmail.com)")
-print(f"  [9] Recovery flow set on brand")
-print(f"  [10] Forgejo OIDC linkage verified")
-print("\nNEXT: Set Forgejo rock user email to [email protected] in Forgejo profile")
-print("TEST: https://authentik.riotpiao.com/if/flow/password-recovery/")
-print("="*70)
@@ -1,216 +0,0 @@
-#!/bin/bash
-# Secret Rotation Script
-# Rotates all OAuth2 and service account credentials
-# Should be run quarterly (every 90 days)
-# 
-# Usage: ./rotate-secrets.sh [--dry-run]
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
-DRY_RUN=${1:-}
-
-# Color output
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-log() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*"; }
-warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
-error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }
-
-log "=== Secret Rotation Script ==="
-log "Rotation Date: $(date -u +"%Y-%m-%dT%H:%M:%SZ")"
-
-if [[ -n "$DRY_RUN" ]]; then
-  log "Running in DRY-RUN mode (no changes will be applied)"
-fi
-
-# Verify prerequisites
-log "Checking prerequisites..."
-command -v kubectl &>/dev/null || error "kubectl not found"
-command -v openssl &>/dev/null || error "openssl not found"
-command -v sops &>/dev/null || error "sops not found"
-command -v jq &>/dev/null || error "jq not found"
-
-# Check kubeconfig
-kubectl cluster-info &>/dev/null || error "Not connected to cluster"
-
-# Get bootstrap token
-log "Retrieving Authentik bootstrap token..."
-BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
-  -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' 2>/dev/null | base64 -d) || \
-  error "Failed to get bootstrap token"
-
-# Generate new secrets (13 OAuth2 + service accounts)
-log "Generating 13 new secrets (256-bit)..."
-
-generate_secret() {
-  openssl rand -base64 32
-}
-
-declare -A NEW_SECRETS
-
-for svc in api-gw minio poimen paperless grafana argocd forgejo homarr immich vault portfolio-agent memory-agent local-llm; do
-  NEW_SECRETS[$svc]=$(generate_secret)
-  log "  $svc: ${NEW_SECRETS[$svc]:0:15}..."
-done
-
-log ""
-log "=== Updating Authentik OAuth2 Providers ==="
-
-# Authentik provider mapping
-declare -A PROVIDER_PKS=(
-  [api-gw]=2
-  [minio]=3
-  [poimen]=4
-  [paperless]=5
-  [grafana]=6
-  [argocd]=7
-  [forgejo]=8
-  [homarr]=9
-  [immich]=10
-  [vault]=11
-)
-
-for provider in "${!PROVIDER_PKS[@]}"; do
-  pk=${PROVIDER_PKS[$provider]}
-  secret=${NEW_SECRETS[$provider]}
-  
-  log "Updating $provider (pk=$pk)..."
-  
-  if [[ -z "$DRY_RUN" ]]; then
-    response=$(curl -s -X PATCH "https://authentik.riotpiao.com/api/v3/providers/oauth2/$pk/" \
-      -H "Authorization: Bearer $BOOTSTRAP_TOKEN" \
-      -H "Content-Type: application/json" \
-      -d "{\"client_secret\": \"$secret\"}")
-    
-    if echo "$response" | jq -e '.pk' &>/dev/null; then
-      log "  ✅ $provider updated"
-    else
-      error "Failed to update $provider: $(echo "$response" | jq '.detail // .')"
-    fi
-  fi
-done
-
-log ""
-log "=== Updating k8s Secrets ==="
-
-# Update api-gw
-if [[ -z "$DRY_RUN" ]]; then
-  log "Patching api/api-gateway-oauth2-creds..."
-  kubectl -n api patch secret api-gateway-oauth2-creds \
-    -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[api-gw]}" | base64)\"}}" --type=merge 2>/dev/null || true
-fi
-
-# Update minio (skip if namespace doesn't exist)
-if kubectl get ns minio &>/dev/null 2>&1; then
-  if [[ -z "$DRY_RUN" ]]; then
-    log "Patching minio/minio-oauth2-creds..."
-    kubectl -n minio patch secret minio-oauth2-creds \
-      -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[minio]}" | base64)\"}}" --type=merge 2>/dev/null || true
-  fi
-fi
-
-# Update poimen
-if [[ -z "$DRY_RUN" ]]; then
-  log "Patching poimen/poimen-oauth2-creds..."
-  kubectl -n poimen patch secret poimen-oauth2-creds \
-    -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[poimen]}" | base64)\"}}" --type=merge 2>/dev/null || true
-fi
-
-# Update paperless
-if [[ -z "$DRY_RUN" ]]; then
-  log "Patching paperless/paperless-oauth2-creds..."
-  kubectl -n paperless patch secret paperless-oauth2-creds \
-    -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[paperless]}" | base64)\"}}" --type=merge 2>/dev/null || true
-fi
-
-# Update logging/grafana
-if [[ -z "$DRY_RUN" ]]; then
-  log "Patching logging/grafana-oauth2-creds..."
-  kubectl -n logging patch secret grafana-oauth2-creds \
-    -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[grafana]}" | base64)\"}}" --type=merge 2>/dev/null || true
-fi
-
-# Update service accounts
-if [[ -z "$DRY_RUN" ]]; then
-  log "Patching portfolio/portfolio-agent-oidc..."
-  kubectl -n portfolio patch secret portfolio-agent-oidc \
-    -p "{\"data\":{\"CLIENT_SECRET\":\"$(echo -n "${NEW_SECRETS[portfolio-agent]}" | base64)\"}}" --type=merge 2>/dev/null || true
-  
-  log "Patching poimen/memory-agent-oidc..."
-  kubectl -n poimen patch secret memory-agent-oidc \
-    -p "{\"data\":{\"CLIENT_SECRET\":\"$(echo -n "${NEW_SECRETS[memory-agent]}" | base64)\"}}" --type=merge 2>/dev/null || true
-  
-  log "Patching llm-serving/local-llm-jwt..."
-  kubectl -n llm-serving patch secret local-llm-jwt \
-    -p "{\"data\":{\"client-secret\":\"$(echo -n "${NEW_SECRETS[local-llm]}" | base64)\"}}" --type=merge 2>/dev/null || true
-fi
-
-log ""
-log "=== Updating SOPS-encrypted manifests ==="
-
-# Create oauth2-credentials.enc.yaml
-cat > "$REPO_ROOT/k8s/argocd/secrets/oauth2-credentials.yaml" << 'OAUTH_EOF'
-apiVersion: v1
-kind: Secret
-metadata:
-  name: oauth2-credentials
-  namespace: iam
-type: Opaque
-data:
-OAUTH_EOF
-
-for svc in api-gw minio poimen paperless grafana; do
-  echo "  ${svc}-client-secret: $(echo -n "${NEW_SECRETS[$svc]}" | base64)" >> \
-    "$REPO_ROOT/k8s/argocd/secrets/oauth2-credentials.yaml"
-done
-
-if [[ -z "$DRY_RUN" ]]; then
-  log "Encrypting oauth2-credentials.yaml with SOPS..."
-  sops -e "$REPO_ROOT/k8s/argocd/secrets/oauth2-credentials.yaml" > \
-    "$REPO_ROOT/k8s/argocd/secrets/oauth2-credentials.enc.yaml"
-  rm "$REPO_ROOT/k8s/argocd/secrets/oauth2-credentials.yaml"
-  log "  ✅ oauth2-credentials.enc.yaml created"
-fi
-
-# Create memory-agent-oidc.enc.yaml
-cat > "$REPO_ROOT/k8s/argocd/secrets/memory-agent-oidc.yaml" << AGENT_EOF
-apiVersion: v1
-kind: Secret
-metadata:
-  name: memory-agent-oidc
-  namespace: poimen
-type: Opaque
-data:
-  CLIENT_ID: bWVtb3J5LWFnZW50
-  CLIENT_SECRET: $(echo -n "${NEW_SECRETS[memory-agent]}" | base64)
-  ISSUER: aHR0cHM6Ly9hdXRoZW50aWsucmlvdHBpYW8uY29tL2FwcGxpY2F0aW9uL28v
-  TOKEN_URL: aHR0cHM6Ly9hdXRoZW50aWsucmlvdHBpYW8uY29tL2FwcGxpY2F0aW9uL28vdG9rZW4v
-AGENT_EOF
-
-if [[ -z "$DRY_RUN" ]]; then
-  log "Encrypting memory-agent-oidc.yaml with SOPS..."
-  sops -e "$REPO_ROOT/k8s/argocd/secrets/memory-agent-oidc.yaml" > \
-    "$REPO_ROOT/k8s/argocd/secrets/memory-agent-oidc.enc.yaml"
-  rm "$REPO_ROOT/k8s/argocd/secrets/memory-agent-oidc.yaml"
-  log "  ✅ memory-agent-oidc.enc.yaml created"
-fi
-
-log ""
-log "=== Summary ==="
-log "Rotated 13 credentials:"
-log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, homarr, immich, vault"
-log "  Service Accounts: portfolio-agent, memory-agent, local-llm"
-
-log ""
-log "Next steps:"
-log "  1. Review changes: git diff k8s/argocd/secrets/"
-log "  2. Commit: git add k8s/argocd/secrets/oauth2-credentials.enc.yaml"
-log "  3. Commit message: 'chore: rotate OAuth2 secrets (quarterly)'"
-log "  4. Push: git push"
-log ""
-log "✅ Rotation complete!"
@@ -37,10 +37,6 @@
    rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-    rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local

    kubernetes cluster.local in-addr.arpa ip6.arpa {
@@ -1,90 +0,0 @@
-# RECOVERED from live cluster state via talosctl get machineconfig.
-# Regenerated after the original tfvars.local was lost/corrupted.
-
-cluster_id  = "Rtc4g2av9EP0mOdxA4M0-QQitzHLWICz-rLBNfrOjgw="
-cluster_secret = "8E0CAeylAPKmmUEQmIGmHQAhQf+8c7NUf43CdrQZ+vg="
-bootstrap_token = "b2q7lh.9w7hwgrulrd65gr3"
-machine_token = "hq9wlf.96l9z46efd79jtr2"
-machine_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBMkUwRWZqbG41L1J3eTVjMVJmNkNSakFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qWXdOekE1TURVd056VTRXaGNOTXpZd056QTJNRFV3TnpVNFdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUNwR3NQZkVHdEU4anVmNG9JTkNHNnlCQmN6aURFaEpLbDV3CnJib0hSR0tUbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkdmaExZUUdaOGYzd3lZZAo0SFI3KzlONnZKQXJNQVVHQXl0bGNBTkJBQ1ZQVlAwcGYxMkdUakYvSHJMZmcwZHBLemdBMlE1OGR5Y0paY0ZvClQvNDdPQk8ra29VZm11dXNjVVNNQnBXS0VuWHNYMFNocmNab3hQd1FHM3diblFFPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-machine_ca_key = "LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM0Q0FRQXdCUVlESzJWd0JDSUVJSlR4MXRqZEVOTTg1cGNRRFR0WWRtMFd0QXNBd0tzL1VESlZLN0ZqYU5uUgotLS0tLUVORCBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0K"
-kubernetes_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVRDZ0F3SUJBZ0lSQUtwT1ZaK29CRzljNHFycEpwaUc4TkV3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOakEzTURrd05UQTNOVGhhRncwek5qQTNNRFl3TlRBMwpOVGhhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVQ1VjJvNkliMUpRZkcza3lCTTBCeTRkd2FLbnlGY2tVdjNVem9yOG1Ba3RaN2JrT2ZKZDlmL2VmcVkKYXBzUFpLV1RHQnYxM3JZbFdvOGtuU3ZnNm83Um8yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGQ1dPZVJUVHdnN2tnNVNWMG9raFQwY0VDNGwzTUFvR0NDcUdTTTQ5QkFNQ0EwY0FNRVFDSURURCtNYXUKb2JXdkp6UkNMVEdJaHJHc1daalFnalVmRWl2MnhCTXB6RnNVQWlCQzNNWkYwMjVqVHk4Uno2YjI5czVJQmpkNgpZQTVWd0kvNVFieXlwSGFXQlE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-kubernetes_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUUrSEdPcUJJYXpJMzdqNmJJUlA1emVxeXEwZzhBU2xZeGplZUlJcEpIcXBvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1ZkcU9pRzlTVUh4dDVNZ1ROQWN1SGNHaXA4aFhKRkw5MU02Sy9KZ0pMV2UyNURueVhmWAovM242bUdxYkQyU2xreGdiOWQ2MkpWcVBKSjByNE9xTzBRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-etcd_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmVENDQVNTZ0F3SUJBZ0lSQVAyaHkwdUhvTm5vSzQyZE9LTk1nU2t3Q2dZSUtvWkl6ajBFQXdJd0R6RU4KTUFzR0ExVUVDaE1FWlhSalpEQWVGdzB5TmpBM01Ea3dOVEEzTlRoYUZ3MHpOakEzTURZd05UQTNOVGhhTUE4eApEVEFMQmdOVkJBb1RCR1YwWTJRd1dUQVRCZ2NxaGtqT1BRSUJCZ2dxaGtqT1BRTUJCd05DQUFRVGlKYUJpSEJPCmJha3JuL0dVdkUxVFd5czRVUkVzVGtVNEM4OG1EdWZYQURjV0NnN2RTRzc0QjkzOGFwSWgybGc4UDhKRDFFRUoKN0RkU1lQVG5zYWh1bzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSApBd0VHQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkliYTBLaEhmM3hhClBQNk1hb3dESUNWVXBLdDVNQW9HQ0NxR1NNNDlCQU1DQTBjQU1FUUNJRzZMYUJGeGgvcys2WXpGdVlwaUxUWlYKS2prOGh5UVNvMmVTZTJTUy81MG5BaUI0a0RNWnR4b1UzTitHc3BEOHVEbXFrNlhxbzZoeE0vbG0yU21OMzlPNAovdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
-etcd_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUUwZ3JiMk0yblA4c1hxWjVhd3NzNThwbUdvb1FlSWg3a3RoWVlxNzhZZThvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFRTRpV2dZaHdUbTJwSzUveGxMeE5VMXNyT0ZFUkxFNUZPQXZQSmc3bjF3QTNGZ29PM1VodQorQWZkL0dxU0lkcFlQRC9DUTlSQkNldzNVbUQwNTdHb2JnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-aggregator_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJZVENDQVFhZ0F3SUJBZ0lSQUpxeFI3alBzcmhzRUp2cmtEWndYR3N3Q2dZSUtvWkl6ajBFQXdJd0FEQWUKRncweU5qQTNNRGt3TlRBM05UaGFGdzB6TmpBM01EWXdOVEEzTlRoYU1BQXdXVEFUQmdjcWhrak9QUUlCQmdncQpoa2pPUFFNQkJ3TkNBQVFBWWlieHY1ZDVFRGdTbWhlRHlhYjJLZGh4ZFZnZ3lQc2pNcjBET0JMVmxtQmpMcXFqClpNSkk2d1ZmV2hkUjBRVGxVdEFLZDJvREJZLy9TeDBzQi9qY28yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXcKSFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4dwpIUVlEVlIwT0JCWUVGSTkrWm1EYVBybDhrTnhXUUxOT3ErTmVzeEh0TUFvR0NDcUdTTTQ5QkFNQ0Ewa0FNRVlDCklRRC9DRUZ2SUVHMW9qcmRZRVUzUldmTklLV1FwVXlZQWJ1ZGE0T0ZWQS9yOUFJaEFPS1VOZFF6bUk2WVZOdzgKeklDejlLblRxazQrT3dvV3RjNVl5SmlKR29zUgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-aggregator_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUFoOEwycXFLbUxoYkpmczJ2M3ZRWXBFZHF6Ri9hTGZhSmoraEZRazdPY2NvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQUdJbThiK1hlUkE0RXBvWGc4bW05aW5ZY1hWWUlNajdJeks5QXpnUzFaWmdZeTZxbzJUQwpTT3NGWDFvWFVkRUU1VkxRQ25kcUF3V1AvMHNkTEFmNDNBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-service_account_key = "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKSndJQkFBS0NBZ0VBd3NNZU40eE1YQkZ0VVE2dVZ2NEZFU1dNT2ZLc1RKdmxHa0ptVG1URjJIazg1SlJvCkhuQU1nMEkvMEJiMGFUQTJ3MjlkSEUrdUN6dFd5eVVqTzV5eWdJSDQ0Q2RtVnNHVzVua1Z0TmFGNzNpcFllbjYKaG11bmRoWDY1VndqOHpVUmFYZUxtUEc0Y2d4dk5SOXdGYUZXS0ZZdzQzdGtlWjg5S0F5QTNibjFXN3JHTloyOQovcXRYWEcrQnFSSlVuUzhXQmt4dkErUktyRE1OVHF4bjVGOEl1VkpTWkw5bDVEd3lSUnNqMjVMZVdRZUFYenl6CllWcmFKZmVubSt6L3hLZjhLdHFZMSs5U1pwbkZsS2N6TGlWVzg4WkJ5TnR0KytYZjB4N3FSL2lQOHhRQXV0ckIKV1pTbytoa01FRjV5YU0wQWdxcG5mbDBsUmhsM0I0UUx3NHkyTUpoTEpyQ1ltQjl1QllLWU5oRE5NOXk0Z1lrZgpacmdEdm9NdGpsSUJUUHRyVjMyemxYN05Od3d5UjdXOXp5ZlBSVjEvbjhpMk9KR0szeittdGNCR2Y0TS8veDFsClJRNEhzWWhDUVJRaE8vQWp2U2QvQlhFRnB3dEhsb1VYRU5nQ2lPSW00MnFrZTNWb3lLbjJ2Q3g2QlhodlJVZDAKUHgvK0JpM0d4RmRoa2Uxa0doelJLQTdySGhEMVBkVGsyM04wUXV3WHNNVDZRWXVrRDdPZG5KdFZtZVN4TkxqcwptdVpybmpwRzZsaHRDSmdvMGdVeXBYQ2RlWGVnZ0dIc0JCMks4NjIrdThVa0dwMk9IQUhGeldsdDF5L2VXTW9PCmhneWx1SEVLcjkzUU5uNDZsTGtZTDViWjlVTUZ0NXBMRFVrSFQxNkV5dTh0V21ET2ZoQ3Z3M1lmRWY4Q0F3RUEKQVFLQ0FnQUdSbUlSV1JoV3VRc0VHd3g3NmdoQXdxeHZhNFdvbkRjMzd0Njc5Tnc0K3NMKy9GY1ViL2kvTytHeApjeVBoeGJkbCtZOE82L1JJRVZ2ZEJLL0xhbU9INTJnYzFMZ2o0RzNidEJnQ2NRejBwN2NSWEFnQno3TWdCMXBECmpJSHVBbzR5anpMMHRRa0R4Nm5IbE9FNEdUQWM4WlgycGxHWTU0d0JYOUhCRXc0NEs5N1orR0NZTlc0Rm9PUVYKRGUyaStOTGxWZzRYbW9IYlpYT3V6cmcwTCttb2l1SHp0QVQwNHdtZGwxL0M0Y3IvSkZJNi8wb3FQMUthK1kweApaV1BpTXFWWnZoeEJqTWpqWEYzMHlhUkkvdFA3MjYzZjZrM3pXVGNxWnFzV3NZZjF4WFcyajNpK1NaOWVHM043CmpZZHpIL085d2Y2K29BS2s3UW9jT0dGbXBnQnlwdm9JbytlSFdjZlpZNFNXNloyeTRacUl0M2xTSWFHMEtmQjEKUnVrSVBtMlYzNDNlc3Azdy9TV2liRVU2elhvd01sWjlZc0dPalY1MTZDQmJZK0lQcmY2RXY0UFhDWjlnUERxcgpnUFFIZ0lFS294aUdYK2dENG9pYWdUMWVVSk1iYWZkaEkzSHJMWk9YbUpBZU40RHpOZXR1SGRsTk13YnpZZHN5CnpyMllSMkhqNTZydk9hcWZZUFJKV1NmS3ozRlhQdDNDMHJRVmNRekVqV1ZZbm9MZG9yR3FkaU1uMyt1NzF2RW4KSnluZksvN3VRSDY5VTU3NGNKK3FjOFNtNTlPeHBnN1RoODljMUZTeSszTk03bTE2czlJcjQvcG94Q2pWaHNWeQpKRmN2cHE4UTMwek9CQk0waUxyblNKcVRXaXJDRU8xNHlRY0VLbFBVd2VGUWdJK05pUUtDQVFFQTMwMWNrTnZuCmpaRHduRXFQaWp4SlBDSUNEeHcxYWtnVTZUNUpMVkt5SUFnK3BYNHlPZDZtendvV2hpWE5oVUQ5eWxPL21lTEsKSDRPUXVzeWdJUDdkOVNKZi9ZVTV5a1RZaVZLMUdzM1loZXovRmhRQXdBQUg4UmdBdW8rZU9UTUcyT0IxM2loLworQlFYL1lrOG9VR1M0YlpsZWNGWG5pSTZ5S0g1MUJUQ3g0ZTZZcStvdGMrMFc2RzBRVHVBT3JWcjdXWklGY1htClVDdElReXRJN2s2UVd6SXU0K3dnVnU3OUdrUEJTMHNBaXhzU0ppYXRTNzk1TjlhYjYwRzNDeVFWZDZWajJaVmEKWGpCR1oyQXREalpEWS9MMDlZQTlRVUxDbWVqT2hYVFlGNkJuNnRkYmVjeVlZTHdNS2MzWEhqNEt2U2ZaQ01HTwpvUXROdnFoUFFlSWpxUUtDQVFFQTMwZnFCSThDK2QwdXJlM1JhNFFRdUlzUC9xVTMyejIzMUdzbTZOOXlQd0hUClM4ZzlrS0ZDYzhMVGlmWFdnajhIRWxTVjVLSXJseERwZEVWQ1pEM05qdy9GVDdHcHV5SVplN1E3VlhiNld3MnYKY1I0M1FkdUQvOEc4ZkdlZGkrZ1BnKzlzeURCQXFIVzFGSHl1RmVaWGNsVTF1cndOV3V3TlFKUnZHczFoK1NuQQpXVmJxd1ZUL01GMmYxTjlBaFN0alhkM0Nscm1rb0o4ek05YW0zVWg5VUprQ2xSZi9mZVRONXZndE1odS9NNGtGCmVQYUlBMTVqT3h1cEFMN3ptVVVxZ0h1NE9yMm5mYTVNNHMzWTR6TVRYYm9VZGNVT09oRTRzU0J2bGlaak5KL3QKV2pSTmJmUWhxbWRBTTNZZjFGUC8vRW9CYTBPejBxMHNXci92cEhDUlp3S0NBUUFxMlkySnZxa1FZVi9LbmdRdApZcVFyQmR1ZlNxcDFXcCtvb21zb1oxWUhENDMxOCtGdmVXcEpFSWFCOTM4WXN3QUFjMUd4RmZQeldDdk5yTGFOCm5scTVUMzljQnRTd0c4WHhsQTFzdDFOMVg2VVRkNE10Vk5ReFQ0blVRdnI1dnZEeGJTRXhJRlJ1Sm16MEdnR28KY0F6ZmcwQzF2SVF6dEIzVG9rRnVrUTFQZkp3bms4MnNGYzltUmdGeEF4bjRLaGdyMWhTL0dOcTVSNVQyVHJnUQpBc053dkpDQzdDeklnZFBQMW5DaElpTllqamxOV042b1NuWFlZVFpLVHJIeFVWdE5PaytPMFRvbUdOMXB1T3JzCmJ6MC9VTC93M0VyazJ3cTh2Zy9qVENpclgveVE5QUo1dk9rQXB4VXVjSEYzUERDVFc3SXFHL3Bpck9pZVRXM28KRnAwQkFvSUJBRjBxa3JsSU8wT3JTUmtHRE1aQ0d3QUY5cXlZb0EvNVZzVnAySmgrOUJyYVZpSmU4V0Z5Q0ZwcApSdjlmOXh2dDFMT1BXK1JFenMrQUhRbUpCTVR6RE56UEJkUFZIQytiY09xdkw3cmZwR050K0hESTNPRzhDUDRsCkJ0TWFJU0VKdWIraG5kQ0NZZGhwRlIveFRtcVE3SmdtZWY3ckROK05jNUlvM1p0ZmE2d2VBY2JGZjdzZ0RrTk8KTGEwVFlzYXViZzN5eElsRCtTK1VmamI1TUROUlZnalZiOEJxZlE4NDg3bVdnTFZSNHB4TVpsNHM4R0FIZUh4bgpkRU45YWdQZ1duVzJLZzlJcDZUSG9BbGJQMDYrTnl4NndxTEprTUFtQTNQVlJ2cHVGaU1WUUdMTlJDbkhIbTBPCkhEbmM1amNndmNXMTA1WEFjRDVPU0IydHpQN2VnYTBDZ2dFQUZhTHJxMDJ6M2tXaW1iVzdoNi9Pby9YRHdjZ2YKSmdXMDYyYWdWUlpEMjM3Mm4valZSL25kMnFybmwxTFdWaXpzWW91b3hvY2N6NUwvQ2h3MktCOC9Rc1Zxai9KOApOUWh1ZDJ2ZUxjQlUvVzNseVhENnJpajJZMythNzRvM1A4b1psOGdDQmJEck1jajNsMHRZMXRWbm9nOWo0eHQ3Ckp3UXA3djNXb3ArSFVuRWVjbGpscTdlN1VuTGNwZlRDOE5PUmdxbjBGSUtCbXdFNlpJQnQxUzB6NmxSKytJdVMKQk1oTTZTSkZmeUFOVVdjTCt2cHJoRzBjWUZmcUYzajlPZ1dXRnBGTHFWQmdSTlZBVDRoM0U1Ymh4L0lsMTVHMAo2USs2aWlkRElqMEhNNFAzcjhja2lRUlVPdFI1R2NVS1RYZGh1TjkwMjZGRTdEckRvSURVZ0F5dGVnPT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K"
-secretbox_encryption_secret = "RLkR4RmeaLmH/abyK3eYf6q22umJ/byLhheVCnh/yrA="
-
-controlplane_configs = {
-  "talos-cp-1" = {
-    hostname     = "talos-cp-1"
-    lan_ip       = "192.168.1.166"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/nvme0n1"
-    longhorn_disks = []
-    zone             = "az-a"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-  "talos-cp-2" = {
-    hostname     = "talos-cp-2"
-    lan_ip       = "192.168.1.214"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
-    longhorn_disks = [
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
-      ]
-    zone             = "az-b"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-  "talos-cp-3" = {
-    hostname     = "talos-cp-3"
-    lan_ip       = "192.168.1.162"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/nvme0n1"
-    longhorn_disks = [
-        { device = "/dev/disk/by-id/usb-Seagate_One_Touch_w_PW_00000000NABV3H34-0:0", mountpoint = "/var/lib/longhorn-paperless-media" },
-      ]
-    zone             = "az-c"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-}
-
-worker_configs = {
-  "worker-1" = {
-    hostname          = "worker-1"
-    lan_ip            = "192.168.1.223"
-    lan_subnet        = "192.168.1.0/24"
-    lan_gateway       = "192.168.1.254"
-    install_disk      = "/dev/nvme0n1"
-    network_interface = "enp28s0f0np0"
-    zone              = "az-a"
-    gpu_count         = 4
-    node_labels       = {}
-    node_taints       = []
-    factory_image     = "factory.talos.dev/metal-installer/0a2153a6dc099a371bf2f63d6c3c22d275c876bf6302dd154c5813072924cb3f:v1.13.3"
-    swap_size         = "64GiB"
-    ephemeral_max_size = "700GiB"
-    extra_disks       = []
-  }
-}
-
-cluster_config = {
-  controlplane_ip = "192.168.1.166"
-  pod_subnets     = ["10.244.0.0/16"]
-  service_subnets = ["10.96.0.0/12"]
-  dns_servers     = ["8.8.8.8", "1.1.1.1"]
-  dns_domain      = "cluster.local"
-}