test: add e2e auth flow test script

Validates JWT auth, identity claims, X-Service dispatch, and
auth endpoints against live cluster. 16 tests across 5 scenarios.

15/16 pass (1 skipped: poimen-memory pods down).

Closes #14

Co-authored-by: poimen <[email protected]>

.sops.yaml

@@ -2,5 +2,4 @@ creation_rules:
   # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
   # and was committed in plaintext to a public remote.
   - path_regex: k8s/.*secrets?.*\.ya?ml
-    encrypted_regex: ^(data|stringData)$
     age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla

AUTH_INTEGRATION.md

@@ -0,0 +1,206 @@
+# Authentik Auth Integration for NextJS
+
+## Current State
+
+### Gateway Auth Status
+
+| Endpoint | Auth Status | Notes |
+|----------|-------------|-------|
+| `/v1/chat/completions` | ❌ **OFF** | LLM routes have no auth middleware |
+| `/v1/embeddings` | ❌ **OFF** | Same - no auth |
+| `/v1/rerank` | ❌ **OFF** | Same - no auth |
+| `X-Service: sqs` | ✅ **ON** | JWT validated via `internal/auth/jwt.go` |
+| `/workflow` | ❌ **OFF** | Pass-through to Temporal |
+
+**Auth module exists** at `homelab-frontend/internal/auth/jwt.go` but only wired for SQS.
+LLM routes in `internal/proxy/proxy.go` have no auth middleware.
+
+### Authentik App
+
+Authentik app `local-llm` exists for LLM API auth:
+- **Client ID**: `local-llm`
+- **Client Secret**: `kubectl -n llm-serving get secret local-llm-jwt -o jsonpath='{.data.client-secret}' | base64 -d`
+- **Token endpoint**: `https://authentik.riotpiao.com/application/o/token/`
+- **Userinfo endpoint**: `https://authentik.riotpiao.com/application/o/userinfo/`
+- **OIDC discovery**: `https://authentik.riotpiao.com/application/o/local-llm/.well-known/openid-configuration`
+
+## Sign-in Methods
+
+### 1. Resource Owner Password Credentials (ROPC)
+
+Direct username/password login. Server-side only (needs client_secret).
+
+```typescript
+// API Route: app/api/auth/login/route.ts
+const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+  method: 'POST',
+  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+  body: new URLSearchParams({
+    grant_type: 'password',
+    client_id: 'local-llm',
+    client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+    username: '[email protected]',
+    password: 'userpassword',
+    scope: 'openid email profile groups',
+  }),
+});
+
+const tokens = await response.json();
+// { access_token, refresh_token, expires_in, token_type }
+```
+
+### 2. Authorization Code Flow (Browser Redirect)
+
+Requires adding redirect URIs to `local-llm` Authentik app:
+
+```python
+# In k8s/infra/iam/scripts/authentik-provision.py, update:
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",  # dev
+        "https://your-nextjs-app.com/api/auth/callback",  # prod
+    ],
+}
+```
+
+Then standard OIDC flow:
+1. Redirect to `https://authentik.riotpiao.com/application/o/authorize/?client_id=local-llm&redirect_uri=...&response_type=code&scope=openid email profile groups`
+2. User logs in via Authentik UI
+3. Callback receives `code`, exchange for tokens
+
+## JWT Token Persistence
+
+### Browser (localStorage)
+
+```typescript
+const TOKEN_KEY = 'llm_auth_token';
+
+// Save
+localStorage.setItem(TOKEN_KEY, JSON.stringify({
+  access_token: tokens.access_token,
+  refresh_token: tokens.refresh_token,
+  expires_at: Date.now() + tokens.expires_in * 1000,
+}));
+
+// Load
+const stored = JSON.parse(localStorage.getItem(TOKEN_KEY) || 'null');
+if (stored && stored.expires_at > Date.now()) {
+  // Token valid
+}
+
+// Clear (logout)
+localStorage.removeItem(TOKEN_KEY);
+```
+
+### Server-side (HTTP-only cookies)
+
+```typescript
+// app/api/auth/login/route.ts
+import { cookies } from 'next/headers';
+
+// After successful login
+cookies().set('llm_auth_token', JSON.stringify(tokens), {
+  httpOnly: true,
+  secure: process.env.NODE_ENV === 'production',
+  sameSite: 'lax',
+  maxAge: tokens.expires_in,
+  path: '/',
+});
+
+// Read in middleware or API routes
+const tokenCookie = cookies().get('llm_auth_token');
+const tokens = JSON.parse(tokenCookie?.value || 'null');
+```
+
+## Token Refresh
+
+```typescript
+async function refreshAccessToken(refresh_token: string) {
+  const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'refresh_token',
+      client_id: 'local-llm',
+      client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+      refresh_token,
+    }),
+  });
+  return response.json();
+}
+```
+
+## Environment Variables
+
+```bash
+# .env.local
+AUTHENTIK_URL=https://authentik.riotpiao.com
+AUTHENTIK_CLIENT_ID=local-llm
+AUTHENTIK_CLIENT_SECRET=<from-secret>
+
+# For client-side (public)
+NEXT_PUBLIC_AUTHENTIK_URL=https://authentik.riotpiao.com
+NEXT_PUBLIC_AUTHENTIK_CLIENT_ID=local-llm
+```
+
+## Using Token with LLM API
+
+```typescript
+const token = await getValidToken(); // from localStorage or cookie
+
+const response = await fetch('https://api.riotpiao.com/v1/chat/completions', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${token}`,  // JWT from Authentik
+  },
+  body: JSON.stringify({
+    model: 'reasoning',
+    messages: [{ role: 'user', content: 'Hello' }],
+  }),
+});
+```
+
+## TODO
+
+### Gateway-side (homelab-frontend)
+
+- [ ] Wire `internal/auth/jwt.go` into LLM proxy handler (`internal/proxy/proxy.go`)
+- [ ] Add `authRequired: true` to model config or create LLM-specific middleware
+- [ ] Example pattern from SQS (in `internal/serviceadapter/router.go`):
+
+```go
+// In proxy.go ServeHTTP, before dispatching to LLM upstream:
+if strings.HasPrefix(r.URL.Path, "/v1/") {
+    authHeader := r.Header.Get("Authorization")
+    claims, err := llmJWTAuth.ValidateBearerToken(authHeader)
+    if err != nil {
+        // Return 401/403
+    }
+    if !llmJWTAuth.CheckPermissions(claims, "llm:inference", "*") {
+        // Return 403 insufficient permissions
+    }
+}
+```
+
+### Authentik-side
+
+- [ ] Enable ROPC grant in Authentik provider settings (if not already)
+- [ ] Add redirect URIs to `local-llm` app if browser OAuth flow needed:
+
+```python
+# k8s/infra/iam/scripts/authentik-provision.py
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",
+        "https://your-app.com/api/auth/callback",
+    ],
+}
+```
+
+### NextJS-side
+
+- [ ] Until gateway auth is wired, LLM API works without token
+- [ ] Once wired, add `Authorization: Bearer <token>` to all LLM requests

k8s/apps/comfyui/deployment.yaml

@@ -1,82 +0,0 @@
-# ComfyUI — GPU-accelerated image generation on worker-1.
-# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: comfyui
-  template:
-    metadata:
-      labels:
-        app: comfyui
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: worker-1
-      runtimeClassName: nvidia
-      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
-      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
-      # Disable service link injection to avoid the collision.
-      enableServiceLinks: false
-      containers:
-        - name: comfyui
-          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
-          ports:
-            - containerPort: 8188
-              protocol: TCP
-          env:
-            - name: NVIDIA_VISIBLE_DEVICES
-              value: "all"
-          resources:
-            requests:
-              cpu: "4"
-              memory: 8Gi
-              nvidia.com/gpu: "1"
-            limits:
-              cpu: "8"
-              memory: 16Gi
-              nvidia.com/gpu: "1"
-          volumeMounts:
-            - mountPath: /workspace/ComfyUI/models
-              name: models
-            - mountPath: /workspace/ComfyUI/output
-              name: output
-          readinessProbe:
-            httpGet:
-              path: /
-              port: 8188
-            periodSeconds: 10
-            initialDelaySeconds: 30
-          startupProbe:
-            httpGet:
-              path: /
-              port: 8188
-            failureThreshold: 120
-            periodSeconds: 10
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: comfyui-models
-        - name: output
-          emptyDir: {}
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: comfyui-models
-  namespace: comfyui
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 50Gi

k8s/apps/comfyui/ingress.yaml

@@ -1,33 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: comfyui
-  namespace: comfyui
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    # WebSocket support for ComfyUI's live preview
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "upgrade";
-spec:
-  ingressClassName: nginx
-  tls:
-    - secretName: comfyui-tls
-      hosts:
-        - comfyui.riotpiao.com
-  rules:
-    - host: comfyui.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: comfyui
-                port:
-                  number: 80

k8s/apps/comfyui/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml

k8s/apps/comfyui/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: comfyui
-  namespace: comfyui
-  labels:
-    app: comfyui
-spec:
-  selector:
-    app: comfyui
-  ports:
-    - port: 80
-      targetPort: 8188
-      protocol: TCP

k8s/apps/gotify/README.md

@@ -1,107 +0,0 @@
-# Gotify — Push Notifications + Email Relay
-
-Self-hosted notification server with SMTP email forwarding sidecar.
-
-## Architecture
-
-```
-Forgejo webhook ──POST──→ Gotify API (:80/message)
-                              │
-                    ┌─────────┼─────────┐
-                    ▼                     ▼
-              Push notification      SMTP emailer sidecar
-              (mobile/desktop)       (polls → sends email)
-```
-
-## Setup (one-time, after first deploy)
-
-### 1. Encrypt secrets before committing
-
-```bash
-# Edit secrets.yaml with real values first, then:
-sops -e -i k8s/apps/gotify/secrets.yaml
-```
-
-### 2. Create Gotify app + client tokens
-
-1. Login to `https://gotify.riotpiao.com` with admin creds
-2. **Applications** → Create `forgejo` → copy **app token**
-3. **Clients** → Create `smtp-emailer` → copy **client token**
-4. Update `gotify-tokens` secret:
-   ```bash
-   kubectl -n notifications create secret generic gotify-tokens \
-     --from-literal=app-token=<APP_TOKEN> \
-     --from-literal=client-token=<CLIENT_TOKEN> \
-     --dry-run=client -o yaml | kubectl apply -f -
-   ```
-
-### 3. Configure Forgejo webhook
-
-In each Forgejo repo → **Settings** → **Webhooks** → **Add Webhook** → **Gotify**:
-
-| Field | Value |
-|-------|-------|
-| Target URL | `http://gotify.notifications.svc.cluster.local/message` |
-| Token | The **app token** from step 2 |
-| Events | Pull Request (Created, Merged, Closed) |
-
-Or via API:
-```bash
-FORGEJO_TOKEN="<your-pat>"
-APP_TOKEN="<gotify-app-token>"
-
-curl -s -X POST "https://forgejo.riotpiao.com/api/v1/repos/rock/homelab/hooks" \
-  -H "Authorization: token $FORGEJO_TOKEN" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "type": "gotify",
-    "active": true,
-    "config": {
-      "content_type": "json",
-      "url": "http://gotify.notifications.svc.cluster.local/message?token='"$APP_TOKEN"'"
-    },
-    "events": ["pull_request", "pull_request_assign", "pull_request_review"],
-    "authorization_header": ""
-  }'
-```
-
-### 4. Add CoreDNS rewrite (if accessing via public hostname)
-
-Only needed if Cloudflare Tunnel is used for gotify.riotpiao.com:
-
-```
-# terraform/files/coredns/Corefile — add rewrite:
-rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
-```
-
-Then: `cd terraform && terraform apply && cd .. && make apply-cp`
-
-### 5. SMTP providers
-
-| Provider | Host | Port | Notes |
-|----------|------|------|-------|
-| Gmail | smtp.gmail.com | 587 | Use App Password (2FA required) |
-| Resend | smtp.resend.com | 587 | Free 100 emails/day |
-| Sendgrid | smtp.sendgrid.net | 587 | Free 100 emails/day |
-| Mailgun | smtp.mailgun.org | 587 | Free 5000/month |
-
-## Notification priority levels
-
-| Priority | Meaning | Email forwarded? |
-|----------|---------|-----------------|
-| 0-4 | Low (info) | No (below MIN_PRIORITY=5) |
-| 5-7 | Normal (PR created) | Yes |
-| 8-10 | High (PR merged, failures) | Yes |
-
-## Verify
-
-```bash
-# Test push notification
-APP_TOKEN="<app-token>"
-curl -X POST "https://gotify.riotpiao.com/message?token=$APP_TOKEN" \
-  -H "Content-Type: application/json" \
-  -d '{"title":"Test","message":"Hello from homelab","priority":5}'
-
-# Check email sidecar logs
-kubectl -n notifications logs deployment/gotify -c smtp-emailer --tail=20
-```

k8s/apps/gotify/db.yaml

@@ -1,35 +0,0 @@
-# CNPG Postgres for Gotify. Lightweight — 2 instances, 2Gi storage.
-# CNPG generates secret `gotify-db-app` + service `gotify-db-rw` in ns notifications.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: gotify-db
-  namespace: notifications
-  annotations:
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-spec:
-  instances: 3
-  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
-  bootstrap:
-    initdb:
-      database: gotify
-      owner: app
-      encoding: UTF8
-      localeCollate: C
-      localeCType: C
-  enableSuperuserAccess: false
-  resources:
-    requests: { memory: "256Mi", cpu: "100m" }
-    limits: { memory: "512Mi", cpu: "500m" }
-  storage:
-    size: 2Gi
-    storageClass: longhorn-cnpg
-  monitoring:
-    enablePodMonitor: true
-  affinity:
-    podAntiAffinityType: preferred
-    topologyKey: kubernetes.io/hostname
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule

k8s/apps/gotify/deployment.yaml

@@ -1,204 +0,0 @@
-# Gotify — self-hosted push notification server + SMTP email relay.
-# Forgejo webhooks → Gotify → push notifications + email forwarding.
-# Runs on control plane (no GPU needed), lightweight.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gotify
-  namespace: notifications
-  labels:
-    app: gotify
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: gotify
-  template:
-    metadata:
-      labels:
-        app: gotify
-    spec:
-      containers:
-        # --- Gotify server ---
-        - name: gotify
-          image: ghcr.io/gotify/server:3.1.0
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              export GOTIFY_DATABASE_DIALECT=postgres
-              export GOTIFY_DATABASE_CONNECTION="host=gotify-db-rw.notifications port=5432 user=${DB_USER} password=${DB_PASS} dbname=gotify sslmode=disable"
-              exec /app/gotify-app
-          ports:
-            - containerPort: 80
-              protocol: TCP
-          env:
-            - name: GOTIFY_DEFAULTUSER_NAME
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-admin
-                  key: username
-            - name: GOTIFY_DEFAULTUSER_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-admin
-                  key: password
-            - name: DB_USER
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-db-app
-                  key: username
-            - name: DB_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-db-app
-                  key: password
-            - name: GOTIFY_SERVER_PORT
-              value: "80"
-            - name: GOTIFY_SERVER_KEEPALIVEPERIODSECONDS
-              value: "0"
-            - name: TZ
-              value: Asia/Tokyo
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 200m
-              memory: 128Mi
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: 80
-            periodSeconds: 30
-            initialDelaySeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /health
-              port: 80
-            periodSeconds: 10
-            initialDelaySeconds: 5
-
-        # --- SMTP emailer sidecar ---
-        # Watches Gotify WebSocket stream, forwards messages as email.
-        # https://github.com/eternal-flame-AD/gotify-broadcast
-        - name: smtp-emailer
-          image: ghcr.io/gotify/server:3.1.0
-          command:
-            - /bin/sh
-            - -c
-            - |
-              # Wait for Gotify to be ready
-              until wget -qO- http://localhost:80/health >/dev/null 2>&1; do
-                echo "Waiting for Gotify..."
-                sleep 2
-              done
-              echo "Gotify is ready, starting email relay..."
-
-              # Poll Gotify messages and forward via SMTP using msmtp
-              # Install msmtp for lightweight SMTP sending
-              apk add --no-cache msmtp curl jq
-
-              # Configure msmtp
-              cat > /tmp/msmtprc <<MSMTP
-              defaults
-              auth           on
-              tls            on
-              tls_trust_file /etc/ssl/certs/ca-certificates.crt
-              logfile        /tmp/msmtp.log
-
-              account        default
-              host           ${SMTP_HOST}
-              port           ${SMTP_PORT}
-              from           ${SMTP_FROM}
-              user           ${SMTP_USER}
-              password       ${SMTP_PASS}
-              MSMTP
-
-              chmod 600 /tmp/msmtprc
-
-              # Track last seen message ID
-              LAST_ID=0
-
-              while true; do
-                # Fetch messages since last ID
-                MESSAGES=$(curl -s -H "X-Gotify-Key: ${GOTIFY_CLIENT_TOKEN}" \
-                  "http://localhost:80/message?since=${LAST_ID}&limit=10" 2>/dev/null)
-
-                if [ -n "$MESSAGES" ]; then
-                  echo "$MESSAGES" | jq -r '.messages[]? | @base64' | while read -r MSG; do
-                    DECODED=$(echo "$MSG" | base64 -d)
-                    ID=$(echo "$DECODED" | jq -r '.id')
-                    TITLE=$(echo "$DECODED" | jq -r '.title // "Notification"')
-                    BODY=$(echo "$DECODED" | jq -r '.message // ""')
-                    PRIORITY=$(echo "$DECODED" | jq -r '.priority // 5')
-                    APP=$(echo "$DECODED" | jq -r '.appid // 0')
-                    DATE=$(echo "$DECODED" | jq -r '.date // ""')
-
-                    # Only forward messages with priority >= configured threshold
-                    if [ "$PRIORITY" -ge "${MIN_PRIORITY:-0}" ]; then
-                      printf "Subject: [Gotify] %s\nFrom: %s\nTo: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n\n---\nPriority: %s\nDate: %s" \
-                        "$TITLE" "$SMTP_FROM" "$NOTIFY_EMAIL" "$BODY" "$PRIORITY" "$DATE" | \
-                        msmtp -C /tmp/msmtprc "$NOTIFY_EMAIL" && \
-                        echo "Email sent for message $ID: $TITLE" || \
-                        echo "Failed to send email for message $ID"
-                    fi
-
-                    # Update last seen ID
-                    if [ "$ID" -gt "$LAST_ID" ]; then
-                      LAST_ID=$ID
-                    fi
-                  done
-                fi
-
-                sleep ${POLL_INTERVAL:-30}
-              done
-          env:
-            - name: GOTIFY_CLIENT_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-tokens
-                  key: client-token
-            - name: SMTP_HOST
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: host
-            - name: SMTP_PORT
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: port
-            - name: SMTP_FROM
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: from
-            - name: SMTP_USER
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: user
-            - name: SMTP_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: password
-            - name: NOTIFY_EMAIL
-              valueFrom:
-                secretKeyRef:
-                  name: gotify-smtp
-                  key: notify-email
-            - name: MIN_PRIORITY
-              value: "5"
-            - name: POLL_INTERVAL
-              value: "15"
-          resources:
-            requests:
-              cpu: 10m
-              memory: 32Mi
-            limits:
-              cpu: 100m
-              memory: 64Mi
-      # No volumes — Postgres handles persistence

k8s/apps/gotify/igotify-deployment.yaml

@@ -1,59 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: igotify
-  template:
-    metadata:
-      labels:
-        app: igotify
-    spec:
-      containers:
-        - name: igotify
-          image: ghcr.io/androidseb25/igotify-notification-assist:v1.6.0.1
-          ports:
-            - containerPort: 8080
-              protocol: TCP
-          env:
-            - name: ENABLE_CONSOLE_LOG
-              value: "true"
-            - name: ENABLE_SCALAR_UI
-              value: "true"
-            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
-            # left unset — not required when Gotify is reachable via public domain
-            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
-          volumeMounts:
-            - name: data
-              mountPath: /app/data
-          livenessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 10
-            periodSeconds: 30
-          readinessProbe:
-            httpGet:
-              path: /Version
-              port: 8080
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 50m
-              memory: 64Mi
-            limits:
-              cpu: 200m
-              memory: 128Mi
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: igotify-data

k8s/apps/gotify/igotify-ingress.yaml

@@ -1,22 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: igotify
-  namespace: notifications
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: igotify.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: igotify
-                port:
-                  number: 80

k8s/apps/gotify/igotify-pvc.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: igotify-data
-  namespace: notifications
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi

k8s/apps/gotify/igotify-service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: igotify
-  namespace: notifications
-  labels:
-    app: igotify
-spec:
-  selector:
-    app: igotify
-  ports:
-    - port: 80
-      targetPort: 8080
-      protocol: TCP

k8s/apps/gotify/ingress.yaml

@@ -1,24 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gotify
-  namespace: notifications
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
-    # WebSocket support for Gotify client connections (/stream endpoint)
-    # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: gotify.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: gotify
-                port:
-                  number: 80

k8s/apps/gotify/kustomization.yaml

@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - namespace.yaml
-  - db.yaml
-  - deployment.yaml
-  - service.yaml
-  - ingress.yaml
-  - igotify-pvc.yaml
-  - igotify-deployment.yaml
-  - igotify-service.yaml
-  - igotify-ingress.yaml

k8s/apps/gotify/namespace.yaml

@@ -1,6 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: notifications
-  labels:
-    kubernetes.io/metadata.name: notifications

k8s/apps/gotify/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gotify
-  namespace: notifications
-  labels:
-    app: gotify
-spec:
-  selector:
-    app: gotify
-  ports:
-    - port: 80
-      targetPort: 80
-      protocol: TCP

k8s/apps/immich/db.yaml

@@ -18,7 +18,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie
   postgresql:
     extensions:

k8s/apps/llm-serving/embeddings.yaml

@@ -45,14 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-embeddings
-        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-serving/kustomization.yaml

@@ -14,7 +14,6 @@ resources:
   - ornith.yaml
   - reasoning.yaml
   - reranker.yaml
-  - qwen-cpu.yaml
   - networkpolicy.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.

k8s/apps/llm-serving/networkpolicy.yaml

@@ -52,20 +52,6 @@ spec:
           port: 8080
         - protocol: TCP
           port: 9000
-    # Allow from paperless namespace (paperless-ai auto-tagging)
-    # Bypasses gateway until service-account JWT token exchange is implemented.
-    # paperless-ai-agent has llm:inference role in Authentik.
-    - from:
-        - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: paperless
-      ports:
-        - protocol: TCP
-          port: 8080
-        - protocol: TCP
-          port: 80
-        - protocol: TCP
-          port: 8000
     # Allow intra-namespace (pod-to-pod within llm-serving)
     - from:
         - podSelector:

k8s/apps/llm-serving/ornith.yaml

@@ -33,8 +33,12 @@ spec:
 
         ollama pull ornith:35b
 
+        ollama pull qwen2.5:3b-instruct
+
         ollama run ornith:35b "ok" >/dev/null 2>&1 || true
 
+        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+
         wait $SERVE_PID
 
         '
@@ -44,13 +48,13 @@ spec:
       - name: OLLAMA_MODELS
         value: /mnt/models/ollama
       - name: OLLAMA_CONTEXT_LENGTH
-        value: '65536'
+        value: '32768'
       - name: OLLAMA_KEEP_ALIVE
         value: '-1'
       - name: OLLAMA_NUM_PARALLEL
         value: '1'
       - name: OLLAMA_MAX_LOADED_MODELS
-        value: '1'
+        value: '2'
       image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
       name: kserve-container
       ports:
@@ -61,7 +65,8 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama ps 2>/dev/null | grep -q ornith
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
         periodSeconds: 10
       resources:
         limits:
@@ -77,26 +82,22 @@ spec:
           command:
           - /bin/sh
           - -c
-          - ollama ps 2>/dev/null | grep -q ornith
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
         failureThreshold: 120
         periodSeconds: 15
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-ornith
-        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
-    # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
-    # Frees 1 GPU for ComfyUI.
-    maxReplicas: 1
-    minReplicas: 1
+    # 2 replicas -- each its own GPU, each loading both ornith:35b and
+    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
+    # get an independent instance instead of contending on one, at the
+    # cost of judge/qwen traffic still sharing whichever replica an
+    # implementer call also lands on.
+    maxReplicas: 2
+    minReplicas: 2
     nodeSelector:
       kubernetes.io/hostname: worker-1
     runtimeClassName: nvidia

k8s/apps/llm-serving/qwen-cpu.yaml

@@ -1,115 +0,0 @@
-# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
-# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
-# than GPU but sufficient for lightweight tasks (summarization,
-# classification, quick answers).
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: qwen-cpu
-  namespace: llm-serving
-  labels:
-    app: qwen-cpu
-    app.kubernetes.io/name: qwen-cpu
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app: qwen-cpu
-  template:
-    metadata:
-      labels:
-        app: qwen-cpu
-        app.kubernetes.io/name: qwen-cpu
-        app.kubernetes.io/part-of: llm-serving
-    spec:
-      nodeSelector:
-        kubernetes.io/hostname: talos-cp-2
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      containers:
-        - name: ollama
-          image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              ollama serve &
-              SERVE_PID=$!
-              until ollama list >/dev/null 2>&1; do sleep 2; done
-              ollama pull qwen2.5:3b-instruct
-              ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
-              wait $SERVE_PID
-          env:
-            - name: OLLAMA_HOST
-              value: "0.0.0.0:8080"
-            - name: OLLAMA_MODELS
-              value: /root/.ollama/models
-            - name: OLLAMA_CONTEXT_LENGTH
-              value: "32768"
-            - name: OLLAMA_KEEP_ALIVE
-              value: "-1"
-            - name: OLLAMA_MAX_LOADED_MODELS
-              value: "1"
-            - name: OLLAMA_NUM_PARALLEL
-              value: "2"
-          ports:
-            - containerPort: 8080
-              protocol: TCP
-          readinessProbe:
-            exec:
-              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
-            periodSeconds: 10
-          startupProbe:
-            exec:
-              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
-            failureThreshold: 60
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: "4"
-              memory: 4Gi
-            limits:
-              cpu: "8"
-              memory: 8Gi
-          volumeMounts:
-            - mountPath: /root/.ollama
-              name: ollama-data
-      volumes:
-        - name: ollama-data
-          persistentVolumeClaim:
-            claimName: qwen-cpu-data
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: qwen-cpu
-  namespace: llm-serving
-  labels:
-    app: qwen-cpu
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  selector:
-    app: qwen-cpu
-  ports:
-    - port: 80
-      targetPort: 8080
-      protocol: TCP
----
-# Small PVC for qwen2.5:3b model weights (~1.9GB).
-# Separate from llm-models PVC which is pinned to worker-1.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: qwen-cpu-data
-  namespace: llm-serving
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 5Gi

k8s/apps/llm-serving/reasoning.yaml

@@ -28,14 +28,9 @@ spec:
       # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
       # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
       # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
-      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
-      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
-      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
-      # metric after the rollout -- max_num_batched_tokens=8192 also raises
-      # activation memory out of the same budget, so the post-boot
-      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
-      # only valid if it still covers 2x65k + 3x8k. The old
+      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
+      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
+      # above the 131072 target with room to spare -- the old
       # OffloadingConnector CPU-DRAM spillover (tuned for the previous
       # model's much smaller 16384 context) is no longer needed and is
       # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -59,37 +54,10 @@ spec:
       - --tensor-parallel-size=1
       - --pipeline-parallel-size=2
       - --max-model-len=131072
-      - --gpu-memory-utilization=0.95
-      - --max-num-seqs=5
-      # Prefill scheduling -- see the measured starvation this fixes.
-      # V1's scheduler has no prefill/decode phase: it walks self.running and
-      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
-      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
-      # per request is applied BEFORE the budget clamp:
-      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
-      #     take = min(take, token_budget)
-      # The loop is `while ... and token_budget > 0`. With the threshold at its
-      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
-      # budget, token_budget hits 0, the loop exits and every other request is
-      # skipped for that step -- measured as 262 tok/s contended prefill and
-      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
-      # Hence threshold must be strictly BELOW the budget or it never binds:
-      # at 8192/8192 a single request is served per step (still starved), while
-      # 8192/2048 serves four. Verified by replaying the loop.
-      - --max-num-batched-tokens=8192
-      - --long-prefill-token-threshold=2048
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
       - --enable-chunked-prefill
       - --enable-prefix-caching
-      #
-      # NOT set, deliberately: --max-num-partial-prefills and
-      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
-      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
-      # rejects any value other than the default for either one, calling
-      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
-      # -> AssertionError, because there is no V0 engine to fall back to.
-      # The Deployment uses strategy Recreate, so setting them is an outage.
-      # V1 does concurrent chunked prefill natively; it needs no flag, only the
-      # budget/threshold ratio above.
       # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
       # blocks.
       - --reasoning-parser=qwen3
@@ -136,14 +104,6 @@ spec:
         name: models
       - mountPath: /dev/shm
         name: shm
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reasoning
-        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     maxReplicas: 1

k8s/apps/llm-serving/reranker.yaml

@@ -45,14 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reranker
-        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/paperless/backup-cronjob.yaml

@@ -48,7 +48,7 @@ spec:
                   mountPath: /backup
           containers:
             - name: mc-mirror
-              image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
+              image: minio/mc:latest
               env:
                 - name: ACCESS_KEY
                   valueFrom:

k8s/apps/paperless/configmap.yaml

@@ -20,7 +20,3 @@ data:
   # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
   # on 'account_confirm_email' without this.
   PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
-  # Auto-connect social accounts to existing users with matching email
-  PAPERLESS_SOCIAL_AUTO_SIGNUP: "true"
-  # Allow automatic linking of social accounts to existing users
-  PAPERLESS_ACCOUNT_ALLOW_SIGNUPS: "true"

k8s/apps/paperless/kustomization.yaml

@@ -4,7 +4,6 @@ namespace: paperless
 resources:
   - pvc.yaml
   - configmap.yaml
-  - paperless-ai-config.yaml
   - redis.yaml
   - deployment.yaml
   - service.yaml
@@ -12,7 +11,6 @@ resources:
   - backup-cronjob.yaml
   - adapter-configmap.yaml
   - rbac.yaml
-  - paperless-ai.yaml
 # postgres: paperless-db CNPG Cluster, deployed by k8s/infra/databases (wave 2,
 # before this app at wave 8) - not duplicated here. Same for the paperless-oidc
 # and paperless-minio-creds Secrets, written by PostSync provisioning Jobs in

k8s/apps/paperless/paperless-ai-config.yaml

@@ -1,49 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-ai-env
-  namespace: paperless
-data:
-  paperless-ai-config.env: |
-    # Paperless-NGX API configuration
-    PAPERLESS_URL=http://paperless.paperless.svc.cluster.local:8000
-    PAPERLESS_API_TOKEN=7b89463e04c141f4172fbcddf78d623547d327ec
-
-    # AI Processing settings
-    ENABLE_AI_PROCESSING=yes
-    AI_MODEL=reasoning
-    LLM_API_URL=https://api.riotpiao.com/v1
-    LLM_API_TOKEN_FILE=/app/data/llm_token.txt
-
-    # Auto-tagging configuration
-    ENABLE_AUTO_TAGGING=yes
-    ENABLE_CORRESPONDENT_EXTRACTION=yes
-    ENABLE_DOCUMENT_TYPE_EXTRACTION=yes
-    ENABLE_TITLE_GENERATION=yes
-    
-    # Advanced tagging with AI
-    ENABLE_INTELLIGENT_TAGGING=yes
-    ENABLE_DATE_EXTRACTION=yes
-    ENABLE_TIMELINE_TAGGING=yes
-    AUTO_CREATE_TAGS=yes
-    
-    # Tagging prompts for custom extraction
-    CUSTOM_EXTRACTION_PROMPT=Extract document date, time period, financial year, and temporal context. Create tags like YYYY, Q1-YYYY, Last-Month, Current-Year, Historic
-    TAG_EXTRACTION_MODEL=reasoning
-
-    # Scanning behavior
-    SCAN_INTERVAL_SECONDS=60
-    PROCESS_EXISTING_DOCUMENTS=no
-    ADD_AI_TAG=yes
-    AI_TAG_NAME=ai-processed
-
-    # Date/Timeline tagging configuration
-    DATE_EXTRACTION_FORMAT=iso8601
-    TIMELINE_TAG_GRANULARITY=year,quarter,month
-    AUTO_TAG_HISTORICAL=yes
-    AUTO_TAG_RECENT=yes
-    
-    # Performance
-    MAX_CONCURRENT_REQUESTS=2
-    REQUEST_TIMEOUT=60
-    DATE_EXTRACTION_TIMEOUT=30

k8s/apps/paperless/paperless-ai-new.yaml

@@ -1,127 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-ai-data
-  namespace: paperless
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 2Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-ai
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-ai
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-ai
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-ai
-    spec:
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-      
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token
-              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              mkdir -p /data
-              echo "$TOKEN" > /data/llm_token.txt
-              echo "[init] Token fetched and stored successfully"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /data
-      
-      containers:
-        - name: paperless-ai
-          image: clusterzx/paperless-ai:latest
-          imagePullPolicy: Always
-          env:
-            # Paperless-ngx connection
-            - name: PAPERLESS_API_URL
-              value: "http://paperless.paperless.svc.cluster.local:8000"
-            - name: PAPERLESS_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: PAPERLESS_API_TOKEN
-            - name: PAPERLESS_USERNAME
-              value: "admin"
-            
-            # LLM API configuration
-            - name: AI_PROVIDER
-              value: "custom"
-            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token file created by init container with JWT from Authentik
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
-            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            
-            # Scanning behavior
-            - name: SCAN_INTERVAL
-              value: "60"
-            - name: PROCESS_PREDEFINED_DOCUMENTS
-              value: "no"
-            
-            # Tagging configuration
-            - name: ADD_AI_TAG
-              value: "yes"
-            - name: AI_TAG_NAME
-              value: "ai-processed"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-            - name: USE_PROMPT_TAGS
-              value: "yes"
-          
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-          
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi

k8s/apps/paperless/paperless-ai.yaml

@@ -1,127 +0,0 @@
-# Secret paperless-ai-config managed via SOPS (argocd/secrets/paperless-ai-secrets.enc.yaml)
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless-ai
-  namespace: paperless
-  labels:
-    app.kubernetes.io/name: paperless-ai
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless-ai
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-ai
-    spec:
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-      volumes:
-        - name: paperless-ai-data
-          persistentVolumeClaim:
-            claimName: paperless-ai-data
-        - name: paperless-ai-env-config
-          configMap:
-            name: paperless-ai-env
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          securityContext:
-            runAsUser: 0
-            fsGroup: 0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
-                -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
-                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              # Write to both locations for compatibility
-              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
-              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
-              echo "[init] Token fetched and stored"
-              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
-              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
-          env:
-            - name: LLM_AUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: LLM_AUTH_CLIENT_SECRET
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-      containers:
-        - name: paperless-ai
-          image: clusterzx/paperless-ai:latest
-          env:
-            # Paperless-ngx connection
-            - name: PAPERLESS_API_URL
-              value: "http://paperless.paperless.svc.cluster.local:8000"
-            - name: PAPERLESS_API_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: PAPERLESS_API_TOKEN
-            - name: PAPERLESS_USERNAME
-              value: "admin"
-            # LLM API — via public gateway with JWT auth
-            - name: AI_PROVIDER
-              value: "custom"
-            - name: CUSTOM_BASE_URL
-              value: "https://api.riotpiao.com/v1"
-            # Token will be read from file at runtime by the application
-            # The init container fetches it and stores in /app/data/llm_token.txt
-            - name: CUSTOM_API_KEY_FILE
-              value: "/app/data/llm_token.txt"
-            - name: CUSTOM_MODEL
-              value: "qwen2.5:3b-instruct"
-            # Behavior - scan for new documents and tag them
-            - name: SCAN_INTERVAL
-              value: "60"
-            - name: PROCESS_PREDEFINED_DOCUMENTS
-              value: "no"
-            - name: ADD_AI_TAG
-              value: "yes"
-            - name: AI_TAG_NAME
-              value: "ai-processed"
-            - name: USE_PROMPT_TAGS
-              value: "yes"
-            - name: ADD_AI_DESCRIPTIONS
-              value: "yes"
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
-            - name: paperless-ai-env-config
-              mountPath: /app/data/.env
-              subPath: paperless-ai-config.env
-          resources:
-            requests:
-              cpu: 100m
-              memory: 512Mi
-            limits:
-              cpu: "1"
-              memory: 2Gi

k8s/apps/poimen-frontend/ingress.yaml

@@ -1,24 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: poimen-frontend
-  namespace: poimen
-  annotations:
-    # Next.js SSE + LLM streaming responses need long timeouts + no buffering.
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-buffering: "off"
-    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: poimen.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: poimen-frontend
-                port:
-                  number: 3000

k8s/apps/poimen-frontend/kustomization.yaml

@@ -1,8 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Edge Ingress for poimen.riotpiao.com.
-# Same pattern as k8s/apps/api/ (api-gw ingress).
-# App manifests (Deployment, Service, NetworkPolicy) live in
-# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
-resources:
-  - ingress.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -40,7 +40,7 @@ spec:
         
         # Git
         - name: GIT_REPO
-          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+          value: https://forgejo.riotpiao.com/rock/homelab.git
         - name: GIT_AUTHOR_EMAIL
           value: [email protected]
         - name: GIT_AUTHOR_NAME

k8s/argocd/apps/-1-projects.yaml

@@ -18,7 +18,7 @@ spec:
       prune: true
       selfHeal: true
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/argocd/projects
   destination:

k8s/argocd/apps/00-secrets.yaml

@@ -17,7 +17,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
     # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the

k8s/argocd/apps/00-substrate.yaml

@@ -21,7 +21,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -93,7 +93,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
     # deterministically. The previous directory.include with bare filenames
@@ -127,7 +127,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/bootstrap/ingress
   destination:
@@ -149,7 +149,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/cluster-maintenance
   destination:
@@ -177,7 +177,7 @@ spec:
       valueFiles:
         - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
   sources:
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -200,7 +200,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/bootstrap/kyverno
   destination:

k8s/argocd/apps/01-image-updater.yaml

@@ -0,0 +1,33 @@
+# ArgoCD Image Updater - auto-updates Application images from registry
+# Watches forgejo.riotpiao.com for new image tags and updates Applications
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-image-updater
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://argoproj.github.io/argo-helm
+      chart: argocd-image-updater
+      targetRevision: "0.11.2"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/argocd-image-updater/values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false

k8s/argocd/apps/06-ci-cd.yaml

@@ -1,55 +0,0 @@
-# Tekton Pipelines v0.68.0
-#
-# Install method: vendored release.yaml in k8s/infra/tekton/
-# downloaded from https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.68.0/release.yaml
-#
-# To upgrade:
-#   1. Download new release.yaml from https://github.com/tektoncd/pipeline/releases
-#   2. Replace k8s/infra/tekton/release.yaml
-#   3. Commit and push — ArgoCD syncs automatically
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: tekton-pipelines
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: tekton-pipelines
-    app.kubernetes.io/part-of: homelab-infra
-    wave: "06"
-spec:
-  project: homelab
-
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/infra/tekton
-
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: tekton-pipelines
-
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m
-
-  ignoreDifferences:
-  - group: admissionregistration.k8s.io
-    kind: ValidatingWebhookConfiguration
-    jsonPointers:
-    - /webhooks/0/clientConfig/caBundle
-    - /webhooks
-  - group: admissionregistration.k8s.io
-    kind: MutatingWebhookConfiguration
-    jsonPointers:
-    - /webhooks/0/clientConfig/caBundle
-    - /webhooks

k8s/argocd/apps/08-secret-rotation.yaml

@@ -9,7 +9,7 @@ spec:
   project: homelab
   
   sources:
-  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     path: k8s/apps/secret-rotation-controller
     targetRevision: main
   

k8s/argocd/apps/10-storage-observability.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/minio/minio-operator-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -41,7 +41,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/minio
   destination:
@@ -66,7 +66,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/longhorn
   destination:
@@ -102,7 +102,7 @@ spec:
         skipCrds: true
         valueFiles:
           - $values/k8s/infra/monitoring/prometheus-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -152,7 +152,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/monitoring/crds
   destination:
@@ -183,7 +183,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/monitoring
   destination:
@@ -213,7 +213,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -237,7 +237,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/tracing
   destination:
@@ -249,26 +249,3 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: local-path-provisioner
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/infra/local-path
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: local-path-storage
-  # No CreateNamespace: the Namespace object (with its privileged PSA label,
-  # which the helper Pods need for their hostPath) is part of the manifest.
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true

k8s/argocd/apps/20-logging.yaml

@@ -19,7 +19,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/loki-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -53,7 +53,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/grafana-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -87,7 +87,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/promtail-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:

k8s/argocd/apps/30-security.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/vault-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -46,7 +46,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/authentik-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/iam
   destination:
@@ -109,7 +109,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -152,12 +152,17 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
   destination:
@@ -175,12 +180,17 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
     helm:
@@ -201,12 +211,17 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
     helm:

k8s/argocd/apps/40-data.yaml

@@ -14,7 +14,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/databases
   destination:

k8s/argocd/apps/50-memory-queues.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/messaging/memory-queues
   destination:

k8s/argocd/apps/50-messaging.yaml

@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/messaging/kafka-cluster
   destination:

k8s/argocd/apps/55-api-gateway.yaml

@@ -36,17 +36,19 @@ metadata:
     app.kubernetes.io/component: gateway
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
-    # → patches this Application's kustomize.images via k8s API.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
+    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   revisionHistoryLimit: 3
   sources:
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab-frontend.git
       targetRevision: main
       path: k8s
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       path: k8s/apps/api
   destination:

k8s/argocd/apps/56-llm-serving.yaml

@@ -20,7 +20,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/llm-serving
   destination:

k8s/argocd/apps/58-comfyui.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: comfyui
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: comfyui
-    app.kubernetes.io/component: image-generation
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/comfyui
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: comfyui
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/apps/59-gotify.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: gotify
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: gotify
-    app.kubernetes.io/component: notifications
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/gotify
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: notifications
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/apps/60-applications.yaml

@@ -19,10 +19,10 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/temporal/temporal-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       path: k8s/apps/temporal
   destination:
@@ -51,7 +51,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/portainer/portainer-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -74,7 +74,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/cloudflared
   destination:
@@ -97,7 +97,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/agent-pod
   destination:
@@ -130,7 +130,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/sms
   destination:
@@ -157,7 +157,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/paperless
   destination:
@@ -189,7 +189,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/apps/immich
   destination:
@@ -220,10 +220,10 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/homarr/homarr-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       ref: values
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
       targetRevision: main
       path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
   destination:
@@ -246,8 +246,11 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "8"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # ArgoCD Image Updater - auto-update on new image push
+    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
+    argocd-image-updater.argoproj.io/app.update-strategy: digest
+    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   source:
@@ -278,7 +281,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/infra/rbac
   destination:

k8s/argocd/apps/60-kmsvc-manage.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
+    repoURL: https://forgejo.riotpiao.com/rock/kmsvc-manage.git
     targetRevision: main
     path: k8s/argocd/apps
     directory:

k8s/argocd/apps/70-poimen.yaml

@@ -1,81 +1,43 @@
-# Poimen — unified ArgoCD Application (wave 7).
-#
-# Consolidates what was previously five separate apps:
-#   poimen            → source 1 (poimen-workflows workers)
-#   poimen-workflows  → duplicate of source 1 (deleted)
-#   poimen-memory     → source 2 (memory service + redis)
-#   poimen-memory-app → duplicate of source 2 (deleted)
-#   poimen-memory-database → source 3 (CNPG cluster, moved to k8s/infra/poimen)
-#
-# poimen-toolkit is intentionally kept as a separate Application because it
-# manages cluster-scoped CRDs (ToolDefinition). A bad CRD can affect the
-# whole cluster; keeping it separate means it has its own sync history,
-# independent rollback, and can be paused without touching app workloads.
-#
-# Sync ordering via argocd sync-wave annotations:
-#   wave -1 → DB (CNPG Cluster) — must be Ready before app pods start
-#   wave  0 → all application Deployments (default)
-#
-# Image Updater tracks all four images. Write-back via argocd (in-cluster
-# annotation patch — no git commit per image update).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
   name: poimen
   namespace: argocd
-  labels:
-    app.kubernetes.io/part-of: poimen
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # ── Image promotion ──────────────────────────────────────────────────────
-    # Forgejo package webhook → api-gw patches kustomize.images per source.
-    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: digest
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
-  # Multi-source: each repo owns its own k8s manifests.
-  # ArgoCD syncs all sources in parallel; sync-wave annotations on individual
-  # resources control ordering within the sync operation.
   sources:
-    # 1. Workers — Temporal activity + workflow workers, pi-worker pool
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-memory.git
       targetRevision: main
-      path: k8s
-
-    # 2. Memory service — poimen-memory API, Redis cache, NetworkPolicies
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-workflows.git
       targetRevision: main
-      path: k8s/app
-
-    # 3. Database infrastructure — CNPG Postgres cluster (wave -1)
-    #    Moved from k8s/infra/databases to isolate poimen DB from other tenants.
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/rock/poimen-frontend.git
       targetRevision: main
-      path: k8s/infra/poimen
-
-    # 4. Frontend app manifests — Deployment, Service, NetworkPolicy.
-    #    Each repo owns its own k8s/ (same pattern as homelab-frontend for api-gw).
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
-      targetRevision: main
-      path: k8s
-
-    # 5. Frontend ingress — poimen.riotpiao.com edge route.
-    #    Lives in homelab (same pattern as k8s/apps/api/ for api-gw).
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      path: k8s/apps/poimen-frontend
-
+      path: k8s/argocd
   destination:
     server: https://kubernetes.default.svc
     namespace: poimen
-
   syncPolicy:
     automated:
       prune: true
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
-      - SkipDryRunOnMissingResource=true
     retry:
       limit: 5
       backoff:

k8s/argocd/apps/71-poimen-toolkit.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: poimen-toolkit
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "7"
-    # Image promotion: Forgejo package webhook → api-gw.
-    # Tracks: poimen-toolkit-controller, pi-worker.
-    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
-    targetRevision: main
-    path: k8s
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: poimen
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/projects/homelab-project.yaml

@@ -17,14 +17,12 @@ spec:
     - https://github.com/Riotpiaole/Poimen-workflows.git
     - https://github.com/Riotpiaole/poimen*.git
     # In-cluster Forgejo repos — explicit allowlist (no wildcard)
-    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
-    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
+    - https://forgejo.riotpiao.com/rock/homelab.git
+    - https://forgejo.riotpiao.com/rock/homelab-frontend.git
+    - https://forgejo.riotpiao.com/rock/kmsvc-manage.git
+    - https://forgejo.riotpiao.com/rock/poimen.git
+    - https://forgejo.riotpiao.com/rock/poimen-memory.git
+    - https://forgejo.riotpiao.com/rock/poimen-workflows.git
     - https://forgejo.riotpiao.com/rock/riotpiao.com.git
     # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
     - https://cloudnative-pg.github.io/charts

k8s/argocd/root/homelab-root.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
     targetRevision: main
     path: k8s/argocd/apps
     directory:

k8s/argocd/secrets/etcd-defrag-creds.enc.yaml

@@ -1,27 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: etcd-defrag-creds
-    namespace: kube-system
-    labels:
-        app.kubernetes.io/name: etcd-defrag
-type: Opaque
-stringData:
-    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
-    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
-    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
-            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
-            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
-            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
-            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-07T05:37:26Z"
-    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/forgejo-registry-pat-api.enc.yaml

@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: forgejo-registry-pat
-    namespace: api
-type: Opaque
-stringData:
-    pat: ENC[AES256_GCM,data:alYPvHBVqnf13NZu6jtgA6WdFM4x46s5mIY8uEnfWdq0Vp+7oKrY1A==,iv:JbUjes5ZB9eKQ1nkV02++7F/BaWIROfT6gjShzajXro=,tag:ApjawJold0QApnX1GUXGhw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6bFZhU3JLSUZEVWhYdEdZ
-            MURoSWNlZjY0RkxyaWlwU2ZTVjJ5V2FnMVVZCis5YlBwQnZrWnVJNFh4UnNldmx6
-            Y2JjdnpkRTVGSjRSZ0U0THdDZHIrNHcKLS0tIHZDd1BDVGRDSHJEQThuTExORWs2
-            b3BxenY4WHhpRkRTZHJGMkNYMm14RnMKH//Xpqbf/ZBNFQ43D0eFUWwRcQWqA3CC
-            RG8U9wyxC6YrlruU2hY6eB2x0OjsVic34ZGaR752Bwyp0Pz49C/k3g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-28T03:10:55Z"
-    mac: ENC[AES256_GCM,data:nLZueHygckw+Gddlea90tRU8h8nEGpe21JUi9nCdpc0ctkSlTDqcR3Q7JFPwbEGHGFU5UJ2xnkBZEDIVmExJryqPBMXT7hQd3ySeQgPwAPGt6zEhlRc5jXhqmmrpJiiN5XUL52AZVDwcWACg7kVzBXIClzihFRW+UqjJZhdtELM=,iv:C0H4AFfdfpsBvP1z2chVvW/VTejxYxwNn/KuA50LeH0=,tag:ZEkTCTKvoZH7DjrcwfL3dw==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/forgejo-smtp-secrets.enc.yaml

@@ -1,24 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:uS8=,iv:EEoo9U+C244eAJMSTOQVkf5AE6BeHrc5DWPjtWnPRdk=,tag:H+YmBSGvcON3wh0p22LXDQ==,type:str]
-kind: ENC[AES256_GCM,data:j1/PFkTS,iv:ja4q8X+nzE/ZczwcY+Qe2DnnsxG64W1fkRPQvOaoqvA=,tag:WiilMGagudEKUlc2JdbGyg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:J9iAIHboMyHu6xn/,iv:p3z3uzlkM7VDzOT3WDCelCdZ2t+QqSPmFtqYfvXPQMs=,tag:rKRtpUexVkuZKOJ34a0Xjw==,type:str]
-    namespace: ENC[AES256_GCM,data:su0FvA==,iv:6SPvwxZ/4aoL0Z07zdPC9r6L7HOHuKv3RodXpVcii04=,tag:njwkj+yvSgN8sliJP8T/Kw==,type:str]
-type: ENC[AES256_GCM,data:Qd6WJN1c,iv:M3fc78LvemcEbWzesuYeQ/GZyIOl7Eg/0EmUe3p0qAk=,tag:Tnzwewn0vwdowxy/EyuPdA==,type:str]
-stringData:
-    user: ENC[AES256_GCM,data:/Z7gPrsTUZOLzfoZ8cZGD10wrZE=,iv:0ydBSeq+yHB2b1J4W7FwIv55Eh+N3qfQ6Q7PwoUAvYo=,tag:CO19F8nElaYZyFiFR6N/Tg==,type:str]
-    password: ENC[AES256_GCM,data:AOl4StpeQIHSLX+oEFnkfg==,iv:mYsZ+5sum6YqyUPndtPCniTraxldKc803ULlKs8Gsaw=,tag:hK3w51ifZ/omAL7XDf8seg==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaMlhKVEM2bFdFOFAwV1lp
-            WUd6SEN4ZTF1TEpGYXhOYmJLK0tpcWZyc2w4ClVzZmI2Qk1KUnV1OXpyTEV2WWFa
-            VlJ2eHRSaEhqUnA2dUJVbWJUcEgxcFkKLS0tIG9IaFVnenNpSFRzdStiWjJvakVL
-            aDk2bTZGR3Zya3ROUS9vd1hEQVRFaG8KbeXA6IebHEaB79N6u795336aHesHOgzO
-            uZvvBUzSBy3t3jfFk8bJP4aH79I33Ha2eK5rsvdsiv/orwCMXUINKg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T21:00:46Z"
-    mac: ENC[AES256_GCM,data:DjzPtR+Ueihh166Bvd3jCLtZFQdrvrxOoF87cv6lxKGWPEptT5vbw/EfuIFJBb6lvEJFDWKRC/r5ASdGwComYsPn0DZs4BPCzeKBtLfP9K2OGCXnAC7eGqt4mzMrrW0CQd1QSGcuXw8CY7uJGkMPGPKe3/0mQWiis2OSpHxTM18=,iv:/QqSEFU9RuX9z5Z6aSaD7KlP/cgGTOYvTH+VJOnDTYM=,tag:yFbFXn+iWqQZx7wW4dKppg==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/gotify-admin-secrets.enc.yaml

@@ -1,24 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:wR4=,iv:cRBzbvu0eUYCYeKeysua1/P3Meli/rQyj/mIV6VWnPM=,tag:oyTPA/mLYNUX+QDkYLlZyQ==,type:str]
-kind: ENC[AES256_GCM,data:bdKQdadW,iv:F3DQiBI9xhSx87jkS1Hyevo48uUCBjsJSjbScIBznKA=,tag:PFakzzBj5S9F65dxu0L2rg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:XHFYrKClPo+IwRbM,iv:ZGuL+cN840Y1bOTC62NhDDcoZpTzDWQ4GfqPJX/QWmI=,tag:hlCVjzvfcxXGOASc3vda/Q==,type:str]
-    namespace: ENC[AES256_GCM,data:0BbhgZBog+1qY/iRJA==,iv:88tiSpEDqIokT5VP/d6bB2+aUyh1kZ7NEHwZWoJW3XU=,tag:CBR01jh2U9h7kNEcK1e1sA==,type:str]
-type: ENC[AES256_GCM,data:Mpv1V73w,iv:BW3r6RpLnwe3XDKwrZySyOjrWUnSwIG5JOoPLzP/5gM=,tag:oyJySL0haOei1m4i+1AJ1g==,type:str]
-stringData:
-    username: ENC[AES256_GCM,data:8xmxLGE=,iv:J/vEvXGoD+ka6FDgnSwDz0fs9IxuJIZW9r7Oyu2qxC8=,tag:dN9jd6NSavMN8+uzvemYWg==,type:str]
-    password: ENC[AES256_GCM,data:vB9PaaUiQZ8FY8pO0cq1fHLi7Gq5t4U=,iv:C0Y1m2SGYP3oTIFoau5JavVmLblj6te/SPMLodIwiZw=,tag:3Ip4YvR4WPzR0bBpTyjytA==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRT3JoTnhVVW5SUUJXbTdz
-            dUpLcmtBWWhyaGk3Y1hBM1ArcVI5eHREbmp3CiswUGVmd0NhejZwQ2UvNEdxS3ow
-            L1RweS9Kb2paeStLZ0tLSFdWbVZqQW8KLS0tIHJHT3hiMmxtM0Q1Ym5KOVpLVFpl
-            enR3NmdNdmdwVitTQVJlRHFWcjR0N2sKQw9ZZs+Ji/Zq/feO3qy4DwaCfWgDOQ/z
-            FVVhcCXweN58tb+9fzCJ+pNi/hSmvUkCMbb1+60qBvEehNzOoMRJ5g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-10T13:31:00Z"
-    mac: ENC[AES256_GCM,data:3HWV/NG0yTbsxY28u41zTRmAPc1kokGb4nCLmAsyq8/uvxcP+evDNyZXYpS93eVdPRfRZ1OqVBzyVGJEnj7JSXQVmlzor9JcWEL2fcpogoLVfJZKTRD6hk6optnBMjj84iQEEOx3A80JrDOdoXsH0pd9bJBABwoUOJwuufbXcIU=,iv:KLZsmAcapQgV08pFhGIvPt5ylsWg3xazAAjPuJYOaXA=,tag:1vKA3fISMIurHzxZ04F3lg==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/gotify-smtp-secrets.enc.yaml

@@ -1,28 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:FOg=,iv:15mfPeWXV5LQEYahaYvNf1z2bHbxk4j5i8DXT6mdG/0=,tag:1f9/jnnu+wGeeiXGNFgKzA==,type:str]
-kind: ENC[AES256_GCM,data:dOFfNQiM,iv:HVLosbRpcCgu4iiYKV8MDLiQ0uhj8DXBfVpRBW2NFNo=,tag:mCbRIzD1OHRhBLc+7xcVug==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:kHoKhGyiIQvCfng=,iv:dAhxjeLlXK3yueMKfrHxmh9YmNA7AYKFBquLikCNzcE=,tag:3Xoaw7YNBCU/GINlaQOpBw==,type:str]
-    namespace: ENC[AES256_GCM,data:fTJMZhbqSQWD3/cnWg==,iv:D7zWa91+Fgerfyrywf8VA5YNzGrThhLz7CvYXucfiq0=,tag:RcIegCl2UR1JsbfIQm928w==,type:str]
-type: ENC[AES256_GCM,data:Qh51bTsM,iv:2Htv0Xze7Hd1m6zkP6rtFXAlg8qm0AKylSjwJxRjpBk=,tag:NCu5iVUTNVfYgErZSvwHIg==,type:str]
-stringData:
-    host: ENC[AES256_GCM,data:aZXSvQ8B7h78q1kHmh0=,iv:uUyL9X3ehIHqztGAtXtAQWgduvbSsSwZBZlTFMdOlBo=,tag:OM/vdO19VKDSa4W5WQAKNQ==,type:str]
-    port: ENC[AES256_GCM,data:5ZBB,iv:1/XAfq+PJKdBsufx+EPvvB/cm9nbEwYigc8eACXjR8g=,tag:WlNyz7gTPh0KMe5oKVd0BA==,type:str]
-    from: ENC[AES256_GCM,data:2/akr8cXgmgQGnqJaFcLJMDcWw==,iv:0rT/6aqyXxn1jIJ5umYCDZR4S8Pbh4vUgaXNrxd3JF8=,tag:k0pQrPOrqWTyE1Xu9oSzVA==,type:str]
-    user: ENC[AES256_GCM,data:d9BLaFYOYm++HZMzlf1gVauKkUQ=,iv:Wd48T5UPVn6z3bS0t02X9GbrnWal3QoIQv2EgM9z9Wg=,tag:cmg2KnLY4Atwco+j2wVdeg==,type:str]
-    password: ENC[AES256_GCM,data:pmqEj9623A6bThKrkCCjuA==,iv:M/QwJ0s//UvuOjOljl67EDhvZt/9Wp8JicvCcows51A=,tag:bfRyIdj1cgIZxU3WPbKteg==,type:str]
-    notify-email: ENC[AES256_GCM,data:+Xo22g8U2wDKsnPnFq/+GKxgYOI=,iv:PE+C1etlp6046ragiv1iMDQbhfo5IULd/5akD0+Sc6A=,tag:cCZmXByA85fuZL9yozVLjw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpVVdxQTZFTHcwMXFFZEFk
-            ckNsYk4rdWtkeXFMVnlyVmdaRDBuRWsvZkQ4CitSSTBXOTZhWVpUZFFlR0JITVV4
-            Uy9lSlNHNjFNaFhHNTN0WnRaRlNNVGcKLS0tIHRlMnZpQVlScGRYYm5nT3Z5TWd6
-            ak1UZ1RobkpQZHBXR3MyenBDcU53Mk0KvH9O6bgwrjay0+1/A6TGX8GhITiDjWoO
-            RNX4fDtqNwhzmCfXbVjK30vlBjOFe+Bb7Z2n+hWMmHHDgFdS0xIAzA==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T20:57:18Z"
-    mac: ENC[AES256_GCM,data:V/wgjnpUBvaV39BCTlecCwQy2/1h+0vixEXleJc/I9y+AOvuwVZNH7M86hdjoAdsKiIoNYySj4Flz+MJ9C8jQoAxBTDPbolP9UwDFWQ5KMJTKPPDoJGLNjy7bUq51WoyePUM6WWAYIiWHIB3OvAxHaSzECzL+ZoAhQy92cPKa1o=,iv:vLtIoD/C5yeXPEr+2Lim6XnWzAj42vr9qQgsxKpuhA8=,tag:PBq8qPHrx4RRgAKCWrxG3Q==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/gotify-tokens-secrets.enc.yaml

@@ -1,27 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:YE0=,iv:s3yNqcBn7/DKUQNgbGGwVmlM1HzxYGLBKyB6Y2ii2WE=,tag:77KxAjOFU3G0fSfrgudKkg==,type:str]
-kind: ENC[AES256_GCM,data:PFW4VV9T,iv:n8gzMjE5C2TAfUTpp/8ex64B+hWUGG1K+2oQ4deN03Q=,tag:18Xqb6Di5izMHmzR5EU+QA==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:Xhg1fQrD4itrbvDW0g==,iv:/THWSXAThb9fj+mm3wcxqzSdzdt7nE06+xOpkCxyImE=,tag:UChokr06VkbDZcFZDcUY2A==,type:str]
-    namespace: ENC[AES256_GCM,data:r25U5MuuzDd8JJ2YjQ==,iv:uDUcS4ZTpZe8HmZMArzkdu7LV5GUoLSP2wIs5HB1gv0=,tag:fci7j30hoxnVKJwPFNfd1Q==,type:str]
-type: ENC[AES256_GCM,data:agEG8Fu8,iv:ckYX0buX8md1CWHXfxbAXQJLzoTxTJI16nlQ9LSzYi0=,tag:4tZWf3REbGOmmBiT14+dew==,type:str]
-stringData:
-    #ENC[AES256_GCM,data:ehjcsmz1R0i/n31f8M0IIUT4KDBwzz6f5ds=,iv:j5swFAKpC67ZvGioiCCC1D651LxUl9gME8GqK5Uc+ys=,tag:BUJ8k9LHs8NAPPZAwPuifA==,type:comment]
-    #ENC[AES256_GCM,data:ndAB00yun636VHDMonqTghO/jCWodNd/hmdbm1QmCvOAi4FvTLl8bY3wYR+ZtlkSQYvFNqH798MJixv5OZwxBj5H,iv:pS+7B60jaS5jhqDRw2LbBFv7mD1HO6+hjjv+iNpenXc=,tag:NU6+gxCHTGxqeDMfvkwJWw==,type:comment]
-    #ENC[AES256_GCM,data:T3mhSm/5q7JTSXNLrjhpP5GhqA7nXz8uAhJcEV1RDctAX0Dyfq8Qn4bNFO51ibwTETx4SnunPuFZVs++AvRnsA==,iv:oRGM8N4iEcwBrMzoEXQAeKqCKzUq+jXTMVq9W2l6oh4=,tag:GrrbC/tkWpA5kp5UYCyRBA==,type:comment]
-    app-token: ENC[AES256_GCM,data:jw+AqbsxyoYRvNrUDrq+GNq/TNfweFCs,iv:67vSSgMZCMV0GG37ZxUxHAWZqOOGMYCmo3J43WgLyNk=,tag:xlrnip4XomXBbMmooW9FKg==,type:str]
-    client-token: ENC[AES256_GCM,data:gsxeCqKh4e+DFjpn9jM5GfemAdBf8dSv,iv:l2bBMdxQUil8jU9XDjXGn3EtvFVrK5ibXDCeGaPbYTY=,tag:ELf5S6cdNQJ84Es5upu1IQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAza3YrRW1VTVNSOGlMK21a
-            bDhzQlEyZDNnZTZrSVNnTUEzU1hSdnM1ZFVzCjdBSUlmWG5EcnlzbSs5bXdDaGkx
-            ME1nMnRqQzF1Vkc3b3FXSUVHT1g0ZDgKLS0tIHpMUytEMjdLQ2E0Unp2di9KS3JQ
-            eHBraDk1clJyanhLY2dGM0tESmJIUFkKHTl3y9uQiEofOFD8j2vH3YK/CVzlq11w
-            GfShIji1yCvvowKGzYYhsQK0UM0FzhzBv0GFMYWQCBq8pGdoPVmO5g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-10T13:31:00Z"
-    mac: ENC[AES256_GCM,data:JhnO0V6cd2QVY/VhwHAJ5J75GeVlOVHBfVBTZstkj/IvpkAcwS/JE2b6jXiCwEC4n/vdA8DJ074noysUBRxh4Y7O4NKuvWcTniQKgqULNL1Hzgj3NdUkcQlWT+Z0HQ7FlvFH97VVXVfasU+CLHG48ShT2fDSj8JusEllb9CwSvg=,iv:PZo2krxvJc1TLJbvx+S9ClthoBe4w7WigUkq7dg0gNk=,tag:2IF5g/WTRP4XdnCZzDbiIA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/paperless-ai-secrets.enc.yaml

@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: paperless-ai-config
-    namespace: paperless
-type: Opaque
-stringData:
-    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
-    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
-            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
-            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
-            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
-            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-14T14:19:39Z"
-    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/pi-sandbox-secrets.enc.yaml

@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: pi-code-agent-api-keys
-    namespace: poimen-sandbox
-stringData:
-    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
-    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
-            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
-            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
-            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
-            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T01:19:49Z"
-    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
-    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -12,7 +12,6 @@ files:
   - authentik-secrets.enc.yaml
   - cloudflare-secrets.enc.yaml
   - forgejo-registry-pat.enc.yaml
-  - forgejo-registry-pat-api.enc.yaml
   - forgejo-registry-pull.enc.yaml
   - forgejo-runner-token.enc.yaml
   - forgejo-secrets.enc.yaml
@@ -28,10 +27,3 @@ files:
   - vault-secrets.enc.yaml
   - vault-unseal-keys.enc.yaml
   - portfolio-secrets.enc.yaml
-  - gotify-admin-secrets.enc.yaml
-  - gotify-tokens-secrets.enc.yaml
-  - gotify-smtp-secrets.enc.yaml
-  - forgejo-smtp-secrets.enc.yaml
-  - paperless-ai-secrets.enc.yaml
-    - pi-sandbox-secrets.enc.yaml
-    - etcd-defrag-creds.enc.yaml

k8s/bootstrap/phase1-storage/longhorn-values.yaml

@@ -12,7 +12,6 @@ defaultSettings:
   replicaSoftAntiAffinity: false  # REQUIRED for true HA
   replicaAutoBalance: best-effort
   storageMinimalAvailablePercentage: 10
-  storageOverProvisioningPercentage: 200  # Actual usage is ~10% of scheduled; 200% unblocks all 3-replica scheduling on cp-1
   
   # Performance tuning
   defaultDataPath: /var/lib/longhorn

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -46,12 +46,6 @@ gitea:
       SSH_PORT: 2222
       SSH_LISTEN_PORT: 2222
       
-    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
-    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
-    # the admin only; admin create-user via UI/API is unaffected by this flag.
-    service:
-      DISABLE_REGISTRATION: true
-
     database:
       DB_TYPE: postgres
       HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -86,14 +80,6 @@ gitea:
     actions:
       ENABLED: true
 
-    mailer:
-      ENABLED: true
-      PROTOCOL: smtp+starttls
-      SMTP_ADDR: smtp.gmail.com
-      SMTP_PORT: 587
-      FROM: "Forgejo <[email protected]>"
-      # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
-
 # Persistence (shared storage for repos)
 persistence:
   enabled: true
@@ -162,18 +148,3 @@ deployment:
         secretKeyRef:
           name: forgejo-db-app
           key: password
-    - name: GITEA__MAILER__USER
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-smtp
-          key: user
-    - name: GITEA__MAILER__PASSWD
-      valueFrom:
-        secretKeyRef:
-          name: forgejo-smtp
-          key: password
-    # Allow webhooks to reach api-gw through the cluster ingress (resolves to
-    # internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
-    # api.riotpiao.com is our own gateway — safe to allowlist.
-    - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
-      value: "external,api.riotpiao.com"

k8s/infra/argocd-image-updater/values.yaml

@@ -0,0 +1,72 @@
+# ArgoCD Image Updater configuration
+# Watches Forgejo registry and updates ArgoCD Applications with new image tags
+
+config:
+  # Registry configuration - Forgejo allows anonymous pulls
+  registries:
+    - name: forgejo
+      api_url: https://forgejo.riotpiao.com
+      prefix: forgejo.riotpiao.com
+      default: true
+      insecure: false
+
+  # Log level
+  logLevel: debug
+
+  # ArgoCD API server
+  argocd:
+    grpcWeb: true
+    serverAddress: argocd-server.argocd.svc.cluster.local
+    insecure: true
+    plaintext: true
+
+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
+# Extra environment variables
+extraEnv:
+  - name: ARGOCD_GRPC_WEB
+    value: "true"
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
+    value: "true"
+
+# Resources
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi

k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml

@@ -1,177 +0,0 @@
-# etcd backend maintenance: compact, defrag, disarm NOSPACE.
-#
-# Why this exists
-# ---------------
-# etcd's backend file only grows. The apiserver compacts every 5m, but
-# compaction marks superseded revisions free *inside* the file; only `defrag`
-# returns that space to the filesystem. Nothing ran defrag, so the file grew
-# until it crossed the 2 GiB quota:
-#
-#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
-#               -> etcd raised NOSPACE on those two members
-#               -> etcd rejected ALL writes cluster-wide
-#               -> kubelets could not post node status: 3 nodes NotReady
-#               -> apiserver writes returned 500, the tunnel API path 502
-#
-# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
-# dead pages. Recovery was `compact <current-revision>` then `defrag` per
-# member then `alarm disarm`. Compaction had also stalled while the alarm was
-# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
-# worked. Order matters: compact first, then defrag.
-#
-# The earlier revision of this file never ran, for two independent reasons:
-#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
-#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
-#      cluster never had the CronJob or the Secret.
-#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
-#      `exec: "/bin/sh": no such file or directory`). Any container that runs
-#      a shell script needs an image that ships one.
-# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
-# 3.6.4, matching the 3.6.11 servers) pinned by digest.
-#
-# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
-# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
-# and the script exits unless the UTC epoch day is a multiple of 15. That gives
-# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
-# on the 1st, 16th and 31st, which puts two runs one day apart).
-#
-# Safety
-# ------
-# 1. Compaction uses the MINIMUM revision across members. Compacting to a
-#    revision a member has not reached wedges that member - measured on
-#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
-#    needed a reboot to rejoin.
-# 2. A member whose raft applied index lags the leader by more than 100000 is
-#    skipped and reported. Defragging a frozen member reclaims nothing.
-# 3. Members are defragged one at a time. A defrag pauses the member's apply
-#    loop for the duration of the file copy.
-#
-# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
-# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
-# Its client certificate is signed by the cluster's etcd CA and expires
-# 2036-10; rotate it before then by re-running the mint and re-encrypting.
-#
-# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
-# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: etcd-defrag
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: etcd-defrag
-    app.kubernetes.io/part-of: cluster-maintenance
-spec:
-  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
-  concurrencyPolicy: Forbid
-  startingDeadlineSeconds: 600
-  successfulJobsHistoryLimit: 3
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      backoffLimit: 2
-      template:
-        spec:
-          restartPolicy: OnFailure
-          tolerations:
-            - key: node-role.kubernetes.io/control-plane
-              operator: Exists
-              effect: NoSchedule
-          containers:
-            - name: defrag
-              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
-              env:
-                - name: ETCDCTL_API
-                  value: "3"
-                - name: ETCDCTL_CACERT
-                  value: /etc/etcd/ca.crt
-                - name: ETCDCTL_CERT
-                  value: /etc/etcd/client.crt
-                - name: ETCDCTL_KEY
-                  value: /etc/etcd/client.key
-                - name: EPS
-                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
-              command: ["/bin/bash", "-c"]
-              args:
-                - |
-                  set -u
-
-                  DAY=$(( $(date -u +%s) / 86400 ))
-                  if [ $(( DAY % 15 )) -ne 0 ]; then
-                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
-                    exit 0
-                  fi
-
-                  echo "== before =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" endpoint status -w table || true
-                  done
-
-                  MINREV=""
-                  MAXAPPLIED=0
-                  for ep in $EPS; do
-                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
-                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
-                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
-                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
-                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
-                      MINREV="$rev"
-                    fi
-                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
-                      MAXAPPLIED="$app"
-                    fi
-                  done
-
-                  if [ -z "$MINREV" ]; then
-                    echo "FATAL: no member returned a revision; nothing attempted"
-                    exit 1
-                  fi
-
-                  echo "== compact to revision $MINREV =="
-                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
-
-                  # Physical deletion of superseded keys runs in the background
-                  # after the compaction commit; give it a moment to drain.
-                  sleep 20
-
-                  echo "== defrag each member (sequential, one at a time) =="
-                  rc=0
-                  for ep in $EPS; do
-                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
-                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
-                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
-                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
-                      rc=1
-                      continue
-                    fi
-                    echo "-- $ep"
-                    if ! etcdctl --endpoints="$ep" defrag; then
-                      echo "DEFRAG-FAILED $ep"
-                      rc=1
-                    fi
-                  done
-
-                  echo "== disarm NOSPACE =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
-                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
-                  done
-
-                  echo "== after =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" endpoint status -w table || true
-                  done
-                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
-                  exit $rc
-              resources:
-                requests: { cpu: 10m, memory: 32Mi }
-                limits:   { cpu: 200m, memory: 128Mi }
-              volumeMounts:
-                - name: creds
-                  mountPath: /etc/etcd
-                  readOnly: true
-          volumes:
-            - name: creds
-              secret:
-                secretName: etcd-defrag-creds
-                defaultMode: 0444

k8s/infra/cluster-maintenance/stale-job-cleanup.yaml

@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs every 2 minutes. Deletes:
-#   - Failed Jobs older than 2m (any namespace)
-#   - Completed standalone Jobs older than 2m with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
+# Runs daily at 04:00 UTC. Deletes:
+#   - Failed Jobs older than 24h (any namespace)
+#   - Completed Jobs older than 72h with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted state older than 1h
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
-# and linger forever. Aggressive schedule keeps the cluster clean.
+# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
+# and linger forever.
 
 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
   labels:
     app: stale-job-cleanup
 spec:
-  schedule: "*/2 * * * *"
+  schedule: "0 4 * * *"
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
@@ -74,9 +74,7 @@ spec:
                   set -e
                   NOW=$(date +%s)
 
-                  TTL=120  # 2 minutes in seconds
-
-                  echo "=== Cleaning failed Jobs older than 2m ==="
+                  echo "=== Cleaning failed Jobs older than 24h ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.conditions[]?.type == "Failed") |
@@ -84,15 +82,15 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 24 ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
+                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.succeeded >= 1) |
@@ -100,20 +98,20 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 72 ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
+                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
+                  # Evicted pods show as Failed with reason Evicted
                   kubectl get pods --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(
                         .status.phase == "Failed" or
-                        .status.phase == "Succeeded" or
                         (.status.reason // "") == "Evicted" or
                         (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                       ) |
@@ -121,9 +119,9 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE=$(( NOW - POD_EPOCH ))
-                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
+                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 1 ]; then
+                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
                       kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                     fi
                   done

k8s/infra/databases/kustomization.yaml

@@ -8,4 +8,6 @@ resources:
   - namespaces.yaml
   - authentik-db.yaml
   - temporal-db.yaml
+  - memory-db.yaml
   - paperless-db.yaml
+  - obsidian-vault-pvc.yaml

k8s/infra/databases/memory-db.yaml

@@ -9,7 +9,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/infra/databases/obsidian-vault-pvc.yaml

@@ -0,0 +1,18 @@
+---
+# Obsidian vault PVC — shared storage for REST API + UI pods
+# ReadWriteMany so both obsidian-server and obsidian-ui can mount simultaneously
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: obsidian-vault
+  namespace: poimen
+  labels:
+    app.kubernetes.io/name: obsidian-server
+    app.kubernetes.io/part-of: poimen-memory
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 10Gi

k8s/infra/databases/paperless-db.yaml

@@ -11,7 +11,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/infra/forgejo-runner/templates/cilium-apiserver-egress.yaml

@@ -1,25 +0,0 @@
-# CiliumNetworkPolicy for kube-apiserver access.
-#
-# Standard K8s NetworkPolicy ipBlock rules don't work for the API server
-# under Cilium — the except clause on 192.168.1.0/24 blocks the post-DNAT
-# destination even when a separate rule re-allows a /32 or subnet.
-#
-# Cilium's native `kube-apiserver` entity resolves this correctly: it
-# tracks the API server endpoints regardless of ClusterIP vs node-IP
-# routing, so the policy stays valid across node changes and NAT paths.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: {{ .Release.Name }}-apiserver
-  namespace: {{ .Release.Namespace }}
-spec:
-  endpointSelector:
-    matchLabels:
-      app: {{ .Release.Name }}
-  egress:
-  - toEntities:
-    - kube-apiserver
-    toPorts:
-    - ports:
-      - port: "6443"
-        protocol: TCP

k8s/infra/iam/authentik-values.yaml

@@ -153,11 +153,9 @@ server:
   # checks aren't treated as failures. (Only these fields are overridden; the
   # chart deep-merges the rest of each probe, incl. the httpGet path.)
   livenessProbe:
-    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
     timeoutSeconds: 15
     failureThreshold: 6
   readinessProbe:
-    initialDelaySeconds: 300  # skip probe until migrations complete
     timeoutSeconds: 15
     failureThreshold: 6
   startupProbe:
@@ -217,13 +215,6 @@ worker:
   podAnnotations:
     configmap.reloader.stakater.com/reload: "homelab-ca"
     homelab.io/restart-at: "2026-06-21T13-40"
-  livenessProbe:
-    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
-  readinessProbe:
-    initialDelaySeconds: 300  # skip probe until migrations complete
-  startupProbe:
-    initialDelaySeconds: 30  # let server finish DB work first
-    failureThreshold: 120
   metrics:
     enabled: true
     serviceMonitor:

k8s/infra/local-path/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# No commonLabels on purpose: this is a vendored third-party manifest and
-# kustomize would also rewrite the provisioner Deployment's selector, which
-# is immutable on re-apply.
-resources:
-  - local-path-storage.yaml

k8s/infra/local-path/local-path-storage.yaml

@@ -1,200 +0,0 @@
-# Vendored from https://github.com/rancher/local-path-provisioner
-#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
-#
-# Local modifications (the only three; re-apply them on any re-vendor):
-#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
-#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
-#      baseline policy, which forbids that;
-#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
-#      filesystem read-only, so upstream's /opt path cannot be created;
-#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: local-path-storage
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: local-path-provisioner-service-account
-  namespace: local-path-storage
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: local-path-provisioner-role
-  namespace: local-path-storage
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: local-path-provisioner-role
-rules:
-  - apiGroups: [""]
-    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["create", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: local-path-provisioner-bind
-  namespace: local-path-storage
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: local-path-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: local-path-provisioner-service-account
-    namespace: local-path-storage
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: local-path-provisioner-bind
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: local-path-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: local-path-provisioner-service-account
-    namespace: local-path-storage
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: local-path-provisioner
-  namespace: local-path-storage
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: local-path-provisioner
-  template:
-    metadata:
-      labels:
-        app: local-path-provisioner
-    spec:
-      serviceAccountName: local-path-provisioner-service-account
-      containers:
-        - name: local-path-provisioner
-          image: docker.io/rancher/local-path-provisioner:v0.0.37
-          imagePullPolicy: IfNotPresent
-          command:
-            - local-path-provisioner
-            - --debug
-            - start
-            - --config
-            - /etc/config/config.json
-          ports:
-            - name: health
-              containerPort: 8080
-              protocol: TCP
-          startupProbe:
-            httpGet:
-              path: /health
-              port: health
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            failureThreshold: 12
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: health
-            initialDelaySeconds: 10
-            periodSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /ready
-              port: health
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          volumeMounts:
-            - name: config-volume
-              mountPath: /etc/config/
-          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CONFIG_MOUNT_PATH
-              value: /etc/config/
-            - name: HEALTH_PORT
-              value: "8080"
-      volumes:
-        - name: config-volume
-          configMap:
-            name: local-path-config
-
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: local-path
-provisioner: rancher.io/local-path
-volumeBindingMode: WaitForFirstConsumer
-reclaimPolicy: Delete
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: local-path-config
-  namespace: local-path-storage
-data:
-  config.json: |-
-    {
-            "nodePathMap":[
-            {
-                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
-                    "paths":["/var/lib/local-path-provisioner"]
-            }
-            ]
-    }
-  setup: |-
-    #!/bin/sh
-    set -eu
-    mkdir -m 0777 -p "$VOL_DIR"
-  teardown: |-
-    #!/bin/sh
-    set -eu
-    rm -rf "$VOL_DIR"
-  helperPod.yaml: |-
-    apiVersion: v1
-    kind: Pod
-    metadata:
-      name: helper-pod
-    spec:
-      priorityClassName: system-node-critical
-      tolerations:
-        - key: node.kubernetes.io/disk-pressure
-          operator: Exists
-          effect: NoSchedule
-      containers:
-      - name: helper-pod
-        image: docker.io/library/busybox:1.36
-        imagePullPolicy: IfNotPresent

k8s/infra/logging/grafana-values.yaml

@@ -73,7 +73,7 @@ grafana.ini:
   auth.generic_oauth:
     enabled: true
     name: Authentik
-    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
+    allow_sign_up: true
     client_id: grafana
     scopes: openid email profile groups
     auth_url: https://authentik.riotpiao.com/application/o/authorize/
@@ -84,13 +84,13 @@ grafana.ini:
     # Authentik doesn't serve — request 404s with "Error getting email address"
     # and the whole OAuth login fails.
     email_attribute_path: email
-    login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
+    login_attribute_path: preferred_username
     name_attribute_path: name
     role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
     allow_assign_grafana_admin: true
     use_pkce: false
     use_refresh_token: false
-    skip_org_role_sync: false  # Sync org roles from OAuth groups
+    skip_org_role_sync: false
     tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
 
 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -41,24 +41,13 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          #
-          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
-          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
-          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
-          # process that touched it blocked in unkillable D state. Those unkillable
-          # processes held Longhorn's per-node rebuild slots forever, so no new
-          # replica could start on talos-cp-2 and every new volume there failed to
-          # attach. It is removed from the Terraform disk list too, so a re-render
-          # does not put it back.
+          # - /var/lib/longhorn-disk4 (sde1, 2TB)
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
-          # existed; leaving the old threshold would re-apply the patch on every
-          # run once disk4 is gone.
-          if [ "$CURRENT_DISKS" -lt 4 ]; then
+          if [ "$CURRENT_DISKS" -lt 5 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -86,6 +75,14 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
+                  },
+                  "disk4": {
+                    "allowScheduling": true,
+                    "diskType": "filesystem",
+                    "evictionRequested": false,
+                    "path": "/var/lib/longhorn-disk4",
+                    "storageReserved": 21474836480,
+                    "tags": []
                   }
                 }
               }
@@ -120,22 +117,6 @@ spec:
             echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
           fi
 
-          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
-          # clear eviction flag that was set when scheduling was disabled.
-          echo "Configuring worker-1 disk..."
-          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
-            "spec": {
-              "disks": {
-                "default-disk-1030400000000": {
-                  "allowScheduling": true,
-                  "evictionRequested": false,
-                  "storageReserved": 429496729600
-                }
-              }
-            }
-          }'
-          echo "  ✓ worker-1 disk configured (400Gi reserved)"
-
           echo
           echo "Waiting for disks to be ready..."
           sleep 10

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -35,5 +35,8 @@ parameters:
   mkfsParams: "-O ^64bit,^metadata_csum"
 mountOptions:
   - "noatime"
+  # Critical: mount with postgres UID/GID (26:26) to avoid permission issues
+  - "uid=26"
+  - "gid=26"
 reclaimPolicy: Delete
 volumeBindingMode: Immediate

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,25 +1,33 @@
-# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
+# Longhorn Node CRDs for cp-2 and cp-3.
+# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
-# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
-# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
-# for general replica scheduling).
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
-# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
-# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
+# What these objects are actually for is `allowScheduling: true` on tainted
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-2, namespace: longhorn-system }
-spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-2
+  namespace: longhorn-system
+spec:
+  name: talos-cp-2
+  allowScheduling: true
+  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata: { name: talos-cp-3, namespace: longhorn-system }
-spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
----
-apiVersion: longhorn.io/v1beta2
-kind: Node
-metadata: { name: worker-1, namespace: longhorn-system }
-spec: { name: worker-1, allowScheduling: true, tags: [] }
+metadata:
+  name: talos-cp-3
+  namespace: longhorn-system
+spec:
+  name: talos-cp-3
+  allowScheduling: true
+  tags: []

k8s/infra/minio/minio-provision-paperless-job.yaml

@@ -52,14 +52,14 @@ spec:
       restartPolicy: Never
       initContainers:
         - name: kubectl-copy
-          image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
+          image: bitnami/kubectl:latest
           command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
           volumeMounts:
             - name: shared
               mountPath: /shared
       containers:
         - name: provision
-          image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
+          image: minio/mc:latest
           volumeMounts:
             - name: shared
               mountPath: /shared
@@ -81,9 +81,6 @@ spec:
               mc alias set m http://minio-cluster-hl.storage.svc.cluster.local:9000 \
                 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
 
-              echo "Ensuring paperless bucket exists..."
-              mc mb --ignore-existing m/paperless
-
               echo "Checking for existing paperless-minio-creds secret..."
               if kubectl -n paperless get secret paperless-minio-creds >/dev/null 2>&1; then
                 ACCESS_KEY=$(kubectl -n paperless get secret paperless-minio-creds -o jsonpath='{.data.ACCESS_KEY}' | base64 -d)

k8s/infra/minio/minio-tenant.yaml

@@ -11,15 +11,7 @@ metadata:
     prometheus.io/port: "9000"
     prometheus.io/scrape: "true"
 spec:
-  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
-  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
-  # backend could not resolve minio-cluster-hl. Official images now need registry auth
-  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
-  # release; swap it for the official image once MinIO registry credentials exist.
-  # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
-  # the pull again. The same digest already sits in the containerd cache on talos-cp-1,
-  # so the first start needs no registry access.
-  image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
+  image: minio/minio:RELEASE.2025-07-23T15-54-02Z
 
   # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
   # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -120,16 +112,10 @@ spec:
           name: minio-oidc
           key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
-      value: "groups"
-    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
-      value: ""
+      value: "policy"
     - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
       value: "https://minio.riotpiao.com/oauth_callback"
     - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
       value: "Authentik"
     - name: MINIO_IDENTITY_OPENID_SCOPES
-      value: "openid,profile,email,groups"
-    - name: MINIO_BROWSER_LOGIN_ANIMATION
-      value: "off"
-    - name: MINIO_BROWSER_REDIRECT_URL
-      value: "https://minio.riotpiao.com"
+      value: "openid,profile,email,minio"

k8s/infra/monitoring/alerts/api-gateway-alerts.yaml

@@ -158,66 +158,3 @@ spec:
             severity: warning
           annotations:
             summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
-
-    - name: api-gateway.admission
-      # The admission gate fails OPEN when its view of the engine gauges goes
-      # stale, so a broken observer degrades silently to the pre-#64 behaviour
-      # of admitting everything. These two series are the only detection —
-      # the gate has no automated tests.
-      rules:
-        # The gate fails OPEN when it cannot read the engine, so "no data" must
-        # alert: otherwise a missing emitter, a renamed metric, or a broken
-        # scrape is indistinguishable from a healthy observer. Both expressions
-        # therefore OR in an absent() guard on the metric itself.
-        - alert: APIGatewayObserverDegraded
-          expr: observer_degraded == 1 or absent(observer_degraded)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
-            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
-
-        # Engine gauges went stale, or are not being reported at all
-        - alert: GatewayEngineStateStale
-          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
-          for: 2m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
-            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
-
-        # The acceptance metric for reasoning.yaml's prefill change. Preemption
-        # of a 65k prompt costs a full re-prefill (~250s at the measured
-        # 262 tok/s), so any sustained rate is a regression, not noise.
-        - alert: LLMPreemptions
-          # Guarded absent() is deliberately NOT used bare here: the engine
-          # restarts (strategy Recreate) and the series legitimately vanishes
-          # for the length of a boot, so a bare absent() would false-fire on
-          # every rollout. It only fires when the metric is missing WHILE the
-          # predictor is up, which is the real "renamed or unscraped" case.
-          expr: |
-            increase(vllm:num_preemptions_total[15m]) > 0
-            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
-          for: 5m
-          labels:
-            severity: warning
-          annotations:
-            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
-            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
-
-        # This change trades single-request latency for fairness: capping a long
-        # prompt at 2048 tokens/step means it takes more steps to finish, so a
-        # lone large prompt can now be slower even while preemptions stay 0.
-        # Without this alert the change has no way to show a regression.
-        - alert: LLMPrefillStarvation
-          expr: |
-            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
-            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
-          for: 10m
-          labels:
-            severity: warning
-          annotations:
-            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
-            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."

k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

@@ -1,76 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: cag-kv-cache-rules
-  namespace: monitoring
-  labels:
-    release: prometheus
-spec:
-  groups:
-    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
-    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
-    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
-    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
-    # prefix_cache_queries_total — so this rule produced no data and
-    # VLLMPrefixCacheHitRateLow could never fire.
-    - name: vllm.recording
-      rules:
-        - record: vllm:prefix_cache_hit_rate
-          expr: |
-            rate(vllm:prefix_cache_hits_total[5m])
-            /
-            clamp_min(
-              rate(vllm:prefix_cache_queries_total[5m]),
-              0.001
-            )
-          labels:
-            job: "reasoning-predictor"
-
-    # Alert rules: CAG pipeline health
-    - name: cag.alerts
-      rules:
-        # KV warmup consistently failing — cache never gets warm
-        - alert: KVWarmupFailureHigh
-          expr: |
-            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
-            description: |
-              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
-              for 5 consecutive minutes. The prefix KV cache is not being warmed;
-              agents will see full prefill latency on every call.
-              Check reasoning-predictor health and poimen-memory NetworkPolicy.
-
-        # Warmup misses: vLLM reachable but not caching
-        - alert: KVWarmupMissHigh
-          expr: |
-            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
-          for: 5m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
-            description: |
-              vLLM is reachable but cached_tokens=0 on warmup responses.
-              Possible causes: prefix caching disabled in vLLM, max-num-seqs
-              saturated during warmup, or context format changed.
-
-        # Prefix cache hit rate low after warmup should be established
-        - alert: VLLMPrefixCacheHitRateLow
-          expr: |
-            vllm:prefix_cache_hit_rate < 0.70
-          for: 10m
-          labels:
-            severity: warning
-            team: poimen
-          annotations:
-            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
-            description: |
-              vLLM prefix cache hit rate below 70% for 10 minutes.
-              Expected >= 85% in steady state with CAG warming active.
-              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.

k8s/infra/monitoring/alerts/cluster-alerts.yaml

@@ -27,24 +27,6 @@ spec:
           annotations:
             summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
 
-        # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
-        - alert: PodImagePullFailure
-          expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
-          for: 10m
-          labels:
-            severity: critical
-          annotations:
-            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
-
-        # Pod not ready
-        - alert: PodNotReady
-          expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
-          for: 15m
-          labels:
-            severity: warning
-          annotations:
-            summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
-
         # CrashLoopBackOff
         - alert: PodCrashLooping
           expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0

k8s/infra/monitoring/dashboards/generate-dashboards.py

@@ -26,23 +26,20 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
     return p
 
 
-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
     targets = []
     for e in exprs:
         if isinstance(e, tuple):
             targets.append({"expr": e[0], "legendFormat": e[1]})
         else:
             targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    p = {
+    return {
         "id": id, "title": title, "type": "timeseries",
         "gridPos": {"h": h, "w": w, "x": x, "y": y},
         "datasource": DS_PROM,
         "fieldConfig": {"defaults": {"unit": unit}},
         "targets": targets,
     }
-    if description:
-        p["description"] = description
-    return p
 
 
 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -268,237 +265,6 @@ def build_api_gateway():
     }
 
 
-# ============================================================================
-# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
-# ============================================================================
-#
-# Two independent metric sources, deliberately both shown:
-#
-#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
-#                               that expose /metrics. Ollama (ornith:35b,
-#                               qwen2.5:3b-instruct) exposes nothing at all.
-#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
-#                               ones, derived from the SSE byte stream: TTFT is
-#                               request-start to first byte, ITL is the gap
-#                               between SSE writes carrying "data: ", so it
-#                               counts SSE chunks rather than true tokens.
-#                               vLLM's own histograms remain authoritative for
-#                               `reasoning`.
-#
-# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
-# verified against the running gateway and Prometheus:
-#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
-#
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
-# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
-# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
-# homelab-frontend#63 fixed exactly that — the wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
-# The "dead code" note that used to live in this header is superseded.
-#
-# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
-# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
-#
-#
-# All metric names below were verified against a live scrape, not assumed:
-#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
-#   NOT cache_query_hit_total / cache_query_miss_total.
-
-def build_llm_observability():
-    def hq(metric, q, by="le"):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
-            f"p{int(q * 100)}",
-        )
-
-    def hq_gw(metric, q):
-        return (
-            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
-            f"p{int(q * 100)} · {{{{model}}}}",
-        )
-
-    def avg_of(metric):
-        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
-        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
-
-    panels = []
-
-    # ── TTFT ────────────────────────────────────────────────────────────────
-    panels.append(row(100, "Latency — Time To First Token", 0, [
-        stat_panel(101, "TTFT p95 (gateway)",
-                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
-                   0, 1, w=4, h=4, unit="s"),
-        stat_panel(102, "TTFT p95 (vLLM)",
-                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
-                   4, 1, w=4, h=4, unit="s"),
-        stat_panel(103, "KV cache in use",
-                   "max(vllm:kv_cache_usage_perc)",
-                   8, 1, w=4, h=4, unit="percentunit"),
-        stat_panel(104, "Requests running",
-                   "sum(vllm:num_requests_running)",
-                   12, 1, w=4, h=4, unit="short"),
-        stat_panel(105, "Requests waiting",
-                   "sum(vllm:num_requests_waiting)",
-                   16, 1, w=4, h=4, unit="short"),
-        stat_panel(106, "Preemptions (5m)",
-                   "increase(vllm:num_preemptions_total[5m])",
-                   20, 1, w=4, h=4, unit="short"),
-        ts_panel(110, "TTFT percentile by model (gateway)",
-                 [hq_gw("llm_ttft_seconds", 0.50),
-                  hq_gw("llm_ttft_seconds", 0.95),
-                  hq_gw("llm_ttft_seconds", 0.99)],
-                 0, 2, w=12, h=8, unit="s"),
-        ts_panel(111, "TTFT percentile — vLLM reasoning",
-                 [hq("vllm:time_to_first_token_seconds", 0.50),
-                  hq("vllm:time_to_first_token_seconds", 0.95),
-                  hq("vllm:time_to_first_token_seconds", 0.99)],
-                 12, 2, w=12, h=8, unit="s"),
-        ts_panel(112, "Prefill vs decode (mean)",
-                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
-                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
-                 0, 3, w=12, h=8, unit="s"),
-        ts_panel(113, "Queue wait before scheduling (mean)",
-                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
-                 12, 3, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Inter-token latency ─────────────────────────────────────────────────
-    panels.append(row(120, "Latency — Inter-Token", 4, [
-        ts_panel(121, "Inter-token latency by model (gateway)",
-                 [hq_gw("llm_itl_seconds", 0.50),
-                  hq_gw("llm_itl_seconds", 0.95),
-                  hq_gw("llm_itl_seconds", 0.99)],
-                 0, 5, w=12, h=8, unit="s"),
-        ts_panel(122, "Inter-token latency — vLLM reasoning",
-                 [hq("vllm:inter_token_latency_seconds", 0.50),
-                  hq("vllm:inter_token_latency_seconds", 0.95),
-                  hq("vllm:inter_token_latency_seconds", 0.99)],
-                 12, 5, w=12, h=8, unit="s"),
-        ts_panel(123, "Decode throughput (tokens/s per request)",
-                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
-                 0, 6, w=12, h=8, unit="short"),
-        ts_panel(124, "End-to-end request latency p50/p95/p99",
-                 [hq("vllm:e2e_request_latency_seconds", 0.50),
-                  hq("vllm:e2e_request_latency_seconds", 0.95),
-                  hq("vllm:e2e_request_latency_seconds", 0.99)],
-                 12, 6, w=12, h=8, unit="s"),
-    ]))
-
-    # ── Batching / scheduling ───────────────────────────────────────────────
-    panels.append(row(130, "Batching & Scheduling", 7, [
-        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
-                 [("sum(vllm:num_requests_running)", "running (in batch)"),
-                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
-                 0, 8, w=12, h=8, unit="short"),
-        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
-                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
-                 12, 8, w=12, h=8, unit="short"),
-        ts_panel(133, "TEI batch size (embeddings / reranker)",
-                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
-                   "{{app}}")],
-                 0, 9, w=12, h=8, unit="short"),
-        ts_panel(134, "TEI queue depth",
-                 [("max(te_queue_size) by (app)", "{{app}}")],
-                 12, 9, w=12, h=8, unit="short"),
-    ]))
-
-    # ── KV cache ────────────────────────────────────────────────────────────
-    panels.append(row(140, "KV Cache", 10, [
-        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
-                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
-                 0, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
-                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
-                 12, 11, w=12, h=8, unit="percentunit"),
-        ts_panel(143, "Prefix cache hits vs queries",
-                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
-                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
-                 0, 12, w=12, h=8, unit="short"),
-        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
-                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
-                 12, 12, w=12, h=8, unit="short"),
-    ]))
-
-    # ── Throughput & health ─────────────────────────────────────────────────
-    panels.append(row(150, "Throughput & Health", 13, [
-        ts_panel(151, "Token throughput",
-                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
-                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
-                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
-                 0, 14, w=12, h=8, unit="short"),
-        ts_panel(152, "Successful requests/s",
-                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
-                  ("sum(rate(te_request_count[5m]))", "TEI")],
-                 12, 14, w=12, h=8, unit="short"),
-        ts_panel(153, "Backpressure — queue-full rejects",
-                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
-                 0, 15, w=8, h=8, unit="short"),
-        ts_panel(154, "KV warmup requests delayed",
-                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
-                 8, 15, w=8, h=8, unit="short"),
-        ts_panel(155, "Embedding / rerank latency p95",
-                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
-                   "{{app}} embed"),
-                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
-                   "{{app}} queue")],
-                 16, 15, w=8, h=8, unit="s"),
-    ]))
-
-    # ── Admission control ───────────────────────────────────────────────────
-    # These four panels read the gateway's admission gate, which is the
-    # companion change in homelab-frontend#64. That emitter is NOT deployed
-    # yet (verified against the live /metrics endpoint: 0 series matching
-    # ^(admission_|observer_|engine_state_)), so they render "No data" until
-    # #64 ships. That is expected, and each panel carries a description
-    # saying so — an unexplained blank panel would read as a broken
-    # dashboard, which is exactly the failure mode this row exists to catch.
-    panels.append(row(160, "Admission Control", 16, [
-        ts_panel(161, "Admission parked (5m)",
-                 [("increase(admission_parked_total[5m])", "parked")],
-                 0, 17, w=8, h=8, unit="short",
-                 description="Requests the gateway held back rather than admitting. "
-                             "Emitted by the admission gate (homelab-frontend#64); "
-                             "empty until that change is deployed."),
-        ts_panel(162, "Admission rejected by reason",
-                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
-                 8, 17, w=8, h=8, unit="short",
-                 description="Admission rejections split by the gate's own reason label. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(163, "Engine state stale",
-                 [("increase(engine_state_stale_total[5m])", "stale")],
-                 16, 17, w=8, h=8, unit="short",
-                 description="Times the gateway's view of the engine gauges went stale. "
-                             "The gate fails OPEN when this happens, so any non-zero rate "
-                             "means admission is silently degraded to admit-everything. "
-                             "Emitted by homelab-frontend#64; empty until deployed."),
-        ts_panel(164, "Gateway view vs engine truth",
-                 [("max(vllm_requests_running)", "gateway: running"),
-                  ("sum(vllm:num_requests_running)", "engine: running"),
-                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
-                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
-                 0, 18, w=24, h=8, unit="short",
-                 description="The gateway's own reading of the engine against the engine's "
-                             "self-reported value. A disagreement between a pair is how a "
-                             "stale observer or a mistuned admission target shows itself, "
-                             "and nothing else in this dashboard reveals it. The two "
-                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
-                             "ship with homelab-frontend#64 and are absent until it is "
-                             "deployed — until then this panel shows the engine's pair only."),
-    ]))
-
-    return {
-        "title": "LLM Observability",
-        "uid": "llm-observability",
-        "schemaVersion": 39,
-        "timezone": "browser",
-        "time": {"from": "now-6h", "to": "now"},
-        "refresh": "30s",
-        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
-        "panels": panels,
-    }
-
-
 # ============================================================================
 # Generate
 # ============================================================================
@@ -511,7 +277,4 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
 
-# Dashboard 4
-write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
-
 print("Done.")

k8s/infra/monitoring/dashboards/llm-observability.yaml

@@ -1,95 +0,0 @@
-apiVersion: v1
-data:
-  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
-    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
-    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
-    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
-    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
-    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
-    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
-    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
-    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
-    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
-    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
-    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
-    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
-    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
-    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
-    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
-    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
-    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
-    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
-    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
-    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
-    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
-    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
-    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
-    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
-    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
-    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
-    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
-    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
-    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
-    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
-    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
-    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
-    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
-    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
-    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
-    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
-    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
-    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
-    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
-    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
-    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
-    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
-    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
-    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
-    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
-    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
-    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
-    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
-    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
-    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
-    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
-    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
-    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
-    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
-    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
-    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
-    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
-    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
-    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
-    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
-    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
-    the engine''s self-reported value. A disagreement between a pair is how a stale
-    observer or a mistuned admission target shows itself, and nothing else in this
-    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
-    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
-    then this panel shows the engine''s pair only."}]}]}'
-kind: ConfigMap
-metadata:
-  annotations:
-    grafana_folder: LLM
-  labels:
-    grafana_dashboard: '1'
-  name: llm-observability-dashboard
-  namespace: logging

k8s/infra/monitoring/kustomization.yaml

@@ -14,14 +14,10 @@ resources:
   - alerts/svc-vault-rules.yaml
   - alerts/cluster-alerts.yaml
   - alerts/api-gateway-alerts.yaml
-  - alerts/cag-kv-cache-rules.yaml
   - servicemonitors/argocd.yaml
   - servicemonitors/authentik.yaml
   - servicemonitors/forgejo.yaml
   - servicemonitors/minio.yaml
   - servicemonitors/ingress-nginx.yaml
-  - servicemonitors/vllm.yaml
-  - servicemonitors/api-gateway.yaml
   - dashboards/cluster-infrastructure.yaml
   - dashboards/api-gateway.yaml
-  - dashboards/llm-observability.yaml

k8s/infra/monitoring/prometheus-values.yaml

@@ -54,8 +54,8 @@ alertmanager:
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:
   prometheusSpec:
-    retention: 30d
-    retentionSize: "95GB"
+    retention: 15d
+    retentionSize: "18GB"
 
     # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
     # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -64,9 +64,10 @@ prometheus:
     scrapeTimeout: 60s
     evaluationInterval: 30s
 
-    # Pin to worker-1 for more storage capacity
+    # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
+    # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
     nodeSelector:
-      kubernetes.io/hostname: worker-1
+      topology.kubernetes.io/zone: az-a
 
     # Persistent storage — metrics survive node reboots and pod restarts.
     # Uses the default `longhorn` StorageClass; nodeSelector above already pins
@@ -78,7 +79,7 @@ prometheus:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: 25Gi
+              storage: 20Gi
 
     resources:
       requests:

k8s/infra/monitoring/servicemonitors/api-gateway.yaml

@@ -1,62 +0,0 @@
-# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
-# three latency series below recorded nothing. internal/proxy/llm_metrics.go
-# declared a capture wrapper (LLMMetricsCapture) that was never constructed
-# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
-# stayed empty.
-#
-# homelab-frontend#63 fixed exactly that. The wrapper is now
-# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
-# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
-# request path. Verified live on 2026-09-29:
-#   llm_ttft_seconds  count=7  sum=598.212s
-#   llm_itl_seconds   count=16116
-#   llm_tokens_total  3 series
-#
-# These derived-from-SSE series cover EVERY model including the two Ollama ones
-# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
-# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
-# between SSE writes, so both are granular to SSE write boundaries rather than
-# to the engine's own token cadence. llm_tokens_total counts `data:` events,
-# and the documented assumption in internal/proxy/llm_metrics.go is that these
-# upstreams emit one such event per token — so it is a token count only as far
-# as that assumption holds. vLLM's own histograms remain authoritative.
-#
-# Also recorded and working (3 series, one per replica):
-#   queue_full_requests_total           counter    backpressure rejects
-#   warmup_requests_delayed_total       counter    vLLM slot contention
-#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
-#
-# NOT working, do not build on it: request_duration_seconds is declared and
-# exported but has 0 samples, and its only writer Metrics.RecordRequest has
-# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
-# the same dead-code shape this branch already corrected for the latency
-# series, so it is listed here rather than left to be rediscovered.
-#
-# Also note the Deployment already carried prometheus.io/scrape annotations,
-# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
-# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
-# collected before this ServiceMonitor existed.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: api-gateway
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: api-gateway
-    app.kubernetes.io/part-of: api
-spec:
-  namespaceSelector:
-    matchNames:
-      - api
-  selector:
-    matchLabels:
-      app: api-gateway
-  endpoints:
-    # api-gateway Service port IS named `http` (unlike the KServe ones), so
-    # the name form works here.
-    - port: http
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -1,49 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: vllm
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: vllm
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  namespaceSelector:
-    matchNames:
-      - llm-serving
-  selector:
-    # Only the three engines that actually serve /metrics.
-    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
-    # endpoint (/metrics returns 404) — selecting them would add permanently
-    # DOWN targets. Ollama-model visibility comes from the api-gateway
-    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
-    # measured at the proxy, so they cover every model including Ollama).
-    matchExpressions:
-      - key: app.kubernetes.io/name
-        operator: In
-        values:
-          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
-          - llm-embeddings  # TEI 1.8.2 — CPU
-          - llm-reranker    # TEI 1.8.2 — CPU
-  endpoints:
-    # targetPort, not port: the KServe-generated Services on llm-serving have
-    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
-    # editing the InferenceServices (predictor restart = full weights reload,
-    # tens of seconds on the vLLM slot), so scrape the pod port directly.
-    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
-    - targetPort: 8080
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-      relabelings:
-        # Prometheus's Kubernetes service discovery already supplies the
-        # namespace, pod and service labels; the only one that needs deriving
-        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
-        # not promote arbitrary pod labels.
-        #
-        # `action: replace` is written explicitly. It is the API server's
-        # default for relabeling rules, so omitting it leaves the live object
-        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
-        - action: replace
-          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
-          targetLabel: app

k8s/infra/poimen/kustomization.yaml

@@ -1,12 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# Poimen infrastructure — CNPG Postgres for memory service.
-# Part of the `poimen` ArgoCD Application (multi-source, source 3).
-# Sync-wave -1 ensures the DB cluster is healthy before application
-# deployments (wave 0) are applied.
-resources:
-  - memory-db.yaml
-  - poimen-sandbox-namespace.yaml
-  - poimen-sandbox-quota.yaml
-  - pi-session-netpol.yaml
-  - pi-agent-workspace-pv.yaml

k8s/infra/poimen/memory-db.yaml

@@ -1,37 +0,0 @@
-# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
-# Uses default longhorn storage class (3 replicas, dataLocality disabled).
-# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: memory-db
-  namespace: poimen
-  annotations:
-    argocd.argoproj.io/sync-wave: "-1"
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
-spec:
-  instances: 3
-  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
-  bootstrap:
-    initdb:
-      database: memory
-      owner: app
-      encoding: UTF8
-      localeCollate: C
-      localeCType: C
-      postInitApplicationSQL:
-        - "CREATE EXTENSION vector;"
-  enableSuperuserAccess: false
-  resources:
-    requests: { memory: "512Mi", cpu: "250m" }
-    limits: { memory: "2Gi", cpu: "1" }
-  storage:
-    size: 20Gi
-    storageClass: longhorn
-  affinity:
-    podAntiAffinityType: preferred
-    topologyKey: kubernetes.io/hostname
-    tolerations:
-      - key: node-role.kubernetes.io/control-plane
-        operator: Exists
-        effect: NoSchedule

k8s/infra/poimen/pi-agent-workspace-pv.yaml

@@ -1,57 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  finalizers:
-  - kubernetes.io/pv-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace-pv
-  annotations:
-    argocd.argoproj.io/sync-wave: '-1'
-spec:
-  accessModes:
-  - ReadWriteOnce
-  capacity:
-    storage: 10Gi
-  claimRef:
-    apiVersion: v1
-    kind: PersistentVolumeClaim
-    name: pi-agent-workspace
-    namespace: poimen-sandbox
-  hostPath:
-    path: /var/lib/pi-agent-sandbox
-    type: DirectoryOrCreate
-  nodeAffinity:
-    required:
-      nodeSelectorTerms:
-      - matchExpressions:
-        - key: kubernetes.io/hostname
-          operator: In
-          values:
-          - talos-cp-2
-  persistentVolumeReclaimPolicy: Retain
-  volumeMode: Filesystem
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  annotations:
-    pv.kubernetes.io/bind-completed: 'yes'
-    argocd.argoproj.io/sync-wave: '-1'
-  finalizers:
-  - kubernetes.io/pvc-protection
-  labels:
-    app: pi-coding-agent-sandbox
-    part-of: poimen-toolkit
-  name: pi-agent-workspace
-  namespace: poimen-sandbox
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi
-  storageClassName: ''
-  volumeMode: Filesystem
-  volumeName: pi-agent-workspace-pv

k8s/infra/poimen/pi-session-netpol.yaml

@@ -1,12 +0,0 @@
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: pi-session-deny-kube-api
-  namespace: poimen-sandbox
-spec:
-  endpointSelector:
-    matchLabels:
-      app: pi-session
-  egressDeny:
-    - toEntities:
-        - kube-apiserver