fix(oidc): match /proc/<pid>/comm instead of cmdline when signalling the API server

Exit code 143 proved the previous job killed itself: the script's own cmdline contains the
string 'kube-apiserver', so it matched its own process. Matching comm (== 'kube-apiserver')
and skipping $$ removes that failure mode. Staggered 2 minutes apart.

.sops.yaml

@@ -2,4 +2,5 @@ creation_rules:
   # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
   # and was committed in plaintext to a public remote.
   - path_regex: k8s/.*secrets?.*\.ya?ml
+    encrypted_regex: ^(data|stringData)$
     age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla

AUTH_INTEGRATION.md

@@ -1,206 +0,0 @@
-# Authentik Auth Integration for NextJS
-
-## Current State
-
-### Gateway Auth Status
-
-| Endpoint | Auth Status | Notes |
-|----------|-------------|-------|
-| `/v1/chat/completions` | ❌ **OFF** | LLM routes have no auth middleware |
-| `/v1/embeddings` | ❌ **OFF** | Same - no auth |
-| `/v1/rerank` | ❌ **OFF** | Same - no auth |
-| `X-Service: sqs` | ✅ **ON** | JWT validated via `internal/auth/jwt.go` |
-| `/workflow` | ❌ **OFF** | Pass-through to Temporal |
-
-**Auth module exists** at `homelab-frontend/internal/auth/jwt.go` but only wired for SQS.
-LLM routes in `internal/proxy/proxy.go` have no auth middleware.
-
-### Authentik App
-
-Authentik app `local-llm` exists for LLM API auth:
-- **Client ID**: `local-llm`
-- **Client Secret**: `kubectl -n llm-serving get secret local-llm-jwt -o jsonpath='{.data.client-secret}' | base64 -d`
-- **Token endpoint**: `https://authentik.riotpiao.com/application/o/token/`
-- **Userinfo endpoint**: `https://authentik.riotpiao.com/application/o/userinfo/`
-- **OIDC discovery**: `https://authentik.riotpiao.com/application/o/local-llm/.well-known/openid-configuration`
-
-## Sign-in Methods
-
-### 1. Resource Owner Password Credentials (ROPC)
-
-Direct username/password login. Server-side only (needs client_secret).
-
-```typescript
-// API Route: app/api/auth/login/route.ts
-const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
-  method: 'POST',
-  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-  body: new URLSearchParams({
-    grant_type: 'password',
-    client_id: 'local-llm',
-    client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
-    username: '[email protected]',
-    password: 'userpassword',
-    scope: 'openid email profile groups',
-  }),
-});
-
-const tokens = await response.json();
-// { access_token, refresh_token, expires_in, token_type }
-```
-
-### 2. Authorization Code Flow (Browser Redirect)
-
-Requires adding redirect URIs to `local-llm` Authentik app:
-
-```python
-# In k8s/infra/iam/scripts/authentik-provision.py, update:
-"local-llm": {
-    ...
-    "redirect_uris": [
-        "http://localhost:3000/api/auth/callback",  # dev
-        "https://your-nextjs-app.com/api/auth/callback",  # prod
-    ],
-}
-```
-
-Then standard OIDC flow:
-1. Redirect to `https://authentik.riotpiao.com/application/o/authorize/?client_id=local-llm&redirect_uri=...&response_type=code&scope=openid email profile groups`
-2. User logs in via Authentik UI
-3. Callback receives `code`, exchange for tokens
-
-## JWT Token Persistence
-
-### Browser (localStorage)
-
-```typescript
-const TOKEN_KEY = 'llm_auth_token';
-
-// Save
-localStorage.setItem(TOKEN_KEY, JSON.stringify({
-  access_token: tokens.access_token,
-  refresh_token: tokens.refresh_token,
-  expires_at: Date.now() + tokens.expires_in * 1000,
-}));
-
-// Load
-const stored = JSON.parse(localStorage.getItem(TOKEN_KEY) || 'null');
-if (stored && stored.expires_at > Date.now()) {
-  // Token valid
-}
-
-// Clear (logout)
-localStorage.removeItem(TOKEN_KEY);
-```
-
-### Server-side (HTTP-only cookies)
-
-```typescript
-// app/api/auth/login/route.ts
-import { cookies } from 'next/headers';
-
-// After successful login
-cookies().set('llm_auth_token', JSON.stringify(tokens), {
-  httpOnly: true,
-  secure: process.env.NODE_ENV === 'production',
-  sameSite: 'lax',
-  maxAge: tokens.expires_in,
-  path: '/',
-});
-
-// Read in middleware or API routes
-const tokenCookie = cookies().get('llm_auth_token');
-const tokens = JSON.parse(tokenCookie?.value || 'null');
-```
-
-## Token Refresh
-
-```typescript
-async function refreshAccessToken(refresh_token: string) {
-  const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-    body: new URLSearchParams({
-      grant_type: 'refresh_token',
-      client_id: 'local-llm',
-      client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
-      refresh_token,
-    }),
-  });
-  return response.json();
-}
-```
-
-## Environment Variables
-
-```bash
-# .env.local
-AUTHENTIK_URL=https://authentik.riotpiao.com
-AUTHENTIK_CLIENT_ID=local-llm
-AUTHENTIK_CLIENT_SECRET=<from-secret>
-
-# For client-side (public)
-NEXT_PUBLIC_AUTHENTIK_URL=https://authentik.riotpiao.com
-NEXT_PUBLIC_AUTHENTIK_CLIENT_ID=local-llm
-```
-
-## Using Token with LLM API
-
-```typescript
-const token = await getValidToken(); // from localStorage or cookie
-
-const response = await fetch('https://api.riotpiao.com/v1/chat/completions', {
-  method: 'POST',
-  headers: {
-    'Content-Type': 'application/json',
-    'Authorization': `Bearer ${token}`,  // JWT from Authentik
-  },
-  body: JSON.stringify({
-    model: 'reasoning',
-    messages: [{ role: 'user', content: 'Hello' }],
-  }),
-});
-```
-
-## TODO
-
-### Gateway-side (homelab-frontend)
-
-- [ ] Wire `internal/auth/jwt.go` into LLM proxy handler (`internal/proxy/proxy.go`)
-- [ ] Add `authRequired: true` to model config or create LLM-specific middleware
-- [ ] Example pattern from SQS (in `internal/serviceadapter/router.go`):
-
-```go
-// In proxy.go ServeHTTP, before dispatching to LLM upstream:
-if strings.HasPrefix(r.URL.Path, "/v1/") {
-    authHeader := r.Header.Get("Authorization")
-    claims, err := llmJWTAuth.ValidateBearerToken(authHeader)
-    if err != nil {
-        // Return 401/403
-    }
-    if !llmJWTAuth.CheckPermissions(claims, "llm:inference", "*") {
-        // Return 403 insufficient permissions
-    }
-}
-```
-
-### Authentik-side
-
-- [ ] Enable ROPC grant in Authentik provider settings (if not already)
-- [ ] Add redirect URIs to `local-llm` app if browser OAuth flow needed:
-
-```python
-# k8s/infra/iam/scripts/authentik-provision.py
-"local-llm": {
-    ...
-    "redirect_uris": [
-        "http://localhost:3000/api/auth/callback",
-        "https://your-app.com/api/auth/callback",
-    ],
-}
-```
-
-### NextJS-side
-
-- [ ] Until gateway auth is wired, LLM API works without token
-- [ ] Once wired, add `Authorization: Bearer <token>` to all LLM requests

k8s/apps/comfyui/deployment.yaml

@@ -22,6 +22,10 @@ spec:
       nodeSelector:
         kubernetes.io/hostname: worker-1
       runtimeClassName: nvidia
+      # k8s Service named 'comfyui' injects COMFYUI_PORT=tcp://... into pod env,
+      # which clobbers ai-dock's own COMFYUI_PORT variable (expects a port number).
+      # Disable service link injection to avoid the collision.
+      enableServiceLinks: false
       containers:
         - name: comfyui
           image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
@@ -31,8 +35,6 @@ spec:
           env:
             - name: NVIDIA_VISIBLE_DEVICES
               value: "all"
-            - name: COMFYUI_FLAGS
-              value: "--listen 0.0.0.0 --port 8188"
           resources:
             requests:
               cpu: "4"
@@ -57,7 +59,7 @@ spec:
             httpGet:
               path: /
               port: 8188
-            failureThreshold: 60
+            failureThreshold: 120
             periodSeconds: 10
       volumes:
         - name: models

k8s/apps/comfyui/ingress.yaml

@@ -4,16 +4,24 @@ metadata:
   name: comfyui
   namespace: comfyui
   annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
     # WebSocket support for ComfyUI's live preview
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    nginx.ingress.kubernetes.io/proxy-set-headers: "Upgrade"
+    nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
 spec:
   ingressClassName: nginx
+  tls:
+    - secretName: comfyui-tls
+      hosts:
+        - comfyui.riotpiao.com
   rules:
-    - host: comfy.riotpiao.com
+    - host: comfyui.riotpiao.com
       http:
         paths:
           - path: /

k8s/apps/gotify/README.md

@@ -0,0 +1,107 @@
+# Gotify — Push Notifications + Email Relay
+
+Self-hosted notification server with SMTP email forwarding sidecar.
+
+## Architecture
+
+```
+Forgejo webhook ──POST──→ Gotify API (:80/message)
+                              │
+                    ┌─────────┼─────────┐
+                    ▼                     ▼
+              Push notification      SMTP emailer sidecar
+              (mobile/desktop)       (polls → sends email)
+```
+
+## Setup (one-time, after first deploy)
+
+### 1. Encrypt secrets before committing
+
+```bash
+# Edit secrets.yaml with real values first, then:
+sops -e -i k8s/apps/gotify/secrets.yaml
+```
+
+### 2. Create Gotify app + client tokens
+
+1. Login to `https://gotify.riotpiao.com` with admin creds
+2. **Applications** → Create `forgejo` → copy **app token**
+3. **Clients** → Create `smtp-emailer` → copy **client token**
+4. Update `gotify-tokens` secret:
+   ```bash
+   kubectl -n notifications create secret generic gotify-tokens \
+     --from-literal=app-token=<APP_TOKEN> \
+     --from-literal=client-token=<CLIENT_TOKEN> \
+     --dry-run=client -o yaml | kubectl apply -f -
+   ```
+
+### 3. Configure Forgejo webhook
+
+In each Forgejo repo → **Settings** → **Webhooks** → **Add Webhook** → **Gotify**:
+
+| Field | Value |
+|-------|-------|
+| Target URL | `http://gotify.notifications.svc.cluster.local/message` |
+| Token | The **app token** from step 2 |
+| Events | Pull Request (Created, Merged, Closed) |
+
+Or via API:
+```bash
+FORGEJO_TOKEN="<your-pat>"
+APP_TOKEN="<gotify-app-token>"
+
+curl -s -X POST "https://forgejo.riotpiao.com/api/v1/repos/rock/homelab/hooks" \
+  -H "Authorization: token $FORGEJO_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "gotify",
+    "active": true,
+    "config": {
+      "content_type": "json",
+      "url": "http://gotify.notifications.svc.cluster.local/message?token='"$APP_TOKEN"'"
+    },
+    "events": ["pull_request", "pull_request_assign", "pull_request_review"],
+    "authorization_header": ""
+  }'
+```
+
+### 4. Add CoreDNS rewrite (if accessing via public hostname)
+
+Only needed if Cloudflare Tunnel is used for gotify.riotpiao.com:
+
+```
+# terraform/files/coredns/Corefile — add rewrite:
+rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+```
+
+Then: `cd terraform && terraform apply && cd .. && make apply-cp`
+
+### 5. SMTP providers
+
+| Provider | Host | Port | Notes |
+|----------|------|------|-------|
+| Gmail | smtp.gmail.com | 587 | Use App Password (2FA required) |
+| Resend | smtp.resend.com | 587 | Free 100 emails/day |
+| Sendgrid | smtp.sendgrid.net | 587 | Free 100 emails/day |
+| Mailgun | smtp.mailgun.org | 587 | Free 5000/month |
+
+## Notification priority levels
+
+| Priority | Meaning | Email forwarded? |
+|----------|---------|-----------------|
+| 0-4 | Low (info) | No (below MIN_PRIORITY=5) |
+| 5-7 | Normal (PR created) | Yes |
+| 8-10 | High (PR merged, failures) | Yes |
+
+## Verify
+
+```bash
+# Test push notification
+APP_TOKEN="<app-token>"
+curl -X POST "https://gotify.riotpiao.com/message?token=$APP_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"title":"Test","message":"Hello from homelab","priority":5}'
+
+# Check email sidecar logs
+kubectl -n notifications logs deployment/gotify -c smtp-emailer --tail=20
+```

k8s/apps/gotify/db.yaml

@@ -0,0 +1,35 @@
+# CNPG Postgres for Gotify. Lightweight — 2 instances, 2Gi storage.
+# CNPG generates secret `gotify-db-app` + service `gotify-db-rw` in ns notifications.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: gotify-db
+  namespace: notifications
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: gotify
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "256Mi", cpu: "100m" }
+    limits: { memory: "512Mi", cpu: "500m" }
+  storage:
+    size: 2Gi
+    storageClass: longhorn-cnpg
+  monitoring:
+    enablePodMonitor: true
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/apps/gotify/deployment.yaml

@@ -0,0 +1,204 @@
+# Gotify — self-hosted push notification server + SMTP email relay.
+# Forgejo webhooks → Gotify → push notifications + email forwarding.
+# Runs on control plane (no GPU needed), lightweight.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: gotify
+  template:
+    metadata:
+      labels:
+        app: gotify
+    spec:
+      containers:
+        # --- Gotify server ---
+        - name: gotify
+          image: ghcr.io/gotify/server:3.1.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              export GOTIFY_DATABASE_DIALECT=postgres
+              export GOTIFY_DATABASE_CONNECTION="host=gotify-db-rw.notifications port=5432 user=${DB_USER} password=${DB_PASS} dbname=gotify sslmode=disable"
+              exec /app/gotify-app
+          ports:
+            - containerPort: 80
+              protocol: TCP
+          env:
+            - name: GOTIFY_DEFAULTUSER_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: username
+            - name: GOTIFY_DEFAULTUSER_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: password
+            - name: DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: username
+            - name: DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: password
+            - name: GOTIFY_SERVER_PORT
+              value: "80"
+            - name: GOTIFY_SERVER_KEEPALIVEPERIODSECONDS
+              value: "0"
+            - name: TZ
+              value: Asia/Tokyo
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 30
+            initialDelaySeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 10
+            initialDelaySeconds: 5
+
+        # --- SMTP emailer sidecar ---
+        # Watches Gotify WebSocket stream, forwards messages as email.
+        # https://github.com/eternal-flame-AD/gotify-broadcast
+        - name: smtp-emailer
+          image: ghcr.io/gotify/server:3.1.0
+          command:
+            - /bin/sh
+            - -c
+            - |
+              # Wait for Gotify to be ready
+              until wget -qO- http://localhost:80/health >/dev/null 2>&1; do
+                echo "Waiting for Gotify..."
+                sleep 2
+              done
+              echo "Gotify is ready, starting email relay..."
+
+              # Poll Gotify messages and forward via SMTP using msmtp
+              # Install msmtp for lightweight SMTP sending
+              apk add --no-cache msmtp curl jq
+
+              # Configure msmtp
+              cat > /tmp/msmtprc <<MSMTP
+              defaults
+              auth           on
+              tls            on
+              tls_trust_file /etc/ssl/certs/ca-certificates.crt
+              logfile        /tmp/msmtp.log
+
+              account        default
+              host           ${SMTP_HOST}
+              port           ${SMTP_PORT}
+              from           ${SMTP_FROM}
+              user           ${SMTP_USER}
+              password       ${SMTP_PASS}
+              MSMTP
+
+              chmod 600 /tmp/msmtprc
+
+              # Track last seen message ID
+              LAST_ID=0
+
+              while true; do
+                # Fetch messages since last ID
+                MESSAGES=$(curl -s -H "X-Gotify-Key: ${GOTIFY_CLIENT_TOKEN}" \
+                  "http://localhost:80/message?since=${LAST_ID}&limit=10" 2>/dev/null)
+
+                if [ -n "$MESSAGES" ]; then
+                  echo "$MESSAGES" | jq -r '.messages[]? | @base64' | while read -r MSG; do
+                    DECODED=$(echo "$MSG" | base64 -d)
+                    ID=$(echo "$DECODED" | jq -r '.id')
+                    TITLE=$(echo "$DECODED" | jq -r '.title // "Notification"')
+                    BODY=$(echo "$DECODED" | jq -r '.message // ""')
+                    PRIORITY=$(echo "$DECODED" | jq -r '.priority // 5')
+                    APP=$(echo "$DECODED" | jq -r '.appid // 0')
+                    DATE=$(echo "$DECODED" | jq -r '.date // ""')
+
+                    # Only forward messages with priority >= configured threshold
+                    if [ "$PRIORITY" -ge "${MIN_PRIORITY:-0}" ]; then
+                      printf "Subject: [Gotify] %s\nFrom: %s\nTo: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n\n---\nPriority: %s\nDate: %s" \
+                        "$TITLE" "$SMTP_FROM" "$NOTIFY_EMAIL" "$BODY" "$PRIORITY" "$DATE" | \
+                        msmtp -C /tmp/msmtprc "$NOTIFY_EMAIL" && \
+                        echo "Email sent for message $ID: $TITLE" || \
+                        echo "Failed to send email for message $ID"
+                    fi
+
+                    # Update last seen ID
+                    if [ "$ID" -gt "$LAST_ID" ]; then
+                      LAST_ID=$ID
+                    fi
+                  done
+                fi
+
+                sleep ${POLL_INTERVAL:-30}
+              done
+          env:
+            - name: GOTIFY_CLIENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-tokens
+                  key: client-token
+            - name: SMTP_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: host
+            - name: SMTP_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: port
+            - name: SMTP_FROM
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: from
+            - name: SMTP_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: user
+            - name: SMTP_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: password
+            - name: NOTIFY_EMAIL
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: notify-email
+            - name: MIN_PRIORITY
+              value: "5"
+            - name: POLL_INTERVAL
+              value: "15"
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      # No volumes — Postgres handles persistence

k8s/apps/gotify/igotify-deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: igotify
+  namespace: notifications
+  labels:
+    app: igotify
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: igotify
+  template:
+    metadata:
+      labels:
+        app: igotify
+    spec:
+      containers:
+        - name: igotify
+          image: ghcr.io/androidseb25/igotify-notification-assist:v1.6.0.1
+          ports:
+            - containerPort: 8080
+              protocol: TCP
+          env:
+            - name: ENABLE_CONSOLE_LOG
+              value: "true"
+            - name: ENABLE_SCALAR_UI
+              value: "true"
+            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
+            # left unset — not required when Gotify is reachable via public domain
+            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
+          volumeMounts:
+            - name: data
+              mountPath: /app/data
+          livenessProbe:
+            httpGet:
+              path: /Version
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /Version
+              port: 8080
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: igotify-data

k8s/apps/gotify/igotify-ingress.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: igotify
+  namespace: notifications
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: igotify.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: igotify
+                port:
+                  number: 80

k8s/apps/gotify/igotify-pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: igotify-data
+  namespace: notifications
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi

k8s/apps/gotify/igotify-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: igotify
+  namespace: notifications
+  labels:
+    app: igotify
+spec:
+  selector:
+    app: igotify
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP

k8s/apps/gotify/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gotify
+  namespace: notifications
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    # WebSocket support for Gotify client connections (/stream endpoint)
+    # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: gotify.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gotify
+                port:
+                  number: 80

k8s/apps/gotify/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - db.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+  - igotify-pvc.yaml
+  - igotify-deployment.yaml
+  - igotify-service.yaml
+  - igotify-ingress.yaml

k8s/apps/gotify/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: notifications
+  labels:
+    kubernetes.io/metadata.name: notifications

k8s/apps/gotify/service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  selector:
+    app: gotify
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP

k8s/apps/immich/db.yaml

@@ -18,7 +18,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 2
+  instances: 3
   imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie
   postgresql:
     extensions:

k8s/apps/llm-serving/embeddings.yaml

@@ -45,6 +45,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-embeddings
+        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-serving/networkpolicy.yaml

@@ -52,6 +52,20 @@ spec:
           port: 8080
         - protocol: TCP
           port: 9000
+    # Allow from paperless namespace (paperless-ai auto-tagging)
+    # Bypasses gateway until service-account JWT token exchange is implemented.
+    # paperless-ai-agent has llm:inference role in Authentik.
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: paperless
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 80
+        - protocol: TCP
+          port: 8000
     # Allow intra-namespace (pod-to-pod within llm-serving)
     - from:
         - podSelector:

k8s/apps/llm-serving/ornith.yaml

@@ -44,7 +44,7 @@ spec:
       - name: OLLAMA_MODELS
         value: /mnt/models/ollama
       - name: OLLAMA_CONTEXT_LENGTH
-        value: '32768'
+        value: '65536'
       - name: OLLAMA_KEEP_ALIVE
         value: '-1'
       - name: OLLAMA_NUM_PARALLEL
@@ -83,6 +83,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-ornith
+        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.

k8s/apps/llm-serving/reasoning.yaml

@@ -28,9 +28,14 @@ spec:
       # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
       # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
       # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
-      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
-      # above the 131072 target with room to spare -- the old
+      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
+      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
+      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
+      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
+      # metric after the rollout -- max_num_batched_tokens=8192 also raises
+      # activation memory out of the same budget, so the post-boot
+      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
+      # only valid if it still covers 2x65k + 3x8k. The old
       # OffloadingConnector CPU-DRAM spillover (tuned for the previous
       # model's much smaller 16384 context) is no longer needed and is
       # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -54,10 +59,37 @@ spec:
       - --tensor-parallel-size=1
       - --pipeline-parallel-size=2
       - --max-model-len=131072
-      - --gpu-memory-utilization=0.90
-      - --max-num-seqs=4
+      - --gpu-memory-utilization=0.95
+      - --max-num-seqs=5
+      # Prefill scheduling -- see the measured starvation this fixes.
+      # V1's scheduler has no prefill/decode phase: it walks self.running and
+      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
+      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
+      # per request is applied BEFORE the budget clamp:
+      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
+      #     take = min(take, token_budget)
+      # The loop is `while ... and token_budget > 0`. With the threshold at its
+      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
+      # budget, token_budget hits 0, the loop exits and every other request is
+      # skipped for that step -- measured as 262 tok/s contended prefill and
+      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
+      # Hence threshold must be strictly BELOW the budget or it never binds:
+      # at 8192/8192 a single request is served per step (still starved), while
+      # 8192/2048 serves four. Verified by replaying the loop.
+      - --max-num-batched-tokens=8192
+      - --long-prefill-token-threshold=2048
       - --enable-chunked-prefill
       - --enable-prefix-caching
+      #
+      # NOT set, deliberately: --max-num-partial-prefills and
+      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
+      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
+      # rejects any value other than the default for either one, calling
+      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
+      # -> AssertionError, because there is no V0 engine to fall back to.
+      # The Deployment uses strategy Recreate, so setting them is an outage.
+      # V1 does concurrent chunked prefill natively; it needs no flag, only the
+      # budget/threshold ratio above.
       # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
       # blocks.
       - --reasoning-parser=qwen3
@@ -104,6 +136,14 @@ spec:
         name: models
       - mountPath: /dev/shm
         name: shm
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-reasoning
+        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     maxReplicas: 1

k8s/apps/llm-serving/reranker.yaml

@@ -45,6 +45,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-reranker
+        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/paperless/backup-cronjob.yaml

@@ -48,7 +48,7 @@ spec:
                   mountPath: /backup
           containers:
             - name: mc-mirror
-              image: minio/mc:latest
+              image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
               env:
                 - name: ACCESS_KEY
                   valueFrom:

k8s/apps/paperless/configmap.yaml

@@ -20,3 +20,7 @@ data:
   # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
   # on 'account_confirm_email' without this.
   PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
+  # Auto-connect social accounts to existing users with matching email
+  PAPERLESS_SOCIAL_AUTO_SIGNUP: "true"
+  # Allow automatic linking of social accounts to existing users
+  PAPERLESS_ACCOUNT_ALLOW_SIGNUPS: "true"

k8s/apps/paperless/kustomization.yaml

@@ -4,6 +4,7 @@ namespace: paperless
 resources:
   - pvc.yaml
   - configmap.yaml
+  - paperless-ai-config.yaml
   - redis.yaml
   - deployment.yaml
   - service.yaml
@@ -11,6 +12,7 @@ resources:
   - backup-cronjob.yaml
   - adapter-configmap.yaml
   - rbac.yaml
+  - paperless-ai.yaml
 # postgres: paperless-db CNPG Cluster, deployed by k8s/infra/databases (wave 2,
 # before this app at wave 8) - not duplicated here. Same for the paperless-oidc
 # and paperless-minio-creds Secrets, written by PostSync provisioning Jobs in

k8s/apps/paperless/paperless-ai-config.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: paperless-ai-env
+  namespace: paperless
+data:
+  paperless-ai-config.env: |
+    # Paperless-NGX API configuration
+    PAPERLESS_URL=http://paperless.paperless.svc.cluster.local:8000
+    PAPERLESS_API_TOKEN=7b89463e04c141f4172fbcddf78d623547d327ec
+
+    # AI Processing settings
+    ENABLE_AI_PROCESSING=yes
+    AI_MODEL=reasoning
+    LLM_API_URL=https://api.riotpiao.com/v1
+    LLM_API_TOKEN_FILE=/app/data/llm_token.txt
+
+    # Auto-tagging configuration
+    ENABLE_AUTO_TAGGING=yes
+    ENABLE_CORRESPONDENT_EXTRACTION=yes
+    ENABLE_DOCUMENT_TYPE_EXTRACTION=yes
+    ENABLE_TITLE_GENERATION=yes
+    
+    # Advanced tagging with AI
+    ENABLE_INTELLIGENT_TAGGING=yes
+    ENABLE_DATE_EXTRACTION=yes
+    ENABLE_TIMELINE_TAGGING=yes
+    AUTO_CREATE_TAGS=yes
+    
+    # Tagging prompts for custom extraction
+    CUSTOM_EXTRACTION_PROMPT=Extract document date, time period, financial year, and temporal context. Create tags like YYYY, Q1-YYYY, Last-Month, Current-Year, Historic
+    TAG_EXTRACTION_MODEL=reasoning
+
+    # Scanning behavior
+    SCAN_INTERVAL_SECONDS=60
+    PROCESS_EXISTING_DOCUMENTS=no
+    ADD_AI_TAG=yes
+    AI_TAG_NAME=ai-processed
+
+    # Date/Timeline tagging configuration
+    DATE_EXTRACTION_FORMAT=iso8601
+    TIMELINE_TAG_GRANULARITY=year,quarter,month
+    AUTO_TAG_HISTORICAL=yes
+    AUTO_TAG_RECENT=yes
+    
+    # Performance
+    MAX_CONCURRENT_REQUESTS=2
+    REQUEST_TIMEOUT=60
+    DATE_EXTRACTION_TIMEOUT=30

k8s/apps/paperless/paperless-ai-new.yaml

@@ -0,0 +1,127 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-ai-data
+  namespace: paperless
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperless-ai
+  namespace: paperless
+  labels:
+    app.kubernetes.io/name: paperless-ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: paperless-ai
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: paperless-ai
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      
+      volumes:
+        - name: paperless-ai-data
+          persistentVolumeClaim:
+            claimName: paperless-ai-data
+      
+      initContainers:
+        - name: fetch-llm-token
+          image: curlimages/curl:8.12.0
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
+              
+              # Get JWT token via client_credentials grant
+              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+                -d "grant_type=client_credentials" \
+                -d "client_id=paperless-ai-agent" \
+                -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
+                -d "scope=openid llm:inference" 2>/dev/null)
+              
+              # Extract token
+              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
+              
+              if [ -z "$TOKEN" ]; then
+                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
+                exit 1
+              fi
+              
+              # Store token in file for main container to read
+              mkdir -p /data
+              echo "$TOKEN" > /data/llm_token.txt
+              echo "[init] Token fetched and stored successfully"
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /data
+      
+      containers:
+        - name: paperless-ai
+          image: clusterzx/paperless-ai:latest
+          imagePullPolicy: Always
+          env:
+            # Paperless-ngx connection
+            - name: PAPERLESS_API_URL
+              value: "http://paperless.paperless.svc.cluster.local:8000"
+            - name: PAPERLESS_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-ai-config
+                  key: PAPERLESS_API_TOKEN
+            - name: PAPERLESS_USERNAME
+              value: "admin"
+            
+            # LLM API configuration
+            - name: AI_PROVIDER
+              value: "custom"
+            - name: CUSTOM_BASE_URL
+              value: "https://api.riotpiao.com/v1"
+            # Token file created by init container with JWT from Authentik
+            - name: CUSTOM_API_KEY_FILE
+              value: "/app/data/llm_token.txt"
+            - name: CUSTOM_MODEL
+              value: "qwen2.5:3b-instruct"
+            
+            # Scanning behavior
+            - name: SCAN_INTERVAL
+              value: "60"
+            - name: PROCESS_PREDEFINED_DOCUMENTS
+              value: "no"
+            
+            # Tagging configuration
+            - name: ADD_AI_TAG
+              value: "yes"
+            - name: AI_TAG_NAME
+              value: "ai-processed"
+            - name: ADD_AI_DESCRIPTIONS
+              value: "yes"
+            - name: USE_PROMPT_TAGS
+              value: "yes"
+          
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /app/data
+          
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi

k8s/apps/paperless/paperless-ai.yaml

@@ -0,0 +1,127 @@
+# Secret paperless-ai-config managed via SOPS (argocd/secrets/paperless-ai-secrets.enc.yaml)
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperless-ai
+  namespace: paperless
+  labels:
+    app.kubernetes.io/name: paperless-ai
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: paperless-ai
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: paperless-ai
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      volumes:
+        - name: paperless-ai-data
+          persistentVolumeClaim:
+            claimName: paperless-ai-data
+        - name: paperless-ai-env-config
+          configMap:
+            name: paperless-ai-env
+      initContainers:
+        - name: fetch-llm-token
+          image: curlimages/curl:8.12.0
+          securityContext:
+            runAsUser: 0
+            fsGroup: 0
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
+              
+              # Get JWT token via client_credentials grant
+              # Client secret sourced from environment (injected from paperless-ai-config Secret)
+              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+                -d "grant_type=client_credentials" \
+                -d "client_id=paperless-ai-agent" \
+                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
+                -d "scope=openid llm:inference" 2>/dev/null)
+              
+              # Extract token from JSON response using sed (handles spaces after colons)
+              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
+              
+              if [ -z "$TOKEN" ]; then
+                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
+                exit 1
+              fi
+              
+              # Store token in file for main container to read
+              # Write to both locations for compatibility
+              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
+              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
+              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
+              echo "[init] Token fetched and stored"
+              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
+              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
+          env:
+            - name: LLM_AUTH_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-ai-config
+                  key: LLM_AUTH_CLIENT_SECRET
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /app/data
+      containers:
+        - name: paperless-ai
+          image: clusterzx/paperless-ai:latest
+          env:
+            # Paperless-ngx connection
+            - name: PAPERLESS_API_URL
+              value: "http://paperless.paperless.svc.cluster.local:8000"
+            - name: PAPERLESS_API_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-ai-config
+                  key: PAPERLESS_API_TOKEN
+            - name: PAPERLESS_USERNAME
+              value: "admin"
+            # LLM API — via public gateway with JWT auth
+            - name: AI_PROVIDER
+              value: "custom"
+            - name: CUSTOM_BASE_URL
+              value: "https://api.riotpiao.com/v1"
+            # Token will be read from file at runtime by the application
+            # The init container fetches it and stores in /app/data/llm_token.txt
+            - name: CUSTOM_API_KEY_FILE
+              value: "/app/data/llm_token.txt"
+            - name: CUSTOM_MODEL
+              value: "qwen2.5:3b-instruct"
+            # Behavior - scan for new documents and tag them
+            - name: SCAN_INTERVAL
+              value: "60"
+            - name: PROCESS_PREDEFINED_DOCUMENTS
+              value: "no"
+            - name: ADD_AI_TAG
+              value: "yes"
+            - name: AI_TAG_NAME
+              value: "ai-processed"
+            - name: USE_PROMPT_TAGS
+              value: "yes"
+            - name: ADD_AI_DESCRIPTIONS
+              value: "yes"
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /app/data
+            - name: paperless-ai-env-config
+              mountPath: /app/data/.env
+              subPath: paperless-ai-config.env
+          resources:
+            requests:
+              cpu: 100m
+              memory: 512Mi
+            limits:
+              cpu: "1"
+              memory: 2Gi

k8s/apps/poimen-frontend/ingress.yaml

@@ -0,0 +1,24 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: poimen-frontend
+  namespace: poimen
+  annotations:
+    # Next.js SSE + LLM streaming responses need long timeouts + no buffering.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: poimen.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: poimen-frontend
+                port:
+                  number: 3000

k8s/apps/poimen-frontend/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Edge Ingress for poimen.riotpiao.com.
+# Same pattern as k8s/apps/api/ (api-gw ingress).
+# App manifests (Deployment, Service, NetworkPolicy) live in
+# riotpiao-poimen/poimen-frontend.git k8s/ — each repo owns its own k8s.
+resources:
+  - ingress.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -40,7 +40,7 @@ spec:
         
         # Git
         - name: GIT_REPO
-          value: https://forgejo.riotpiao.com/rock/homelab.git
+          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
         - name: GIT_AUTHOR_EMAIL
           value: [email protected]
         - name: GIT_AUTHOR_NAME

k8s/argocd/apps/-1-projects.yaml

@@ -18,7 +18,7 @@ spec:
       prune: true
       selfHeal: true
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/argocd/projects
   destination:

k8s/argocd/apps/00-secrets.yaml

@@ -17,7 +17,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
     # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the

k8s/argocd/apps/00-substrate.yaml

@@ -21,7 +21,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -93,7 +93,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
     # deterministically. The previous directory.include with bare filenames
@@ -127,7 +127,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/bootstrap/ingress
   destination:
@@ -149,7 +149,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/cluster-maintenance
   destination:
@@ -177,7 +177,7 @@ spec:
       valueFiles:
         - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
   sources:
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -200,7 +200,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/bootstrap/kyverno
   destination:

k8s/argocd/apps/01-image-updater.yaml

@@ -1,33 +0,0 @@
-# ArgoCD Image Updater - auto-updates Application images from registry
-# Watches forgejo.riotpiao.com for new image tags and updates Applications
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: argocd-image-updater
-  namespace: argocd
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  sources:
-    - repoURL: https://argoproj.github.io/argo-helm
-      chart: argocd-image-updater
-      targetRevision: "0.11.2"
-      helm:
-        valueFiles:
-          - $values/k8s/infra/argocd-image-updater/values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: argocd
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=false

k8s/argocd/apps/06-ci-cd.yaml

@@ -0,0 +1,55 @@
+# Tekton Pipelines v0.68.0
+#
+# Install method: vendored release.yaml in k8s/infra/tekton/
+# downloaded from https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.68.0/release.yaml
+#
+# To upgrade:
+#   1. Download new release.yaml from https://github.com/tektoncd/pipeline/releases
+#   2. Replace k8s/infra/tekton/release.yaml
+#   3. Commit and push — ArgoCD syncs automatically
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: tekton-pipelines
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: tekton-pipelines
+    app.kubernetes.io/part-of: homelab-infra
+    wave: "06"
+spec:
+  project: homelab
+
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/tekton
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: tekton-pipelines
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+
+  ignoreDifferences:
+  - group: admissionregistration.k8s.io
+    kind: ValidatingWebhookConfiguration
+    jsonPointers:
+    - /webhooks/0/clientConfig/caBundle
+    - /webhooks
+  - group: admissionregistration.k8s.io
+    kind: MutatingWebhookConfiguration
+    jsonPointers:
+    - /webhooks/0/clientConfig/caBundle
+    - /webhooks

k8s/argocd/apps/08-secret-rotation.yaml

@@ -9,7 +9,7 @@ spec:
   project: homelab
   
   sources:
-  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     path: k8s/apps/secret-rotation-controller
     targetRevision: main
   

k8s/argocd/apps/10-storage-observability.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/minio/minio-operator-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -41,7 +41,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/minio
   destination:
@@ -66,7 +66,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/longhorn
   destination:
@@ -102,7 +102,7 @@ spec:
         skipCrds: true
         valueFiles:
           - $values/k8s/infra/monitoring/prometheus-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -152,7 +152,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/monitoring/crds
   destination:
@@ -183,7 +183,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/monitoring
   destination:
@@ -213,7 +213,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -237,7 +237,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/tracing
   destination:

k8s/argocd/apps/20-logging.yaml

@@ -19,7 +19,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/loki-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -53,7 +53,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/grafana-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -87,7 +87,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/promtail-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:

k8s/argocd/apps/30-security.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/vault-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -46,7 +46,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/authentik-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/iam
   destination:
@@ -109,7 +109,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -152,17 +152,12 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
   destination:
@@ -180,17 +175,12 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
     helm:
@@ -211,17 +201,12 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
     helm:

k8s/argocd/apps/40-data.yaml

@@ -14,7 +14,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/databases
   destination:

k8s/argocd/apps/50-memory-queues.yaml

@@ -8,7 +8,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/messaging/memory-queues
   destination:

k8s/argocd/apps/50-messaging.yaml

@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/messaging/kafka-cluster
   destination:

k8s/argocd/apps/55-api-gateway.yaml

@@ -36,19 +36,17 @@ metadata:
     app.kubernetes.io/component: gateway
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # ArgoCD Image Updater - auto-update on new image push
-    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
-    argocd-image-updater.argoproj.io/gw.update-strategy: digest
-    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
+    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
+    # → patches this Application's kustomize.images via k8s API.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
   revisionHistoryLimit: 3
   sources:
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab-frontend.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
       targetRevision: main
       path: k8s
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       path: k8s/apps/api
   destination:

k8s/argocd/apps/56-llm-serving.yaml

@@ -20,7 +20,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/llm-serving
   destination:

k8s/argocd/apps/58-comfyui.yaml

@@ -12,7 +12,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/comfyui
   destination:

k8s/argocd/apps/59-gotify.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: gotify
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: gotify
+    app.kubernetes.io/component: notifications
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/apps/gotify
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: notifications
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/60-applications.yaml

@@ -19,10 +19,10 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/temporal/temporal-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       path: k8s/apps/temporal
   destination:
@@ -51,7 +51,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/portainer/portainer-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
   destination:
@@ -74,7 +74,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/cloudflared
   destination:
@@ -97,7 +97,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/agent-pod
   destination:
@@ -130,7 +130,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/sms
   destination:
@@ -157,7 +157,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/paperless
   destination:
@@ -189,7 +189,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/apps/immich
   destination:
@@ -220,10 +220,10 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/homarr/homarr-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       ref: values
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
       path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
   destination:
@@ -246,11 +246,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "8"
-    # ArgoCD Image Updater - auto-update on new image push
-    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
-    argocd-image-updater.argoproj.io/app.update-strategy: digest
-    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
   source:
@@ -281,7 +278,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/infra/rbac
   destination:

k8s/argocd/apps/60-kmsvc-manage.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/kmsvc-manage.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
     targetRevision: main
     path: k8s/argocd/apps
     directory:

k8s/argocd/apps/70-poimen.yaml

@@ -1,43 +1,81 @@
+# Poimen — unified ArgoCD Application (wave 7).
+#
+# Consolidates what was previously five separate apps:
+#   poimen            → source 1 (poimen-workflows workers)
+#   poimen-workflows  → duplicate of source 1 (deleted)
+#   poimen-memory     → source 2 (memory service + redis)
+#   poimen-memory-app → duplicate of source 2 (deleted)
+#   poimen-memory-database → source 3 (CNPG cluster, moved to k8s/infra/poimen)
+#
+# poimen-toolkit is intentionally kept as a separate Application because it
+# manages cluster-scoped CRDs (ToolDefinition). A bad CRD can affect the
+# whole cluster; keeping it separate means it has its own sync history,
+# independent rollback, and can be paused without touching app workloads.
+#
+# Sync ordering via argocd sync-wave annotations:
+#   wave -1 → DB (CNPG Cluster) — must be Ready before app pods start
+#   wave  0 → all application Deployments (default)
+#
+# Image Updater tracks all four images. Write-back via argocd (in-cluster
+# annotation patch — no git commit per image update).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
   name: poimen
   namespace: argocd
+  labels:
+    app.kubernetes.io/part-of: poimen
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # Image Updater: auto-update on new image push (SHA tag filter)
-    argocd-image-updater.argoproj.io/image-list: |
-      memory=forgejo.riotpiao.com/rock/poimen-memory
-      workflows=forgejo.riotpiao.com/rock/poimen-workflows
-      frontend=forgejo.riotpiao.com/rock/poimen-frontend
-    argocd-image-updater.argoproj.io/memory.update-strategy: digest
-    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
-    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
-    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
+    # ── Image promotion ──────────────────────────────────────────────────────
+    # Forgejo package webhook → api-gw patches kustomize.images per source.
+    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
 spec:
   project: homelab
+  # Multi-source: each repo owns its own k8s manifests.
+  # ArgoCD syncs all sources in parallel; sync-wave annotations on individual
+  # resources control ordering within the sync operation.
   sources:
-    - repoURL: https://forgejo.riotpiao.com/rock/poimen-memory.git
+    # 1. Workers — Temporal activity + workflow workers, pi-worker pool
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
       targetRevision: main
-      path: k8s/argocd
-    - repoURL: https://forgejo.riotpiao.com/rock/poimen-workflows.git
+      path: k8s
+
+    # 2. Memory service — poimen-memory API, Redis cache, NetworkPolicies
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
       targetRevision: main
-      path: k8s/argocd
-    - repoURL: https://forgejo.riotpiao.com/rock/poimen-frontend.git
+      path: k8s/app
+
+    # 3. Database infrastructure — CNPG Postgres cluster (wave -1)
+    #    Moved from k8s/infra/databases to isolate poimen DB from other tenants.
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
       targetRevision: main
-      path: k8s/argocd
+      path: k8s/infra/poimen
+
+    # 4. Frontend app manifests — Deployment, Service, NetworkPolicy.
+    #    Each repo owns its own k8s/ (same pattern as homelab-frontend for api-gw).
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
+      targetRevision: main
+      path: k8s
+
+    # 5. Frontend ingress — poimen.riotpiao.com edge route.
+    #    Lives in homelab (same pattern as k8s/apps/api/ for api-gw).
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      path: k8s/apps/poimen-frontend
+
   destination:
     server: https://kubernetes.default.svc
     namespace: poimen
+
   syncPolicy:
     automated:
       prune: true
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+      - SkipDryRunOnMissingResource=true
     retry:
       limit: 5
       backoff:

k8s/argocd/apps/71-poimen-toolkit.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: poimen-toolkit
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+    # Image promotion: Forgejo package webhook → api-gw.
+    # Tracks: poimen-toolkit-controller, pi-worker.
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
+    targetRevision: main
+    path: k8s
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: poimen
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/99-oidc-bootstrap.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: oidc-bootstrap
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/oidc-bootstrap
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/projects/homelab-project.yaml

@@ -17,12 +17,14 @@ spec:
     - https://github.com/Riotpiaole/Poimen-workflows.git
     - https://github.com/Riotpiaole/poimen*.git
     # In-cluster Forgejo repos — explicit allowlist (no wildcard)
-    - https://forgejo.riotpiao.com/rock/homelab.git
-    - https://forgejo.riotpiao.com/rock/homelab-frontend.git
-    - https://forgejo.riotpiao.com/rock/kmsvc-manage.git
-    - https://forgejo.riotpiao.com/rock/poimen.git
-    - https://forgejo.riotpiao.com/rock/poimen-memory.git
-    - https://forgejo.riotpiao.com/rock/poimen-workflows.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-toolkit.git
     - https://forgejo.riotpiao.com/rock/riotpiao.com.git
     # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
     - https://cloudnative-pg.github.io/charts

k8s/argocd/root/homelab-root.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     targetRevision: main
     path: k8s/argocd/apps
     directory:

k8s/argocd/secrets/forgejo-registry-pat-api.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: forgejo-registry-pat
+    namespace: api
+type: Opaque
+stringData:
+    pat: ENC[AES256_GCM,data:alYPvHBVqnf13NZu6jtgA6WdFM4x46s5mIY8uEnfWdq0Vp+7oKrY1A==,iv:JbUjes5ZB9eKQ1nkV02++7F/BaWIROfT6gjShzajXro=,tag:ApjawJold0QApnX1GUXGhw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6bFZhU3JLSUZEVWhYdEdZ
+            MURoSWNlZjY0RkxyaWlwU2ZTVjJ5V2FnMVVZCis5YlBwQnZrWnVJNFh4UnNldmx6
+            Y2JjdnpkRTVGSjRSZ0U0THdDZHIrNHcKLS0tIHZDd1BDVGRDSHJEQThuTExORWs2
+            b3BxenY4WHhpRkRTZHJGMkNYMm14RnMKH//Xpqbf/ZBNFQ43D0eFUWwRcQWqA3CC
+            RG8U9wyxC6YrlruU2hY6eB2x0OjsVic34ZGaR752Bwyp0Pz49C/k3g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-09-28T03:10:55Z"
+    mac: ENC[AES256_GCM,data:nLZueHygckw+Gddlea90tRU8h8nEGpe21JUi9nCdpc0ctkSlTDqcR3Q7JFPwbEGHGFU5UJ2xnkBZEDIVmExJryqPBMXT7hQd3ySeQgPwAPGt6zEhlRc5jXhqmmrpJiiN5XUL52AZVDwcWACg7kVzBXIClzihFRW+UqjJZhdtELM=,iv:C0H4AFfdfpsBvP1z2chVvW/VTejxYxwNn/KuA50LeH0=,tag:ZEkTCTKvoZH7DjrcwfL3dw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/forgejo-registry-token.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: forgejo-registry-token
+    namespace: cicd
+data:
+    username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
+    password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
+            ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
+            cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
+            WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
+            SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/forgejo-smtp-secrets.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: ENC[AES256_GCM,data:uS8=,iv:EEoo9U+C244eAJMSTOQVkf5AE6BeHrc5DWPjtWnPRdk=,tag:H+YmBSGvcON3wh0p22LXDQ==,type:str]
+kind: ENC[AES256_GCM,data:j1/PFkTS,iv:ja4q8X+nzE/ZczwcY+Qe2DnnsxG64W1fkRPQvOaoqvA=,tag:WiilMGagudEKUlc2JdbGyg==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:J9iAIHboMyHu6xn/,iv:p3z3uzlkM7VDzOT3WDCelCdZ2t+QqSPmFtqYfvXPQMs=,tag:rKRtpUexVkuZKOJ34a0Xjw==,type:str]
+    namespace: ENC[AES256_GCM,data:su0FvA==,iv:6SPvwxZ/4aoL0Z07zdPC9r6L7HOHuKv3RodXpVcii04=,tag:njwkj+yvSgN8sliJP8T/Kw==,type:str]
+type: ENC[AES256_GCM,data:Qd6WJN1c,iv:M3fc78LvemcEbWzesuYeQ/GZyIOl7Eg/0EmUe3p0qAk=,tag:Tnzwewn0vwdowxy/EyuPdA==,type:str]
+stringData:
+    user: ENC[AES256_GCM,data:/Z7gPrsTUZOLzfoZ8cZGD10wrZE=,iv:0ydBSeq+yHB2b1J4W7FwIv55Eh+N3qfQ6Q7PwoUAvYo=,tag:CO19F8nElaYZyFiFR6N/Tg==,type:str]
+    password: ENC[AES256_GCM,data:AOl4StpeQIHSLX+oEFnkfg==,iv:mYsZ+5sum6YqyUPndtPCniTraxldKc803ULlKs8Gsaw=,tag:hK3w51ifZ/omAL7XDf8seg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaMlhKVEM2bFdFOFAwV1lp
+            WUd6SEN4ZTF1TEpGYXhOYmJLK0tpcWZyc2w4ClVzZmI2Qk1KUnV1OXpyTEV2WWFa
+            VlJ2eHRSaEhqUnA2dUJVbWJUcEgxcFkKLS0tIG9IaFVnenNpSFRzdStiWjJvakVL
+            aDk2bTZGR3Zya3ROUS9vd1hEQVRFaG8KbeXA6IebHEaB79N6u795336aHesHOgzO
+            uZvvBUzSBy3t3jfFk8bJP4aH79I33Ha2eK5rsvdsiv/orwCMXUINKg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-12T21:00:46Z"
+    mac: ENC[AES256_GCM,data:DjzPtR+Ueihh166Bvd3jCLtZFQdrvrxOoF87cv6lxKGWPEptT5vbw/EfuIFJBb6lvEJFDWKRC/r5ASdGwComYsPn0DZs4BPCzeKBtLfP9K2OGCXnAC7eGqt4mzMrrW0CQd1QSGcuXw8CY7uJGkMPGPKe3/0mQWiis2OSpHxTM18=,iv:/QqSEFU9RuX9z5Z6aSaD7KlP/cgGTOYvTH+VJOnDTYM=,tag:yFbFXn+iWqQZx7wW4dKppg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/gotify-admin-secrets.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: ENC[AES256_GCM,data:wR4=,iv:cRBzbvu0eUYCYeKeysua1/P3Meli/rQyj/mIV6VWnPM=,tag:oyTPA/mLYNUX+QDkYLlZyQ==,type:str]
+kind: ENC[AES256_GCM,data:bdKQdadW,iv:F3DQiBI9xhSx87jkS1Hyevo48uUCBjsJSjbScIBznKA=,tag:PFakzzBj5S9F65dxu0L2rg==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:XHFYrKClPo+IwRbM,iv:ZGuL+cN840Y1bOTC62NhDDcoZpTzDWQ4GfqPJX/QWmI=,tag:hlCVjzvfcxXGOASc3vda/Q==,type:str]
+    namespace: ENC[AES256_GCM,data:0BbhgZBog+1qY/iRJA==,iv:88tiSpEDqIokT5VP/d6bB2+aUyh1kZ7NEHwZWoJW3XU=,tag:CBR01jh2U9h7kNEcK1e1sA==,type:str]
+type: ENC[AES256_GCM,data:Mpv1V73w,iv:BW3r6RpLnwe3XDKwrZySyOjrWUnSwIG5JOoPLzP/5gM=,tag:oyJySL0haOei1m4i+1AJ1g==,type:str]
+stringData:
+    username: ENC[AES256_GCM,data:8xmxLGE=,iv:J/vEvXGoD+ka6FDgnSwDz0fs9IxuJIZW9r7Oyu2qxC8=,tag:dN9jd6NSavMN8+uzvemYWg==,type:str]
+    password: ENC[AES256_GCM,data:vB9PaaUiQZ8FY8pO0cq1fHLi7Gq5t4U=,iv:C0Y1m2SGYP3oTIFoau5JavVmLblj6te/SPMLodIwiZw=,tag:3Ip4YvR4WPzR0bBpTyjytA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRT3JoTnhVVW5SUUJXbTdz
+            dUpLcmtBWWhyaGk3Y1hBM1ArcVI5eHREbmp3CiswUGVmd0NhejZwQ2UvNEdxS3ow
+            L1RweS9Kb2paeStLZ0tLSFdWbVZqQW8KLS0tIHJHT3hiMmxtM0Q1Ym5KOVpLVFpl
+            enR3NmdNdmdwVitTQVJlRHFWcjR0N2sKQw9ZZs+Ji/Zq/feO3qy4DwaCfWgDOQ/z
+            FVVhcCXweN58tb+9fzCJ+pNi/hSmvUkCMbb1+60qBvEehNzOoMRJ5g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:3HWV/NG0yTbsxY28u41zTRmAPc1kokGb4nCLmAsyq8/uvxcP+evDNyZXYpS93eVdPRfRZ1OqVBzyVGJEnj7JSXQVmlzor9JcWEL2fcpogoLVfJZKTRD6hk6optnBMjj84iQEEOx3A80JrDOdoXsH0pd9bJBABwoUOJwuufbXcIU=,iv:KLZsmAcapQgV08pFhGIvPt5ylsWg3xazAAjPuJYOaXA=,tag:1vKA3fISMIurHzxZ04F3lg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/gotify-smtp-secrets.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: ENC[AES256_GCM,data:FOg=,iv:15mfPeWXV5LQEYahaYvNf1z2bHbxk4j5i8DXT6mdG/0=,tag:1f9/jnnu+wGeeiXGNFgKzA==,type:str]
+kind: ENC[AES256_GCM,data:dOFfNQiM,iv:HVLosbRpcCgu4iiYKV8MDLiQ0uhj8DXBfVpRBW2NFNo=,tag:mCbRIzD1OHRhBLc+7xcVug==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:kHoKhGyiIQvCfng=,iv:dAhxjeLlXK3yueMKfrHxmh9YmNA7AYKFBquLikCNzcE=,tag:3Xoaw7YNBCU/GINlaQOpBw==,type:str]
+    namespace: ENC[AES256_GCM,data:fTJMZhbqSQWD3/cnWg==,iv:D7zWa91+Fgerfyrywf8VA5YNzGrThhLz7CvYXucfiq0=,tag:RcIegCl2UR1JsbfIQm928w==,type:str]
+type: ENC[AES256_GCM,data:Qh51bTsM,iv:2Htv0Xze7Hd1m6zkP6rtFXAlg8qm0AKylSjwJxRjpBk=,tag:NCu5iVUTNVfYgErZSvwHIg==,type:str]
+stringData:
+    host: ENC[AES256_GCM,data:aZXSvQ8B7h78q1kHmh0=,iv:uUyL9X3ehIHqztGAtXtAQWgduvbSsSwZBZlTFMdOlBo=,tag:OM/vdO19VKDSa4W5WQAKNQ==,type:str]
+    port: ENC[AES256_GCM,data:5ZBB,iv:1/XAfq+PJKdBsufx+EPvvB/cm9nbEwYigc8eACXjR8g=,tag:WlNyz7gTPh0KMe5oKVd0BA==,type:str]
+    from: ENC[AES256_GCM,data:2/akr8cXgmgQGnqJaFcLJMDcWw==,iv:0rT/6aqyXxn1jIJ5umYCDZR4S8Pbh4vUgaXNrxd3JF8=,tag:k0pQrPOrqWTyE1Xu9oSzVA==,type:str]
+    user: ENC[AES256_GCM,data:d9BLaFYOYm++HZMzlf1gVauKkUQ=,iv:Wd48T5UPVn6z3bS0t02X9GbrnWal3QoIQv2EgM9z9Wg=,tag:cmg2KnLY4Atwco+j2wVdeg==,type:str]
+    password: ENC[AES256_GCM,data:pmqEj9623A6bThKrkCCjuA==,iv:M/QwJ0s//UvuOjOljl67EDhvZt/9Wp8JicvCcows51A=,tag:bfRyIdj1cgIZxU3WPbKteg==,type:str]
+    notify-email: ENC[AES256_GCM,data:+Xo22g8U2wDKsnPnFq/+GKxgYOI=,iv:PE+C1etlp6046ragiv1iMDQbhfo5IULd/5akD0+Sc6A=,tag:cCZmXByA85fuZL9yozVLjw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpVVdxQTZFTHcwMXFFZEFk
+            ckNsYk4rdWtkeXFMVnlyVmdaRDBuRWsvZkQ4CitSSTBXOTZhWVpUZFFlR0JITVV4
+            Uy9lSlNHNjFNaFhHNTN0WnRaRlNNVGcKLS0tIHRlMnZpQVlScGRYYm5nT3Z5TWd6
+            ak1UZ1RobkpQZHBXR3MyenBDcU53Mk0KvH9O6bgwrjay0+1/A6TGX8GhITiDjWoO
+            RNX4fDtqNwhzmCfXbVjK30vlBjOFe+Bb7Z2n+hWMmHHDgFdS0xIAzA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-12T20:57:18Z"
+    mac: ENC[AES256_GCM,data:V/wgjnpUBvaV39BCTlecCwQy2/1h+0vixEXleJc/I9y+AOvuwVZNH7M86hdjoAdsKiIoNYySj4Flz+MJ9C8jQoAxBTDPbolP9UwDFWQ5KMJTKPPDoJGLNjy7bUq51WoyePUM6WWAYIiWHIB3OvAxHaSzECzL+ZoAhQy92cPKa1o=,iv:vLtIoD/C5yeXPEr+2Lim6XnWzAj42vr9qQgsxKpuhA8=,tag:PBq8qPHrx4RRgAKCWrxG3Q==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/gotify-tokens-secrets.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: ENC[AES256_GCM,data:YE0=,iv:s3yNqcBn7/DKUQNgbGGwVmlM1HzxYGLBKyB6Y2ii2WE=,tag:77KxAjOFU3G0fSfrgudKkg==,type:str]
+kind: ENC[AES256_GCM,data:PFW4VV9T,iv:n8gzMjE5C2TAfUTpp/8ex64B+hWUGG1K+2oQ4deN03Q=,tag:18Xqb6Di5izMHmzR5EU+QA==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:Xhg1fQrD4itrbvDW0g==,iv:/THWSXAThb9fj+mm3wcxqzSdzdt7nE06+xOpkCxyImE=,tag:UChokr06VkbDZcFZDcUY2A==,type:str]
+    namespace: ENC[AES256_GCM,data:r25U5MuuzDd8JJ2YjQ==,iv:uDUcS4ZTpZe8HmZMArzkdu7LV5GUoLSP2wIs5HB1gv0=,tag:fci7j30hoxnVKJwPFNfd1Q==,type:str]
+type: ENC[AES256_GCM,data:agEG8Fu8,iv:ckYX0buX8md1CWHXfxbAXQJLzoTxTJI16nlQ9LSzYi0=,tag:4tZWf3REbGOmmBiT14+dew==,type:str]
+stringData:
+    #ENC[AES256_GCM,data:ehjcsmz1R0i/n31f8M0IIUT4KDBwzz6f5ds=,iv:j5swFAKpC67ZvGioiCCC1D651LxUl9gME8GqK5Uc+ys=,tag:BUJ8k9LHs8NAPPZAwPuifA==,type:comment]
+    #ENC[AES256_GCM,data:ndAB00yun636VHDMonqTghO/jCWodNd/hmdbm1QmCvOAi4FvTLl8bY3wYR+ZtlkSQYvFNqH798MJixv5OZwxBj5H,iv:pS+7B60jaS5jhqDRw2LbBFv7mD1HO6+hjjv+iNpenXc=,tag:NU6+gxCHTGxqeDMfvkwJWw==,type:comment]
+    #ENC[AES256_GCM,data:T3mhSm/5q7JTSXNLrjhpP5GhqA7nXz8uAhJcEV1RDctAX0Dyfq8Qn4bNFO51ibwTETx4SnunPuFZVs++AvRnsA==,iv:oRGM8N4iEcwBrMzoEXQAeKqCKzUq+jXTMVq9W2l6oh4=,tag:GrrbC/tkWpA5kp5UYCyRBA==,type:comment]
+    app-token: ENC[AES256_GCM,data:jw+AqbsxyoYRvNrUDrq+GNq/TNfweFCs,iv:67vSSgMZCMV0GG37ZxUxHAWZqOOGMYCmo3J43WgLyNk=,tag:xlrnip4XomXBbMmooW9FKg==,type:str]
+    client-token: ENC[AES256_GCM,data:gsxeCqKh4e+DFjpn9jM5GfemAdBf8dSv,iv:l2bBMdxQUil8jU9XDjXGn3EtvFVrK5ibXDCeGaPbYTY=,tag:ELf5S6cdNQJ84Es5upu1IQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAza3YrRW1VTVNSOGlMK21a
+            bDhzQlEyZDNnZTZrSVNnTUEzU1hSdnM1ZFVzCjdBSUlmWG5EcnlzbSs5bXdDaGkx
+            ME1nMnRqQzF1Vkc3b3FXSUVHT1g0ZDgKLS0tIHpMUytEMjdLQ2E0Unp2di9KS3JQ
+            eHBraDk1clJyanhLY2dGM0tESmJIUFkKHTl3y9uQiEofOFD8j2vH3YK/CVzlq11w
+            GfShIji1yCvvowKGzYYhsQK0UM0FzhzBv0GFMYWQCBq8pGdoPVmO5g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:JhnO0V6cd2QVY/VhwHAJ5J75GeVlOVHBfVBTZstkj/IvpkAcwS/JE2b6jXiCwEC4n/vdA8DJ074noysUBRxh4Y7O4NKuvWcTniQKgqULNL1Hzgj3NdUkcQlWT+Z0HQ7FlvFH97VVXVfasU+CLHG48ShT2fDSj8JusEllb9CwSvg=,iv:PZo2krxvJc1TLJbvx+S9ClthoBe4w7WigUkq7dg0gNk=,tag:2IF5g/WTRP4XdnCZzDbiIA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/memory-agent-oidc.enc.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: memory-agent-oidc
+    namespace: poimen
+data:
+    CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
+    CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
+    ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
+    TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
+            Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
+            K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
+            cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
+            es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: oauth2-credentials
+    namespace: iam
+data:
+    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
+    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
+    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
+    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
+            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
+            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
+            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
+            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/paperless-ai-secrets.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: paperless-ai-config
+    namespace: paperless
+type: Opaque
+stringData:
+    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
+    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
+            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
+            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
+            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
+            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-09-14T14:19:39Z"
+    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/pi-sandbox-secrets.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: pi-code-agent-api-keys
+    namespace: poimen-sandbox
+stringData:
+    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
+    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
+            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
+            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
+            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
+            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T01:19:49Z"
+    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -12,6 +12,7 @@ files:
     - authentik-secrets.enc.yaml
     - cloudflare-secrets.enc.yaml
     - forgejo-registry-pat.enc.yaml
+    - forgejo-registry-pat-api.enc.yaml
     - forgejo-registry-pull.enc.yaml
     - forgejo-runner-token.enc.yaml
     - forgejo-secrets.enc.yaml
@@ -27,3 +28,12 @@ files:
     - vault-secrets.enc.yaml
     - vault-unseal-keys.enc.yaml
     - portfolio-secrets.enc.yaml
+    - gotify-admin-secrets.enc.yaml
+    - gotify-tokens-secrets.enc.yaml
+    - gotify-smtp-secrets.enc.yaml
+    - forgejo-smtp-secrets.enc.yaml
+    - paperless-ai-secrets.enc.yaml
+    - memory-agent-oidc.enc.yaml
+    - oauth2-credentials.enc.yaml
+    - forgejo-registry-token.enc.yaml
+    - pi-sandbox-secrets.enc.yaml

k8s/bootstrap/phase1-storage/longhorn-values.yaml

@@ -12,6 +12,7 @@ defaultSettings:
   replicaSoftAntiAffinity: false  # REQUIRED for true HA
   replicaAutoBalance: best-effort
   storageMinimalAvailablePercentage: 10
+  storageOverProvisioningPercentage: 200  # Actual usage is ~10% of scheduled; 200% unblocks all 3-replica scheduling on cp-1
   
   # Performance tuning
   defaultDataPath: /var/lib/longhorn

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -46,6 +46,12 @@ gitea:
       SSH_PORT: 2222
       SSH_LISTEN_PORT: 2222
 
+    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
+    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
+    # the admin only; admin create-user via UI/API is unaffected by this flag.
+    service:
+      DISABLE_REGISTRATION: true
+
     database:
       DB_TYPE: postgres
       HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -80,6 +86,14 @@ gitea:
     actions:
       ENABLED: true
 
+    mailer:
+      ENABLED: true
+      PROTOCOL: smtp+starttls
+      SMTP_ADDR: smtp.gmail.com
+      SMTP_PORT: 587
+      FROM: "Forgejo <[email protected]>"
+      # USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
+
 # Persistence (shared storage for repos)
 persistence:
   enabled: true
@@ -148,3 +162,18 @@ deployment:
         secretKeyRef:
           name: forgejo-db-app
           key: password
+    - name: GITEA__MAILER__USER
+      valueFrom:
+        secretKeyRef:
+          name: forgejo-smtp
+          key: user
+    - name: GITEA__MAILER__PASSWD
+      valueFrom:
+        secretKeyRef:
+          name: forgejo-smtp
+          key: password
+    # Allow webhooks to reach api-gw through the cluster ingress (resolves to
+    # internal ClusterIP 10.101.x.x which is blocked by default ALLOWED_HOST_LIST).
+    # api.riotpiao.com is our own gateway — safe to allowlist.
+    - name: GITEA__WEBHOOK__ALLOWED_HOST_LIST
+      value: "external,api.riotpiao.com"

k8s/infra/argocd-image-updater/values.yaml

@@ -1,72 +0,0 @@
-# ArgoCD Image Updater configuration
-# Watches Forgejo registry and updates ArgoCD Applications with new image tags
-
-config:
-  # Registry configuration - Forgejo allows anonymous pulls
-  registries:
-    - name: forgejo
-      api_url: https://forgejo.riotpiao.com
-      prefix: forgejo.riotpiao.com
-      default: true
-      insecure: false
-
-  # Log level
-  logLevel: debug
-
-  # ArgoCD API server
-  argocd:
-    grpcWeb: true
-    serverAddress: argocd-server.argocd.svc.cluster.local
-    insecure: true
-    plaintext: true
-
-  # Git write-back configuration (for multi-source Applications)
-  git:
-    # Commit author for image updates
-    user:
-      name: "ArgoCD Image Updater"
-      email: "[email protected]"
-    # Use SSH keys from ArgoCD's known hosts + credentials
-    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
-
-# Mount ArgoCD's git credentials for write-back
-extraVolumes:
-  - name: argocd-ssh-known-hosts-cm
-    configMap:
-      name: argocd-ssh-known-hosts-cm
-      defaultMode: 0644
-  - name: argocd-gpg-keys-cm
-    configMap:
-      name: argocd-gpg-keys-cm
-      optional: true
-      defaultMode: 0644
-  - name: argocd-gpg-pubring
-    configMap:
-      name: argocd-gpg-pubring-cm
-      optional: true
-      defaultMode: 0644
-
-extraVolumeMounts:
-  - name: argocd-ssh-known-hosts-cm
-    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
-    subPath: ssh_known_hosts
-  - name: argocd-gpg-keys-cm
-    mountPath: /etc/gpg/source
-  - name: argocd-gpg-pubring
-    mountPath: /etc/gpg/pubring
-
-# Extra environment variables
-extraEnv:
-  - name: ARGOCD_GRPC_WEB
-    value: "true"
-  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
-    value: "true"
-
-# Resources
-resources:
-  requests:
-    cpu: 50m
-    memory: 64Mi
-  limits:
-    cpu: 200m
-    memory: 128Mi

k8s/infra/cluster-maintenance/stale-job-cleanup.yaml

@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs daily at 04:00 UTC. Deletes:
-#   - Failed Jobs older than 24h (any namespace)
-#   - Completed Jobs older than 72h with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted state older than 1h
+# Runs every 2 minutes. Deletes:
+#   - Failed Jobs older than 2m (any namespace)
+#   - Completed standalone Jobs older than 2m with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
-# and linger forever.
+# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
+# and linger forever. Aggressive schedule keeps the cluster clean.
 
 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
   labels:
     app: stale-job-cleanup
 spec:
-  schedule: "0 4 * * *"
+  schedule: "*/2 * * * *"
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
@@ -74,7 +74,9 @@ spec:
                   set -e
                   NOW=$(date +%s)
 
-                  echo "=== Cleaning failed Jobs older than 24h ==="
+                  TTL=120  # 2 minutes in seconds
+
+                  echo "=== Cleaning failed Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.conditions[]?.type == "Failed") |
@@ -82,15 +84,15 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 24 ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
+                    AGE=$(( NOW - JOB_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
+                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.succeeded >= 1) |
@@ -98,20 +100,20 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 72 ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
+                    AGE=$(( NOW - JOB_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
-                  # Evicted pods show as Failed with reason Evicted
+                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
                   kubectl get pods --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(
                         .status.phase == "Failed" or
+                        .status.phase == "Succeeded" or
                         (.status.reason // "") == "Evicted" or
                         (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                       ) |
@@ -119,9 +121,9 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 1 ]; then
-                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
+                    AGE=$(( NOW - POD_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
                       kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                     fi
                   done

k8s/infra/databases/kustomization.yaml

@@ -8,6 +8,4 @@ resources:
   - namespaces.yaml
   - authentik-db.yaml
   - temporal-db.yaml
-  - memory-db.yaml
   - paperless-db.yaml
-  - obsidian-vault-pvc.yaml

k8s/infra/databases/memory-db.yaml

@@ -9,7 +9,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 2
+  instances: 3
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/infra/databases/obsidian-vault-pvc.yaml

@@ -1,18 +0,0 @@
----
-# Obsidian vault PVC — shared storage for REST API + UI pods
-# ReadWriteMany so both obsidian-server and obsidian-ui can mount simultaneously
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: obsidian-vault
-  namespace: poimen
-  labels:
-    app.kubernetes.io/name: obsidian-server
-    app.kubernetes.io/part-of: poimen-memory
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: longhorn
-  resources:
-    requests:
-      storage: 10Gi

k8s/infra/databases/paperless-db.yaml

@@ -11,7 +11,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 2
+  instances: 3
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/infra/forgejo-runner/templates/cilium-apiserver-egress.yaml

@@ -0,0 +1,25 @@
+# CiliumNetworkPolicy for kube-apiserver access.
+#
+# Standard K8s NetworkPolicy ipBlock rules don't work for the API server
+# under Cilium — the except clause on 192.168.1.0/24 blocks the post-DNAT
+# destination even when a separate rule re-allows a /32 or subnet.
+#
+# Cilium's native `kube-apiserver` entity resolves this correctly: it
+# tracks the API server endpoints regardless of ClusterIP vs node-IP
+# routing, so the policy stays valid across node changes and NAT paths.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: {{ .Release.Name }}-apiserver
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpointSelector:
+    matchLabels:
+      app: {{ .Release.Name }}
+  egress:
+  - toEntities:
+    - kube-apiserver
+    toPorts:
+    - ports:
+      - port: "6443"
+        protocol: TCP

k8s/infra/iam/authentik-values.yaml

@@ -153,9 +153,11 @@ server:
   # checks aren't treated as failures. (Only these fields are overridden; the
   # chart deep-merges the rest of each probe, incl. the httpGet path.)
   livenessProbe:
+    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
     timeoutSeconds: 15
     failureThreshold: 6
   readinessProbe:
+    initialDelaySeconds: 300  # skip probe until migrations complete
     timeoutSeconds: 15
     failureThreshold: 6
   startupProbe:
@@ -215,6 +217,13 @@ worker:
   podAnnotations:
     configmap.reloader.stakater.com/reload: "homelab-ca"
     homelab.io/restart-at: "2026-06-21T13-40"
+  livenessProbe:
+    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
+  readinessProbe:
+    initialDelaySeconds: 300  # skip probe until migrations complete
+  startupProbe:
+    initialDelaySeconds: 30  # let server finish DB work first
+    failureThreshold: 120
   metrics:
     enabled: true
     serviceMonitor:

k8s/infra/logging/grafana-values.yaml

@@ -73,7 +73,7 @@ grafana.ini:
   auth.generic_oauth:
     enabled: true
     name: Authentik
-    allow_sign_up: true
+    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
     client_id: grafana
     scopes: openid email profile groups
     auth_url: https://authentik.riotpiao.com/application/o/authorize/
@@ -84,13 +84,13 @@ grafana.ini:
     # Authentik doesn't serve — request 404s with "Error getting email address"
     # and the whole OAuth login fails.
     email_attribute_path: email
-    login_attribute_path: preferred_username
+    login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
     name_attribute_path: name
     role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
     allow_assign_grafana_admin: true
     use_pkce: false
     use_refresh_token: false
-    skip_org_role_sync: false
+    skip_org_role_sync: false  # Sync org roles from OAuth groups
     tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
 
 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -117,6 +117,22 @@ spec:
             echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
           fi
 
+          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
+          # clear eviction flag that was set when scheduling was disabled.
+          echo "Configuring worker-1 disk..."
+          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
+            "spec": {
+              "disks": {
+                "default-disk-1030400000000": {
+                  "allowScheduling": true,
+                  "evictionRequested": false,
+                  "storageReserved": 429496729600
+                }
+              }
+            }
+          }'
+          echo "  ✓ worker-1 disk configured (400Gi reserved)"
+
           echo
           echo "Waiting for disks to be ready..."
           sleep 10

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -35,8 +35,5 @@ parameters:
   mkfsParams: "-O ^64bit,^metadata_csum"
 mountOptions:
   - "noatime"
-  # Critical: mount with postgres UID/GID (26:26) to avoid permission issues
-  - "uid=26"
-  - "gid=26"
 reclaimPolicy: Delete
 volumeBindingMode: Immediate

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,33 +1,25 @@
-# Longhorn Node CRDs for cp-2 and cp-3.
-# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
-# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
+# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
 #
-# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
-# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
-# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
-# `default-disk` key here never matched the live one, so the Application sat
-# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
-# same /var/lib/longhorn path — which is worse than the drift it was fixing.
+# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
+# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
+# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
+# for general replica scheduling).
 #
-# What these objects are actually for is `allowScheduling: true` on tainted
-# control-plane nodes. That is all they need to declare.
+# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
+# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
+# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
-  name: talos-cp-2
-  namespace: longhorn-system
-spec:
-  name: talos-cp-2
-  allowScheduling: true
-  tags: []
+metadata: { name: talos-cp-2, namespace: longhorn-system }
+spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
-  name: talos-cp-3
-  namespace: longhorn-system
-spec:
-  name: talos-cp-3
-  allowScheduling: true
-  tags: []
+metadata: { name: talos-cp-3, namespace: longhorn-system }
+spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
+---
+apiVersion: longhorn.io/v1beta2
+kind: Node
+metadata: { name: worker-1, namespace: longhorn-system }
+spec: { name: worker-1, allowScheduling: true, tags: [] }

k8s/infra/minio/minio-provision-paperless-job.yaml

@@ -52,14 +52,14 @@ spec:
       restartPolicy: Never
       initContainers:
         - name: kubectl-copy
-          image: bitnami/kubectl:latest
+          image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
           command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
           volumeMounts:
             - name: shared
               mountPath: /shared
       containers:
         - name: provision
-          image: minio/mc:latest
+          image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
           volumeMounts:
             - name: shared
               mountPath: /shared
@@ -81,6 +81,9 @@ spec:
               mc alias set m http://minio-cluster-hl.storage.svc.cluster.local:9000 \
                 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
 
+              echo "Ensuring paperless bucket exists..."
+              mc mb --ignore-existing m/paperless
+
               echo "Checking for existing paperless-minio-creds secret..."
               if kubectl -n paperless get secret paperless-minio-creds >/dev/null 2>&1; then
                 ACCESS_KEY=$(kubectl -n paperless get secret paperless-minio-creds -o jsonpath='{.data.ACCESS_KEY}' | base64 -d)

k8s/infra/minio/minio-tenant.yaml

@@ -11,7 +11,15 @@ metadata:
     prometheus.io/port: "9000"
     prometheus.io/scrape: "true"
 spec:
-  image: minio/minio:RELEASE.2025-07-23T15-54-02Z
+  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
+  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
+  # backend could not resolve minio-cluster-hl. Official images now need registry auth
+  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
+  # release; swap it for the official image once MinIO registry credentials exist.
+  # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
+  # the pull again. The same digest already sits in the containerd cache on talos-cp-1,
+  # so the first start needs no registry access.
+  image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
 
   # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
   # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -112,10 +120,16 @@ spec:
           name: minio-oidc
           key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
     - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
-      value: "policy"
+      value: "groups"
+    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
+      value: ""
     - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
       value: "https://minio.riotpiao.com/oauth_callback"
     - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
       value: "Authentik"
     - name: MINIO_IDENTITY_OPENID_SCOPES
-      value: "openid,profile,email,minio"
+      value: "openid,profile,email,groups"
+    - name: MINIO_BROWSER_LOGIN_ANIMATION
+      value: "off"
+    - name: MINIO_BROWSER_REDIRECT_URL
+      value: "https://minio.riotpiao.com"

k8s/infra/monitoring/alerts/api-gateway-alerts.yaml

@@ -158,3 +158,66 @@ spec:
             severity: warning
           annotations:
             summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
+
+    - name: api-gateway.admission
+      # The admission gate fails OPEN when its view of the engine gauges goes
+      # stale, so a broken observer degrades silently to the pre-#64 behaviour
+      # of admitting everything. These two series are the only detection —
+      # the gate has no automated tests.
+      rules:
+        # The gate fails OPEN when it cannot read the engine, so "no data" must
+        # alert: otherwise a missing emitter, a renamed metric, or a broken
+        # scrape is indistinguishable from a healthy observer. Both expressions
+        # therefore OR in an absent() guard on the metric itself.
+        - alert: APIGatewayObserverDegraded
+          expr: observer_degraded == 1 or absent(observer_degraded)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
+            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
+
+        # Engine gauges went stale, or are not being reported at all
+        - alert: GatewayEngineStateStale
+          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
+            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
+
+        # The acceptance metric for reasoning.yaml's prefill change. Preemption
+        # of a 65k prompt costs a full re-prefill (~250s at the measured
+        # 262 tok/s), so any sustained rate is a regression, not noise.
+        - alert: LLMPreemptions
+          # Guarded absent() is deliberately NOT used bare here: the engine
+          # restarts (strategy Recreate) and the series legitimately vanishes
+          # for the length of a boot, so a bare absent() would false-fire on
+          # every rollout. It only fires when the metric is missing WHILE the
+          # predictor is up, which is the real "renamed or unscraped" case.
+          expr: |
+            increase(vllm:num_preemptions_total[15m]) > 0
+            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
+            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
+
+        # This change trades single-request latency for fairness: capping a long
+        # prompt at 2048 tokens/step means it takes more steps to finish, so a
+        # lone large prompt can now be slower even while preemptions stay 0.
+        # Without this alert the change has no way to show a regression.
+        - alert: LLMPrefillStarvation
+          expr: |
+            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
+            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
+            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."

k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

@@ -0,0 +1,76 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cag-kv-cache-rules
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
+    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
+    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
+    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
+    # prefix_cache_queries_total — so this rule produced no data and
+    # VLLMPrefixCacheHitRateLow could never fire.
+    - name: vllm.recording
+      rules:
+        - record: vllm:prefix_cache_hit_rate
+          expr: |
+            rate(vllm:prefix_cache_hits_total[5m])
+            /
+            clamp_min(
+              rate(vllm:prefix_cache_queries_total[5m]),
+              0.001
+            )
+          labels:
+            job: "reasoning-predictor"
+
+    # Alert rules: CAG pipeline health
+    - name: cag.alerts
+      rules:
+        # KV warmup consistently failing — cache never gets warm
+        - alert: KVWarmupFailureHigh
+          expr: |
+            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
+            description: |
+              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
+              for 5 consecutive minutes. The prefix KV cache is not being warmed;
+              agents will see full prefill latency on every call.
+              Check reasoning-predictor health and poimen-memory NetworkPolicy.
+
+        # Warmup misses: vLLM reachable but not caching
+        - alert: KVWarmupMissHigh
+          expr: |
+            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
+            description: |
+              vLLM is reachable but cached_tokens=0 on warmup responses.
+              Possible causes: prefix caching disabled in vLLM, max-num-seqs
+              saturated during warmup, or context format changed.
+
+        # Prefix cache hit rate low after warmup should be established
+        - alert: VLLMPrefixCacheHitRateLow
+          expr: |
+            vllm:prefix_cache_hit_rate < 0.70
+          for: 10m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
+            description: |
+              vLLM prefix cache hit rate below 70% for 10 minutes.
+              Expected >= 85% in steady state with CAG warming active.
+              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.

k8s/infra/monitoring/alerts/cluster-alerts.yaml

@@ -27,6 +27,24 @@ spec:
           annotations:
             summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
 
+        # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
+        - alert: PodImagePullFailure
+          expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
+
+        # Pod not ready
+        - alert: PodNotReady
+          expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
+
         # CrashLoopBackOff
         - alert: PodCrashLooping
           expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0

k8s/infra/monitoring/dashboards/generate-dashboards.py

@@ -26,20 +26,23 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
     return p
 
 
-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
     targets = []
     for e in exprs:
         if isinstance(e, tuple):
             targets.append({"expr": e[0], "legendFormat": e[1]})
         else:
             targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    return {
+    p = {
         "id": id, "title": title, "type": "timeseries",
         "gridPos": {"h": h, "w": w, "x": x, "y": y},
         "datasource": DS_PROM,
         "fieldConfig": {"defaults": {"unit": unit}},
         "targets": targets,
     }
+    if description:
+        p["description"] = description
+    return p
 
 
 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -265,6 +268,237 @@ def build_api_gateway():
     }
 
 
+# ============================================================================
+# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
+# ============================================================================
+#
+# Two independent metric sources, deliberately both shown:
+#
+#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
+#                               that expose /metrics. Ollama (ornith:35b,
+#                               qwen2.5:3b-instruct) exposes nothing at all.
+#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
+#                               ones, derived from the SSE byte stream: TTFT is
+#                               request-start to first byte, ITL is the gap
+#                               between SSE writes carrying "data: ", so it
+#                               counts SSE chunks rather than true tokens.
+#                               vLLM's own histograms remain authoritative for
+#                               `reasoning`.
+#
+# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
+# verified against the running gateway and Prometheus:
+#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
+#
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
+# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
+# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
+# homelab-frontend#63 fixed exactly that — the wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
+# The "dead code" note that used to live in this header is superseded.
+#
+# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
+# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
+#
+#
+# All metric names below were verified against a live scrape, not assumed:
+#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
+#   NOT cache_query_hit_total / cache_query_miss_total.
+
+def build_llm_observability():
+    def hq(metric, q, by="le"):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
+            f"p{int(q * 100)}",
+        )
+
+    def hq_gw(metric, q):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
+            f"p{int(q * 100)} · {{{{model}}}}",
+        )
+
+    def avg_of(metric):
+        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
+        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
+
+    panels = []
+
+    # ── TTFT ────────────────────────────────────────────────────────────────
+    panels.append(row(100, "Latency — Time To First Token", 0, [
+        stat_panel(101, "TTFT p95 (gateway)",
+                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
+                   0, 1, w=4, h=4, unit="s"),
+        stat_panel(102, "TTFT p95 (vLLM)",
+                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
+                   4, 1, w=4, h=4, unit="s"),
+        stat_panel(103, "KV cache in use",
+                   "max(vllm:kv_cache_usage_perc)",
+                   8, 1, w=4, h=4, unit="percentunit"),
+        stat_panel(104, "Requests running",
+                   "sum(vllm:num_requests_running)",
+                   12, 1, w=4, h=4, unit="short"),
+        stat_panel(105, "Requests waiting",
+                   "sum(vllm:num_requests_waiting)",
+                   16, 1, w=4, h=4, unit="short"),
+        stat_panel(106, "Preemptions (5m)",
+                   "increase(vllm:num_preemptions_total[5m])",
+                   20, 1, w=4, h=4, unit="short"),
+        ts_panel(110, "TTFT percentile by model (gateway)",
+                 [hq_gw("llm_ttft_seconds", 0.50),
+                  hq_gw("llm_ttft_seconds", 0.95),
+                  hq_gw("llm_ttft_seconds", 0.99)],
+                 0, 2, w=12, h=8, unit="s"),
+        ts_panel(111, "TTFT percentile — vLLM reasoning",
+                 [hq("vllm:time_to_first_token_seconds", 0.50),
+                  hq("vllm:time_to_first_token_seconds", 0.95),
+                  hq("vllm:time_to_first_token_seconds", 0.99)],
+                 12, 2, w=12, h=8, unit="s"),
+        ts_panel(112, "Prefill vs decode (mean)",
+                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
+                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
+                 0, 3, w=12, h=8, unit="s"),
+        ts_panel(113, "Queue wait before scheduling (mean)",
+                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
+                 12, 3, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Inter-token latency ─────────────────────────────────────────────────
+    panels.append(row(120, "Latency — Inter-Token", 4, [
+        ts_panel(121, "Inter-token latency by model (gateway)",
+                 [hq_gw("llm_itl_seconds", 0.50),
+                  hq_gw("llm_itl_seconds", 0.95),
+                  hq_gw("llm_itl_seconds", 0.99)],
+                 0, 5, w=12, h=8, unit="s"),
+        ts_panel(122, "Inter-token latency — vLLM reasoning",
+                 [hq("vllm:inter_token_latency_seconds", 0.50),
+                  hq("vllm:inter_token_latency_seconds", 0.95),
+                  hq("vllm:inter_token_latency_seconds", 0.99)],
+                 12, 5, w=12, h=8, unit="s"),
+        ts_panel(123, "Decode throughput (tokens/s per request)",
+                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
+                 0, 6, w=12, h=8, unit="short"),
+        ts_panel(124, "End-to-end request latency p50/p95/p99",
+                 [hq("vllm:e2e_request_latency_seconds", 0.50),
+                  hq("vllm:e2e_request_latency_seconds", 0.95),
+                  hq("vllm:e2e_request_latency_seconds", 0.99)],
+                 12, 6, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Batching / scheduling ───────────────────────────────────────────────
+    panels.append(row(130, "Batching & Scheduling", 7, [
+        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
+                 [("sum(vllm:num_requests_running)", "running (in batch)"),
+                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
+                 0, 8, w=12, h=8, unit="short"),
+        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
+                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
+                 12, 8, w=12, h=8, unit="short"),
+        ts_panel(133, "TEI batch size (embeddings / reranker)",
+                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
+                   "{{app}}")],
+                 0, 9, w=12, h=8, unit="short"),
+        ts_panel(134, "TEI queue depth",
+                 [("max(te_queue_size) by (app)", "{{app}}")],
+                 12, 9, w=12, h=8, unit="short"),
+    ]))
+
+    # ── KV cache ────────────────────────────────────────────────────────────
+    panels.append(row(140, "KV Cache", 10, [
+        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
+                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
+                 0, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
+                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
+                 12, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(143, "Prefix cache hits vs queries",
+                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
+                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
+                 0, 12, w=12, h=8, unit="short"),
+        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
+                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
+                 12, 12, w=12, h=8, unit="short"),
+    ]))
+
+    # ── Throughput & health ─────────────────────────────────────────────────
+    panels.append(row(150, "Throughput & Health", 13, [
+        ts_panel(151, "Token throughput",
+                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
+                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
+                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
+                 0, 14, w=12, h=8, unit="short"),
+        ts_panel(152, "Successful requests/s",
+                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
+                  ("sum(rate(te_request_count[5m]))", "TEI")],
+                 12, 14, w=12, h=8, unit="short"),
+        ts_panel(153, "Backpressure — queue-full rejects",
+                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
+                 0, 15, w=8, h=8, unit="short"),
+        ts_panel(154, "KV warmup requests delayed",
+                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
+                 8, 15, w=8, h=8, unit="short"),
+        ts_panel(155, "Embedding / rerank latency p95",
+                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
+                   "{{app}} embed"),
+                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
+                   "{{app}} queue")],
+                 16, 15, w=8, h=8, unit="s"),
+    ]))
+
+    # ── Admission control ───────────────────────────────────────────────────
+    # These four panels read the gateway's admission gate, which is the
+    # companion change in homelab-frontend#64. That emitter is NOT deployed
+    # yet (verified against the live /metrics endpoint: 0 series matching
+    # ^(admission_|observer_|engine_state_)), so they render "No data" until
+    # #64 ships. That is expected, and each panel carries a description
+    # saying so — an unexplained blank panel would read as a broken
+    # dashboard, which is exactly the failure mode this row exists to catch.
+    panels.append(row(160, "Admission Control", 16, [
+        ts_panel(161, "Admission parked (5m)",
+                 [("increase(admission_parked_total[5m])", "parked")],
+                 0, 17, w=8, h=8, unit="short",
+                 description="Requests the gateway held back rather than admitting. "
+                             "Emitted by the admission gate (homelab-frontend#64); "
+                             "empty until that change is deployed."),
+        ts_panel(162, "Admission rejected by reason",
+                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
+                 8, 17, w=8, h=8, unit="short",
+                 description="Admission rejections split by the gate's own reason label. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(163, "Engine state stale",
+                 [("increase(engine_state_stale_total[5m])", "stale")],
+                 16, 17, w=8, h=8, unit="short",
+                 description="Times the gateway's view of the engine gauges went stale. "
+                             "The gate fails OPEN when this happens, so any non-zero rate "
+                             "means admission is silently degraded to admit-everything. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(164, "Gateway view vs engine truth",
+                 [("max(vllm_requests_running)", "gateway: running"),
+                  ("sum(vllm:num_requests_running)", "engine: running"),
+                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
+                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
+                 0, 18, w=24, h=8, unit="short",
+                 description="The gateway's own reading of the engine against the engine's "
+                             "self-reported value. A disagreement between a pair is how a "
+                             "stale observer or a mistuned admission target shows itself, "
+                             "and nothing else in this dashboard reveals it. The two "
+                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
+                             "ship with homelab-frontend#64 and are absent until it is "
+                             "deployed — until then this panel shows the engine's pair only."),
+    ]))
+
+    return {
+        "title": "LLM Observability",
+        "uid": "llm-observability",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
+        "panels": panels,
+    }
+
+
 # ============================================================================
 # Generate
 # ============================================================================
@@ -277,4 +511,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
 
+# Dashboard 4
+write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
+
 print("Done.")

k8s/infra/monitoring/dashboards/llm-observability.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+data:
+  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
+    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
+    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
+    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
+    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
+    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
+    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
+    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
+    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
+    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
+    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
+    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
+    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
+    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
+    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
+    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
+    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
+    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
+    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
+    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
+    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
+    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
+    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
+    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
+    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
+    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
+    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
+    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
+    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
+    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
+    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
+    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
+    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
+    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
+    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
+    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
+    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
+    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
+    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
+    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
+    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
+    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
+    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
+    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
+    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
+    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
+    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
+    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
+    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
+    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
+    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
+    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
+    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
+    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
+    the engine''s self-reported value. A disagreement between a pair is how a stale
+    observer or a mistuned admission target shows itself, and nothing else in this
+    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
+    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
+    then this panel shows the engine''s pair only."}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: LLM
+  labels:
+    grafana_dashboard: '1'
+  name: llm-observability-dashboard
+  namespace: logging

k8s/infra/monitoring/kustomization.yaml

@@ -14,10 +14,14 @@ resources:
   - alerts/svc-vault-rules.yaml
   - alerts/cluster-alerts.yaml
   - alerts/api-gateway-alerts.yaml
+  - alerts/cag-kv-cache-rules.yaml
   - servicemonitors/argocd.yaml
   - servicemonitors/authentik.yaml
   - servicemonitors/forgejo.yaml
   - servicemonitors/minio.yaml
   - servicemonitors/ingress-nginx.yaml
+  - servicemonitors/vllm.yaml
+  - servicemonitors/api-gateway.yaml
   - dashboards/cluster-infrastructure.yaml
   - dashboards/api-gateway.yaml
+  - dashboards/llm-observability.yaml

k8s/infra/monitoring/prometheus-values.yaml

@@ -54,8 +54,8 @@ alertmanager:
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:
   prometheusSpec:
-    retention: 15d
-    retentionSize: "18GB"
+    retention: 30d
+    retentionSize: "95GB"
 
     # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
     # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -64,10 +64,9 @@ prometheus:
     scrapeTimeout: 60s
     evaluationInterval: 30s
 
-    # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't
-    # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io).
+    # Pin to worker-1 for more storage capacity
     nodeSelector:
-      topology.kubernetes.io/zone: az-a
+      kubernetes.io/hostname: worker-1
 
     # Persistent storage — metrics survive node reboots and pod restarts.
     # Uses the default `longhorn` StorageClass; nodeSelector above already pins
@@ -79,7 +78,7 @@ prometheus:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: 20Gi
+              storage: 25Gi
 
     resources:
       requests:

k8s/infra/monitoring/servicemonitors/api-gateway.yaml

@@ -0,0 +1,62 @@
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
+# three latency series below recorded nothing. internal/proxy/llm_metrics.go
+# declared a capture wrapper (LLMMetricsCapture) that was never constructed
+# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
+# stayed empty.
+#
+# homelab-frontend#63 fixed exactly that. The wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
+# request path. Verified live on 2026-09-29:
+#   llm_ttft_seconds  count=7  sum=598.212s
+#   llm_itl_seconds   count=16116
+#   llm_tokens_total  3 series
+#
+# These derived-from-SSE series cover EVERY model including the two Ollama ones
+# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
+# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
+# between SSE writes, so both are granular to SSE write boundaries rather than
+# to the engine's own token cadence. llm_tokens_total counts `data:` events,
+# and the documented assumption in internal/proxy/llm_metrics.go is that these
+# upstreams emit one such event per token — so it is a token count only as far
+# as that assumption holds. vLLM's own histograms remain authoritative.
+#
+# Also recorded and working (3 series, one per replica):
+#   queue_full_requests_total           counter    backpressure rejects
+#   warmup_requests_delayed_total       counter    vLLM slot contention
+#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
+#
+# NOT working, do not build on it: request_duration_seconds is declared and
+# exported but has 0 samples, and its only writer Metrics.RecordRequest has
+# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
+# the same dead-code shape this branch already corrected for the latency
+# series, so it is listed here rather than left to be rediscovered.
+#
+# Also note the Deployment already carried prometheus.io/scrape annotations,
+# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
+# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
+# collected before this ServiceMonitor existed.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: api-gateway
+    app.kubernetes.io/part-of: api
+spec:
+  namespaceSelector:
+    matchNames:
+      - api
+  selector:
+    matchLabels:
+      app: api-gateway
+  endpoints:
+    # api-gateway Service port IS named `http` (unlike the KServe ones), so
+    # the name form works here.
+    - port: http
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -0,0 +1,49 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vllm
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: vllm
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  namespaceSelector:
+    matchNames:
+      - llm-serving
+  selector:
+    # Only the three engines that actually serve /metrics.
+    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
+    # endpoint (/metrics returns 404) — selecting them would add permanently
+    # DOWN targets. Ollama-model visibility comes from the api-gateway
+    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
+    # measured at the proxy, so they cover every model including Ollama).
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
+          - llm-embeddings  # TEI 1.8.2 — CPU
+          - llm-reranker    # TEI 1.8.2 — CPU
+  endpoints:
+    # targetPort, not port: the KServe-generated Services on llm-serving have
+    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
+    # editing the InferenceServices (predictor restart = full weights reload,
+    # tens of seconds on the vLLM slot), so scrape the pod port directly.
+    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
+    - targetPort: 8080
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+      relabelings:
+        # Prometheus's Kubernetes service discovery already supplies the
+        # namespace, pod and service labels; the only one that needs deriving
+        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
+        # not promote arbitrary pod labels.
+        #
+        # `action: replace` is written explicitly. It is the API server's
+        # default for relabeling rules, so omitting it leaves the live object
+        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+          targetLabel: app

k8s/infra/oidc-bootstrap/bootstrap.yaml

@@ -0,0 +1,59 @@
+# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
+# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
+# tokens were never accepted -> every login got 401). Delete this whole directory after.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: oidc-bootstrap
+    namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: oidc-bootstrap
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-bootstrap-restart-apiserver-v2
+  namespace: kube-system
+spec:
+  backoffLimit: 4
+  ttlSecondsAfterFinished: 7200
+  template:
+    spec:
+      serviceAccountName: oidc-bootstrap
+      restartPolicy: Never
+      containers:
+        - name: restart
+          image: docker.io/curlimages/curl:8.22.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -u
+              TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+              CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
+                curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN"                   -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0"                   -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
+              done
+              echo "deletes requested; kubelet recreates each static pod in ~20-40s"

k8s/infra/oidc-bootstrap/restart-apiserver.yaml

@@ -0,0 +1,141 @@
+# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
+#
+# Two earlier mechanisms failed and are documented so nobody repeats them:
+#   1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
+#   2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
+#      so every pod SIGTERMed itself (exit 143) before reaching the real process.
+# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
+# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
+# three control planes by 2 minutes each, so only one API server is down at a time.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp1-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 10s before restarting the API server on talos-cp-1"
+              sleep 10
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp2-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 120s before restarting the API server on talos-cp-2"
+              sleep 120
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp3-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 240s before restarting the API server on talos-cp-3"
+              sleep 240
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+

k8s/infra/poimen/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Poimen infrastructure — CNPG Postgres for memory service.
+# Part of the `poimen` ArgoCD Application (multi-source, source 3).
+# Sync-wave -1 ensures the DB cluster is healthy before application
+# deployments (wave 0) are applied.
+resources:
+  - memory-db.yaml
+  - poimen-sandbox-namespace.yaml
+  - poimen-sandbox-quota.yaml
+  - pi-session-netpol.yaml
+  - pi-agent-workspace-pv.yaml

k8s/infra/poimen/memory-db.yaml

@@ -0,0 +1,37 @@
+# Dedicated CNPG Postgres for Poimen Memory (GitOps, wave 2).
+# Uses default longhorn storage class (3 replicas, dataLocality disabled).
+# CNPG generates secret `memory-db-app` + service `memory-db-rw` in ns poimen.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: memory-db
+  namespace: poimen
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: memory
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+      postInitApplicationSQL:
+        - "CREATE EXTENSION vector;"
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 20Gi
+    storageClass: longhorn
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/infra/poimen/pi-agent-workspace-pv.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  finalizers:
+  - kubernetes.io/pv-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace-pv
+  annotations:
+    argocd.argoproj.io/sync-wave: '-1'
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: pi-agent-workspace
+    namespace: poimen-sandbox
+  hostPath:
+    path: /var/lib/pi-agent-sandbox
+    type: DirectoryOrCreate
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - talos-cp-2
+  persistentVolumeReclaimPolicy: Retain
+  volumeMode: Filesystem
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    pv.kubernetes.io/bind-completed: 'yes'
+    argocd.argoproj.io/sync-wave: '-1'
+  finalizers:
+  - kubernetes.io/pvc-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace
+  namespace: poimen-sandbox
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: ''
+  volumeMode: Filesystem
+  volumeName: pi-agent-workspace-pv

k8s/infra/poimen/pi-session-netpol.yaml

@@ -0,0 +1,12 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pi-session-deny-kube-api
+  namespace: poimen-sandbox
+spec:
+  endpointSelector:
+    matchLabels:
+      app: pi-session
+  egressDeny:
+    - toEntities:
+        - kube-apiserver

k8s/infra/poimen/poimen-sandbox-namespace.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: poimen-sandbox
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

k8s/infra/poimen/poimen-sandbox-quota.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: sandbox-quota
+  namespace: poimen-sandbox
+spec:
+  hard:
+    pods: "20"
+    requests.cpu: "10"
+    requests.memory: 20Gi
+    limits.cpu: "20"
+    limits.memory: 40Gi

k8s/infra/rbac/kustomization.yaml

@@ -16,6 +16,7 @@ resources:
   - temporal-operator-role.yaml
   - llm-serving-operator-role.yaml
   - poimen-memory-operator-role.yaml
+  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until

k8s/infra/rbac/sandbox-observer-role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sandbox-observer
+  namespace: poimen-sandbox
+rules:
+  - apiGroups: [""]
+    resources: [pods, pods/log, services]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: poimen-devs-sandbox
+  namespace: poimen-sandbox
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: "oidc:poimen-devs"
+roleRef:
+  kind: Role
+  name: sandbox-observer
+  apiGroup: rbac.authorization.k8s.io