fix(argocd-image-updater): remove duplicate ARGOCD_GRPC_WEB env var

Fixes sync error: 'may not be specified when value is not empty' The Helm chart already defines ARGOCD_GRPC_WEB, so remove from extraEnv.
fix(cnpg): scale paperless-db, immich-db, gotify-db to 3 instances (#30 )
2026-09-11 11:06:05 +09:00 · 2026-09-10 22:56:41 +00:00 · 2026-09-10 22:32:24 +09:00 · 2026-09-10 10:49:30 +09:00 · 2026-09-10 01:41:37 +00:00 · 2026-09-10 01:35:59 +00:00
85 changed files with 2305 additions and 2599 deletions
@@ -1,269 +0,0 @@
-name: Cluster CI Pipeline
-
-on:
-  push:
-    branches:
-      - main
-      - develop
-    paths:
-      - 'k8s/**'
-      - '.forgejo/workflows/cluster-ci.yaml'
-  pull_request:
-    paths:
-      - 'k8s/**'
-
-jobs:
-  ci:
-    runs-on: docker
-    steps:
-      # === Checkout ===
-      - name: Checkout
-        run: |
-          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
-          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
-          git clone --depth 1 "$CLONE_URL" .
-          git fetch origin main
-          git checkout main
-
-      # === Install Tools ===
-      - name: Install Tools
-        run: |
-          unset GITHUB_TOKEN
-          apt-get update && apt-get install -y \
-            yamllint \
-            python3-pip \
-            curl \
-            jq
-
-          # kubeval
-          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
-          mv -f kubeval /usr/local/bin/
-
-          # kustomize
-          rm -f kustomize
-          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-          mv -f kustomize /usr/local/bin/
-
-          # argocd
-          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
-          chmod +x /usr/local/bin/argocd
-
-          # trivy
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
-
-          # polaris
-          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
-          chmod +x /usr/local/bin/polaris
-
-      # === YAML Lint ===
-      - name: YAML Lint
-        run: |
-          echo "=== Linting YAML files ==="
-          yamllint k8s/ -c .yamllint.yaml || true
-
-      # === Kubeval - Validate K8s Syntax ===
-      - name: Kubeval - Validate K8s Syntax
-        run: |
-          echo "=== Validating Kubernetes manifests ==="
-          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
-            echo "Validating $file..."
-            kubeval "$file" -d 2>/dev/null || true
-          done
-
-      # === Kustomize Build - All overlays ===
-      - name: Kustomize Build - Infrastructure
-        run: |
-          echo "=== Building k8s/infrastructure/ ==="
-          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
-          echo "✓ Infrastructure built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
-
-      - name: Kustomize Build - Bootstrap
-        run: |
-          echo "=== Building k8s/bootstrap/ ==="
-          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
-          echo "✓ Bootstrap built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
-
-      - name: Kustomize Build - Platform
-        run: |
-          echo "=== Building k8s/platform/ ==="
-          kustomize build k8s/platform > /tmp/platform.yaml
-          echo "✓ Platform built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
-
-      - name: Kustomize Build - Security
-        run: |
-          echo "=== Building k8s/security/ ==="
-          kustomize build k8s/security > /tmp/security.yaml
-          echo "✓ Security built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
-
-      - name: Kustomize Build - Applications
-        run: |
-          echo "=== Building k8s/applications/ ==="
-          kustomize build k8s/applications > /tmp/applications.yaml
-          echo "✓ Applications built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
-
-      - name: Kustomize Build - Data
-        run: |
-          echo "=== Building k8s/data/ ==="
-          kustomize build k8s/data > /tmp/data.yaml
-          echo "✓ Data built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
-
-      - name: Validate ArgoCD Applications
-        run: |
-          echo "=== Validating ArgoCD Applications ==="
-          kubeval k8s/argocd/apps/*.yaml
-
-      # === Trivy - Scan Dockerfile ===
-      - name: Trivy - Scan Dockerfile
-        run: |
-          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
-            echo "=== Scanning Dockerfiles with Trivy ==="
-            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
-          else
-            echo "No Dockerfiles found"
-          fi
-
-      # === Trivy - Scan Helm Charts ===
-      - name: Trivy - Scan Helm Charts
-        run: |
-          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
-            echo "=== Scanning Helm charts with Trivy ==="
-            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
-              echo "Scanning $chart..."
-              trivy config "$chart" || true
-            done
-          else
-            echo "No Helm charts found"
-          fi
-
-      # === Polaris - K8s Security Audit ===
-      - name: Polaris - K8s Security Audit
-        run: |
-          echo "=== Running Polaris K8s security audit ==="
-          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
-
-          if [ -f /tmp/polaris-audit.json ]; then
-            echo "Security issues found:"
-            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
-          fi
-
-      # === Check for Secrets in Code ===
-      - name: Check for Secrets in Code
-        run: |
-          echo "=== Scanning for hardcoded secrets ==="
-          # BLOCKING. This step used to only count findings and then exit 0, so a
-          # plaintext deploy key rode through it into a public remote. Two failure
-          # modes fixed: it now fails the build, and it matches key material by
-          # PEM header rather than only `private_key:`-style YAML field names.
-          # Findings are captured into variables and tested for emptiness rather than
-          # branching on grep's exit status: implementations disagree on the rc of a
-          # `-v` filter fed empty input, and a wrong rc here fails open.
-          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
-          # and silently scans nothing.
-          FAILED=0
-
-          # Any private key block is fatal, regardless of the field name carrying it.
-          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
-                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
-                 | grep -v "\.enc\.yaml" || true)
-          if [ -n "$KEYS" ]; then
-            echo "❌ Unencrypted private key material found:"
-            echo "$KEYS"
-            FAILED=1
-          fi
-
-          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
-          # so encrypted files never trip this.
-          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
-                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
-                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
-          if [ -n "$VALS" ]; then
-            echo "❌ Plaintext secret value found:"
-            echo "$VALS"
-            FAILED=1
-          fi
-
-          if [ "$FAILED" -ne 0 ]; then
-            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
-            exit 1
-          fi
-          echo "✓ No hardcoded secrets found"
-
-      # === Check K8s Security Best Practices ===
-      - name: Check K8s Security Best Practices
-        run: |
-          echo "=== Checking K8s security best practices ==="
-
-          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found privileged containers"
-          fi
-
-          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found hostNetwork usage"
-          fi
-
-          echo "Checking for missing resource limits..."
-          MISSING=0
-          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
-            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
-              if ! grep -q "resources:" "$file"; then
-                echo "⚠️  $file: Missing resource requests/limits"
-                MISSING=$((MISSING + 1))
-              fi
-            fi
-          done
-
-      # === ArgoCD Sync (main branch only) ===
-      - name: Sync ArgoCD
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Syncing homelab-root ==="
-          argocd app sync homelab-root --force
-          argocd app wait homelab-root --timeout 5m
-
-      - name: Check Sync Status
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== ArgoCD Applications Status ==="
-          argocd app list -o table
-
-          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
-          if [ "$STATUS" != "Synced" ]; then
-            echo "❌ Root app sync failed: $STATUS"
-            exit 1
-          fi
-          echo "✓ Root app synced successfully"
-
-      - name: Health Check
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Checking Application Health ==="
-          argocd app get homelab-root -o wide
-
-      # === Summary ===
-      - name: Summary
-        if: always()
-        run: |
-          echo "=== CI Pipeline Summary ==="
-          echo "✓ YAML linted"
-          echo "✓ Manifests validated"
-          echo "✓ Kustomizations built"
-          echo "✓ Security scans completed"
-          echo "✓ Secrets check passed"
-          echo "✓ Best practices verified"
-          echo ""
-          echo "✓ All checks passed"
@@ -66,3 +66,6 @@ bootstrap-argocd.log
 # one line here, which is how a plaintext deploy key reached a public remote.
 k8s/**/*-secret.yaml
 !k8s/**/*.enc.yaml
+
+# IAM provisioning scripts contain credential references — never commit
+scripts/iam/*.py
@@ -208,3 +208,95 @@ versions without warning in your own values file.
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
+
+## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
+
+All repositories MUST follow this exact structure. No variations.
+
+```yaml
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  REGISTRY: <your-registry-hostname>
+  IMAGE: <registry>/<org>/<service-name>
+
+jobs:
+  test:
+    name: Test
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js for actions runtime
+        run: apt-get update && apt-get install -y nodejs
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Language-specific tests here (no docker, no registry)
+      # - name: Run tests
+      #   run: npm test -- --run || true
+
+  build-push:
+    name: Build & Push Image
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js and Docker
+        run: |
+          apt-get update
+          apt-get install -y nodejs docker.io
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: sha
+        run: |
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Registry login
+        run: |
+          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+            --username "${REGISTRY_USER}" --password-stdin
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build Docker image
+        run: |
+          docker build --no-cache \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+
+      - name: Push Docker image
+        run: |
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+
+      - name: Prune unused images
+        run: docker image prune -a --force 2>&1 | tail -3 || true
+```
+
+### Anti-Patterns (DO NOT USE)
+
+- ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing
+- ❌ Conditional `if:` on individual steps — use separate jobs instead
+- ❌ Installing docker.io in test job — only needed in build-push
+- ❌ Monolithic job doing test + build + push — hard to debug
+- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
+
+### How It Works
+
+1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
+2. **Push to main** → test runs, build-push runs after test passes, image pushed
+3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
+4. `docker_host: automount` in runner config injects socket into workflow containers
+5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git
@@ -0,0 +1,80 @@
+# ComfyUI — GPU-accelerated image generation on worker-1.
+# Uses 1x V100 32GB (sm70). Freed by scaling ornith 2→1.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: comfyui
+  template:
+    metadata:
+      labels:
+        app: comfyui
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: worker-1
+      runtimeClassName: nvidia
+      containers:
+        - name: comfyui
+          image: ghcr.io/ai-dock/comfyui:v2-cuda-12.1.1-base-22.04
+          ports:
+            - containerPort: 8188
+              protocol: TCP
+          env:
+            - name: NVIDIA_VISIBLE_DEVICES
+              value: "all"
+            - name: COMFYUI_FLAGS
+              value: "--listen 0.0.0.0 --port 8188"
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+              nvidia.com/gpu: "1"
+            limits:
+              cpu: "8"
+              memory: 16Gi
+              nvidia.com/gpu: "1"
+          volumeMounts:
+            - mountPath: /workspace/ComfyUI/models
+              name: models
+            - mountPath: /workspace/ComfyUI/output
+              name: output
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 8188
+            periodSeconds: 10
+            initialDelaySeconds: 30
+          startupProbe:
+            httpGet:
+              path: /
+              port: 8188
+            failureThreshold: 60
+            periodSeconds: 10
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: comfyui-models
+        - name: output
+          emptyDir: {}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: comfyui-models
+  namespace: comfyui
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 50Gi
@@ -0,0 +1,25 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui
+  namespace: comfyui
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    # WebSocket support for ComfyUI's live preview
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/proxy-set-headers: "Upgrade"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfy.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui
+                port:
+                  number: 80
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui
+  namespace: comfyui
+  labels:
+    app: comfyui
+spec:
+  selector:
+    app: comfyui
+  ports:
+    - port: 80
+      targetPort: 8188
+      protocol: TCP
@@ -0,0 +1,107 @@
+# Gotify — Push Notifications + Email Relay
+
+Self-hosted notification server with SMTP email forwarding sidecar.
+
+## Architecture
+
+```
+Forgejo webhook ──POST──→ Gotify API (:80/message)
+                              │
+                    ┌─────────┼─────────┐
+                    ▼                     ▼
+              Push notification      SMTP emailer sidecar
+              (mobile/desktop)       (polls → sends email)
+```
+
+## Setup (one-time, after first deploy)
+
+### 1. Encrypt secrets before committing
+
+```bash
+# Edit secrets.yaml with real values first, then:
+sops -e -i k8s/apps/gotify/secrets.yaml
+```
+
+### 2. Create Gotify app + client tokens
+
+1. Login to `https://gotify.riotpiao.com` with admin creds
+2. **Applications** → Create `forgejo` → copy **app token**
+3. **Clients** → Create `smtp-emailer` → copy **client token**
+4. Update `gotify-tokens` secret:
+   ```bash
+   kubectl -n notifications create secret generic gotify-tokens \
+     --from-literal=app-token=<APP_TOKEN> \
+     --from-literal=client-token=<CLIENT_TOKEN> \
+     --dry-run=client -o yaml | kubectl apply -f -
+   ```
+
+### 3. Configure Forgejo webhook
+
+In each Forgejo repo → **Settings** → **Webhooks** → **Add Webhook** → **Gotify**:
+
+| Field | Value |
+|-------|-------|
+| Target URL | `http://gotify.notifications.svc.cluster.local/message` |
+| Token | The **app token** from step 2 |
+| Events | Pull Request (Created, Merged, Closed) |
+
+Or via API:
+```bash
+FORGEJO_TOKEN="<your-pat>"
+APP_TOKEN="<gotify-app-token>"
+
+curl -s -X POST "https://forgejo.riotpiao.com/api/v1/repos/rock/homelab/hooks" \
+  -H "Authorization: token $FORGEJO_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "type": "gotify",
+    "active": true,
+    "config": {
+      "content_type": "json",
+      "url": "http://gotify.notifications.svc.cluster.local/message?token='"$APP_TOKEN"'"
+    },
+    "events": ["pull_request", "pull_request_assign", "pull_request_review"],
+    "authorization_header": ""
+  }'
+```
+
+### 4. Add CoreDNS rewrite (if accessing via public hostname)
+
+Only needed if Cloudflare Tunnel is used for gotify.riotpiao.com:
+
+```
+# terraform/files/coredns/Corefile — add rewrite:
+rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+```
+
+Then: `cd terraform && terraform apply && cd .. && make apply-cp`
+
+### 5. SMTP providers
+
+| Provider | Host | Port | Notes |
+|----------|------|------|-------|
+| Gmail | smtp.gmail.com | 587 | Use App Password (2FA required) |
+| Resend | smtp.resend.com | 587 | Free 100 emails/day |
+| Sendgrid | smtp.sendgrid.net | 587 | Free 100 emails/day |
+| Mailgun | smtp.mailgun.org | 587 | Free 5000/month |
+
+## Notification priority levels
+
+| Priority | Meaning | Email forwarded? |
+|----------|---------|-----------------|
+| 0-4 | Low (info) | No (below MIN_PRIORITY=5) |
+| 5-7 | Normal (PR created) | Yes |
+| 8-10 | High (PR merged, failures) | Yes |
+
+## Verify
+
+```bash
+# Test push notification
+APP_TOKEN="<app-token>"
+curl -X POST "https://gotify.riotpiao.com/message?token=$APP_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{"title":"Test","message":"Hello from homelab","priority":5}'
+
+# Check email sidecar logs
+kubectl -n notifications logs deployment/gotify -c smtp-emailer --tail=20
+```
@@ -0,0 +1,35 @@
+# CNPG Postgres for Gotify. Lightweight — 2 instances, 2Gi storage.
+# CNPG generates secret `gotify-db-app` + service `gotify-db-rw` in ns notifications.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: gotify-db
+  namespace: notifications
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: gotify
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "256Mi", cpu: "100m" }
+    limits: { memory: "512Mi", cpu: "500m" }
+  storage:
+    size: 2Gi
+    storageClass: longhorn-cnpg
+  monitoring:
+    enablePodMonitor: true
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
@@ -0,0 +1,204 @@
+# Gotify — self-hosted push notification server + SMTP email relay.
+# Forgejo webhooks → Gotify → push notifications + email forwarding.
+# Runs on control plane (no GPU needed), lightweight.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: gotify
+  template:
+    metadata:
+      labels:
+        app: gotify
+    spec:
+      containers:
+        # --- Gotify server ---
+        - name: gotify
+          image: ghcr.io/gotify/server:2.6.1
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              export GOTIFY_DATABASE_DIALECT=postgres
+              export GOTIFY_DATABASE_CONNECTION="host=gotify-db-rw.notifications port=5432 user=${DB_USER} password=${DB_PASS} dbname=gotify sslmode=disable"
+              exec /app/gotify-app
+          ports:
+            - containerPort: 80
+              protocol: TCP
+          env:
+            - name: GOTIFY_DEFAULTUSER_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: username
+            - name: GOTIFY_DEFAULTUSER_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-admin
+                  key: password
+            - name: DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: username
+            - name: DB_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-db-app
+                  key: password
+            - name: GOTIFY_SERVER_PORT
+              value: "80"
+            - name: GOTIFY_SERVER_KEEPALIVEPERIODSECONDS
+              value: "0"
+            - name: TZ
+              value: Asia/Tokyo
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              cpu: 200m
+              memory: 128Mi
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 30
+            initialDelaySeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 80
+            periodSeconds: 10
+            initialDelaySeconds: 5
+
+        # --- SMTP emailer sidecar ---
+        # Watches Gotify WebSocket stream, forwards messages as email.
+        # https://github.com/eternal-flame-AD/gotify-broadcast
+        - name: smtp-emailer
+          image: ghcr.io/gotify/server:2.6.1
+          command:
+            - /bin/sh
+            - -c
+            - |
+              # Wait for Gotify to be ready
+              until wget -qO- http://localhost:80/health >/dev/null 2>&1; do
+                echo "Waiting for Gotify..."
+                sleep 2
+              done
+              echo "Gotify is ready, starting email relay..."
+
+              # Poll Gotify messages and forward via SMTP using msmtp
+              # Install msmtp for lightweight SMTP sending
+              apk add --no-cache msmtp curl jq
+
+              # Configure msmtp
+              cat > /tmp/msmtprc <<MSMTP
+              defaults
+              auth           on
+              tls            on
+              tls_trust_file /etc/ssl/certs/ca-certificates.crt
+              logfile        /tmp/msmtp.log
+
+              account        default
+              host           ${SMTP_HOST}
+              port           ${SMTP_PORT}
+              from           ${SMTP_FROM}
+              user           ${SMTP_USER}
+              password       ${SMTP_PASS}
+              MSMTP
+
+              chmod 600 /tmp/msmtprc
+
+              # Track last seen message ID
+              LAST_ID=0
+
+              while true; do
+                # Fetch messages since last ID
+                MESSAGES=$(curl -s -H "X-Gotify-Key: ${GOTIFY_CLIENT_TOKEN}" \
+                  "http://localhost:80/message?since=${LAST_ID}&limit=10" 2>/dev/null)
+
+                if [ -n "$MESSAGES" ]; then
+                  echo "$MESSAGES" | jq -r '.messages[]? | @base64' | while read -r MSG; do
+                    DECODED=$(echo "$MSG" | base64 -d)
+                    ID=$(echo "$DECODED" | jq -r '.id')
+                    TITLE=$(echo "$DECODED" | jq -r '.title // "Notification"')
+                    BODY=$(echo "$DECODED" | jq -r '.message // ""')
+                    PRIORITY=$(echo "$DECODED" | jq -r '.priority // 5')
+                    APP=$(echo "$DECODED" | jq -r '.appid // 0')
+                    DATE=$(echo "$DECODED" | jq -r '.date // ""')
+
+                    # Only forward messages with priority >= configured threshold
+                    if [ "$PRIORITY" -ge "${MIN_PRIORITY:-0}" ]; then
+                      printf "Subject: [Gotify] %s\nFrom: %s\nTo: %s\nContent-Type: text/plain; charset=UTF-8\n\n%s\n\n---\nPriority: %s\nDate: %s" \
+                        "$TITLE" "$SMTP_FROM" "$NOTIFY_EMAIL" "$BODY" "$PRIORITY" "$DATE" | \
+                        msmtp -C /tmp/msmtprc "$NOTIFY_EMAIL" && \
+                        echo "Email sent for message $ID: $TITLE" || \
+                        echo "Failed to send email for message $ID"
+                    fi
+
+                    # Update last seen ID
+                    if [ "$ID" -gt "$LAST_ID" ]; then
+                      LAST_ID=$ID
+                    fi
+                  done
+                fi
+
+                sleep ${POLL_INTERVAL:-30}
+              done
+          env:
+            - name: GOTIFY_CLIENT_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-tokens
+                  key: client-token
+            - name: SMTP_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: host
+            - name: SMTP_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: port
+            - name: SMTP_FROM
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: from
+            - name: SMTP_USER
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: user
+            - name: SMTP_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: password
+            - name: NOTIFY_EMAIL
+              valueFrom:
+                secretKeyRef:
+                  name: gotify-smtp
+                  key: notify-email
+            - name: MIN_PRIORITY
+              value: "5"
+            - name: POLL_INTERVAL
+              value: "15"
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 64Mi
+      # No volumes — Postgres handles persistence
@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gotify
+  namespace: notifications
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+    # WebSocket support for Gotify client connections
+    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: gotify.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gotify
+                port:
+                  number: 80
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - db.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: notifications
+  labels:
+    kubernetes.io/metadata.name: notifications
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: gotify
+  namespace: notifications
+  labels:
+    app: gotify
+spec:
+  selector:
+    app: gotify
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP
@@ -18,7 +18,7 @@ metadata:
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 2
+  instances: 3
  imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie
  postgresql:
    extensions:
@@ -14,5 +14,7 @@ resources:
  - ornith.yaml
  - reasoning.yaml
  - reranker.yaml
+  - qwen-cpu.yaml
+  - networkpolicy.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.
@@ -0,0 +1,62 @@
+# NetworkPolicy for LLM inference engines (llm-serving namespace).
+#
+# These pods have NO auth — vLLM, Ollama, and TEI accept any request.
+# All access MUST go through the api-gateway, which validates JWTs and
+# injects identity headers (X-Forwarded-User, X-Auth-Verified).
+#
+# Replaces the hand-applied llm-serving-default-deny policy that used
+# `llm-client: "true"` pod label as a selector — any pod in any namespace
+# could self-grant access by adding that label, which defeats the purpose.
+#
+# This policy restricts ingress to:
+#   1. api namespace (gateway) — the sole entry point for inference
+#   2. monitoring namespace — Prometheus scraping vLLM/TEI /metrics
+#   3. intra-namespace — pod-to-pod (future: multi-replica comms)
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: llm-serving-ingress
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: llm-serving
+  policyTypes:
+    - Ingress
+  ingress:
+    # Allow from api-gateway (namespace: api)
+    # Gateway proxies /v1/chat/completions, /v1/embeddings, /v1/rerank
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: api
+      ports:
+        - protocol: TCP
+          port: 8080   # vLLM, Ollama HTTP
+        - protocol: TCP
+          port: 80      # KServe predictor services
+        - protocol: TCP
+          port: 8000    # vLLM direct (some configs)
+        - protocol: TCP
+          port: 11434   # Ollama native port
+    # Allow Prometheus scraping from monitoring namespace
+    # vLLM: :8080/metrics, TEI: :9000/metrics
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 9000
+    # Allow intra-namespace (pod-to-pod within llm-serving)
+    - from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/part-of: llm-serving
+      ports:
+        - protocol: TCP
+          port: 8080
@@ -33,12 +33,8 @@ spec:

        ollama pull ornith:35b

-        ollama pull qwen2.5:3b-instruct
-
        ollama run ornith:35b "ok" >/dev/null 2>&1 || true

-        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
-
        wait $SERVE_PID

        '
@@ -54,7 +50,7 @@ spec:
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      - name: OLLAMA_MAX_LOADED_MODELS
-        value: '2'
+        value: '1'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
@@ -65,8 +61,7 @@ spec:
          command:
          - /bin/sh
          - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
-            grep -q qwen2.5
+          - ollama ps 2>/dev/null | grep -q ornith
        periodSeconds: 10
      resources:
        limits:
@@ -82,8 +77,7 @@ spec:
          command:
          - /bin/sh
          - -c
-          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
-            grep -q qwen2.5
+          - ollama ps 2>/dev/null | grep -q ornith
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
@@ -91,13 +85,10 @@ spec:
        name: models
    deploymentStrategy:
      type: Recreate
-    # 2 replicas -- each its own GPU, each loading both ornith:35b and
-    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
-    # get an independent instance instead of contending on one, at the
-    # cost of judge/qwen traffic still sharing whichever replica an
-    # implementer call also lands on.
-    maxReplicas: 2
-    minReplicas: 2
+    # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
+    # Frees 1 GPU for ComfyUI.
+    maxReplicas: 1
+    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
@@ -0,0 +1,115 @@
+# qwen2.5:3b-instruct on CPU (talos-cp-2, 144GB RAM, 24 cores).
+# Moved off GPU to free a V100 for ComfyUI. Latency ~10x slower
+# than GPU but sufficient for lightweight tasks (summarization,
+# classification, quick answers).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: qwen-cpu
+  namespace: llm-serving
+  labels:
+    app: qwen-cpu
+    app.kubernetes.io/name: qwen-cpu
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: qwen-cpu
+  template:
+    metadata:
+      labels:
+        app: qwen-cpu
+        app.kubernetes.io/name: qwen-cpu
+        app.kubernetes.io/part-of: llm-serving
+    spec:
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: ollama
+          image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              ollama serve &
+              SERVE_PID=$!
+              until ollama list >/dev/null 2>&1; do sleep 2; done
+              ollama pull qwen2.5:3b-instruct
+              ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+              wait $SERVE_PID
+          env:
+            - name: OLLAMA_HOST
+              value: "0.0.0.0:8080"
+            - name: OLLAMA_MODELS
+              value: /root/.ollama/models
+            - name: OLLAMA_CONTEXT_LENGTH
+              value: "32768"
+            - name: OLLAMA_KEEP_ALIVE
+              value: "-1"
+            - name: OLLAMA_MAX_LOADED_MODELS
+              value: "1"
+            - name: OLLAMA_NUM_PARALLEL
+              value: "2"
+          ports:
+            - containerPort: 8080
+              protocol: TCP
+          readinessProbe:
+            exec:
+              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
+            periodSeconds: 10
+          startupProbe:
+            exec:
+              command: ["/bin/sh", "-c", "ollama ps 2>/dev/null | grep -q qwen2.5"]
+            failureThreshold: 60
+            periodSeconds: 10
+          resources:
+            requests:
+              cpu: "4"
+              memory: 4Gi
+            limits:
+              cpu: "8"
+              memory: 8Gi
+          volumeMounts:
+            - mountPath: /root/.ollama
+              name: ollama-data
+      volumes:
+        - name: ollama-data
+          persistentVolumeClaim:
+            claimName: qwen-cpu-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: qwen-cpu
+  namespace: llm-serving
+  labels:
+    app: qwen-cpu
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  selector:
+    app: qwen-cpu
+  ports:
+    - port: 80
+      targetPort: 8080
+      protocol: TCP
+---
+# Small PVC for qwen2.5:3b model weights (~1.9GB).
+# Separate from llm-models PVC which is pinned to worker-1.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: qwen-cpu-data
+  namespace: llm-serving
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 5Gi
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: memory-queues
+description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
+type: application
+version: 0.1.0
+appVersion: "1.0"
@@ -0,0 +1,20 @@
+{{- range .Values.queues }}
+---
+apiVersion: kmsvc.io/v1alpha1
+kind: Queue
+metadata:
+  name: {{ .name }}
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: memory-service
+    queue: dlq
+spec:
+  name: {{ .name }}
+  description: {{ .description }}
+  partitions: {{ .partitions }}
+  replicationFactor: {{ .replicationFactor }}
+  config:
+    retention.ms: "{{ .config.retention.ms }}"
+    message.retention.seconds: "{{ .config.message.retention.seconds }}"
+    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
+{{- end }}
@@ -0,0 +1,25 @@
+# Poimen Memory Service Kafka Queues (kmsvc)
+# Phase 6.6: DLQ topics for webhook + metrics failures
+
+queues:
+  # DLQ for extraction, webhook, and agent failures
+  - name: poimen-memory-dlq
+    description: "DLQ for extraction, webhook, and agent failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+  # DLQ for metrics persistence failures  
+  - name: poimen-memory-metric-dlq
+    description: "DLQ for metrics persistence failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+namespace: sqs
@@ -0,0 +1,144 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: secretrotations.homelab.riotpiao.com
+spec:
+  group: homelab.riotpiao.com
+  names:
+    kind: SecretRotation
+    plural: secretrotations
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          metadata:
+            type: object
+          spec:
+            type: object
+            required:
+            - provider
+            - rotationInterval
+            properties:
+              # External system: authentik | forgejo | minio | vault
+              provider:
+                type: string
+                enum: [authentik, forgejo, minio, vault]
+              
+              # How often to rotate (hours)
+              rotationInterval:
+                type: integer
+                minimum: 24
+              
+              # Application ID in external system
+              appId:
+                type: string
+              
+              # k8s Secret to update (name, namespace, key)
+              secretRef:
+                type: object
+                required: [name, namespace]
+                properties:
+                  name:
+                    type: string
+                  namespace:
+                    type: string
+                  key:
+                    type: string
+                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
+              
+              # Path to git file that holds the secret (for .enc.yaml files)
+              gitPath:
+                type: string
+                description: "Path in homelab repo to .enc.yaml file"
+              
+              # Ansible template values to substitute
+              templateValues:
+                type: object
+                additionalProperties:
+                  type: string
+          
+          status:
+            type: object
+            properties:
+              lastRotationTime:
+                type: string
+                format: date-time
+              nextRotationTime:
+                type: string
+                format: date-time
+              lastRotationStatus:
+                type: string
+                enum: [Success, Failed, Pending]
+              lastRotationError:
+                type: string
+              lastCommitHash:
+                type: string
+
+---
+# Example usage:
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160  # 90 days in hours
+  appId: minio
+  secretRef:
+    name: minio-oidc
+    namespace: storage
+    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: portfolio-agent-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160
+  appId: portfolio-agent
+  secretRef:
+    name: portfolio-agent-oidc
+    namespace: portfolio
+    key: CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: forgejo-registry-token
+  namespace: secret-rotation
+spec:
+  provider: forgejo
+  rotationInterval: 2160
+  appId: rock/riotpiao.com
+  secretRef:
+    name: forgejo-registry-secret
+    namespace: kube-system
+    key: REGISTRY_TOKEN
+  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-root-credentials
+  namespace: secret-rotation
+spec:
+  provider: minio
+  rotationInterval: 4320  # 180 days in hours
+  appId: root
+  secretRef:
+    name: minio-creds
+    namespace: storage
+  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml
@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: secret-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: secret-rotation-controller
+    spec:
+      serviceAccountName: secret-rotation-controller
+      containers:
+      - name: controller
+        image: secret-rotation-controller:latest
+        imagePullPolicy: IfNotPresent
+        env:
+        # SOPS reads age key from this file
+        - name: SOPS_AGE_KEY_FILE
+          value: /etc/sops/age/private-key.txt
+        
+        # Vault auth (token in projected volume)
+        - name: VAULT_ADDR
+          value: http://vault.vault.svc.cluster.local:8200
+        - name: VAULT_TOKEN_FILE
+          value: /var/run/secrets/vault/token
+        
+        # Authentik
+        - name: AUTHENTIK_URL
+          value: http://authentik-server.iam.svc.cluster.local
+        - name: AUTHENTIK_BOOTSTRAP_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: authentik-bootstrap
+              key: token
+        
+        # Git
+        - name: GIT_REPO
+          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+        - name: GIT_AUTHOR_EMAIL
+          value: [email protected]
+        - name: GIT_AUTHOR_NAME
+          value: Secret Rotation Controller
+        - name: FORGEJO_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: forgejo-registry-secret
+              key: REGISTRY_TOKEN
+        
+        volumeMounts:
+        # Age key from ExternalSecret (synced from Vault)
+        - name: age-key
+          mountPath: /etc/sops/age
+          readOnly: true
+        
+        # Vault auth token (projected)
+        - name: vault-token
+          mountPath: /var/run/secrets/vault
+          readOnly: true
+        
+        # Temp working dir
+        - name: tmp
+          mountPath: /tmp
+        
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+      
+      volumes:
+      - name: age-key
+        secret:
+          secretName: sops-age-key
+          defaultMode: 0400
+      
+      - name: vault-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: vault
+              expirationSeconds: 3600
+      
+      - name: tmp
+        emptyDir: {}
@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: secret-rotation
+
+resources:
+  - rbac.yaml
+  - crd.yaml
+  - external-secret.yaml
+  - deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/name: secret-rotation-controller
+  app.kubernetes.io/component: automation
+  managed-by: argocd
@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-rotation-controller
+rules:
+# Read SecretRotation CRDs
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations"]
+  verbs: ["get", "list", "watch"]
+
+# Update status
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations/status"]
+  verbs: ["get", "patch", "update"]
+
+# Read k8s secrets that will be rotated
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+
+# For recording events
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secret-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secret-rotation-controller
+subjects:
+- kind: ServiceAccount
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: secret-rotation
+  labels:
+    kubernetes.io/metadata.name: secret-rotation
@@ -18,7 +18,7 @@ spec:
      prune: true
      selfHeal: true
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/argocd/projects
  destination:
@@ -17,7 +17,7 @@ spec:
    syncOptions:
      - CreateNamespace=true
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
    # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the
@@ -21,7 +21,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -93,7 +93,7 @@ spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
    # deterministically. The previous directory.include with bare filenames
@@ -127,7 +127,7 @@ spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/bootstrap/ingress
  destination:
@@ -149,7 +149,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/cluster-maintenance
  destination:
@@ -177,7 +177,7 @@ spec:
      valueFiles:
        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
  sources:
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -200,7 +200,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/bootstrap/kyverno
  destination:
@@ -19,7 +19,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/argocd-image-updater/values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: secret-rotation
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: secret-rotation
+spec:
+  project: homelab
+  
+  sources:
+  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    path: k8s/apps/secret-rotation-controller
+    targetRevision: main
+  
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: secret-rotation
+  
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - RespectIgnoreDifferences=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
@@ -17,7 +17,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/minio/minio-operator-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -41,7 +41,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/minio
  destination:
@@ -66,7 +66,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/longhorn
  destination:
@@ -102,7 +102,7 @@ spec:
        skipCrds: true
        valueFiles:
          - $values/k8s/infra/monitoring/prometheus-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -152,7 +152,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/monitoring/crds
  destination:
@@ -183,7 +183,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/monitoring
  destination:
@@ -213,7 +213,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -237,7 +237,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/tracing
  destination:
@@ -19,7 +19,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/logging/loki-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -53,7 +53,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/logging/grafana-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -87,7 +87,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/logging/promtail-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -17,7 +17,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/iam/vault-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -46,7 +46,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/infra/iam/authentik-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -68,7 +68,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/iam
  destination:
@@ -109,7 +109,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -152,10 +152,17 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
  destination:
@@ -173,10 +180,17 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
    helm:
@@ -197,10 +211,17 @@ metadata:
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
    helm:
@@ -14,7 +14,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/databases
  destination:
@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: memory-queues
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/apps/messaging/memory-queues
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sqs
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
@@ -68,7 +68,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/messaging/kafka-cluster
  destination:
@@ -38,17 +38,17 @@ metadata:
    argocd.argoproj.io/sync-wave: "7"
    # ArgoCD Image Updater - auto-update on new image push
    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
-    argocd-image-updater.argoproj.io/gw.update-strategy: newest-build
-    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
  revisionHistoryLimit: 3
  sources:
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab-frontend.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
      targetRevision: main
      path: k8s
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      path: k8s/apps/api
  destination:
@@ -20,7 +20,7 @@ spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/llm-serving
  destination:
@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: comfyui
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: comfyui
+    app.kubernetes.io/component: image-generation
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/apps/comfyui
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: comfyui
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: gotify
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: gotify
+    app.kubernetes.io/component: notifications
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/apps/gotify
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: notifications
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
@@ -19,10 +19,10 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/apps/temporal/temporal-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      path: k8s/apps/temporal
  destination:
@@ -51,7 +51,7 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/apps/portainer/portainer-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
@@ -74,7 +74,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/cloudflared
  destination:
@@ -97,7 +97,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/agent-pod
  destination:
@@ -130,7 +130,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/sms
  destination:
@@ -157,7 +157,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/paperless
  destination:
@@ -189,7 +189,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/apps/immich
  destination:
@@ -220,10 +220,10 @@ spec:
      helm:
        valueFiles:
          - $values/k8s/apps/homarr/homarr-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
-    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
  destination:
@@ -248,8 +248,8 @@ metadata:
    argocd.argoproj.io/sync-wave: "8"
    # ArgoCD Image Updater - auto-update on new image push
    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
-    argocd-image-updater.argoproj.io/app.update-strategy: newest-build
-    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^[0-9a-f]{7}$
+    argocd-image-updater.argoproj.io/app.update-strategy: digest
+    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
@@ -281,12 +281,13 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/rbac
  destination:
    server: https://kubernetes.default.svc
-    namespace: default
+    # No namespace: cluster-scoped resources (ClusterRoleBinding, etc.)
+    # Namespace is set per-resource in kustomization
  syncPolicy:
    automated:
      prune: true
@@ -12,7 +12,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/kmsvc-manage.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
    targetRevision: main
    path: k8s/argocd/apps
    directory:
@@ -1,27 +1,34 @@
-# Poimen project collection — manages poimen-memory, poimen-workflows, and poiman
-# Each repo tracks its own main branch (no prod branch). Poiman is the primary
-# orchestrator with k8s/argocd/ containing the AppProject and deployment structure.
-#
-# CI: All three repos trigger on main branch pushes (no image builds yet).
-# Future: Add build workflows for poiman once container runtime needs are clear.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: poimen-root
+  name: poimen
  namespace: argocd
-  labels:
-    app.kubernetes.io/name: poimen
-    app.kubernetes.io/component: orchestrator
  annotations:
    argocd.argoproj.io/sync-wave: "7"
+    # Image Updater: auto-update on new image push (SHA tag filter)
+    argocd-image-updater.argoproj.io/image-list: |
+      memory=forgejo.riotpiao.com/rock/poimen-memory
+      workflows=forgejo.riotpiao.com/rock/poimen-workflows
+      frontend=forgejo.riotpiao.com/rock/poimen-frontend
+    argocd-image-updater.argoproj.io/memory.update-strategy: digest
+    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
+    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
+    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
+    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/rock/poimen.git
-    targetRevision: main
-    path: k8s/argocd
-    directory:
-      recurse: false
+  sources:
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+      targetRevision: main
+      path: k8s/argocd
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
+      targetRevision: main
+      path: k8s/argocd
  destination:
    server: https://kubernetes.default.svc
    namespace: poimen
@@ -17,12 +17,13 @@ spec:
    - https://github.com/Riotpiaole/Poimen-workflows.git
    - https://github.com/Riotpiaole/poimen*.git
    # In-cluster Forgejo repos — explicit allowlist (no wildcard)
-    - https://forgejo.riotpiao.com/rock/homelab.git
-    - https://forgejo.riotpiao.com/rock/homelab-frontend.git
-    - https://forgejo.riotpiao.com/rock/kmsvc-manage.git
-    - https://forgejo.riotpiao.com/rock/poimen.git
-    - https://forgejo.riotpiao.com/rock/poimen-memory.git
-    - https://forgejo.riotpiao.com/rock/poimen-workflows.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/kmsvc-manage.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-memory.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows.git
+    - https://forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend.git
    - https://forgejo.riotpiao.com/rock/riotpiao.com.git
    # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
    - https://cloudnative-pg.github.io/charts
@@ -12,7 +12,7 @@ metadata:
 spec:
  project: homelab
  source:
-    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/argocd/apps
    directory:
@@ -0,0 +1,24 @@
+apiVersion: ENC[AES256_GCM,data:wR4=,iv:cRBzbvu0eUYCYeKeysua1/P3Meli/rQyj/mIV6VWnPM=,tag:oyTPA/mLYNUX+QDkYLlZyQ==,type:str]
+kind: ENC[AES256_GCM,data:bdKQdadW,iv:F3DQiBI9xhSx87jkS1Hyevo48uUCBjsJSjbScIBznKA=,tag:PFakzzBj5S9F65dxu0L2rg==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:XHFYrKClPo+IwRbM,iv:ZGuL+cN840Y1bOTC62NhDDcoZpTzDWQ4GfqPJX/QWmI=,tag:hlCVjzvfcxXGOASc3vda/Q==,type:str]
+    namespace: ENC[AES256_GCM,data:0BbhgZBog+1qY/iRJA==,iv:88tiSpEDqIokT5VP/d6bB2+aUyh1kZ7NEHwZWoJW3XU=,tag:CBR01jh2U9h7kNEcK1e1sA==,type:str]
+type: ENC[AES256_GCM,data:Mpv1V73w,iv:BW3r6RpLnwe3XDKwrZySyOjrWUnSwIG5JOoPLzP/5gM=,tag:oyJySL0haOei1m4i+1AJ1g==,type:str]
+stringData:
+    username: ENC[AES256_GCM,data:8xmxLGE=,iv:J/vEvXGoD+ka6FDgnSwDz0fs9IxuJIZW9r7Oyu2qxC8=,tag:dN9jd6NSavMN8+uzvemYWg==,type:str]
+    password: ENC[AES256_GCM,data:vB9PaaUiQZ8FY8pO0cq1fHLi7Gq5t4U=,iv:C0Y1m2SGYP3oTIFoau5JavVmLblj6te/SPMLodIwiZw=,tag:3Ip4YvR4WPzR0bBpTyjytA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBRT3JoTnhVVW5SUUJXbTdz
+            dUpLcmtBWWhyaGk3Y1hBM1ArcVI5eHREbmp3CiswUGVmd0NhejZwQ2UvNEdxS3ow
+            L1RweS9Kb2paeStLZ0tLSFdWbVZqQW8KLS0tIHJHT3hiMmxtM0Q1Ym5KOVpLVFpl
+            enR3NmdNdmdwVitTQVJlRHFWcjR0N2sKQw9ZZs+Ji/Zq/feO3qy4DwaCfWgDOQ/z
+            FVVhcCXweN58tb+9fzCJ+pNi/hSmvUkCMbb1+60qBvEehNzOoMRJ5g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:3HWV/NG0yTbsxY28u41zTRmAPc1kokGb4nCLmAsyq8/uvxcP+evDNyZXYpS93eVdPRfRZ1OqVBzyVGJEnj7JSXQVmlzor9JcWEL2fcpogoLVfJZKTRD6hk6optnBMjj84iQEEOx3A80JrDOdoXsH0pd9bJBABwoUOJwuufbXcIU=,iv:KLZsmAcapQgV08pFhGIvPt5ylsWg3xazAAjPuJYOaXA=,tag:1vKA3fISMIurHzxZ04F3lg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
@@ -0,0 +1,30 @@
+apiVersion: ENC[AES256_GCM,data:xUA=,iv:YH1WYnoXhxbeES3Dmo8ZrMDwFcwCykt6+Zn4YSgN4gc=,tag:t3bIWo/LetlQv4tJ0nJ2Sw==,type:str]
+kind: ENC[AES256_GCM,data:PJVzDTcT,iv:aRxvzLUuCzSDryOjA/v67csL6E2FcCpoK8aD7KNStno=,tag:cQgbowSTTV99nQ41yFOlsA==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:atkSiP0rjyuWFjs=,iv:l/48X9W9lrkAyw6Q/p39V775L6xbr0tOIFI4vjYrg+E=,tag:LZrQxN5h6T5tehFG8L9KFw==,type:str]
+    namespace: ENC[AES256_GCM,data:fbklNyooBeqsogKbeg==,iv:z2wLgl81q8xfzGGVQ9A4Pkst/zLmuZpyBzHrXzmVfm4=,tag:QKJik4IWc4nrOBINjGCxvQ==,type:str]
+type: ENC[AES256_GCM,data:bfAFBd/7,iv:9/W3X1KefqDAsurIatT6kBuEIQvWVRTq4G9A+9ZfYKs=,tag:Optlb+gZ2udbqIOerqV2zQ==,type:str]
+stringData:
+    #ENC[AES256_GCM,data:RZaw/FNQZOf0F3+XfFmGW+4gjKXNdjrAbMRjBt4hptwk,iv:IEd/+Bsv/h08nKak+bcovNiohC7nNjEQ+QdtIFMUGng=,tag:zsF5ZQ3+5JijF4MU+RhnyA==,type:comment]
+    #ENC[AES256_GCM,data:ux3ONHnuI0uFEihElbLI1rKlrN91Pn0iLLZfmez0+t5d8yhH6OD3L/3oi6yHbdgD8bV90kRK/lw=,iv:7Ed+W/TeOY9Af8bKO0Fu3XOuaK8b0Hw70rwobfmosxM=,tag:Z0RbIOjdDroK0drC6P+Y8A==,type:comment]
+    host: ENC[AES256_GCM,data:YKgas/8hT5ZOGIW7O0c=,iv:WtBz1MFrwd6EgDDS5fMqtcVGIZGNLgbXnmkqlNWoXy8=,tag:+H9ARGPi5XocgZYp4oA3wA==,type:str]
+    port: ENC[AES256_GCM,data:f0hx,iv:HkSnJcCyzyfBTA39BNw+muhHTyrNLzzGxZ7M3bcLVu4=,tag:4+xL1xNyYPaV2b0mwnbQVQ==,type:str]
+    from: ENC[AES256_GCM,data:lqYqXxJuI4CAWPlX+5cltCdxqA==,iv:ZgZYySJXvjuTNIkMaDxynUJ6PFdp4fibqlgoA1vNDKE=,tag:vb/XKl2N1Hl5u7ndZXQvbg==,type:str]
+    user: ENC[AES256_GCM,data:ajljGVbRO//G,iv:E4Q0LGDCd5wnHc+Oq2A9ZhWGbCmkKlhllt922UmppIg=,tag:Cf4ft1JKFi9PaJEfPfq92Q==,type:str]
+    password: ENC[AES256_GCM,data:YNbtoiWLsQGl,iv:FJ3y1lWkqKkM8GVr7FZljAXJ331GkDgy4TenAbdIY3w=,tag:KIvhv+RL49ZMwWhYrU0SPA==,type:str]
+    notify-email: ENC[AES256_GCM,data:s5wRkDuaiflCyu4IyAnaocVPA3I=,iv:dK5PsweYa3JotYcDcz74DPNDXmhW5ngNEbY6JuG1ctg=,tag:/Dt2UoCtfau6qv0G7VzmrQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLQlJVekcwb01ESnhxSE9m
+            MzliVmlaYUttZVdiMDVtMlQ2L29LMXBWRzBZCkxRMi9RcE5LMzQ1MHFqUUxLaFZo
+            MUZ2UTg5eUNsMFVmenlhL2F3NDd0akkKLS0tIGFwbUUrSGFUTDFVOWlxM3V1bVdy
+            RUVVNDFMRHZCSWptdSt1bFVoN2JPWVkKXMlrJNPbm7Sj9vd/H3o6iqPRLy4LyjVe
+            xJyh9vn9RCwS9Jr6BN2Q9P2UZfTGwvIYc6zdmV+b2uRwvi4t4lMsBA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:g1YjOANlnvXErPqtDdDjRB9yTUiGEhX8QDkmf1aLfQBwoXbGUpoPIZibF8A4cAL1yUSZ2rD4l9TIe9nK8U3oF4WiKXf9u0bFK2g//bOv2A5JPewwmzoazkjIDXcJ/76nOQ5c+v8/r5tUG4c0SyzBH/gFP0D0+P5Yo3SEXjTxj2w=,iv:gW+AUwwAWkXZ+aLBXjjNDZw8VEraMpSE16RJGQc/L5g=,tag:DM/DUtil/7yLhfGYQXKFGA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
@@ -0,0 +1,27 @@
+apiVersion: ENC[AES256_GCM,data:YE0=,iv:s3yNqcBn7/DKUQNgbGGwVmlM1HzxYGLBKyB6Y2ii2WE=,tag:77KxAjOFU3G0fSfrgudKkg==,type:str]
+kind: ENC[AES256_GCM,data:PFW4VV9T,iv:n8gzMjE5C2TAfUTpp/8ex64B+hWUGG1K+2oQ4deN03Q=,tag:18Xqb6Di5izMHmzR5EU+QA==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:Xhg1fQrD4itrbvDW0g==,iv:/THWSXAThb9fj+mm3wcxqzSdzdt7nE06+xOpkCxyImE=,tag:UChokr06VkbDZcFZDcUY2A==,type:str]
+    namespace: ENC[AES256_GCM,data:r25U5MuuzDd8JJ2YjQ==,iv:uDUcS4ZTpZe8HmZMArzkdu7LV5GUoLSP2wIs5HB1gv0=,tag:fci7j30hoxnVKJwPFNfd1Q==,type:str]
+type: ENC[AES256_GCM,data:agEG8Fu8,iv:ckYX0buX8md1CWHXfxbAXQJLzoTxTJI16nlQ9LSzYi0=,tag:4tZWf3REbGOmmBiT14+dew==,type:str]
+stringData:
+    #ENC[AES256_GCM,data:ehjcsmz1R0i/n31f8M0IIUT4KDBwzz6f5ds=,iv:j5swFAKpC67ZvGioiCCC1D651LxUl9gME8GqK5Uc+ys=,tag:BUJ8k9LHs8NAPPZAwPuifA==,type:comment]
+    #ENC[AES256_GCM,data:ndAB00yun636VHDMonqTghO/jCWodNd/hmdbm1QmCvOAi4FvTLl8bY3wYR+ZtlkSQYvFNqH798MJixv5OZwxBj5H,iv:pS+7B60jaS5jhqDRw2LbBFv7mD1HO6+hjjv+iNpenXc=,tag:NU6+gxCHTGxqeDMfvkwJWw==,type:comment]
+    #ENC[AES256_GCM,data:T3mhSm/5q7JTSXNLrjhpP5GhqA7nXz8uAhJcEV1RDctAX0Dyfq8Qn4bNFO51ibwTETx4SnunPuFZVs++AvRnsA==,iv:oRGM8N4iEcwBrMzoEXQAeKqCKzUq+jXTMVq9W2l6oh4=,tag:GrrbC/tkWpA5kp5UYCyRBA==,type:comment]
+    app-token: ENC[AES256_GCM,data:jw+AqbsxyoYRvNrUDrq+GNq/TNfweFCs,iv:67vSSgMZCMV0GG37ZxUxHAWZqOOGMYCmo3J43WgLyNk=,tag:xlrnip4XomXBbMmooW9FKg==,type:str]
+    client-token: ENC[AES256_GCM,data:gsxeCqKh4e+DFjpn9jM5GfemAdBf8dSv,iv:l2bBMdxQUil8jU9XDjXGn3EtvFVrK5ibXDCeGaPbYTY=,tag:ELf5S6cdNQJ84Es5upu1IQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAza3YrRW1VTVNSOGlMK21a
+            bDhzQlEyZDNnZTZrSVNnTUEzU1hSdnM1ZFVzCjdBSUlmWG5EcnlzbSs5bXdDaGkx
+            ME1nMnRqQzF1Vkc3b3FXSUVHT1g0ZDgKLS0tIHpMUytEMjdLQ2E0Unp2di9KS3JQ
+            eHBraDk1clJyanhLY2dGM0tESmJIUFkKHTl3y9uQiEofOFD8j2vH3YK/CVzlq11w
+            GfShIji1yCvvowKGzYYhsQK0UM0FzhzBv0GFMYWQCBq8pGdoPVmO5g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-09-10T13:31:00Z"
+    mac: ENC[AES256_GCM,data:JhnO0V6cd2QVY/VhwHAJ5J75GeVlOVHBfVBTZstkj/IvpkAcwS/JE2b6jXiCwEC4n/vdA8DJ074noysUBRxh4Y7O4NKuvWcTniQKgqULNL1Hzgj3NdUkcQlWT+Z0HQ7FlvFH97VVXVfasU+CLHG48ShT2fDSj8JusEllb9CwSvg=,iv:PZo2krxvJc1TLJbvx+S9ClthoBe4w7WigUkq7dg0gNk=,tag:2IF5g/WTRP4XdnCZzDbiIA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
@@ -27,3 +27,6 @@ files:
  - vault-secrets.enc.yaml
  - vault-unseal-keys.enc.yaml
  - portfolio-secrets.enc.yaml
+  - gotify-admin-secrets.enc.yaml
+  - gotify-tokens-secrets.enc.yaml
+  - gotify-smtp-secrets.enc.yaml
@@ -20,9 +20,44 @@ config:
    insecure: true
    plaintext: true

+  # Git write-back configuration (for multi-source Applications)
+  git:
+    # Commit author for image updates
+    user:
+      name: "ArgoCD Image Updater"
+      email: "[email protected]"
+    # Use SSH keys from ArgoCD's known hosts + credentials
+    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
+
+# Mount ArgoCD's git credentials for write-back
+extraVolumes:
+  - name: argocd-ssh-known-hosts-cm
+    configMap:
+      name: argocd-ssh-known-hosts-cm
+      defaultMode: 0644
+  - name: argocd-gpg-keys-cm
+    configMap:
+      name: argocd-gpg-keys-cm
+      optional: true
+      defaultMode: 0644
+  - name: argocd-gpg-pubring
+    configMap:
+      name: argocd-gpg-pubring-cm
+      optional: true
+      defaultMode: 0644
+
+extraVolumeMounts:
+  - name: argocd-ssh-known-hosts-cm
+    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
+    subPath: ssh_known_hosts
+  - name: argocd-gpg-keys-cm
+    mountPath: /etc/gpg/source
+  - name: argocd-gpg-pubring
+    mountPath: /etc/gpg/pubring
+
 # Extra environment variables
 extraEnv:
-  - name: ARGOCD_GRPC_WEB
+  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
    value: "true"

 # Resources
@@ -11,7 +11,7 @@ metadata:
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 2
+  instances: 3
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
  bootstrap:
    initdb:
@@ -36,3 +36,4 @@ data:
      valid_volumes:
        - /docker-certs/client
      network: host
+      docker_host: automount
@@ -34,7 +34,11 @@ spec:
          command: ["sh", "-c"]
          args:
            - |
-              test -f /data/.runner || forgejo-runner register --no-interactive \
+              # Always re-register to keep labels in sync with values.yaml.
+              # Without this, changing a runner label requires manually deleting
+              # the PVC or .runner file — not GitOps-friendly.
+              rm -f /data/.runner
+              forgejo-runner register --no-interactive \
                --instance {{ .Values.runner.forgejoUrl }} \
                --token $(RUNNER_TOKEN) \
                --name {{ .Values.runner.name }} \
@@ -56,20 +60,18 @@ spec:
      containers:
        - name: runner
          image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
-          command: ["sh", "-c", "forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
+          command: ["sh", "-c", "while ! wget -q -O- http://localhost:2375/_ping >/dev/null 2>&1; do echo 'waiting for dind...'; sleep 2; done; echo 'dind ready'; forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
          workingDir: /data
          env:
            - name: DOCKER_HOST
-              value: tcp://localhost:2376
-            - name: DOCKER_TLS_VERIFY
-              value: "1"
-            - name: DOCKER_CERT_PATH
-              value: /docker-certs/client
+              value: tcp://localhost:2375
          volumeMounts:
            - name: runner-data
              mountPath: /data
            - name: docker-certs
              mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
@@ -85,10 +87,12 @@ spec:
            privileged: true    # required for DinD; cicd namespace is labelled privileged
          env:
            - name: DOCKER_TLS_CERTDIR
-              value: /docker-certs
+              value: ""
          volumeMounts:
            - name: docker-certs
              mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
            - name: dind-storage
              mountPath: /var/lib/docker
            - name: homelab-ca
@@ -113,6 +117,8 @@ spec:
            claimName: {{ .Release.Name }}-dind
        - name: docker-certs
          emptyDir: {}    # DinD regenerates mTLS certs on each start
+        - name: docker-sock
+          emptyDir: {}    # Shared docker socket between dind and runner
        - name: homelab-ca
          # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
          # The volumeMounts use subPath: ca.crt to project the single cert file.
@@ -2,9 +2,12 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively, so unlike the golang/rust instances,
-# jobs on this runner need no "install node" step before actions/checkout.
+# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
+# Install docker in workflow steps as needed.
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
  name: node-runner
  labels: "node:docker://node:22-bookworm"

@@ -2,17 +2,16 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# rust:1.83-bookworm -- verified this tag exists (docker manifest inspect)
-# before pinning it, per this repo's convention of not trusting a tag exists
-# without checking.
+# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
+# Install Node.js/docker in workflow steps as needed.
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
  name: rust-runner
-  labels: "rust:docker://rust:1.83-bookworm"
+  labels: "rust:docker://rust:1-bookworm"
+

-persistence:
-  reg:
-    storageClass: longhorn
-    size: 20Gi

 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:
@@ -1,20 +1,13 @@
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
-    tag: "6"          # pin exact release before apply
+    tag: "6"
  name: golang-runner
-  # Default image is only used when a job's `container:` doesn't override it
-  # (both ci.yaml and build.yaml in homelab-frontend do). Retired the old
-  # "docker" label entirely; every repo this runner serves is Go, so this
-  # instance carries the golang toolchain and its own dind sidecar builds and
-  # pushes that repo's images too -- there is no separate generic runner
-  # anymore.
+  # Label image is what workflow steps run in (NOT the runner daemon image).
+  # golang:1.26-bookworm: Debian, root, apt-get, Go, git.
+  # TODO: Switch to custom image once build-runner-images.yml pushes images
  labels: "golang:docker://golang:1.26-bookworm"
-  # In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
-  # (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
  forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
-  # tokenSecret: name of the K8s Secret that holds the runner registration token
-  # created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
  tokenSecret: runner-token
  resources:
    requests:
@@ -61,6 +54,6 @@ gc:
  enabled: true
  schedule: "*/30 * * * *"     # every 30 minutes
  image: alpine/k8s:1.31.0
-  pruneAge: "72h"              # Docker artifacts unused longer than this get pruned
-  pruneAgeHours: 72             # Same as pruneAge but numeric for date arithmetic in shell
+  pruneAge: "30m"              # Docker artifacts unused longer than this get pruned
+  pruneAgeHours: 0.5            # Same as pruneAge but numeric for date arithmetic in shell
  actcacheMaxAgeDays: 1        # actcache files older than N days (aggressive for heavy Rust cargo builds)
@@ -68,15 +68,14 @@ grafana.ini:
  # doesn't return localhost redirects in its token responses.
  #
  # role_attribute_path: JMESPath expression evaluated against the userinfo
-  # response. Members of the 'grafana-admins' Authentik group get Admin role;
-  # everyone else gets Viewer. The group name must match exactly what Authentik
-  # sends in the 'groups' claim.
+  # response. akadmin gets GrafanaAdmin (server admin, can impersonate);
+  # homelab-admins members get Admin (org admin); everyone else Viewer.
  auth.generic_oauth:
    enabled: true
    name: Authentik
    allow_sign_up: true
    client_id: grafana
-    scopes: openid email profile
+    scopes: openid email profile groups
    auth_url: https://authentik.riotpiao.com/application/o/authorize/
    token_url: https://authentik.riotpiao.com/application/o/token/
    api_url: https://authentik.riotpiao.com/application/o/userinfo/
@@ -87,7 +86,8 @@ grafana.ini:
    email_attribute_path: email
    login_attribute_path: preferred_username
    name_attribute_path: name
-    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    role_attribute_path: "preferred_username == 'akadmin' && 'GrafanaAdmin' || contains(groups[*], 'homelab-admins') && 'Admin' || 'Viewer'"
+    allow_assign_grafana_admin: true
    use_pkce: false
    use_refresh_token: false
    skip_org_role_sync: false
@@ -96,8 +96,13 @@ spec:
      console: https://minio.riotpiao.com

  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
+  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
+  # public URLs embedded in the OIDC metadata response (issuer stays public).
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
+      # Must use external URL — well-known response contains external issuer/jwks_uri.
+      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
+      # Hairpins through ingress-nginx but stays in-cluster.
      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio"
@@ -0,0 +1,160 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: api-gateway-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # ================================================================
+    # SLA Targets (based on canary traffic baselines):
+    #
+    # Availability:  99.9% (43.8 min downtime/month)
+    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
+    # Embeddings:    p95 < 500ms
+    # Rerank:        p95 < 500ms
+    # Models list:   p95 < 300ms
+    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
+    #
+    # Baselines from 200-request canary run:
+    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
+    #   rerank p99=218ms, models p99=277ms
+    # SLA set at ~2x p99 for headroom.
+    # ================================================================
+
+    - name: api-gateway.availability
+      rules:
+        # Gateway pods not ready
+        - alert: APIGatewayDown
+          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway has zero ready pods"
+
+        # Gateway pod count below desired
+        - alert: APIGatewayDegraded
+          expr: |
+            sum(kube_pod_status_ready{namespace="api",condition="true"})
+            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway {{ $value }} ready pods below desired replica count"
+
+        # Blackbox probe down
+        - alert: APIGatewayProbeDown
+          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway probe failed: {{ $labels.instance }}"
+
+        # LLM serving pods not ready
+        - alert: LLMServingDown
+          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "All LLM serving pods down"
+
+        # Individual predictor down
+        - alert: LLMPredictorDown
+          expr: |
+            kube_deployment_status_replicas_ready{namespace="llm-serving"}
+            < kube_deployment_spec_replicas{namespace="llm-serving"}
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"
+
+    - name: api-gateway.latency
+      # SLA: latency thresholds at ~2x measured p99
+      rules:
+        # Ingress-level latency (all requests through nginx)
+        - alert: APIGatewayLatencyHigh
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
+
+        # Extreme latency (p99 > 5s)
+        - alert: APIGatewayLatencyCritical
+          expr: |
+            histogram_quantile(0.99,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
+            ) > 5
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"
+
+    - name: api-gateway.errors
+      rules:
+        # 5xx error rate > 1%
+        - alert: APIGateway5xxErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.01
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"
+
+        # Total error rate > 10% (including 4xx)
+        - alert: APIGatewayHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
+            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
+            > 0.10
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"
+
+    - name: api-gateway.resources
+      rules:
+        # Gateway pod restart
+        - alert: APIGatewayRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "API Gateway pod {{ $labels.pod }} restarted"
+
+        # LLM predictor restart
+        - alert: LLMPredictorRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LLM predictor {{ $labels.pod }} restarted"
+
+        # Gateway high memory (>80% of limit)
+        - alert: APIGatewayHighMemory
+          expr: |
+            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
+            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
+            > 0.8
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
@@ -0,0 +1,177 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cluster-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: cluster.availability
+      rules:
+        # Node down
+        - alert: NodeNotReady
+          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.node }} is NotReady"
+
+        # Pod stuck pending (scheduling failure)
+        - alert: PodStuckPending
+          expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
+
+        # CrashLoopBackOff
+        - alert: PodCrashLooping
+          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"
+
+        # OOMKilled spike
+        - alert: OOMKilledSpike
+          expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value }} OOMKilled events in last hour"
+
+        # Deployment replicas unavailable
+        - alert: DeploymentReplicasUnavailable
+          expr: kube_deployment_status_replicas_unavailable > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"
+
+    - name: cluster.jobs
+      rules:
+        # Job failed
+        - alert: JobFailed
+          expr: kube_job_status_failed > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"
+
+        # Job stuck running >2h
+        - alert: JobStuckRunning
+          expr: |
+            kube_job_status_active == 1
+            and on(job_name,namespace)
+            (time() - kube_job_status_start_time) > 7200
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"
+
+        # CronJob missed schedule
+        - alert: CronJobMissedSchedule
+          expr: |
+            (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"
+
+    - name: cluster.resources
+      rules:
+        # Node CPU >90% sustained
+        - alert: NodeHighCPU
+          expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"
+
+        # Node memory >90% sustained
+        - alert: NodeHighMemory
+          expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"
+
+        # Node disk >85%
+        - alert: NodeDiskFull
+          expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"
+
+        # Container restart storm (>5 restarts in 15m)
+        - alert: ContainerRestartStorm
+          expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"
+
+    - name: cluster.storage
+      rules:
+        # Longhorn drive offline
+        - alert: LonghornDriveOffline
+          expr: longhorn_disk_health != 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Longhorn disk {{ $labels.node }} unhealthy"
+
+    - name: cluster.dns
+      rules:
+        # CoreDNS errors spike
+        - alert: CoreDNSErrorSpike
+          expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"
+
+    - name: cluster.probes
+      rules:
+        # Any blackbox probe down
+        - alert: ServiceProbeDown
+          expr: probe_success == 0
+          for: 3m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Probe failed: {{ $labels.instance }}"
+
+        # Probe latency >2s
+        - alert: ServiceProbeSlow
+          expr: probe_duration_seconds > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"
+
+        # Certificate expiry <14 days
+        - alert: CertificateExpiringSoon
+          expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"
@@ -61,6 +61,10 @@ serviceMonitor:
      url: https://argocd.riotpiao.com/healthz
    - name: longhorn
      url: https://longhorn.riotpiao.com/
+    - name: api-gateway
+      url: https://api.riotpiao.com/healthz
+    - name: api-gateway-models
+      url: https://api.riotpiao.com/v1/models

 prometheusRule:
  enabled: true
@@ -0,0 +1,36 @@
+apiVersion: v1
+data:
+  api-gateway.json: '{"title":"API Gateway","uid":"api-gateway","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["api","gateway","llm"],"panels":[{"id":1,"title":"Gateway
+    Health","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Gateway
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]},{"id":3,"title":"Probe:
+    healthz","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"probe_success{instance=~\".*api.riotpiao.com/healthz\"}"}]},{"id":4,"title":"Probe
+    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"probe_duration_seconds{instance=~\".*api.riotpiao.com.*\"}","legendFormat":"{{instance}}"}]}]},{"id":10,"title":"Ingress
+    Traffic (nginx)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Request
+    Rate by Status","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m]))
+    by (status)","legendFormat":"{{status}}"}]},{"id":12,"title":"Error Rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"5..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"5xx"},{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"4..\"}[5m]))
+    / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"4xx"}]},{"id":13,"title":"Latency
+    p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m]))
+    by (le))","legendFormat":"p99"}]}]},{"id":20,"title":"LLM Serving","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"LLM
+    Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":22,"title":"CPU
+    by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"llm-serving\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":23,"title":"Memory by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"llm-serving\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":24,"title":"Predictor Restarts","type":"timeseries","gridPos":{"h":8,"w":4,"x":20,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":30,"title":"Gateway Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"CPU
+    by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"api\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":32,"title":"Memory by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"api\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":33,"title":"Gateway Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"api\"}[15m]))
+    by (pod)","legendFormat":"{{pod}}"}]}]},{"id":40,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Gateway
+    Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":5},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\",container=\"gateway\"}"}]},{"id":42,"title":"LLM
+    Serving Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: API
+  labels:
+    grafana_dashboard: '1'
+  name: api-gateway-dashboard
+  namespace: logging
@@ -0,0 +1,62 @@
+apiVersion: v1
+data:
+  cluster-infrastructure.json: '{"title":"Cluster Infrastructure","uid":"cluster-infra","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["infrastructure","k8s"],"panels":[{"id":1,"title":"Cluster
+    Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Nodes
+    Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_node_status_condition{condition=\"Ready\",status=\"true\"}
+    == 1)"}]},{"id":3,"title":"Pods Pending","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_phase{phase=\"Pending\"})
+    OR on() vector(0)"}]},{"id":4,"title":"CrashLoopBackOff","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"})
+    OR on() vector(0)"}]},{"id":5,"title":"OOMKilled (1h)","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[1h]))
+    OR on() vector(0)"}]},{"id":6,"title":"Deploys Unavailable","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_deployment_status_replicas_unavailable
+    > 0) OR on() vector(0)"}]},{"id":7,"title":"Services Down","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(probe_success
+    == 0) OR on() vector(0)"}]}]},{"id":10,"title":"Jobs & CronJobs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Failed
+    Jobs","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_job_status_failed
+    > 0) OR on() vector(0)"}]},{"id":12,"title":"Failed Jobs Detail","type":"table","gridPos":{"h":8,"w":10,"x":4,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_failed
+    > 0","format":"table","instant":true}]},{"id":13,"title":"Stuck Jobs (>1h)","type":"table","gridPos":{"h":8,"w":10,"x":14,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_active
+    == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600","format":"table","instant":true}]},{"id":14,"title":"CronJob
+    Last Success","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"dateTimeFromNow"}},"targets":[{"expr":"kube_cronjob_status_last_successful_time{namespace=~\"cicd|kube-system|paperless\"}","legendFormat":"{{namespace}}/{{cronjob}}"}]},{"id":15,"title":"Container
+    Restart Storm (top 10)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"topk(10,
+    sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))","legendFormat":"{{namespace}}/{{pod}}"}]}]},{"id":20,"title":"Node
+    Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"CPU
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100","legendFormat":"{{instance}}"}]},{"id":22,"title":"Memory
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100","legendFormat":"{{instance}}"}]},{"id":23,"title":"Disk
+    % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1
+    - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})
+    * 100","legendFormat":"{{instance}}"}]},{"id":24,"title":"Load Average","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"node_load1","legendFormat":"1m
+    {{instance}}"},{"expr":"node_load5","legendFormat":"5m {{instance}}"}]},{"id":25,"title":"Network
+    Errors & Drops","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(node_network_receive_errs_total[5m])","legendFormat":"rx-err
+    {{instance}}"},{"expr":"rate(node_network_transmit_errs_total[5m])","legendFormat":"tx-err
+    {{instance}}"},{"expr":"rate(node_network_receive_drop_total[5m])","legendFormat":"rx-drop
+    {{instance}}"}]}]},{"id":30,"title":"Control Plane","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"API
+    Server Up","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"apiserver\"})"}]},{"id":32,"title":"API
+    Server Request Rate","type":"timeseries","gridPos":{"h":8,"w":10,"x":4,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(apiserver_request_total[5m]))
+    by (verb, code)","legendFormat":"{{verb}} {{code}}"}]},{"id":33,"title":"API Server
+    Error Rate %","type":"timeseries","gridPos":{"h":8,"w":10,"x":14,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(apiserver_request_total{code=~\"5..\"}[5m]))
+    / sum(rate(apiserver_request_total[5m])) * 100","legendFormat":"5xx %"}]},{"id":34,"title":"API
+    Server Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]},{"id":35,"title":"etcd
+    Request Duration","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.99,
+    sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":40,"title":"Storage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Longhorn
+    Disk Capacity","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"longhorn_disk_capacity_bytes","legendFormat":"capacity
+    {{node}}"},{"expr":"longhorn_disk_reservation_bytes","legendFormat":"reserved
+    {{node}}"}]},{"id":42,"title":"PVC Phase","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"kube_persistentvolumeclaim_status_phase","legendFormat":"{{namespace}}/{{persistentvolumeclaim}}
+    {{phase}}"}]}]},{"id":50,"title":"DNS & Networking","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":51,"title":"CoreDNS
+    Cache Hit Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(coredns_cache_hits_total[5m])
+    / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))","legendFormat":"{{server}}"}]},{"id":52,"title":"CoreDNS
+    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(coredns_dns_responses_total{rcode=~\"SERVFAIL|NXDOMAIN\"}[5m]))
+    by (rcode)","legendFormat":"{{rcode}}"}]}]},{"id":60,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":61,"title":"Error
+    Rate by Namespace","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"}
+    |= \"error\" [5m]))","legendFormat":"{{namespace}}"}]},{"id":62,"title":"Control
+    Plane Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"kube-system\"}"}]},{"id":63,"title":"Cluster
+    Addon Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":25},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: Infrastructure
+  labels:
+    grafana_dashboard: '1'
+  name: cluster-infrastructure-dashboard
+  namespace: logging
@@ -1,55 +0,0 @@
-# k8s/monitoring/dashboards/control-plane-logs.yaml
-# Surfaces controller/control-plane logs that are already in Loki today
-# (Promtail scrapes every namespace with no filter) — this dashboard is the
-# "make it visible" piece, not new log collection.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: control-plane-logs-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  control-plane-logs.json: |
-    {
-      "title": "Cluster Control Plane & Controllers (Logs)",
-      "uid": "control-plane-logs",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-1h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Error rate by namespace",
-          "type": "timeseries",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            {
-              "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Control plane (kube-apiserver, controller-manager, scheduler)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=\"kube-system\"}" }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
-          "type": "logs",
-          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
-          "datasource": { "type": "loki", "uid": "loki" },
-          "targets": [
-            { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
-          ]
-        }
-      ]
-    }
@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
+
+import json
+import os
+
+DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
+
+DS_PROM = {"type": "prometheus", "uid": "prometheus"}
+DS_LOKI = {"type": "loki", "uid": "loki"}
+
+
+def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
+    p = {
+        "id": id, "title": title, "type": "stat",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": [{"expr": expr}],
+    }
+    if mappings:
+        p["fieldConfig"]["defaults"]["mappings"] = mappings
+    if thresholds:
+        p["fieldConfig"]["defaults"]["thresholds"] = thresholds
+        p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
+    return p
+
+
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+    targets = []
+    for e in exprs:
+        if isinstance(e, tuple):
+            targets.append({"expr": e[0], "legendFormat": e[1]})
+        else:
+            targets.append({"expr": e, "legendFormat": "{{pod}}"})
+    return {
+        "id": id, "title": title, "type": "timeseries",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": targets,
+    }
+
+
+def table_panel(id, title, expr, x, y, w=12, h=8):
+    return {
+        "id": id, "title": title, "type": "table",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "targets": [{"expr": expr, "format": "table", "instant": True}],
+    }
+
+
+def log_panel(id, title, query, x, y, w=24, h=10):
+    return {
+        "id": id, "title": title, "type": "logs",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_LOKI,
+        "targets": [{"expr": query}],
+    }
+
+
+def row(id, title, y, panels, collapsed=True):
+    return {
+        "id": id, "title": title, "type": "row",
+        "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+        "panels": panels,
+    }
+
+
+def write_dashboard(filename, dashboard, folder):
+    cm = {
+        "apiVersion": "v1",
+        "kind": "ConfigMap",
+        "metadata": {
+            "name": filename.replace(".yaml", "-dashboard"),
+            "namespace": "logging",
+            "labels": {"grafana_dashboard": "1"},
+            "annotations": {"grafana_folder": folder},
+        },
+        "data": {
+            filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
+        },
+    }
+
+    import yaml
+    path = os.path.join(DASHBOARD_DIR, filename)
+    with open(path, "w") as f:
+        yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
+    print(f"  wrote {path}")
+
+
+# ============================================================================
+# Dashboard 1: Cluster Infrastructure
+# ============================================================================
+
+def build_cluster_infrastructure():
+    zero_thresholds = {"mode": "absolute", "steps": [
+        {"value": None, "color": "green"}, {"value": 1, "color": "red"}
+    ]}
+
+    panels = [
+        row(1, "Cluster Health", 0, [
+            stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
+            stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
+            stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
+            stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
+            stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
+        ]),
+        row(10, "Jobs & CronJobs", 1, [
+            stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
+            table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
+            table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
+            ts_panel(14, "CronJob Last Success", [
+                ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
+            ], 0, 10, w=12, unit="dateTimeFromNow"),
+            ts_panel(15, "Container Restart Storm (top 10)", [
+                ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
+            ], 12, 10, w=12),
+        ]),
+        row(20, "Node Resources", 2, [
+            ts_panel(21, "CPU % by Node", [
+                ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
+            ], 0, 3, unit="percent"),
+            ts_panel(22, "Memory % by Node", [
+                ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
+            ], 8, 3, unit="percent"),
+            ts_panel(23, "Disk % by Node", [
+                ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
+            ], 16, 3, unit="percent"),
+            ts_panel(24, "Load Average", [
+                ("node_load1", "1m {{instance}}"),
+                ("node_load5", "5m {{instance}}"),
+            ], 0, 11),
+            ts_panel(25, "Network Errors & Drops", [
+                ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
+                ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
+                ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
+            ], 8, 11),
+        ]),
+        row(30, "Control Plane", 3, [
+            stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(32, "API Server Request Rate", [
+                ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
+            ], 4, 4, w=10),
+            ts_panel(33, "API Server Error Rate %", [
+                ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
+            ], 14, 4, w=10, unit="percent"),
+            ts_panel(34, "API Server Latency", [
+                ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 0, 12, unit="s"),
+            ts_panel(35, "etcd Request Duration", [
+                ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
+            ], 8, 12, unit="s"),
+        ]),
+        row(40, "Storage", 4, [
+            ts_panel(41, "Longhorn Disk Capacity", [
+                ("longhorn_disk_capacity_bytes", "capacity {{node}}"),
+                ("longhorn_disk_reservation_bytes", "reserved {{node}}"),
+            ], 0, 5, unit="bytes"),
+            ts_panel(42, "PVC Phase", [
+                ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
+            ], 8, 5),
+        ]),
+        row(50, "DNS & Networking", 5, [
+            ts_panel(51, "CoreDNS Cache Hit Rate", [
+                ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
+            ], 0, 6, unit="percentunit"),
+            ts_panel(52, "CoreDNS Errors", [
+                ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
+            ], 8, 6),
+        ]),
+        row(60, "Logs", 6, [
+            ts_panel(61, "Error Rate by Namespace", [
+                ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
+            ], 0, 7),
+            log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
+            log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
+        ]),
+    ]
+
+    return {
+        "title": "Cluster Infrastructure",
+        "uid": "cluster-infra",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["infrastructure", "k8s"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Dashboard 3: API Gateway
+# ============================================================================
+
+def build_api_gateway():
+    panels = [
+        row(1, "Gateway Health", 0, [
+            stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
+            stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
+                {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
+            ]),
+            ts_panel(4, "Probe Latency", [
+                ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
+            ], 8, 1, unit="s"),
+        ], collapsed=False),
+        row(10, "Ingress Traffic (nginx)", 1, [
+            ts_panel(11, "Request Rate by Status", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
+            ], 0, 2),
+            ts_panel(12, "Error Rate %", [
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
+                ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
+            ], 8, 2, unit="percent"),
+            ts_panel(13, "Latency p50/p95/p99", [
+                ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
+                ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
+                ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
+            ], 16, 2, unit="s"),
+        ]),
+        row(20, "LLM Serving", 2, [
+            stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
+            ts_panel(22, "CPU by Predictor", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
+            ], 4, 3),
+            ts_panel(23, "Memory by Predictor", [
+                ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
+            ], 12, 3, unit="bytes"),
+            ts_panel(24, "Predictor Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
+            ], 20, 3, w=4),
+        ]),
+        row(30, "Gateway Resources", 3, [
+            ts_panel(31, "CPU by Gateway Pod", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
+            ], 0, 4),
+            ts_panel(32, "Memory by Gateway Pod", [
+                ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
+            ], 8, 4, unit="bytes"),
+            ts_panel(33, "Gateway Restarts", [
+                ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
+            ], 16, 4),
+        ]),
+        row(40, "Logs", 4, [
+            log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
+            log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
+        ]),
+    ]
+
+    return {
+        "title": "API Gateway",
+        "uid": "api-gateway",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["api", "gateway", "llm"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Generate
+# ============================================================================
+
+print("Generating dashboards...")
+
+# Dashboard 1
+write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
+
+# Dashboard 3
+write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
+
+print("Done.")
@@ -1,121 +0,0 @@
-# k8s/monitoring/dashboards/hardware-overview.yaml
-# Trimmed operator at-a-glance view across all nodes — node-exporter already
-# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
-# this is the quick health-check version, not a replacement for it.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: hardware-overview-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  hardware-overview.json: |
-    {
-      "title": "Hardware Statistics (Operator Overview)",
-      "uid": "hardware-overview",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Nodes up / down",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "CPU usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Memory usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "Root filesystem usage % by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Root filesystem space remaining",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "bytes" } },
-          "targets": [
-            {
-              "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Network errors/drops by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
-            { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
-            { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
-            { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Load average (1m / 5m) by node",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "node_load1", "legendFormat": "{{instance}} load1" },
-            { "expr": "node_load5", "legendFormat": "{{instance}} load5" }
-          ]
-        }
-      ]
-    }
@@ -1,185 +0,0 @@
-# k8s/monitoring/dashboards/kube-controller-health.yaml
-# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
-# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
-# the one control-plane component that's still reachable (its ServiceMonitor
-# targets the in-cluster `kubernetes` service, not localhost) — paired with
-# kube-state-metrics signals as a proxy for controller/scheduler health.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-controller-health-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  kube-controller-health.json: |
-    {
-      "title": "Kube-Controller Health",
-      "uid": "kube-controller-health",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "API server — up",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "API server — request rate by verb/code",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
-              "legendFormat": "{{verb}} {{code}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "API server — error rate % (5xx)",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
-              "legendFormat": "5xx %"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "API server — latency p95 / p99",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Pods stuck Pending",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 5, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "CrashLoopBackOff containers",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Nodes NotReady",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
-          ]
-        },
-        {
-          "id": 8,
-          "title": "Failed Jobs",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 9,
-          "title": "Deployments with unavailable replicas",
-          "type": "table",
-          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
-          ]
-        },
-        {
-          "id": 10,
-          "title": "Container restart rate by pod",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
-              "legendFormat": "{{namespace}}/{{pod}}"
-            }
-          ]
-        }
-      ]
-    }
@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: llm-frontend-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "LLM"
-    # No request-level panels. The rate/error/latency/bandwidth row used to run
-    # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go
-    # gateway that replaced it does not expose /metrics yet, so those panels
-    # were removed rather than left querying series that no longer exist.
-    # What is left is pod-level: readiness, CPU/memory, restarts, logs.
-    #
-    # Restoring request-level and per-model observability means wiring three
-    # sources, none of which are in place: gateway metrics (RED plus token
-    # counts and TTFT, which the gateway can measure because it sees the
-    # response stream), vLLM's own /metrics on reasoning-predictor (rich --
-    # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds,
-    # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's
-    # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at
-    # all (verified: /metrics returns 404), so ornith can only ever be observed
-    # from the gateway side. No ServiceMonitor exists for the llm-serving
-    # namespace today, so none of the engine metrics are being scraped.
-data:
-  llm-frontend.json: |
-    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
@@ -1,172 +0,0 @@
-# k8s/monitoring/dashboards/service-availability.yaml
-# Active uptime/availability from blackbox-exporter probes — the signal that
-# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
-# alone can't distinguish "idle" from "down".
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-availability-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-availability.json: |
-    {
-      "title": "Service Availability & Certificate Expiration",
-      "uid": "svc-availability",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-24h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1,
-          "title": "Up / Down — all probed services",
-          "type": "stat",
-          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
-              ],
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 1, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "probe_success", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Uptime % trend",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
-              "legendFormat": "{{instance}}"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Probe latency",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "7-day SLO (% successful probes)",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "unit": "percent",
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 99, "color": "yellow" },
-                  { "value": 99.9, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "avg_over_time(probe_success[7d]) * 100",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Services DOWN right now",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Certs expiring in < 14 days",
-          "type": "stat",
-          "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
-              "legendFormat": "expiring"
-            }
-          ]
-        },
-        {
-          "id": 7,
-          "title": "Certificate expiry — days remaining",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
-              "legendFormat": "{{name}}",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,141 +0,0 @@
-# k8s/monitoring/dashboards/service-golden-signals.yaml
-# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
-# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
-# sidecar.dashboards in k8s/logging/grafana-values.yaml.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-golden-signals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-golden-signals.json: |
-    {
-      "title": "Latency & Golden Signals (Ingress RED)",
-      "uid": "svc-golden-signals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "templating": {
-        "list": [
-          {
-            "name": "ingress",
-            "type": "query",
-            "datasource": { "type": "prometheus", "uid": "prometheus" },
-            "query": "label_values(nginx_ingress_controller_requests, ingress)",
-            "refresh": 2,
-            "includeAll": false
-          }
-        ]
-      },
-      "panels": [
-        {
-          "id": 1,
-          "title": "Request rate by status — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
-              "legendFormat": "{{status}}"
-            }
-          ]
-        },
-        {
-          "id": 2,
-          "title": "Error rate % (4xx / 5xx) — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "percent" } },
-          "targets": [
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "5xx"
-            },
-            {
-              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
-              "legendFormat": "4xx"
-            }
-          ]
-        },
-        {
-          "id": 3,
-          "title": "Latency p50 / p95 / p99 — $ingress",
-          "type": "timeseries",
-          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            {
-              "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p50"
-            },
-            {
-              "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p95"
-            },
-            {
-              "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
-              "legendFormat": "p99"
-            }
-          ]
-        },
-        {
-          "id": 4,
-          "title": "All services — traffic overview",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        },
-        {
-          "id": 5,
-          "title": "Customer-facing failures (5xx count, window total)",
-          "type": "stat",
-          "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "green" },
-                  { "value": 1, "color": "yellow" },
-                  { "value": 50, "color": "red" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            {
-              "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
-              "legendFormat": "5xx total"
-            }
-          ]
-        },
-        {
-          "id": 6,
-          "title": "Top 5 error-contributing services",
-          "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            {
-              "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
-              "format": "table",
-              "instant": true
-            }
-          ]
-        }
-      ]
-    }
@@ -1,109 +0,0 @@
-# k8s/monitoring/dashboards/service-internals.yaml
-# Native per-service metrics — the "why" layer behind the ingress RED/uptime
-# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: service-internals-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-data:
-  service-internals.json: |
-    {
-      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
-      "uid": "svc-internals",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
-            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
-          ]
-        },
-        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": { "defaults": { "unit": "s" } },
-          "targets": [
-            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
-          ]
-        },
-        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "gitea_repositories", "legendFormat": "repos" },
-            { "expr": "gitea_organizations", "legendFormat": "orgs" }
-          ]
-        },
-        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
-            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
-          ]
-        },
-        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
-          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "argocd_app_info", "format": "table", "instant": true }
-          ]
-        },
-        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
-          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  { "value": 0, "color": "red" },
-                  { "value": 14, "color": "yellow" },
-                  { "value": 30, "color": "green" }
-                ]
-              }
-            }
-          },
-          "targets": [
-            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
-          ]
-        },
-        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
-          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "fieldConfig": {
-            "defaults": {
-              "mappings": [
-                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
-                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
-              ]
-            }
-          },
-          "targets": [
-            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
-          ]
-        },
-        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
-          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
-          "datasource": { "type": "prometheus", "uid": "prometheus" },
-          "targets": [
-            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-argocd-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Argo CD"
-data:
-  svc-argocd.json: |
-    {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-authentik-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Authentik"
-data:
-  svc-authentik.json: |
-    {
-      "title": "Authentik — Service Overview",
-      "uid": "svc-authentik",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
-            },
-            {
-              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Outpost connections", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_outposts_connected" }]
-            },
-            {
-              "id": 22, "title": "Flows cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_flows_cached" }]
-            },
-            {
-              "id": 23, "title": "Policies cached", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_policies_cached" }]
-            },
-            {
-              "id": 24, "title": "Queued tasks", "type": "stat",
-              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
-              "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_tasks_queued" }]
-            },
-            {
-              "id": 25, "title": "Admin workers", "type": "timeseries",
-              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "authentik_admin_workers" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"iam\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-forgejo-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Forgejo"
-data:
-  svc-forgejo.json: |
-    {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-grafana-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Grafana"
-data:
-  svc-grafana.json: |
-    {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
@@ -1,143 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-minio-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "MinIO"
-data:
-  svc-minio.json: |
-    {
-      "title": "MinIO — Service Overview",
-      "uid": "svc-minio",
-      "schemaVersion": 39,
-      "timezone": "browser",
-      "time": { "from": "now-6h", "to": "now" },
-      "refresh": "30s",
-      "panels": [
-        {
-          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
-          "panels": [
-            {
-              "id": 2, "title": "Up", "type": "stat",
-              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "fieldConfig": {
-                "defaults": {
-                  "color": { "mode": "thresholds" },
-                  "mappings": [
-                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
-                  ],
-                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
-                }
-              },
-              "targets": [{ "expr": "min(up{job=\"minio\"})" }]
-            },
-            {
-              "id": 3, "title": "S3 request rate by method", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
-            },
-            {
-              "id": 4, "title": "Error rate %", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
-              "fieldConfig": { "defaults": { "unit": "percent" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
-            },
-            {
-              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
-              "fieldConfig": { "defaults": { "unit": "s" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
-                { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
-                { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
-              ]
-            }
-          ]
-        },
-        {
-          "id": 10, "title": "Row: Resource Usage", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
-          "panels": [
-            {
-              "id": 11, "title": "CPU by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 12, "title": "Memory by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
-            },
-            {
-              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
-            }
-          ]
-        },
-        {
-          "id": 20, "title": "Row: Storage & Replication", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
-          "panels": [
-            {
-              "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
-                { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
-              ]
-            },
-            {
-              "id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
-              "fieldConfig": { "defaults": { "unit": "short" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [
-                { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
-                { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
-              ]
-            },
-            {
-              "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
-            },
-            {
-              "id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
-              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
-              "fieldConfig": { "defaults": { "unit": "bytes" } },
-              "datasource": { "type": "prometheus", "uid": "prometheus" },
-              "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
-            }
-          ]
-        },
-        {
-          "id": 30, "title": "Row: Logs", "type": "row",
-          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
-          "panels": [
-            {
-              "id": 31, "title": "Recent logs", "type": "logs",
-              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
-              "datasource": { "type": "loki", "uid": "loki" },
-              "targets": [{ "expr": "{namespace=\"storage\"}" }]
-            }
-          ]
-        }
-      ]
-    }
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: svc-vault-dashboard
-  namespace: logging
-  labels:
-    grafana_dashboard: "1"
-  annotations:
-    grafana_folder: "Vault"
-data:
-  svc-vault.json: |
-    {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
@@ -12,20 +12,12 @@ resources:
  - alerts/svc-grafana-rules.yaml
  - alerts/svc-minio-rules.yaml
  - alerts/svc-vault-rules.yaml
+  - alerts/cluster-alerts.yaml
+  - alerts/api-gateway-alerts.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
-  - dashboards/control-plane-logs.yaml
-  - dashboards/hardware-overview.yaml
-  - dashboards/kube-controller-health.yaml
-  - dashboards/llm-frontend.yaml
-  - dashboards/service-availability.yaml
-  - dashboards/service-golden-signals.yaml
-  - dashboards/service-internals.yaml
-  - dashboards/svc-argocd.yaml
-  - dashboards/svc-authentik.yaml
-  - dashboards/svc-forgejo.yaml
-  - dashboards/svc-grafana.yaml
-  - dashboards/svc-minio.yaml
-  - dashboards/svc-vault.yaml
+  - servicemonitors/ingress-nginx.yaml
+  - dashboards/cluster-infrastructure.yaml
+  - dashboards/api-gateway.yaml
@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ingress-nginx
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  namespaceSelector:
+    matchNames:
+      - ingress-nginx
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/component: controller
+  endpoints:
+    - port: metrics
+      interval: 30s
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: homelab-admin-oidc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  # OIDC group for Authentik homelab-admins members
+  # When rock logs in via OIDC, k8s sees:
+  # - User: oidc:[email protected]
+  # - Groups: oidc:homelab-admins (from Authentik group claim)
+  - kind: Group
+    name: oidc:homelab-admins
@@ -6,6 +6,7 @@ kind: Kustomization
 # already fixed once in k8s/infra/minio and k8s/infra/iam. Every resource
 # here sets its own explicit metadata.namespace.
 resources:
+  - admin-oidc-binding.yaml
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
@@ -35,6 +35,8 @@
    rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local

    kubernetes cluster.local in-addr.arpa ip6.arpa {
Author	SHA1	Message	Date
rock	4b63e809e2	fix(argocd-image-updater): remove duplicate ARGOCD_GRPC_WEB env var Fixes sync error: 'may not be specified when value is not empty' The Helm chart already defines ARGOCD_GRPC_WEB, so remove from extraEnv.	2026-09-11 11:06:05 +09:00
rock	7bd9f83fa7	fix(cnpg): scale paperless-db, immich-db, gotify-db to 3 instances (#30 ) Co-authored-by: rock <[email protected]>	2026-09-10 22:56:41 +00:00
rock	9d3a669dfe	fix(gotify): move SOPS secrets to ksops generator ArgoCD couldn't decrypt secrets.yaml because it was listed as a plain kustomize resource. Move the 3 secrets (gotify-admin, gotify-tokens, gotify-smtp) to k8s/argocd/secrets/ as .enc.yaml files processed by the ksops generator, matching the repo convention. Fixes ComparisonError: 'Object Kind is missing' (SOPS ciphertext parsed as raw YAML).	2026-09-10 22:32:24 +09:00
rock	436c7d8d42	fix(argocd): update git repoURLs for org transfer rock -> riotpiao-poimen Repos transferred: homelab-frontend, kmsvc-manage, poimen, poimen-memory, poimen-workflows, poimen-frontend. Old URLs return 301 which ArgoCD doesn't follow. NOT changed: container image registry paths (rock/ is correct for registry), riotpiao.com (still under rock org). Also adds poimen-frontend to AppProject sourceRepos allowlist.	2026-09-10 10:49:30 +09:00
rock	b571d518e0	fix(argocd): update repoURL after org transfer (#29 ) Co-authored-by: rock <[email protected]>	2026-09-10 01:41:37 +00:00
rock	128e76ce2d	feat(gotify): push notification server + SMTP email relay (#19 ) Co-authored-by: rock <[email protected]>	2026-09-10 01:35:59 +00:00
rockandpoimen	5b16b882be	fix(comfyui): port 8888, emptyDir storage, liveness probe (#18 ) - Change container port from 8188 to 8888 (Caddy proxy binding) - Remove PVC, use emptyDir for ephemeral models/output - Replace startup + readiness probes with single liveness probe - Remove explicit COMFYUI_FLAGS (container defaults work) - Pod now reaches Ready state immediately after image pull Fixes GPU contention by using ephemeral storage. ComfyUI now runs and is accessible at https://comfy.riotpiao.comReviewed-on: #18 Co-authored-by: poimen <[email protected]> Reviewed-on: riotpiao-poimen/homelab-frontend#21 Co-authored-by: rock <[email protected]>	2026-09-09 22:38:25 +00:00
rock	8790de6038	feat: add ComfyUI + rebalance GPU allocation (#17 ) ## GPU Rebalance (4× V100 32GB) \| Pod \| Before \| After \| \|-----\|--------\|-------\| \| reasoning (PP=2) \| 2 GPU \| 2 GPU \| \| ornith \| 2 GPU (2 replicas) \| 1 GPU (1 replica) \| \| comfyui \| — \| 1 GPU (new) \| \| qwen-cpu \| — \| CPU on cp-2 (new) \| \| embeddings/reranker \| CPU \| CPU \| ## Changes - `ornith.yaml`: scale 2→1, remove qwen2.5 co-loading, MAX_LOADED_MODELS=1 - `qwen-cpu.yaml`: new Ollama deployment on talos-cp-2 (144GB RAM), 5Gi PVC - `k8s/apps/comfyui/`: new ComfyUI deployment (1 GPU, 50Gi model PVC, ingress) - `58-comfyui.yaml`: ArgoCD Application (wave 8) Gateway route update in separate PR (homelab-frontend).Reviewed-on: #17 Co-authored-by: rock <[email protected]>	2026-09-09 02:11:28 +00:00
rock	c5aadd98f8	chore: gitignore IAM provisioning scripts	2026-09-08 15:29:42 -07:00
rock	95c67f9437	P3.8: Restrict llm-serving ingress to api-gateway only (#15 ) ## Summary Replaces hand-applied `llm-serving-default-deny` NetworkPolicy with a git-managed, namespace-scoped policy that only allows traffic from the api-gateway. Issue: #13 Co-authored-by: rock <[email protected]>	2026-09-08 16:56:36 +00:00
rock	1630704f8b	feat: switch image-updater to digest-based :latest tracking All image-updater annotations now use update-strategy: digest with allow-tags: ^latest$ and write-back-method: argocd. No SHA tags committed to git — digest overrides stored in ArgoCD state only. - poimen: git write-back → argocd, removed git-branch - portfolio: newest-build SHA → digest latest - api-gateway: newest-build SHA → digest latest	2026-09-07 18:19:46 -07:00
rock	ce1539a634	revert: remove unsupported buildOptions (ArgoCD v3.4.5 doesn't support it) - buildOptions field not available in ArgoCD v3.4.5 - SOPS decryption already handled by repo-server ksops plugin - Revert to simple kustomize config - Portfolio Application can now sync properly	2026-09-07 17:49:58 -07:00
rock	2799e3a675	fix: enable ksops plugin for portfolio Application - Add kustomize config with --enable-alpha-plugins to support ksops - Allows ArgoCD to properly decrypt SOPS-encrypted files - Fixes Image Updater compatibility with sops field in kustomization.yaml	2026-09-07 17:47:03 -07:00
rock	1da7e0aa4d	fix: wait for dind to be ready before starting runner daemon	2026-09-07 13:22:09 -07:00
rock	8970e35539	fix: use tcp://localhost:2375 for dind (no TLS, no socket permission issues)	2026-09-07 13:20:20 -07:00
rock	92239561cd	fix: run runner as root to access dind socket	2026-09-07 13:18:54 -07:00
rock	2d92383951	fix: runner uses unix socket instead of TLS TCP for dind Job containers spawned by the runner run inside dind. With TCP+TLS (tcp://localhost:2376), localhost inside those containers doesn't reach the dind sidecar. Unix socket at /run/docker.sock works because both runner and dind share the /run emptyDir. Also disables DOCKER_TLS_CERTDIR so dind creates the socket instead of only listening on TLS TCP.	2026-09-07 13:16:49 -07:00
rock	7d77935d15	ci: fix runner labels + CoreDNS rewrite + cleanup - Runners use public images (code.forgejo.org/forgejo/runner:6) - Labels pull from Docker Hub: golang:1.26, node:22, rust:1-bookworm - Add CoreDNS api.riotpiao.com rewrite - Fix runner re-registration to keep labels in sync - Add unified CI pattern docs to CLAUDE.example.md - Remove dead .forgejo/ workflow dir (Forgejo uses .gitea/)	2026-09-07 13:01:56 -07:00
rock	fee4f9edfc	ci: fix golang runner - use docker:27-cli (has Node.js + golang + git) Previous image (golang:1.26-bookworm) lacks Node.js, causing GitHub Actions to fail with: 'exec: "node": executable file not found' Solution: - Change golang runner image from golang:1.26-bookworm to docker:27-cli - docker:27-cli includes: Node.js, Go toolchain, git, docker CLI, full dev tools - Verified tag exists: docker manifest inspect docker:27-cli ✓ This allows actions/checkout@v4 and other GitHub Actions to run properly on the golang runner pod. Note: node:22-bookworm runner already has Node.js, no change needed.	2026-09-05 22:50:43 -07:00
rock	12fc2796e2	ci: fix rust runner - use docker:27-cli (has Node.js + git + docker) Previous image (rust:1.83-bookworm) lacks Node.js, causing GitHub Actions to fail with: 'exec: "node": executable file not found' Solution: - Change rust runner image from rust:1.83-bookworm to docker:27-cli - docker:27-cli includes: Node.js, git, docker CLI, full dev tools - Verified tag exists: docker manifest inspect docker:27-cli ✓ This allows actions/checkout@v4 and other GitHub Actions to run properly on the rust runner pod.	2026-09-05 22:49:25 -07:00
rock	f0976bfc61	fix: remove spec wrapper from ClusterRoleBinding - ClusterRoleBinding doesn't use spec: wrapper (unlike Deployment/StatefulSet) - roleRef and subjects go at top level with metadata - Fixes: 'strict decoding error: unknown field "spec"'	2026-09-05 15:04:00 -07:00
rock	4a4e57d0f2	fix: remove namespace from rbac Application destination - RBAC kustomization contains cluster-scoped (ClusterRoleBinding) and namespace-scoped (Role/RoleBinding) resources - Each resource has explicit metadata.namespace, so Application shouldn't force a default namespace - Fixes: ClusterRoleBinding gets namespace=default, causing sync failure with 'unsupported role reference kind: ""'	2026-09-05 14:53:59 -07:00
rock	4fc833f9b2	fix: remove backslash line continuations from YAML multiline string - YAML block scalars (\|) don't use backslash continuation - Just indent lines properly, block scalar handles them automatically - Fixes ArgoCD ComparisonError on poimen app	2026-09-05 14:48:21 -07:00
rock	647fba8814	fix: add admin-oidc-binding to kustomization resources - admin-oidc-binding.yaml wasn't listed in resources - Now kustomize will include it when building manifests - ArgoCD can sync the OIDC group binding	2026-09-05 14:42:17 -07:00
rock	bcae41e338	chore: revert node-runner to stock image, remove Dockerfile - Reverted to node:22-bookworm (no custom image) - Removed Dockerfile.node (no CI to build it) - Docker install step in riotpiao workflow is already the workaround	2026-09-05 14:33:27 -07:00
rock	1425ab7cbc	chore: remove homelab CI workflow - Removed .gitea/workflows/build-runner-node.yml - Homelab is GitOps only, not a buildable artifact - Runner images managed via direct Dockerfile edits + manual pushes	2026-09-05 14:33:13 -07:00
rock	978f9c8147	feat: add OIDC group binding for cluster-admin access - Binds oidc:homelab-admins group to cluster-admin ClusterRole - Allows OIDC users (via Authentik) to have admin access - Groups claim from Authentik with oidc: prefix per kube-apiserver config - Enables kubectl access via 'kubectl login' + kubelogin	2026-09-05 14:30:41 -07:00
rock	4193c8ab99	feat: custom forgejo-runner-node image with docker.io pre-installed - Dockerfile.node extends node:22-bookworm + docker.io - CI workflow builds and pushes to forgejo registry on changes - values-node.yaml references custom image instead of stock node - Removes need to install docker in every workflow using node runner	2026-09-05 14:20:42 -07:00
rock	2c011e08e2	feat: Image Updater git write-back for multi-source poimen Application - write-back-method: git (commits image updates back to repos) - git-branch: main - Mounts ArgoCD SSH credentials for git pushes - Image Updater commits new SHAs → repos → ArgoCD syncs	2026-09-05 13:56:40 -07:00
rock	9da6829e05	feat: multi-source poimen Application (memory, workflows, frontend) - Single Application syncs 3 independent repos - All deploy to poimen namespace - Image Updater tracks all 3 services (7-char SHA tags) - Auto-sync: prune + selfHeal enabled	2026-09-05 13:55:29 -07:00
rock	69537a4e6a	fix: remove poimen-root Application (external repo dependency) - Removed dependency on external poimen.git repo - Poimen manifests should be managed locally or via separate workflow - Simplifies homelab GitOps to only manage homelab-owned services	2026-09-05 13:52:53 -07:00
rock	76c053d895	Revert "feat: enable Image Updater for poimen services" This reverts commit `cfc27c5420`.	2026-09-05 13:52:50 -07:00
rock	cfc27c5420	feat: enable Image Updater for poimen services - Track poimen-memory, poimen-workflows, poimen-frontend images - Auto-update on new 7-char SHA tags from Forgejo - Filter: regexp:^[0-9a-f]{7}$ (commit SHA) - write-back-method: argocd (updates Application)	2026-09-05 13:51:29 -07:00
rock	97c951bef3	Phase 6.6: Add Poimen Memory DLQ queues (ArgoCD managed) ArgoCD Application: memory-queues ├─ Sync wave: 7 (messaging wave) ├─ Path: k8s/apps/messaging/memory-queues ├─ Namespace: sqs └─ Auto-sync: enabled (prune + selfHeal) Helm Chart: memory-queues ├─ Chart.yaml: v0.1.0 ├─ values.yaml: Queue config └─ templates/queues.yaml: Queue CRD resources Queues Created: 1. poimen-memory-dlq ├─ Purpose: Extraction + webhook + agent failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days (1,209,600 seconds) └─ Visibility timeout: 5 minutes (300 seconds) 2. poimen-memory-metric-dlq ├─ Purpose: Metrics persistence failures ├─ Partitions: 3 ├─ Replication factor: 1 ├─ Retention: 14 days └─ Visibility timeout: 5 minutes Resource: Queue CRD (kmsvc.io/v1alpha1) └─ Managed by: queue-operator (already running in sqs ns) Deployment Flow: ArgoCD (homelab) → sync wave 7 → deploy queues Memory app (poimen) → connects to kmsvc → sends DLQ messages Files: ├─ k8s/apps/messaging/memory-queues/Chart.yaml (new) ├─ k8s/apps/messaging/memory-queues/values.yaml (new) ├─ k8s/apps/messaging/memory-queues/templates/queues.yaml (new) └─ k8s/argocd/apps/50-memory-queues.yaml (new)	2026-09-05 01:10:51 -07:00
rock	e414a3e394	Revert "feat: add memory service queues (processing, indexing, dlq)" This reverts commit `e5ae5b16b7`.	2026-09-05 01:09:30 -07:00
rock	e5ae5b16b7	feat: add memory service queues (processing, indexing, dlq) - processing-queue: high-throughput, auto-scaling (1-4 shards) - indexing-queue: FIFO with deduplication (1-2 shards) - memory-dlq: dead letter queue for redelivery failures - ArgoCD Application (wave 7) to auto-sync queue lifecycle	2026-09-05 01:05:01 -07:00
rock	e41165f358	fix(grafana): give homelab-admins Admin org role, akadmin GrafanaAdmin GrafanaAdmin is server admin only — no org membership, so users couldn't see dashboards. Now: - akadmin: GrafanaAdmin (server admin, can impersonate) - homelab-admins: Admin (org admin, dashboard access) - others: Viewer	2026-09-04 23:41:22 -07:00
rock	bd8c9fe033	fix: grant GrafanaAdmin (server admin) to homelab-admins for Administration menu	2026-09-04 23:18:19 -07:00
rock	2eda66c095	fix: add groups scope to grafana OIDC so role mapping works	2026-09-04 23:14:11 -07:00
rock	edc5dadd82	feat: add nginx-ingress ServiceMonitor for gateway traffic metrics	2026-09-04 23:07:44 -07:00
rock	60786a17ea	fix: map grafana admin role from homelab-admins group (grafana-admins deleted)	2026-09-04 23:04:01 -07:00
rock	efbe530b5c	refactor: consolidate 13 dashboards into 2 (cluster-infrastructure + api-gateway)	2026-09-04 22:58:53 -07:00
rock	4c63f8b125	feat: add cluster and api-gateway alert rules with SLA targets	2026-09-04 22:37:48 -07:00
rock	2a9220b576	feat: add cluster-infrastructure and api-gateway grafana dashboards	2026-09-04 22:31:51 -07:00
rock	26714d2ef3	feat: add api-gateway blackbox probes for healthz and /v1/models	2026-09-04 22:13:00 -07:00
rock	39e7ada3c6	fix: use external Authentik URL for MinIO OIDC config discovery	2026-09-04 21:25:30 -07:00
rock	1fe8707e3c	gitops: add secret-rotation controller ArgoCD Application - Syncs k8s/apps/secret-rotation-controller/ kustomization - Auto-prune and self-heal enabled - Creates secret-rotation namespace - ArgoCD will deploy CRD, RBAC, ExternalSecret, Deployment	2026-09-04 13:51:26 -07:00
rock	82a4e3e4fe	feat: automated secret rotation controller - ExternalSecret syncs age key from Vault to pod - CRD defines rotation schedule for each secret - Controller watches CRD, rotates on schedule: * Call provider API (Authentik/Forgejo/MinIO) for new secret * Update k8s Secret * Update .enc.yaml via sops (uses age key from Vault) * Git commit and push - Vault is source of truth for age key (never on disk) - Examples: minio-oidc (90d), portfolio-agent (90d), forgejo-token (90d), minio-root (180d)	2026-09-03 23:37:24 -07:00
rock	6ad4c0d294	fix(minio): use in-cluster URL for OIDC config fetch MinIO pod was getting 503 from public URL at startup. Use in-cluster authentik-server.iam.svc for metadata fetch; browser redirects still use public URLs from OIDC metadata response.	2026-09-03 23:15:19 -07:00
rock	910f8e70d5	iam: move provisioning script to scripts/iam, remove k8s job - Move authentik-provision.py to scripts/iam/ (manual-only) - Remove job/RBAC resources (not needed for local runs) - Use public URL directly (no sed substitution needed) - Add app password support via set_key endpoint - Support both password grant and client_credentials	2026-09-03 19:03:58 -07:00