fix(cluster): grant queue-operator watches and retire the macOS workload

queue-operator could not list temporalworkers.kmsvc.io or deployments.apps at cluster scope, so its informers failed on every retry and it reconciled nothing. Add those rules to the ClusterRole. macos-bluebubbles pins itself to a node labelled workload=imessage, and no such node exists. Its PVC macos-disk uses StorageClass longhorn-imessage-local and stayed unbound for 71 days. Scale the Deployment to 0 and record why.
Merge pull request 'fix(sops): drop the orphan forgejo-registry-pat file' (#65 ) from fix/sops-drop-orphan-forgejo-pat into main
2026-10-24 04:03:02 -07:00 · 2026-10-24 10:47:14 +00:00 · 2026-10-24 03:46:29 -07:00 · 2026-10-24 10:41:17 +00:00 · 2026-10-24 10:41:09 +00:00 · 2026-10-24 03:38:21 -07:00
61 changed files with 3002 additions and 105 deletions
@@ -31,6 +31,17 @@ AUTHENTIK_BOOTSTRAP_TOKEN=
 #   openssl rand -base64 24
 AUTHENTIK_PG_PASSWORD=

+# ── ZITADEL (Authentik replacement) ─────────────────────────────────────────────
+# Instance URL, same hostname as the OIDC issuer.
+ZITADEL_URL="https://zitadel.riotpiao.com"
+
+# PAT of the 'iam-admin' machine user, created by the chart's setup job on the
+# first sync. Leave empty until then. Read it with:
+#   kubectl -n iam get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d
+# scripts/iam/zitadel-provision.py reads this variable first, then the cluster
+# Secret. The masterkey and the client secrets never live in this file.
+ZITADEL_BOOTSTRAP_TOKEN=
+
 # ── MinIO ──────────────────────────────────────────────────────────────────────
 MINIO_ROOT_USER=
 MINIO_ROOT_PASSWORD=
@@ -47,6 +47,11 @@ k8s/storage/test/test
 terraform/.terraform/
 terraform/*.tfstate
 terraform/*.tfstate.*
+# The Zitadel root module keeps its own state (see terraform/zitadel/providers.tf).
+# That state carries generated client secrets and machine keys.
+terraform/zitadel/*.tfstate
+terraform/zitadel/*.tfstate.*
+terraform/zitadel/*.tfvars
 terraform.tfvars.local
 skills-lock.json
 secrets-plaintext.yaml
@@ -69,3 +74,10 @@ k8s/**/*-secret.yaml

 # IAM provisioning scripts contain credential references — never commit
 scripts/iam/*.py
+# Exception: this one reads every credential from the cluster at run time and
+# embeds none. Keep it reviewable in git.
+!scripts/iam/zitadel-provision.py
+
+# Runtime credential export from the old Authentik provisioning run. Holds live
+# M2M tokens; never commit it.
+/authentik-m2m-tokens.env
@@ -4,7 +4,9 @@ metadata:
  name: comfyui
  namespace: comfyui
  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation and no tls: block. This host is covered by
+    # the *.riotpiao.com wildcard, which nginx serves as its
+    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
@@ -16,10 +18,6 @@ metadata:
      proxy_set_header Connection "upgrade";
 spec:
  ingressClassName: nginx
-  tls:
-    - secretName: comfyui-tls
-      hosts:
-        - comfyui.riotpiao.com
  rules:
    - host: comfyui.riotpiao.com
      http:
@@ -28,9 +28,14 @@ spec:
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
-      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
-      # above the 131072 target with room to spare -- the old
+      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
+      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
+      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
+      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
+      # metric after the rollout -- max_num_batched_tokens=8192 also raises
+      # activation memory out of the same budget, so the post-boot
+      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
+      # only valid if it still covers 2x65k + 3x8k. The old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -54,10 +59,37 @@ spec:
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
-      - --gpu-memory-utilization=0.90
-      - --max-num-seqs=4
+      - --gpu-memory-utilization=0.95
+      - --max-num-seqs=5
+      # Prefill scheduling -- see the measured starvation this fixes.
+      # V1's scheduler has no prefill/decode phase: it walks self.running and
+      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
+      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
+      # per request is applied BEFORE the budget clamp:
+      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
+      #     take = min(take, token_budget)
+      # The loop is `while ... and token_budget > 0`. With the threshold at its
+      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
+      # budget, token_budget hits 0, the loop exits and every other request is
+      # skipped for that step -- measured as 262 tok/s contended prefill and
+      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
+      # Hence threshold must be strictly BELOW the budget or it never binds:
+      # at 8192/8192 a single request is served per step (still starved), while
+      # 8192/2048 serves four. Verified by replaying the loop.
+      - --max-num-batched-tokens=8192
+      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
+      #
+      # NOT set, deliberately: --max-num-partial-prefills and
+      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
+      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
+      # rejects any value other than the default for either one, calling
+      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
+      # -> AssertionError, because there is no V0 engine to fall back to.
+      # The Deployment uses strategy Recreate, so setting them is an outage.
+      # V1 does concurrent chunked prefill natively; it needs no flag, only the
+      # budget/threshold ratio above.
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
@@ -12,6 +12,19 @@ rules:
  - apiGroups: ["kmsvc.io"]
    resources: ["queues"]
    verbs: ["get", "list", "watch", "update", "patch"]
+  # The operator watches TemporalWorker objects and creates Deployments for
+  # them. Without these two rules the controller logs
+  # "cannot list resource temporalworkers/deployments ... at the cluster scope"
+  # and never reconciles.
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
    resources: ["queues/status"]
    verbs: ["get", "update", "patch"]
@@ -48,7 +48,7 @@ spec:
                  mountPath: /backup
          containers:
            - name: mc-mirror
-              image: quay.io/minio/mc:latest
+              image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
              env:
                - name: ACCESS_KEY
                  valueFrom:
@@ -34,7 +34,12 @@ metadata:
    app.kubernetes.io/name: macos-bluebubbles
    app.kubernetes.io/part-of: sms
 spec:
-  replicas: 1
+  # Scaled to 0 on 2026-10-24. The pod pinned itself to a node labelled
+  # workload=imessage, and no such node exists any more. The PVC macos-disk also
+  # uses StorageClass longhorn-imessage-local, which pins the same dead node and
+  # stayed unbound for 71 days. The macOS guest therefore cannot start. Raise
+  # this to 1 again only after a KVM-capable node carries that label and taint.
+  replicas: 0
  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
  # concurrently — two QEMU processes on one image corrupts it.
  strategy:
@@ -249,3 +249,26 @@ spec:
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: local-path-provisioner
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/local-path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: local-path-storage
+  # No CreateNamespace: the Namespace object (with its privileged PSA label,
+  # which the helper Pods need for their hostPath) is part of the manifest.
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
@@ -57,6 +57,55 @@ spec:
      prune: true
      selfHeal: true
 ---
+# ZITADEL — the Authentik replacement. Coexistence, not cutover: Authentik keeps
+# serving in the same `iam` namespace until the last consumer moves, then this
+# becomes the only IdP. Migration plan (outside git):
+# ~/workplace/homelab-doc/ZITADEL_MIGRATION.html
+#
+# Wave 3 ordering: sops-secrets (wave 0) creates the `zitadel-masterkey` Secret,
+# databases (wave 2) creates the `zitadel-db` CNPG cluster and its
+# `zitadel-db-app` DSN secret. Both must exist before the init job starts.
+#
+# automated sync is deliberately absent for the first sync — same reason as
+# forgejo. The chart ships `pre-install,pre-upgrade` Helm hooks (zitadel-init,
+# zitadel-setup), which ArgoCD maps to PreSync and re-runs on every sync, manual
+# or automated. A human reads the first sync's hook output before any repeat can
+# run. The init job runs `init zitadel` only (CNPG owns the database and the app
+# user), and the setup job creates the first org and the `iam-admin` machine key
+# that Terraform consumes. Read the SECOND-SYNC HAZARD block at the top of
+# k8s/infra/iam/zitadel-values.yaml before enabling automated sync.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: zitadel
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://charts.zitadel.com
+      chart: zitadel
+      # Pinned: the chart trails the upstream app release, and the verified
+      # masterkey/passwap facts were read at v4.19.x. A floating tag would
+      # change the app version without a diff anyone reviews.
+      targetRevision: 10.4.0
+      helm:
+        valueFiles:
+          - $values/k8s/infra/iam/zitadel-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: iam
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+---
 # Raw IAM manifests: key-rotation cronjob + authentik migration job.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: oidc-bootstrap
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/oidc-bootstrap
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: etcd-defrag-creds
+    namespace: kube-system
+    labels:
+        app.kubernetes.io/name: etcd-defrag
+type: Opaque
+stringData:
+    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
+    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
+    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
+            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
+            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
+            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
+            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T05:37:26Z"
+    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
+    version: 3.13.2
@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:ECE=,iv:bISz4HovH++X7DW1Qj8Cw0L6+EvPB0+68hh7tfyW5C0=,tag:w+SjD7MsfeIuSf62n+Zl7Q==,type:str]
-kind: ENC[AES256_GCM,data:AxXaV3Mb,iv:xfJb354Rrjz3zLctW2i6hl40yh9EsfIqKxDvmn8jqnU=,tag:1xQ/RR3KhPLKCTWCYjebJg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:gdQ4OoMYKUPEbsUeA5OI4i1xnh8=,iv:IA2kaCyqQmkYpLolvcTF4aleh+yd/ImXJMhRMvpGCgo=,tag:8qyYYw4EhBKKPzEmpepAeQ==,type:str]
-    namespace: ENC[AES256_GCM,data:/mOSyXWmRg==,iv:cpeHUTlMqlJzJttGtuR3DoiMtvVqFmDS0/5Tl7K7c2M=,tag:q4GxWgWn2wQJxJHqnq4WQA==,type:str]
-type: ENC[AES256_GCM,data:92OcrsuV,iv:Z1XBy6iZ6unGrK4/SSdDa58pbPL3022gU/f1AOM4uvc=,tag:cd4wWRq/ohm6BJ/Eo6HOIw==,type:str]
-stringData:
-    REGISTRY_PAT: ENC[AES256_GCM,data:zDKruOXhzsIcFZTTp4r4r9rwHih9uf1cCp06KZ6eIXSIZDvR6aQY4A==,iv:5Vch5Z7xn1KkRxrgOs2p/M58nd6WhtcuUhoZDCi8LSY=,tag:14SAZIWGOxJMqLBtOorjYQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIaG5QVUYyNjhvaENPcUFy
-            K3RSSVM2N2hTeFhNK0M5YWZmODhzZU1HaHlFCmNGZWdFcFEvanVwcXpGSmVvRHVx
-            aG45c2lBY1RuSTYwbDZTVk1QeHNWOTQKLS0tIDg4TnVMNjNtaU1VQk5zQjUvU2hM
-            R245ZVdqc0ZWWVhJb3dOZ3lpU3JjUlUKawSg09ZPq8FKx5tvOVZZ+K4yh7eTQsUp
-            be8mWUpS0+eEmNqh35BwU3HrETMQFA6a1kjVp30JOMtqa5rbYlzF7w==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-23T23:14:53Z"
-    mac: ENC[AES256_GCM,data:yzi6woUaUCi6w9pG/eKnU7k/VfZgoXg/tW8p9joG8p7iabLWMdlXlx23m/CItw/NE0zeGWA5iZPPFZXOit2vN36VzK3kQNoFW6QbhvYLZi+78C/RWIdcpZkxB/tPIRG0vq8Q1SA+rwGjG/0xeAFh+R7k+YBTMd2XWBH3P7EI5T4=,iv:kf6MDaAVrtDvPIEjHMMLxXDSDRC3I1GpsPeJFUYppiw=,tag:NukVbGLa9EoMYmRsa4nBtA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2
@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: forgejo-registry-token
+    namespace: cicd
+data:
+    username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
+    password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
+            ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
+            cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
+            WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
+            SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
+    version: 3.13.2
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: memory-agent-oidc
+    namespace: poimen
+data:
+    CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
+    CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
+    ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
+    TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
+            Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
+            K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
+            cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
+            es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
+    version: 3.13.2
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: oauth2-credentials
+    namespace: iam
+data:
+    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
+    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
+    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
+    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
+            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
+            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
+            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
+            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
+    version: 3.13.2
@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: pi-code-agent-api-keys
+    namespace: poimen-sandbox
+stringData:
+    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
+    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
+            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
+            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
+            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
+            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T01:19:49Z"
+    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
+    version: 3.13.2
@@ -7,29 +7,34 @@ metadata:
      exec:
        path: ksops
 files:
-  - agent-pod-models.enc.yaml
-  - agent-pod-ssh-key.enc.yaml
-  - authentik-secrets.enc.yaml
-  - cloudflare-secrets.enc.yaml
-  - forgejo-registry-pat.enc.yaml
-  - forgejo-registry-pat-api.enc.yaml
-  - forgejo-registry-pull.enc.yaml
-  - forgejo-runner-token.enc.yaml
-  - forgejo-secrets.enc.yaml
-  - grafana-oidc-secrets.enc.yaml
-  - grafana-secrets.enc.yaml
-  - homarr-auth-oidc.enc.yaml
-  - homarr-db-encryption.enc.yaml
-  - homarr-secrets.enc.yaml
-  - homelab-ca-secrets.enc.yaml
-  - loki-secrets.enc.yaml
-  - minio-secrets.enc.yaml
-  - paperless-secrets.enc.yaml
-  - vault-secrets.enc.yaml
-  - vault-unseal-keys.enc.yaml
-  - portfolio-secrets.enc.yaml
-  - gotify-admin-secrets.enc.yaml
-  - gotify-tokens-secrets.enc.yaml
-  - gotify-smtp-secrets.enc.yaml
-  - forgejo-smtp-secrets.enc.yaml
-  - paperless-ai-secrets.enc.yaml
+    - agent-pod-models.enc.yaml
+    - agent-pod-ssh-key.enc.yaml
+    - authentik-secrets.enc.yaml
+    - cloudflare-secrets.enc.yaml
+    - forgejo-registry-pat-api.enc.yaml
+    - forgejo-registry-pull.enc.yaml
+    - forgejo-runner-token.enc.yaml
+    - forgejo-secrets.enc.yaml
+    - grafana-oidc-secrets.enc.yaml
+    - grafana-secrets.enc.yaml
+    - homarr-auth-oidc.enc.yaml
+    - homarr-db-encryption.enc.yaml
+    - homarr-secrets.enc.yaml
+    - homelab-ca-secrets.enc.yaml
+    - loki-secrets.enc.yaml
+    - minio-secrets.enc.yaml
+    - paperless-secrets.enc.yaml
+    - vault-secrets.enc.yaml
+    - vault-unseal-keys.enc.yaml
+    - portfolio-secrets.enc.yaml
+    - gotify-admin-secrets.enc.yaml
+    - gotify-tokens-secrets.enc.yaml
+    - gotify-smtp-secrets.enc.yaml
+    - forgejo-smtp-secrets.enc.yaml
+    - paperless-ai-secrets.enc.yaml
+    - memory-agent-oidc.enc.yaml
+    - oauth2-credentials.enc.yaml
+    - forgejo-registry-token.enc.yaml
+    - pi-sandbox-secrets.enc.yaml
+    - etcd-defrag-creds.enc.yaml
+  - zitadel-masterkey.enc.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: zitadel-masterkey
+    namespace: iam
+type: Opaque
+stringData:
+    masterkey: ENC[AES256_GCM,data:dGp0wSCUJORP6W5PHLKYaQEqrZsgiAwU1GqcuvuZV+g=,iv:Wwt6E/XXwsoVEQxe4gG1m5oJLOcNUXngftADQvZQepw=,tag:1Vget5gs+t455U9zLavdwg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBld0tUWHNWem13cDlxRG14
+            QWRlTjA5SzZnS2pBd2M0WWkzQWxKZW9HZWdJCk00QWJJcWZFM1ZTSUlReUlGZG4z
+            RFBSUWRlVUx2NGN2OFAwazBYRTVVNkEKLS0tIHpSSW9IWHhNa0d3SmEydXU4akVj
+            MGJkN2U4WmtLbVFMZm55U1hEZElZMlUKKV9twnC9GC7MSmNLxEBHbPl4RN5Hsd+z
+            NViCKxUoQAB8apn2XVHZcDnyfKNfr1P3aSa3IIHyVY3prgo1OFDpdw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-24T08:57:27Z"
+    mac: ENC[AES256_GCM,data:hXDAddVmnlf/yinY2c2Q8yfsjqi6n9cRbEbwykqYScd6mDPfdF7BriTiu9K+CV/IvIT4zcMTZbhBeQ/vTCAh0TZDp1lOVglwtNH/imgagyzPmcCvbCIkZKwF5vwWQIGHcDI6pk4/zVQkqt7zKTf8+Qj0pNh5UrAjVeEFE7cuJkQ=,iv:oIQvROFGbiESmMTthHOJiNM+Gy5BCfM5XtDWbszOPmc=,tag:AaYrJrwXuKD7w0p+TTa7FA==,type:str]
+    version: 3.13.2
@@ -8,6 +8,19 @@
 # depend on them existing first.
 installCRDs: true

+# Point the ACME DNS-01 self-check at the public resolvers directly.
+#
+# Without these flags cert-manager resolves through the pod resolver
+# (CoreDNS at 10.96.0.10). The self-check then never saw the TXT record and
+# reported "Waiting for DNS-01 challenge propagation" forever. The wildcard
+# certificate renewal stalled on 2026-09-23 and the cert expired on
+# 2026-10-23, which broke every Cloudflare Tunnel origin with
+# "x509: certificate has expired".
+#
+extraArgs:
+  - --dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53
+  - --dns01-recursive-nameservers-only
+
 # Tolerate the control-plane taint (talos-cp-1 is the only schedulable node).
 tolerations:
  - key: node-role.kubernetes.io/control-plane
@@ -1,13 +1,21 @@
 # k8s/bootstrap/ingress/riotpiao-com-cert.yaml
 # Public wildcard cert for *.riotpiao.com, signed by Let's Encrypt (DNS-01 via
-# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml). Distinct from
-# homelab-tls (self-signed homelab-ca, internal *.riotpiao.com) —
-# reference riotpiao-com-tls explicitly on ingresses that expose riotpiao.com
-# hosts to the internet; it is not nginx's default-ssl-certificate.
+# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml).
 #
-# issuerRef starts on letsencrypt-staging. Switch to letsencrypt-prod once the
-# staging cert issues cleanly, then delete this Certificate's Secret to force
-# a re-issue against prod (staging certs aren't browser-trusted).
+# nginx serves this secret as its default-ssl-certificate
+# (../ingress/nginx-values.yaml), so every Ingress that omits a tls: block uses
+# it. Do not add per-host tls: blocks for *.riotpiao.com hosts: they create a
+# second, single-host certificate and split the wildcard coverage.
+#
+# Renewal is automatic and does not drop traffic:
+#   - cert-manager starts a new order 30 days before expiry (renewBefore).
+#   - cert-manager writes the renewed pair into the same secret.
+#   - ingress-nginx watches the secret and hot-reloads. No pod restart.
+#   - The old pair stays valid until the new one is written, so no gap exists.
+#
+# The DNS-01 self-check must reach the public resolvers directly. Without the
+# --dns01-recursive-nameservers flags in ../../cert-manager/cert-manager-values.yaml
+# the self-check never sees the TXT record and renewal stalls until expiry.
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
@@ -45,7 +45,13 @@ gitea:
      SSH_DOMAIN: git.riotpiao.com
      SSH_PORT: 2222
      SSH_LISTEN_PORT: 2222
-      
+
+    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
+    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
+    # the admin only; admin create-user via UI/API is unaffected by this flag.
+    service:
+      DISABLE_REGISTRATION: true
+
    database:
      DB_TYPE: postgres
      HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -101,7 +107,9 @@ ingress:
  enabled: true
  className: nginx
  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation. forgejo.riotpiao.com is covered by the
+    # *.riotpiao.com wildcard, served as nginx's default-ssl-certificate.
+    #
    # This chart Ingress and the hand-written one in
    # k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com.
    # ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's
@@ -119,10 +127,9 @@ ingress:
      paths:
        - path: /
          pathType: Prefix
-  tls:
-    - secretName: forgejo-tls
-      hosts:
-        - forgejo.riotpiao.com
+  # tls: intentionally omitted. The chart would create a second, single-host
+  # certificate (forgejo-tls) that expired on 2026-10-23 and competed with the
+  # wildcard. The wildcard covers this host.

 # Resources
 resources:
@@ -0,0 +1,177 @@
+# etcd backend maintenance: compact, defrag, disarm NOSPACE.
+#
+# Why this exists
+# ---------------
+# etcd's backend file only grows. The apiserver compacts every 5m, but
+# compaction marks superseded revisions free *inside* the file; only `defrag`
+# returns that space to the filesystem. Nothing ran defrag, so the file grew
+# until it crossed the 2 GiB quota:
+#
+#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
+#               -> etcd raised NOSPACE on those two members
+#               -> etcd rejected ALL writes cluster-wide
+#               -> kubelets could not post node status: 3 nodes NotReady
+#               -> apiserver writes returned 500, the tunnel API path 502
+#
+# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
+# dead pages. Recovery was `compact <current-revision>` then `defrag` per
+# member then `alarm disarm`. Compaction had also stalled while the alarm was
+# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
+# worked. Order matters: compact first, then defrag.
+#
+# The earlier revision of this file never ran, for two independent reasons:
+#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
+#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
+#      cluster never had the CronJob or the Secret.
+#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
+#      `exec: "/bin/sh": no such file or directory`). Any container that runs
+#      a shell script needs an image that ships one.
+# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
+# 3.6.4, matching the 3.6.11 servers) pinned by digest.
+#
+# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
+# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
+# and the script exits unless the UTC epoch day is a multiple of 15. That gives
+# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
+# on the 1st, 16th and 31st, which puts two runs one day apart).
+#
+# Safety
+# ------
+# 1. Compaction uses the MINIMUM revision across members. Compacting to a
+#    revision a member has not reached wedges that member - measured on
+#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
+#    needed a reboot to rejoin.
+# 2. A member whose raft applied index lags the leader by more than 100000 is
+#    skipped and reported. Defragging a frozen member reclaims nothing.
+# 3. Members are defragged one at a time. A defrag pauses the member's apply
+#    loop for the duration of the file copy.
+#
+# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
+# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
+# Its client certificate is signed by the cluster's etcd CA and expires
+# 2036-10; rotate it before then by re-running the mint and re-encrypting.
+#
+# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
+# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-defrag
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: etcd-defrag
+    app.kubernetes.io/part-of: cluster-maintenance
+spec:
+  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 600
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: defrag
+              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
+              env:
+                - name: ETCDCTL_API
+                  value: "3"
+                - name: ETCDCTL_CACERT
+                  value: /etc/etcd/ca.crt
+                - name: ETCDCTL_CERT
+                  value: /etc/etcd/client.crt
+                - name: ETCDCTL_KEY
+                  value: /etc/etcd/client.key
+                - name: EPS
+                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  set -u
+
+                  DAY=$(( $(date -u +%s) / 86400 ))
+                  if [ $(( DAY % 15 )) -ne 0 ]; then
+                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
+                    exit 0
+                  fi
+
+                  echo "== before =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+
+                  MINREV=""
+                  MAXAPPLIED=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
+                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
+                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
+                      MINREV="$rev"
+                    fi
+                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
+                      MAXAPPLIED="$app"
+                    fi
+                  done
+
+                  if [ -z "$MINREV" ]; then
+                    echo "FATAL: no member returned a revision; nothing attempted"
+                    exit 1
+                  fi
+
+                  echo "== compact to revision $MINREV =="
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
+
+                  # Physical deletion of superseded keys runs in the background
+                  # after the compaction commit; give it a moment to drain.
+                  sleep 20
+
+                  echo "== defrag each member (sequential, one at a time) =="
+                  rc=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
+                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
+                      rc=1
+                      continue
+                    fi
+                    echo "-- $ep"
+                    if ! etcdctl --endpoints="$ep" defrag; then
+                      echo "DEFRAG-FAILED $ep"
+                      rc=1
+                    fi
+                  done
+
+                  echo "== disarm NOSPACE =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
+                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
+                  done
+
+                  echo "== after =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
+                  exit $rc
+              resources:
+                requests: { cpu: 10m, memory: 32Mi }
+                limits:   { cpu: 200m, memory: 128Mi }
+              volumeMounts:
+                - name: creds
+                  mountPath: /etc/etcd
+                  readOnly: true
+          volumes:
+            - name: creds
+              secret:
+                secretName: etcd-defrag-creds
+                defaultMode: 0444
@@ -7,5 +7,6 @@ kind: Kustomization
 resources:
  - namespaces.yaml
  - authentik-db.yaml
+  - zitadel-db.yaml
  - temporal-db.yaml
  - paperless-db.yaml
@@ -0,0 +1,42 @@
+# Dedicated CNPG Postgres for ZITADEL (GitOps, wave 2 — before the zitadel
+# chart at wave 3). Same shape as authentik-db.yaml: CNPG generates the secret
+# `zitadel-db-app` and the service `zitadel-db-rw` in ns iam, and
+# k8s/infra/iam/zitadel-values.yaml reads the DSN from that secret locally (no
+# cross-namespace secret copy).
+#
+# ZITADEL requires PostgreSQL 14+; the repo convention image is 16.2.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: zitadel-db
+  namespace: iam
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: zitadel
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 10Gi
+    storageClass: longhorn-cnpg
+  monitoring:
+    enablePodMonitor: true
+  affinity:
+    # preferred + tolerations: HA across nodes without deadlocking a partly-
+    # schedulable 3-CP (same as authentik-db and forgejo-db).
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
@@ -0,0 +1,134 @@
+# k8s/infra/iam/zitadel-values.yaml
+# ZITADEL — Authentik replacement. Coexistence is deliberate: Authentik keeps
+# running, read-only, in the same `iam` namespace until the last consumer moves.
+# Migration plan: ~/workplace/homelab-doc/ZITADEL_MIGRATION.html (outside git).
+#
+# Chart: zitadel/zitadel 10.4.0, app v4.19.2. The plan cites v4.19.4; the chart
+# has not published it yet. Pin the chart, never `*`, until the v4.19.4 chart
+# exists — a silent app bump invalidates the verified masterkey/passwap facts.
+#
+# URL: https://zitadel.riotpiao.com. TLS terminates at ingress-nginx, which
+# serves the wildcard riotpiao-com-tls as its default-ssl-certificate, so
+# ExternalSecure=true and TLS.Enabled=false (no per-rule tls: block needed).
+#
+# Database: CNPG cluster zitadel-db (wave 2). The DSN comes from CNPG's
+# generated `zitadel-db-app` secret. When ZITADEL_DATABASE_POSTGRES_DSN is set,
+# ZITADEL ignores the discrete Database.Postgres.* fields, so neither a
+# password nor a host is written into this file.
+#
+# Masterkey: Secret `zitadel-masterkey` (k8s/argocd/secrets/
+# zitadel-masterkey.enc.yaml), key `masterkey`, exactly 32 bytes. It wraps the
+# rows in system.encryption_keys, which encrypt every client secret and private
+# key. Losing it makes those values unreadable: keep the offline sealed copy.
+#
+# Ingress uses the chart's own objects (controller: nginx adds the gRPC backend
+# annotation ZITADEL needs for its HTTP/2 API). Do not hand-write a second
+# Ingress for this host.
+
+zitadel:
+  configmapConfig:
+    ExternalSecure: true
+    ExternalDomain: zitadel.riotpiao.com
+    TLS:
+      Enabled: false
+
+  # Rendered into a Secret by the chart; read by pods and init jobs alike.
+  masterkeySecretName: zitadel-masterkey
+
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+
+  # FirstInstance defaults stay for the FIRST sync: the chart creates org
+  # `zitadel` plus the `iam-admin` machine user and its machine-key Secret.
+  # Terraform consumes that key to declare projects, roles, OIDC apps and the
+  # groups action.
+  #
+  # SECOND-SYNC HAZARD — read before enabling automated sync.
+  # The chart marks eight objects as `pre-install,pre-upgrade` hooks (two
+  # ServiceAccounts, two ConfigMaps, Role, RoleBinding, zitadel-init,
+  # zitadel-setup), and one cleanup Job as `post-delete`. ArgoCD maps the first
+  # eight to PreSync with `before-hook-creation`: it deletes and re-creates them
+  # on EVERY sync, not only on a chart change.
+  #   - zitadel-init  (`init zitadel`) — schema step
+  #   - zitadel-setup (`setup`)        — first instance + iam-admin machine key
+  # On an initialized instance the setup step can fail, which aborts the sync
+  # before the Deployment is applied. If it succeeds instead, its
+  # machinekeyWriter container re-applies the iam-admin key Secret that
+  # Terraform reads, which silently rotates a credential in use.
+  # After the first sync reports healthy:
+  #   1. set configmapConfig.FirstInstance.Skip: true here — permanently: the
+  #      setup steps are schema-only once the instance exists, and Skip alone
+  #      stops the machinekeyWriter rotation
+  #   2. set initJob.enabled: false below
+  #   3. commit, sync once, then enable automated+selfHeal on the Application
+  # For a later schema upgrade, re-enable initJob.enabled only — never set
+  # FirstInstance.Skip back to false.
+  # ArgoCD never prunes a hook Job, so a Job that failed before step 2 stays and
+  # keeps ZitadelSetupJobFailed firing until a human deletes it.
+
+# DSN mode (chart renders this through the zitadel.dbEnv helper). CNPG's `uri`
+# key is a postgresql:// URL, which ZITADEL accepts as a DSN.
+env:
+  - name: ZITADEL_DATABASE_POSTGRES_DSN
+    valueFrom:
+      secretKeyRef:
+        name: zitadel-db-app
+        key: uri
+
+# Bundled Bitnami Postgres is disabled: the CNPG cluster above is the system of
+# record. Enabling it would create a second, unmanaged database.
+postgresql:
+  enabled: false
+
+# CNPG already created the database, the `app` user and its grants, so the init
+# job runs only the ZITADEL-internal step. Running the full default command
+# would try to create a database and a superuser it does not own.
+initJob:
+  command: "zitadel"
+  activeDeadlineSeconds: 600
+
+# Two API replicas: the read path survives one node drain, and both share the
+# same masterkey and database. The login UI stays at one replica — a session
+# cookie is not shared, so a second pod buys nothing.
+replicaCount: 2
+revisionHistoryLimit: 3
+
+ingress:
+  enabled: true
+  className: nginx
+  controller: nginx
+
+login:
+  enabled: true
+  replicaCount: 1
+  ingress:
+    enabled: true
+    className: nginx
+
+# Chart-owned ServiceMonitor. The cluster's Prometheus runs with
+# serviceMonitorSelectorNilUsesHelmValues=false, so the release label is not
+# required for discovery. Endpoint is /debug/metrics on port http2-server.
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    scrapeInterval: 30s
+    scrapeTimeout: 10s
+
+# The 3-CP / 2-worker cluster mixes control-plane taints with worker capacity.
+# Without the toleration a rolling upgrade can leave both replicas Pending.
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+affinity:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: zitadel
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No commonLabels on purpose: this is a vendored third-party manifest and
+# kustomize would also rewrite the provisioner Deployment's selector, which
+# is immutable on re-apply.
+resources:
+  - local-path-storage.yaml
@@ -0,0 +1,200 @@
+# Vendored from https://github.com/rancher/local-path-provisioner
+#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
+#
+# Local modifications (the only three; re-apply them on any re-vendor):
+#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
+#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
+#      baseline policy, which forbids that;
+#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
+#      filesystem read-only, so upstream's /opt path cannot be created;
+#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: local-path-provisioner-role
+  namespace: local-path-storage
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-path-provisioner-bind
+  namespace: local-path-storage
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+        - name: local-path-provisioner
+          image: docker.io/rancher/local-path-provisioner:v0.0.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - local-path-provisioner
+            - --debug
+            - start
+            - --config
+            - /etc/config/config.json
+          ports:
+            - name: health
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 12
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config/
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_MOUNT_PATH
+              value: /etc/config/
+            - name: HEALTH_PORT
+              value: "8080"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-path
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["/var/lib/local-path-provisioner"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node.kubernetes.io/disk-pressure
+          operator: Exists
+          effect: NoSchedule
+      containers:
+      - name: helper-pod
+        image: docker.io/library/busybox:1.36
+        imagePullPolicy: IfNotPresent
@@ -41,13 +41,24 @@ spec:
          # - /var/lib/longhorn-disk1 (sdb1, 600GB)
          # - /var/lib/longhorn-disk2 (sdc1, 500GB)
          # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          # - /var/lib/longhorn-disk4 (sde1, 2TB)
+          #
+          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
+          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
+          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
+          # process that touched it blocked in unkillable D state. Those unkillable
+          # processes held Longhorn's per-node rebuild slots forever, so no new
+          # replica could start on talos-cp-2 and every new volume there failed to
+          # attach. It is removed from the Terraform disk list too, so a re-render
+          # does not put it back.
          
          echo "Checking talos-cp-2..."
          CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
          echo "  Current disk count: $CURRENT_DISKS"
          
-          if [ "$CURRENT_DISKS" -lt 5 ]; then
+          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
+          # existed; leaving the old threshold would re-apply the patch on every
+          # run once disk4 is gone.
+          if [ "$CURRENT_DISKS" -lt 4 ]; then
            echo "  Adding extra disks to talos-cp-2..."
            kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
              "spec": {
@@ -75,14 +86,6 @@ spec:
                    "path": "/var/lib/longhorn-disk3",
                    "storageReserved": 5368709120,
                    "tags": []
-                  },
-                  "disk4": {
-                    "allowScheduling": true,
-                    "diskType": "filesystem",
-                    "evictionRequested": false,
-                    "path": "/var/lib/longhorn-disk4",
-                    "storageReserved": 21474836480,
-                    "tags": []
                  }
                }
              }
@@ -52,14 +52,14 @@ spec:
      restartPolicy: Never
      initContainers:
        - name: kubectl-copy
-          image: bitnami/kubectl:latest
+          image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
          command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
          volumeMounts:
            - name: shared
              mountPath: /shared
      containers:
        - name: provision
-          image: quay.io/minio/mc:latest
+          image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
          volumeMounts:
            - name: shared
              mountPath: /shared
@@ -11,7 +11,15 @@ metadata:
    prometheus.io/port: "9000"
    prometheus.io/scrape: "true"
 spec:
-  image: minio/minio:RELEASE.2025-07-23T15-54-02Z
+  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
+  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
+  # backend could not resolve minio-cluster-hl. Official images now need registry auth
+  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
+  # release; swap it for the official image once MinIO registry credentials exist.
+  # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
+  # the pull again. The same digest already sits in the containerd cache on talos-cp-1,
+  # so the first start needs no registry access.
+  image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9

  # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
  # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -158,3 +158,66 @@ spec:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
+
+    - name: api-gateway.admission
+      # The admission gate fails OPEN when its view of the engine gauges goes
+      # stale, so a broken observer degrades silently to the pre-#64 behaviour
+      # of admitting everything. These two series are the only detection —
+      # the gate has no automated tests.
+      rules:
+        # The gate fails OPEN when it cannot read the engine, so "no data" must
+        # alert: otherwise a missing emitter, a renamed metric, or a broken
+        # scrape is indistinguishable from a healthy observer. Both expressions
+        # therefore OR in an absent() guard on the metric itself.
+        - alert: APIGatewayObserverDegraded
+          expr: observer_degraded == 1 or absent(observer_degraded)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
+            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
+
+        # Engine gauges went stale, or are not being reported at all
+        - alert: GatewayEngineStateStale
+          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
+            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
+
+        # The acceptance metric for reasoning.yaml's prefill change. Preemption
+        # of a 65k prompt costs a full re-prefill (~250s at the measured
+        # 262 tok/s), so any sustained rate is a regression, not noise.
+        - alert: LLMPreemptions
+          # Guarded absent() is deliberately NOT used bare here: the engine
+          # restarts (strategy Recreate) and the series legitimately vanishes
+          # for the length of a boot, so a bare absent() would false-fire on
+          # every rollout. It only fires when the metric is missing WHILE the
+          # predictor is up, which is the real "renamed or unscraped" case.
+          expr: |
+            increase(vllm:num_preemptions_total[15m]) > 0
+            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
+            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
+
+        # This change trades single-request latency for fairness: capping a long
+        # prompt at 2048 tokens/step means it takes more steps to finish, so a
+        # lone large prompt can now be slower even while preemptions stay 0.
+        # Without this alert the change has no way to show a regression.
+        - alert: LLMPrefillStarvation
+          expr: |
+            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
+            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
+            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."
@@ -7,19 +7,24 @@ metadata:
    release: prometheus
 spec:
  groups:
-    # Recording rules: derive vLLM prefix cache hit rate from raw counters
+    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
+    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
+    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
+    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
+    # prefix_cache_queries_total — so this rule produced no data and
+    # VLLMPrefixCacheHitRateLow could never fire.
    - name: vllm.recording
      rules:
        - record: vllm:prefix_cache_hit_rate
          expr: |
-            rate(vllm:cache_query_hit_total[5m])
+            rate(vllm:prefix_cache_hits_total[5m])
            /
            clamp_min(
-              rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]),
+              rate(vllm:prefix_cache_queries_total[5m]),
              0.001
            )
          labels:
-            job: "vllm"
+            job: "reasoning-predictor"

    # Alert rules: CAG pipeline health
    - name: cag.alerts
@@ -27,6 +27,24 @@ spec:
          annotations:
            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"

+        # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
+        - alert: PodImagePullFailure
+          expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
+
+        # Pod not ready
+        - alert: PodNotReady
+          expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
+
        # CrashLoopBackOff
        - alert: PodCrashLooping
          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
@@ -0,0 +1,68 @@
+# ZITADEL alert parity with the Authentik rules it will replace.
+#
+# Only two signals are ported at this point. The metrics endpoint
+# (/debug/metrics, scraped by the chart's own ServiceMonitor in ns iam) exposes
+# no request or error counters that this cluster has verified, so no error-rate
+# rule is written here. Writing one against a guessed series name would produce
+# a permanently-firing or permanently-silent alert — both worse than none.
+# AuthentikOutpostDown is deliberately not ported: it monitors an outpost that
+# carries zero providers and is deleted with the migration, not migrated.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: zitadel-rules
+  namespace: iam
+spec:
+  groups:
+    - name: zitadel.rules
+      interval: 30s
+      rules:
+        - alert: ZitadelDown
+          # absent() is required: equality alone cannot fire when the series is
+          # gone (ServiceMonitor deleted, Service selector changed, pods scaled
+          # to zero). Those are exactly the outages a target-down rule exists for.
+          expr: absent(up{job="zitadel"}) or up{job="zitadel"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "ZITADEL API target is down"
+            description: "Prometheus cannot scrape the ZITADEL API for 10 minutes. Token issuance and every OIDC login depend on it. Check the zitadel pods and the zitadel-db cluster in namespace iam. Value: {{ $value }}"
+
+        - alert: ZitadelLoginUIDown
+          expr: absent(kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"}) or kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"} == 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ZITADEL login UI has no available replica"
+            description: "The login UI serves /ui/v2/login. With zero available replicas a browser login completes at the IdP but cannot render. Value: {{ $value }}"
+
+        - alert: ZitadelRestarting
+          # pod!~"zitadel-db-.*" matters: the CNPG database pods share the
+          # `zitadel` name prefix, and a database restart must not page as an
+          # IdP crash loop.
+          expr: absent(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}) or increase(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}[1h]) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ZITADEL pod restarts more than 3 times per hour"
+            description: "A crash loop on the IdP is a cluster-wide login outage. Check the pod logs and the masterkey/database secret references. Value: {{ $value }}"
+
+        # The chart's zitadel-init and zitadel-setup Jobs are PreSync hooks with
+        # restartPolicy: Never, so their restart counter stays 0. Without this
+        # rule a failed schema init or first-instance setup reaches nobody but
+        # the person who runs `kubectl -n iam get job`.
+        # No absent() guard here on purpose: the chart ships only
+        # `before-hook-creation`, so a successful hook Job is replaced at the
+        # next PreSync. Once the jobs are retired, absent() would fire on a
+        # healthy cluster.
+        - alert: ZitadelSetupJobFailed
+          expr: kube_job_status_failed{namespace="iam",job_name=~"zitadel-(init|setup).*"} > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "ZITADEL init/setup job failed"
+            description: "A PreSync job failed, so ArgoCD aborted the sync before the Deployment was applied. Read the job pod logs, then delete the Job object to let the next sync retry. Value: {{ $value }}"
@@ -26,20 +26,23 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
    return p


-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
    targets = []
    for e in exprs:
        if isinstance(e, tuple):
            targets.append({"expr": e[0], "legendFormat": e[1]})
        else:
            targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    return {
+    p = {
        "id": id, "title": title, "type": "timeseries",
        "gridPos": {"h": h, "w": w, "x": x, "y": y},
        "datasource": DS_PROM,
        "fieldConfig": {"defaults": {"unit": unit}},
        "targets": targets,
    }
+    if description:
+        p["description"] = description
+    return p


 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -265,6 +268,237 @@ def build_api_gateway():
    }


+# ============================================================================
+# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
+# ============================================================================
+#
+# Two independent metric sources, deliberately both shown:
+#
+#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
+#                               that expose /metrics. Ollama (ornith:35b,
+#                               qwen2.5:3b-instruct) exposes nothing at all.
+#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
+#                               ones, derived from the SSE byte stream: TTFT is
+#                               request-start to first byte, ITL is the gap
+#                               between SSE writes carrying "data: ", so it
+#                               counts SSE chunks rather than true tokens.
+#                               vLLM's own histograms remain authoritative for
+#                               `reasoning`.
+#
+# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
+# verified against the running gateway and Prometheus:
+#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
+#
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
+# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
+# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
+# homelab-frontend#63 fixed exactly that — the wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
+# The "dead code" note that used to live in this header is superseded.
+#
+# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
+# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
+#
+#
+# All metric names below were verified against a live scrape, not assumed:
+#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
+#   NOT cache_query_hit_total / cache_query_miss_total.
+
+def build_llm_observability():
+    def hq(metric, q, by="le"):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
+            f"p{int(q * 100)}",
+        )
+
+    def hq_gw(metric, q):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
+            f"p{int(q * 100)} · {{{{model}}}}",
+        )
+
+    def avg_of(metric):
+        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
+        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
+
+    panels = []
+
+    # ── TTFT ────────────────────────────────────────────────────────────────
+    panels.append(row(100, "Latency — Time To First Token", 0, [
+        stat_panel(101, "TTFT p95 (gateway)",
+                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
+                   0, 1, w=4, h=4, unit="s"),
+        stat_panel(102, "TTFT p95 (vLLM)",
+                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
+                   4, 1, w=4, h=4, unit="s"),
+        stat_panel(103, "KV cache in use",
+                   "max(vllm:kv_cache_usage_perc)",
+                   8, 1, w=4, h=4, unit="percentunit"),
+        stat_panel(104, "Requests running",
+                   "sum(vllm:num_requests_running)",
+                   12, 1, w=4, h=4, unit="short"),
+        stat_panel(105, "Requests waiting",
+                   "sum(vllm:num_requests_waiting)",
+                   16, 1, w=4, h=4, unit="short"),
+        stat_panel(106, "Preemptions (5m)",
+                   "increase(vllm:num_preemptions_total[5m])",
+                   20, 1, w=4, h=4, unit="short"),
+        ts_panel(110, "TTFT percentile by model (gateway)",
+                 [hq_gw("llm_ttft_seconds", 0.50),
+                  hq_gw("llm_ttft_seconds", 0.95),
+                  hq_gw("llm_ttft_seconds", 0.99)],
+                 0, 2, w=12, h=8, unit="s"),
+        ts_panel(111, "TTFT percentile — vLLM reasoning",
+                 [hq("vllm:time_to_first_token_seconds", 0.50),
+                  hq("vllm:time_to_first_token_seconds", 0.95),
+                  hq("vllm:time_to_first_token_seconds", 0.99)],
+                 12, 2, w=12, h=8, unit="s"),
+        ts_panel(112, "Prefill vs decode (mean)",
+                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
+                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
+                 0, 3, w=12, h=8, unit="s"),
+        ts_panel(113, "Queue wait before scheduling (mean)",
+                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
+                 12, 3, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Inter-token latency ─────────────────────────────────────────────────
+    panels.append(row(120, "Latency — Inter-Token", 4, [
+        ts_panel(121, "Inter-token latency by model (gateway)",
+                 [hq_gw("llm_itl_seconds", 0.50),
+                  hq_gw("llm_itl_seconds", 0.95),
+                  hq_gw("llm_itl_seconds", 0.99)],
+                 0, 5, w=12, h=8, unit="s"),
+        ts_panel(122, "Inter-token latency — vLLM reasoning",
+                 [hq("vllm:inter_token_latency_seconds", 0.50),
+                  hq("vllm:inter_token_latency_seconds", 0.95),
+                  hq("vllm:inter_token_latency_seconds", 0.99)],
+                 12, 5, w=12, h=8, unit="s"),
+        ts_panel(123, "Decode throughput (tokens/s per request)",
+                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
+                 0, 6, w=12, h=8, unit="short"),
+        ts_panel(124, "End-to-end request latency p50/p95/p99",
+                 [hq("vllm:e2e_request_latency_seconds", 0.50),
+                  hq("vllm:e2e_request_latency_seconds", 0.95),
+                  hq("vllm:e2e_request_latency_seconds", 0.99)],
+                 12, 6, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Batching / scheduling ───────────────────────────────────────────────
+    panels.append(row(130, "Batching & Scheduling", 7, [
+        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
+                 [("sum(vllm:num_requests_running)", "running (in batch)"),
+                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
+                 0, 8, w=12, h=8, unit="short"),
+        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
+                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
+                 12, 8, w=12, h=8, unit="short"),
+        ts_panel(133, "TEI batch size (embeddings / reranker)",
+                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
+                   "{{app}}")],
+                 0, 9, w=12, h=8, unit="short"),
+        ts_panel(134, "TEI queue depth",
+                 [("max(te_queue_size) by (app)", "{{app}}")],
+                 12, 9, w=12, h=8, unit="short"),
+    ]))
+
+    # ── KV cache ────────────────────────────────────────────────────────────
+    panels.append(row(140, "KV Cache", 10, [
+        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
+                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
+                 0, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
+                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
+                 12, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(143, "Prefix cache hits vs queries",
+                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
+                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
+                 0, 12, w=12, h=8, unit="short"),
+        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
+                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
+                 12, 12, w=12, h=8, unit="short"),
+    ]))
+
+    # ── Throughput & health ─────────────────────────────────────────────────
+    panels.append(row(150, "Throughput & Health", 13, [
+        ts_panel(151, "Token throughput",
+                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
+                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
+                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
+                 0, 14, w=12, h=8, unit="short"),
+        ts_panel(152, "Successful requests/s",
+                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
+                  ("sum(rate(te_request_count[5m]))", "TEI")],
+                 12, 14, w=12, h=8, unit="short"),
+        ts_panel(153, "Backpressure — queue-full rejects",
+                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
+                 0, 15, w=8, h=8, unit="short"),
+        ts_panel(154, "KV warmup requests delayed",
+                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
+                 8, 15, w=8, h=8, unit="short"),
+        ts_panel(155, "Embedding / rerank latency p95",
+                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
+                   "{{app}} embed"),
+                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
+                   "{{app}} queue")],
+                 16, 15, w=8, h=8, unit="s"),
+    ]))
+
+    # ── Admission control ───────────────────────────────────────────────────
+    # These four panels read the gateway's admission gate, which is the
+    # companion change in homelab-frontend#64. That emitter is NOT deployed
+    # yet (verified against the live /metrics endpoint: 0 series matching
+    # ^(admission_|observer_|engine_state_)), so they render "No data" until
+    # #64 ships. That is expected, and each panel carries a description
+    # saying so — an unexplained blank panel would read as a broken
+    # dashboard, which is exactly the failure mode this row exists to catch.
+    panels.append(row(160, "Admission Control", 16, [
+        ts_panel(161, "Admission parked (5m)",
+                 [("increase(admission_parked_total[5m])", "parked")],
+                 0, 17, w=8, h=8, unit="short",
+                 description="Requests the gateway held back rather than admitting. "
+                             "Emitted by the admission gate (homelab-frontend#64); "
+                             "empty until that change is deployed."),
+        ts_panel(162, "Admission rejected by reason",
+                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
+                 8, 17, w=8, h=8, unit="short",
+                 description="Admission rejections split by the gate's own reason label. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(163, "Engine state stale",
+                 [("increase(engine_state_stale_total[5m])", "stale")],
+                 16, 17, w=8, h=8, unit="short",
+                 description="Times the gateway's view of the engine gauges went stale. "
+                             "The gate fails OPEN when this happens, so any non-zero rate "
+                             "means admission is silently degraded to admit-everything. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(164, "Gateway view vs engine truth",
+                 [("max(vllm_requests_running)", "gateway: running"),
+                  ("sum(vllm:num_requests_running)", "engine: running"),
+                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
+                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
+                 0, 18, w=24, h=8, unit="short",
+                 description="The gateway's own reading of the engine against the engine's "
+                             "self-reported value. A disagreement between a pair is how a "
+                             "stale observer or a mistuned admission target shows itself, "
+                             "and nothing else in this dashboard reveals it. The two "
+                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
+                             "ship with homelab-frontend#64 and are absent until it is "
+                             "deployed — until then this panel shows the engine's pair only."),
+    ]))
+
+    return {
+        "title": "LLM Observability",
+        "uid": "llm-observability",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
+        "panels": panels,
+    }
+
+
 # ============================================================================
 # Generate
 # ============================================================================
@@ -277,4 +511,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")

+# Dashboard 4
+write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
+
 print("Done.")
@@ -0,0 +1,95 @@
+apiVersion: v1
+data:
+  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
+    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
+    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
+    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
+    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
+    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
+    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
+    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
+    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
+    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
+    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
+    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
+    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
+    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
+    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
+    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
+    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
+    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
+    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
+    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
+    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
+    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
+    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
+    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
+    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
+    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
+    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
+    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
+    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
+    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
+    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
+    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
+    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
+    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
+    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
+    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
+    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
+    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
+    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
+    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
+    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
+    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
+    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
+    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
+    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
+    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
+    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
+    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
+    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
+    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
+    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
+    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
+    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
+    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
+    the engine''s self-reported value. A disagreement between a pair is how a stale
+    observer or a mistuned admission target shows itself, and nothing else in this
+    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
+    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
+    then this panel shows the engine''s pair only."}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: LLM
+  labels:
+    grafana_dashboard: '1'
+  name: llm-observability-dashboard
+  namespace: logging
@@ -8,6 +8,7 @@ resources:
  - alerts/ingress-alerts.yaml
  - alerts/svc-argocd-rules.yaml
  - alerts/svc-authentik-rules.yaml
+  - alerts/svc-zitadel-rules.yaml
  - alerts/svc-forgejo-rules.yaml
  - alerts/svc-grafana-rules.yaml
  - alerts/svc-minio-rules.yaml
@@ -20,5 +21,8 @@ resources:
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
  - servicemonitors/ingress-nginx.yaml
+  - servicemonitors/vllm.yaml
+  - servicemonitors/api-gateway.yaml
  - dashboards/cluster-infrastructure.yaml
  - dashboards/api-gateway.yaml
+  - dashboards/llm-observability.yaml
@@ -78,7 +78,7 @@ prometheus:
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
-              storage: 100Gi
+              storage: 25Gi

    resources:
      requests:
@@ -0,0 +1,62 @@
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
+# three latency series below recorded nothing. internal/proxy/llm_metrics.go
+# declared a capture wrapper (LLMMetricsCapture) that was never constructed
+# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
+# stayed empty.
+#
+# homelab-frontend#63 fixed exactly that. The wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
+# request path. Verified live on 2026-09-29:
+#   llm_ttft_seconds  count=7  sum=598.212s
+#   llm_itl_seconds   count=16116
+#   llm_tokens_total  3 series
+#
+# These derived-from-SSE series cover EVERY model including the two Ollama ones
+# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
+# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
+# between SSE writes, so both are granular to SSE write boundaries rather than
+# to the engine's own token cadence. llm_tokens_total counts `data:` events,
+# and the documented assumption in internal/proxy/llm_metrics.go is that these
+# upstreams emit one such event per token — so it is a token count only as far
+# as that assumption holds. vLLM's own histograms remain authoritative.
+#
+# Also recorded and working (3 series, one per replica):
+#   queue_full_requests_total           counter    backpressure rejects
+#   warmup_requests_delayed_total       counter    vLLM slot contention
+#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
+#
+# NOT working, do not build on it: request_duration_seconds is declared and
+# exported but has 0 samples, and its only writer Metrics.RecordRequest has
+# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
+# the same dead-code shape this branch already corrected for the latency
+# series, so it is listed here rather than left to be rediscovered.
+#
+# Also note the Deployment already carried prometheus.io/scrape annotations,
+# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
+# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
+# collected before this ServiceMonitor existed.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: api-gateway
+    app.kubernetes.io/part-of: api
+spec:
+  namespaceSelector:
+    matchNames:
+      - api
+  selector:
+    matchLabels:
+      app: api-gateway
+  endpoints:
+    # api-gateway Service port IS named `http` (unlike the KServe ones), so
+    # the name form works here.
+    - port: http
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+
@@ -11,20 +11,39 @@ spec:
    matchNames:
      - llm-serving
  selector:
-    matchLabels:
-      app.kubernetes.io/part-of: llm-serving
+    # Only the three engines that actually serve /metrics.
+    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
+    # endpoint (/metrics returns 404) — selecting them would add permanently
+    # DOWN targets. Ollama-model visibility comes from the api-gateway
+    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
+    # measured at the proxy, so they cover every model including Ollama).
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
+          - llm-embeddings  # TEI 1.8.2 — CPU
+          - llm-reranker    # TEI 1.8.2 — CPU
  endpoints:
-    - port: http
+    # targetPort, not port: the KServe-generated Services on llm-serving have
+    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
+    # editing the InferenceServices (predictor restart = full weights reload,
+    # tens of seconds on the vLLM slot), so scrape the pod port directly.
+    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
+    - targetPort: 8080
      interval: 30s
      scrapeTimeout: 10s
      path: /metrics
      scheme: http
      relabelings:
-        - sourceLabels: [__meta_kubernetes_namespace]
-          targetLabel: namespace
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          targetLabel: pod
-        - sourceLabels: [__meta_kubernetes_service_name]
-          targetLabel: service
-        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        # Prometheus's Kubernetes service discovery already supplies the
+        # namespace, pod and service labels; the only one that needs deriving
+        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
+        # not promote arbitrary pod labels.
+        #
+        # `action: replace` is written explicitly. It is the API server's
+        # default for relabeling rules, so omitting it leaves the live object
+        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
          targetLabel: app
@@ -0,0 +1,59 @@
+# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
+# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
+# tokens were never accepted -> every login got 401). Delete this whole directory after.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: oidc-bootstrap
+    namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: oidc-bootstrap
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-bootstrap-restart-apiserver-v2
+  namespace: kube-system
+spec:
+  backoffLimit: 4
+  ttlSecondsAfterFinished: 7200
+  template:
+    spec:
+      serviceAccountName: oidc-bootstrap
+      restartPolicy: Never
+      containers:
+        - name: restart
+          image: docker.io/curlimages/curl:8.22.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -u
+              TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+              CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
+                curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN"                   -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0"                   -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
+              done
+              echo "deletes requested; kubelet recreates each static pod in ~20-40s"
@@ -0,0 +1,141 @@
+# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
+#
+# Two earlier mechanisms failed and are documented so nobody repeats them:
+#   1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
+#   2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
+#      so every pod SIGTERMed itself (exit 143) before reaching the real process.
+# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
+# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
+# three control planes by 2 minutes each, so only one API server is down at a time.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp1-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 10s before restarting the API server on talos-cp-1"
+              sleep 10
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp2-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 120s before restarting the API server on talos-cp-2"
+              sleep 120
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp3-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 240s before restarting the API server on talos-cp-3"
+              sleep 240
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
@@ -6,3 +6,7 @@ kind: Kustomization
 # deployments (wave 0) are applied.
 resources:
  - memory-db.yaml
+  - poimen-sandbox-namespace.yaml
+  - poimen-sandbox-quota.yaml
+  - pi-session-netpol.yaml
+  - pi-agent-workspace-pv.yaml
@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  finalizers:
+  - kubernetes.io/pv-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace-pv
+  annotations:
+    argocd.argoproj.io/sync-wave: '-1'
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: pi-agent-workspace
+    namespace: poimen-sandbox
+  hostPath:
+    path: /var/lib/pi-agent-sandbox
+    type: DirectoryOrCreate
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - talos-cp-2
+  persistentVolumeReclaimPolicy: Retain
+  volumeMode: Filesystem
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    pv.kubernetes.io/bind-completed: 'yes'
+    argocd.argoproj.io/sync-wave: '-1'
+  finalizers:
+  - kubernetes.io/pvc-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace
+  namespace: poimen-sandbox
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: ''
+  volumeMode: Filesystem
+  volumeName: pi-agent-workspace-pv
@@ -0,0 +1,12 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pi-session-deny-kube-api
+  namespace: poimen-sandbox
+spec:
+  endpointSelector:
+    matchLabels:
+      app: pi-session
+  egressDeny:
+    - toEntities:
+        - kube-apiserver
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: poimen-sandbox
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: sandbox-quota
+  namespace: poimen-sandbox
+spec:
+  hard:
+    pods: "20"
+    requests.cpu: "10"
+    requests.memory: 20Gi
+    limits.cpu: "20"
+    limits.memory: 40Gi
@@ -16,6 +16,7 @@ resources:
  - temporal-operator-role.yaml
  - llm-serving-operator-role.yaml
  - poimen-memory-operator-role.yaml
+  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until
@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sandbox-observer
+  namespace: poimen-sandbox
+rules:
+  - apiGroups: [""]
+    resources: [pods, pods/log, services]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: poimen-devs-sandbox
+  namespace: poimen-sandbox
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: "oidc:poimen-devs"
+roleRef:
+  kind: Role
+  name: sandbox-observer
+  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,488 @@
+#!/usr/bin/env python3
+"""Provision ZITADEL end to end: machine key, Terraform, SOPS secrets, users.
+
+One command replaces the manual sequence the Authentik script
+(scripts/iam/provision-rbac.py) used to do in a single pass. Phases run in a
+fixed order and every phase is safe to re-run:
+
+  keys         extract the iam-admin machine key and PAT from namespace iam
+  apply        terraform apply terraform/zitadel (project, roles, apps, machines)
+  secrets      write generated client ids/secrets back into the SOPS files
+  users        import one human user, converting the Django password hash
+  verify       check discovery, client count and the groups-claim gate
+
+`all` runs keys, apply, secrets, verify. It never runs users: the password hash
+must be supplied by a human, one user at a time.
+
+Nothing here talks to Authentik. Authentik keeps serving until a consumer is
+moved deliberately.
+
+Usage:
+  scripts/iam/zitadel-provision.py keys
+  scripts/iam/zitadel-provision.py apply --org-id <id>
+  scripts/iam/zitadel-provision.py secrets --dry-run
+  scripts/iam/zitadel-provision.py secrets --apply-config
+  scripts/iam/zitadel-provision.py users --django-hash '<pbkdf2_sha256$...>'
+  scripts/iam/zitadel-provision.py verify
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import re
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import yaml
+
+REPO = Path(__file__).resolve().parents[2]
+TF_DIR = REPO / "terraform" / "zitadel"
+SOPS_DIR = REPO / "k8s" / "argocd" / "secrets"
+SOPS_CONFIG = REPO / ".sops.yaml"
+CONF_DIR = Path(os.environ.get("ZITADEL_CONF_DIR", Path.home() / ".config" / "zitadel"))
+NAMESPACE = os.environ.get("ZITADEL_NAMESPACE", "iam")
+BASE_URL = os.environ.get("ZITADEL_BASE_URL", "https://zitadel.riotpiao.com")
+
+_TEMP_FILES: set[str] = set()
+
+
+def _cleanup_temps(*_a) -> None:
+    for path in list(_TEMP_FILES):
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+        _TEMP_FILES.discard(path)
+
+
+for _sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
+    signal.signal(_sig, lambda s, f: (_cleanup_temps(), sys.exit(130)))
+
+
+def die(msg: str) -> None:
+    _cleanup_temps()
+    print(f"FATAL: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def run(cmd: list[str], **kw) -> subprocess.CompletedProcess:
+    return subprocess.run(cmd, check=False, text=True, capture_output=True, **kw)
+
+
+def need(binary: str) -> None:
+    if subprocess.run(["sh", "-c", f"command -v {binary}"], capture_output=True).returncode != 0:
+        die(f"{binary} is not on PATH")
+
+
+def write_private(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        os.fchmod(fd, 0o600)  # open() mode applies at creation only
+        with os.fdopen(fd, "wb") as fh:
+            fh.write(data)
+    finally:
+        pass
+
+
+def secret_value(name: str, key: str) -> str:
+    cp = run(["kubectl", "-n", NAMESPACE, "get", "secret", name, "-o", f"jsonpath={{{key}}}"])
+    if cp.returncode != 0:
+        die(f"cannot read secret {NAMESPACE}/{name}: {cp.stderr.strip()}")
+    raw = cp.stdout.strip()
+    if not raw:
+        die(f"secret {NAMESPACE}/{name} has no key {key}")
+    return base64.b64decode(raw).decode()
+
+
+def load_env_file(path: Path = REPO / ".env") -> None:
+    """Load KEY=VALUE lines from .env without overriding the real environment.
+
+    The file is git-ignored and holds the bootstrap token a human pastes in.
+    Sourcing it in the shell would work too, but then every phase depends on the
+    caller remembering to do it.
+    """
+    if not path.exists():
+        return
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        key = key.strip()
+        value = value.strip().strip('"').strip("'")
+        if key and key not in os.environ:
+            os.environ[key] = value
+
+
+def bootstrap_token() -> str | None:
+    """Resolve the iam-admin PAT: environment, then cached file, then cluster.
+
+    Order matters. ZITADEL_BOOTSTRAP_TOKEN in .env is filled by a human after the
+    first sync and wins. The cached file under ~/.config/zitadel keeps a run
+    working when the API server is unreachable. The cluster Secret is the last
+    resort and the only source that always reflects a rotation.
+    """
+    token = os.environ.get("ZITADEL_BOOTSTRAP_TOKEN", "").strip()
+    if token:
+        return token
+    cached = CONF_DIR / "iam-admin.pat"
+    if cached.exists():
+        return cached.read_text().strip()
+    try:
+        return secret_value("iam-admin-pat", ".data.pat").strip()
+    except SystemExit:
+        return None
+
+
+def curl_json(method: str, url: str, token: str | None = None, body: dict | None = None,
+              timeout: int = 20) -> tuple[int, str]:
+    """Call the API without putting a secret in the process argv.
+
+    `curl --config -` reads the options from stdin, so neither the bearer token
+    nor the password hash of a user import shows up in `ps`.
+    """
+    def esc(value: str) -> str:
+        return value.replace("\\", "\\\\").replace('"', '\\"')
+
+    cfg = [f'url = "{esc(url)}"', f'request = "{method}"', "silent", "show-error",
+           f"max-time = {timeout}", 'write-out = "\\n%{http_code}"',
+           'header = "Content-Type: application/json"']
+    if token:
+        cfg.append(f'header = "Authorization: Bearer {esc(token)}"')
+    if body is not None:
+        cfg.append(f'data = "{esc(json.dumps(body))}"')
+    cp = run(["curl", "--config", "-"], input="\n".join(cfg) + "\n")
+    if cp.returncode != 0:
+        die(f"{method} {url} failed: {cp.stderr.strip()}")
+    payload, _, code = cp.stdout.rpartition("\n")
+    return int(code.strip() or 0), payload
+
+
+# ── phase: keys ────────────────────────────────────────────────────────────
+
+def phase_keys(_args: argparse.Namespace) -> int:
+    write_private(CONF_DIR / "iam-admin.json", secret_value("iam-admin", ".data.iam-admin\\.json").encode())
+    write_private(CONF_DIR / "iam-admin.pat", secret_value("iam-admin-pat", ".data.pat").strip().encode())
+    print(f"machine key  -> {CONF_DIR / 'iam-admin.json'} (0600)")
+    print(f"PAT          -> {CONF_DIR / 'iam-admin.pat'} (0600)")
+    print("both files stay outside the repo; never commit them")
+    return 0
+
+
+# ── phase: apply ───────────────────────────────────────────────────────────
+
+def phase_apply(args: argparse.Namespace) -> int:
+    need("terraform")
+    if not args.org_id:
+        die("apply requires --org-id (ZITADEL console -> Organization settings -> Resource ID)")
+    key = CONF_DIR / "iam-admin.json"
+    if not key.exists():
+        die(f"{key} missing — run the keys phase first")
+    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
+
+    for cmd in (["init", "-input=false"], ["fmt", "-check", "-recursive"]):
+        cp = run(["terraform", f"-chdir={TF_DIR}", *cmd], env=env)
+        if cp.returncode != 0:
+            die(f"terraform {' '.join(cmd)} failed: {cp.stderr.strip() or cp.stdout.strip()}")
+
+    cp = run(["terraform", f"-chdir={TF_DIR}", "apply", "-input=false", "-auto-approve",
+              f"-var=org_id={args.org_id}"], env=env)
+    sys.stdout.write(cp.stdout)
+    sys.stderr.write(cp.stderr)
+    if cp.returncode != 0:
+        die("terraform apply failed — no SOPS value was written. Objects created before the "
+            "failure stay in ZITADEL; re-run apply once the cause is fixed")
+    return 0
+
+
+# ── phase: secrets ─────────────────────────────────────────────────────────
+
+# client -> [(sops file, Secret name inside the file, key, kind)]
+# Only secrets with a git-tracked SOPS file appear here. Every other client is
+# printed as manual: guessing a path writes a credential into the wrong Secret.
+SECRET_MAP = {
+    "grafana": [("grafana-oidc-secrets.enc.yaml", "grafana-oidc",
+                 "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "secret")],
+    "minio": [("minio-secrets.enc.yaml", "minio-oidc",
+               "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "secret")],
+    "paperless-ai-agent": [("paperless-ai-secrets.enc.yaml", "paperless-ai-config",
+                            "LLM_AUTH_CLIENT_SECRET", "secret")],
+}
+
+# Clients whose client_id lives in a plain (unencrypted) manifest. They need the
+# id update as much as the secret: ZITADEL generates a new client_id for every
+# application, so a secret-only re-point yields invalid_client.
+CLIENT_ID_MAP = {
+    "grafana": ("k8s/infra/logging/grafana-values.yaml",
+                r"(?m)^(\s*client_id:\s*)\S+\s*$", r"\g<1>{value}"),
+    "minio": ("k8s/infra/minio/minio-tenant.yaml",
+              r"(?m)^(\s*value:\s*)\"minio\"\s*$", r'\g<1>"{value}"'),
+    "paperless-ai-agent": ("k8s/apps/paperless/paperless-ai.yaml",
+                           r"client_id=paperless-ai-agent", "client_id={value}"),
+}
+
+# Secrets that a live workload mounts but no SOPS file in git defines. The phase
+# cannot re-point them safely, so it names them.
+MANUAL_ONLY = {
+    "homarr": "Secret homarr-oidc (keys client-id, client-secret) has no SOPS file in git — "
+              "adopt it first, then add it to SECRET_MAP. The repo also carries an orphan "
+              "homarr-auth-oidc.enc.yaml that nothing consumes.",
+}
+
+
+def terraform_outputs() -> dict:
+    need("terraform")
+    key = CONF_DIR / "iam-admin.json"
+    if not key.exists():
+        die(f"{key} missing — run the keys phase first")
+    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
+    cp = run(["terraform", f"-chdir={TF_DIR}", "output", "-json"], env=env)
+    if cp.returncode != 0:
+        die(f"terraform output failed: {cp.stderr.strip()}")
+    return json.loads(cp.stdout or "{}")
+
+
+def sops_env() -> dict:
+    env = dict(os.environ)
+    env.setdefault("SOPS_AGE_KEY_FILE", str(Path.home() / ".sops" / "key.txt"))
+    return env
+
+
+def write_secret(rel_path: str, secret_name: str, key: str, value: str,
+                 dry_run: bool) -> None:
+    path = SOPS_DIR / rel_path
+    if not path.exists():
+        die(f"{path} does not exist — the mapping and the repo disagree")
+    env = sops_env()
+    dec = run(["sops", "--config", str(SOPS_CONFIG), "decrypt", str(path)], env=env)
+    if dec.returncode != 0:
+        die(f"cannot decrypt {rel_path}: {dec.stderr.strip()}")
+    docs = [d for d in yaml.safe_load_all(dec.stdout) if d]
+    matching = [d for d in docs if (d.get("metadata") or {}).get("name") == secret_name]
+    if len(matching) != 1:
+        die(f"{rel_path}: expected exactly one document named {secret_name}, found {len(matching)}")
+
+    current = (matching[0].get("stringData") or {}).get(key)
+    if current == value:
+        print(f"  unchanged {rel_path}:{secret_name}:{key}")
+        return
+    if dry_run:
+        print(f"  would set {rel_path}:{secret_name}:{key} (len {len(value)})")
+        return
+
+    if len(docs) == 1:
+        cp = run(["sops", "--config", str(SOPS_CONFIG), "set", "--value-stdin", "--idempotent",
+                  str(path), f'["stringData"]["{key}"]'], env=env, input=json.dumps(value))
+        if cp.returncode != 0:
+            die(f"sops set failed on {rel_path}: {cp.stderr.strip()}")
+        print(f"  wrote {rel_path}:{secret_name}:{key}")
+        return
+
+    # Multi-document file. `sops set` cannot address one document, so the whole
+    # file is rewritten. sops encrypt reads the creation rule from the FILENAME,
+    # so the temp path is overridden with the repo-relative path; without that
+    # override sops finds no matching rule and refuses.
+    matching[0].setdefault("stringData", {})[key] = value
+    plain = "---\n".join(yaml.safe_dump(d, sort_keys=False) for d in docs)
+    fd, tmp = tempfile.mkstemp(suffix=".yaml", dir=tempfile.gettempdir())
+    _TEMP_FILES.add(tmp)
+    try:
+        with os.fdopen(fd, "w") as fh:
+            fh.write(plain)
+        enc = run(["sops", "--config", str(SOPS_CONFIG), "encrypt",
+                   "--filename-override", str(path), tmp], env=env)
+        if enc.returncode != 0:
+            die(f"sops encrypt failed for {rel_path}: {enc.stderr.strip()}")
+        mode = stat.S_IMODE(path.stat().st_mode)
+        fd2, staged = tempfile.mkstemp(suffix=".enc.yaml", dir=str(path.parent))
+        try:
+            with os.fdopen(fd2, "w") as fh:
+                fh.write(enc.stdout)
+            os.chmod(staged, mode)
+            os.replace(staged, path)  # same directory: atomic, never EXDEV
+        finally:
+            if os.path.exists(staged):
+                os.unlink(staged)
+        print(f"  wrote {rel_path}:{secret_name}:{key} (multi-document rewrite)")
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        _TEMP_FILES.discard(tmp)
+
+
+def update_client_id(client: str, value: str, dry_run: bool, apply_config: bool) -> None:
+    spec = CLIENT_ID_MAP.get(client)
+    if not spec:
+        return
+    rel_path, pattern, repl = spec
+    path = REPO / rel_path
+    if not path.exists():
+        die(f"{path} does not exist — the client-id mapping is stale")
+    text = path.read_text()
+    matches = list(re.finditer(pattern, text))
+    if len(matches) != 1:
+        die(f"{rel_path}: expected 1 match for the {client} client id, found {len(matches)}")
+    new_line = matches[0].expand(repl.format(value=value))
+    if matches[0].group(0) == new_line:
+        print(f"  unchanged {rel_path}: {client} client id")
+        return
+    if dry_run or not apply_config:
+        print(f"  would change {rel_path}: {matches[0].group(0).strip()} -> {new_line.strip()}")
+        if not apply_config:
+            print("    re-run with --apply-config to write this plaintext manifest edit")
+        return
+    path.write_text(text[:matches[0].start()] + new_line + text[matches[0].end():])
+    print(f"  wrote {rel_path}: {client} client id")
+
+
+def phase_secrets(args: argparse.Namespace) -> int:
+    need("sops")
+    out = terraform_outputs()
+    ids = out.get("client_ids", {}).get("value", {})
+    secrets = out.get("client_secrets", {}).get("value", {})
+    machine_keys = out.get("machine_keys", {}).get("value", {})
+
+    if not ids and not machine_keys:
+        die("terraform has no ZITADEL outputs yet — run the apply phase first")
+
+    print("[1/3] client secrets into SOPS")
+    for client, entries in SECRET_MAP.items():
+        for rel_path, secret_name, key, kind in entries:
+            value = ids.get(client) if kind == "id" else secrets.get(client)
+            if value:
+                write_secret(rel_path, secret_name, key, value, args.dry_run)
+
+    print("[2/3] client ids in plain manifests")
+    for client in list(CLIENT_ID_MAP) + list(SECRET_MAP):
+        if ids.get(client):
+            update_client_id(client, ids[client], args.dry_run, args.apply_config)
+
+    print("[3/3] machine keys")
+    key_dir = CONF_DIR / "machine-keys"
+    for name, key_json in machine_keys.items():
+        if args.dry_run:
+            print(f"  would write {key_dir / (name + '.json')}")
+        else:
+            write_private(key_dir / f"{name}.json", key_json.encode())
+            print(f"  wrote {key_dir / (name + '.json')} (0600)")
+
+    unhandled = sorted(set(ids) - set(SECRET_MAP) - set(CLIENT_ID_MAP) - set(MANUAL_ONLY))
+    if MANUAL_ONLY or unhandled:
+        print("\n  MANUAL — no safe write path exists for these clients:")
+        for client, why in MANUAL_ONLY.items():
+            print(f"    {client}: {why}")
+        for client in unhandled:
+            print(f"    {client}: set its client_id and secret in its own manifest or Secret file")
+    return 0
+
+
+# ── phase: users ───────────────────────────────────────────────────────────
+
+HASH_RE = re.compile(r"^pbkdf2_sha256\$(?P<rounds>\d+)\$(?P<salt>[^$]+)\$(?P<hash>[^$]+)$")
+
+
+def convert_hash(django_hash: str) -> str:
+    """Django pbkdf2_sha256 -> passwap MCF $pbkdf2-sha256$...
+
+    ZITADEL imports one MCF string via hashedPassword.hash (max 200 chars) and
+    pins passwap, whose pbkdf2 format is $pbkdf2-sha256$<rounds>$<salt>$<hash>
+    with alt-base64 (standard base64, + -> ., no padding). Rounds, salt and hash
+    bytes are unchanged, so the password still verifies.
+    """
+    m = HASH_RE.match(django_hash.strip())
+    if not m:
+        die("unsupported hash format — only pbkdf2_sha256 is convertible")
+    alt = m.group("hash").rstrip("=").replace("+", ".")
+    mcf = f"$pbkdf2-sha256${m.group('rounds')}${m.group('salt')}${alt}"
+    if len(mcf) > 200:
+        die(f"converted hash is {len(mcf)} chars, ZITADEL accepts 200")
+    return mcf
+
+
+def phase_users(args: argparse.Namespace) -> int:
+    token = bootstrap_token()
+    if not token:
+        die("no bootstrap token — set ZITADEL_BOOTSTRAP_TOKEN in .env, or run the keys phase")
+    if not args.django_hash:
+        die("pass --django-hash with the pbkdf2_sha256 value exported from Authentik")
+    body = {
+        "username": args.username,
+        "profile": {"givenName": args.given_name, "familyName": args.family_name},
+        "email": {"email": args.email, "isVerified": True},
+        "hashedPassword": {"hash": convert_hash(args.django_hash)},
+        "passwordChangeRequired": False,
+    }
+    code, payload = curl_json("POST", f"{BASE_URL}/v2/users/human", token=token, body=body)
+    if code in (200, 201):
+        print(f"  {args.username}: created")
+        return 0
+    if code == 409:
+        print(f"  {args.username}: already exists — the password was NOT updated")
+        return 1
+    die(f"create {args.username} -> {code} {payload[:300]}")
+
+
+# ── phase: verify ──────────────────────────────────────────────────────────
+
+def phase_verify(_args: argparse.Namespace) -> int:
+    code, _ = curl_json("GET", f"{BASE_URL}/.well-known/openid-configuration", timeout=10)
+    print(f"discovery      : {code}")
+    if code != 200:
+        die("discovery endpoint is not 200 — the instance or the ingress is not ready")
+
+    out = terraform_outputs()
+    ids = out.get("client_ids", {}).get("value", {})
+    print(f"client apps    : {len(ids)}")
+    enabled = bool(out.get("groups_claim_enabled", {}).get("value"))
+    print(f"groups action  : {enabled}")
+    if not enabled:
+        print("BLOCKED for cutover: the groups claim action is disabled. Every consumer")
+        print("that reads `groups` (kube-apiserver, MinIO, Grafana) sees no groups.")
+        print("Enable it only after the live gate passes, with the script path set.")
+    return 0
+
+
+PHASES = {"keys": phase_keys, "apply": phase_apply, "secrets": phase_secrets,
+          "users": phase_users, "verify": phase_verify}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Provision ZITADEL end to end.")
+    parser.add_argument("phase", choices=[*PHASES, "all"])
+    parser.add_argument("--org-id", help="ZITADEL org id (required for apply)")
+    parser.add_argument("--dry-run", action="store_true", help="print actions, write nothing")
+    parser.add_argument("--apply-config", action="store_true",
+                        help="also rewrite client ids inside plain manifests")
+    parser.add_argument("--username", default="rock")
+    parser.add_argument("--email", default="[email protected]")
+    parser.add_argument("--given-name", default="Rock")
+    parser.add_argument("--family-name", default="Liang")
+    parser.add_argument("--django-hash", help="pbkdf2_sha256 value exported from Authentik")
+    args = parser.parse_args()
+
+    load_env_file()
+    need("kubectl")
+    if args.phase == "all":
+        if not args.org_id:
+            die("all requires --org-id")
+        for name in ("keys", "apply"):
+            rc = PHASES[name](args)
+            if rc:
+                return rc
+        rc = phase_secrets(args)
+        if rc:
+            return rc
+        return phase_verify(args)
+    return PHASES[args.phase](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
@@ -23,6 +23,7 @@
    # via nginx (not the backend service) is critical: a direct rewrite bypasses
    # nginx TLS and exposes each app's own self-signed cert, trusted by nothing.
    rewrite name authentik.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name zitadel.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name grafana.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name minio.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name minio-api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
@@ -39,6 +40,7 @@
    rewrite name comfy.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name comfyui.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name gotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name pi.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name igotify.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local

@@ -20,6 +20,7 @@ controlplane_configs = {
  "talos-cp-1" = {
    hostname     = "talos-cp-1"
    lan_ip       = "192.168.1.166"
+    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/nvme0n1"
@@ -32,6 +33,7 @@ controlplane_configs = {
  "talos-cp-2" = {
    hostname     = "talos-cp-2"
    lan_ip       = "192.168.1.214"
+    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -39,7 +41,11 @@ controlplane_configs = {
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
+        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
+        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
+        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
+        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
+        # re-add it.
      ]
    zone             = "az-b"
    allow_scheduling = true
@@ -49,6 +55,7 @@ controlplane_configs = {
  "talos-cp-3" = {
    hostname     = "talos-cp-3"
    lan_ip       = "192.168.1.162"
+    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/nvme0n1"
@@ -0,0 +1,58 @@
+# The groups claim — the single load-bearing decision of the migration.
+#
+# ZITADEL does not issue a `groups` claim. Authentik did, and these consumers
+# read it: kube-apiserver RBAC (`--oidc-groups-claim=groups` plus 11 RoleBindings
+# on oidc:<group>), MinIO (`MINIO_IDENTITY_OPENID_CLAIM_NAME=groups`), Grafana
+# role mapping, and every service that filters on a group name.
+#
+# Plan §1.3 Option A keeps those consumers unchanged by synthesizing the claim.
+# Option B refactors all of them. This module takes responsibility only for the
+# ZITADEL side of Option A: the action resource and its trigger.
+#
+# The script body is NOT shipped by default. The go/no-go gate (§1.3, §1.5 task
+# 1.3) is: prove on the live instance that the action runtime can read project
+# roles plus user metadata and write both `groups` and the per-agent claims,
+# for a human token AND a machine-user token. Until that proof exists, enabling
+# this resource would move every consumer onto an unverified claim source:
+#
+#   terraform apply -var enable_groups_claim_action=true \
+#                   -var groups_claim_script=./groups-claim.js
+#
+# After it applies, verify with a real token exchange before moving a consumer:
+#
+#   curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
+#     -d grant_type=refresh_token -d refresh_token=<t> \
+#     -d client_id=kubernetes | jq -R 'split(".")[1] | @base64d | fromjson'
+#
+# The decoded access token must contain `"groups":["homelab-admins", ...]`.
+
+resource "zitadel_action" "groups_claim" {
+  count = var.enable_groups_claim_action ? 1 : 0
+
+  org_id          = data.zitadel_org.default.id
+  name            = "homelab-groups-claim"
+  script          = file(var.groups_claim_script)
+  timeout         = "5s"
+  allowed_to_fail = false
+}
+
+# Both triggers are needed. The access-token trigger fills the JWT that
+# Kubernetes and MinIO read. The userinfo trigger fills the response that
+# Grafana, paperless and Forgejo read.
+resource "zitadel_trigger_actions" "groups_claim_access_token" {
+  count = var.enable_groups_claim_action ? 1 : 0
+
+  org_id       = data.zitadel_org.default.id
+  flow_type    = "FLOW_TYPE_CUSTOMISE_TOKEN"
+  trigger_type = "TRIGGER_TYPE_PRE_ACCESS_TOKEN_CREATION"
+  action_ids   = [zitadel_action.groups_claim[0].id]
+}
+
+resource "zitadel_trigger_actions" "groups_claim_userinfo" {
+  count = var.enable_groups_claim_action ? 1 : 0
+
+  org_id       = data.zitadel_org.default.id
+  flow_type    = "FLOW_TYPE_CUSTOMISE_TOKEN"
+  trigger_type = "TRIGGER_TYPE_PRE_USERINFO_CREATION"
+  action_ids   = [zitadel_action.groups_claim[0].id]
+}
@@ -0,0 +1,18 @@
+# API applications — the machine-token surface.
+#
+# ZITADEL keeps two different objects apart: an OIDC application is for a
+# browser flow, and an API application is what a client_credentials / JWT-profile
+# caller presents. Authentik modelled both with one provider, so the plan §1.1
+# row 1 ("OIDC app + machine user") needs one of each here.
+#
+# The gateway signs its token request with the machine key from machines.tf, so
+# the auth method is PRIVATE_KEY_JWT. No secret is stored for it.
+
+resource "zitadel_application_api" "machine" {
+  for_each = { for name, spec in local.machine_users : name => spec if contains(keys(var.api_app_consumers), name) }
+
+  org_id           = data.zitadel_org.default.id
+  project_id       = zitadel_project_v2.homelab.id
+  name             = "${each.key}-machine"
+  auth_method_type = "API_AUTH_METHOD_TYPE_PRIVATE_KEY_JWT"
+}
@@ -0,0 +1,136 @@
+# OIDC applications — one per consumer that logs a browser in.
+#
+# Client IDs: the plan §3.2 wants every client_id byte-identical after the move,
+# so that a consumer change is only the issuer URL plus the secret.
+#
+# THAT GOAL IS NOT ACHIEVABLE HERE, verified twice on 2026-10-24:
+#   - ZITADEL v4.19.2 CreateOIDCApplicationRequest has no client_id field, and
+#     CreateOIDCApplicationResponse documents the value as "generated"
+#     (proto/zitadel/app/v2beta/app_service.proto:540-650).
+#   - zitadel provider 3.8.7 exposes no settable application id: `id` on
+#     `zitadel_application_v2` is not a valid argument (terraform validate
+#     rejects it), and `client_id` on `zitadel_application_oidc` is computed.
+# Consequence: every consumer takes a NEW client_id as well as a new secret.
+# Budget the extra config change per consumer — this is not the one-line
+# re-point the plan assumed.
+#
+# The only way to keep the old value is to create each app through the v2 REST
+# API with an explicit body id, then import it:
+#   POST /v2/projects/<project_id>/apps
+#   {"id":"<old-client-id>","name":"<old-client-id>","oidcRequest":{...}}
+# Not implemented here: it would put app creation outside Terraform state.
+#
+# Secrets are generated by ZITADEL, never written here. `terraform output -json
+# client_ids` and `client_secrets` feed the SOPS re-point (§3.2, §6.9). Do not
+# commit the tfstate: it carries those values.
+
+locals {
+  oidc_clients = {
+    # Gateway. Browser sessions for the console, plus a machine path
+    # (machines.tf) for callers that need client_credentials.
+    "api-gw" = {
+      redirect_uris = ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "argocd" = {
+      redirect_uris = ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "forgejo" = {
+      redirect_uris = ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "grafana" = {
+      redirect_uris = ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "homarr" = {
+      redirect_uris = ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "immich" = {
+      redirect_uris = ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"]
+      app_type      = "NATIVE"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "kubernetes" = {
+      redirect_uris = ["http://localhost:8000", "http://localhost:18000"]
+      app_type      = "WEB"
+      auth_method   = "NONE"
+      dev_mode      = true
+    }
+    "minio" = {
+      redirect_uris = ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "paperless" = {
+      redirect_uris = ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "poimen" = {
+      redirect_uris = ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "queue-sqs" = {
+      redirect_uris = ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+    "vault" = {
+      redirect_uris = ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"]
+      app_type      = "WEB"
+      auth_method   = "BASIC"
+      dev_mode      = false
+    }
+  }
+}
+
+resource "zitadel_application_v2" "oidc" {
+  for_each = local.oidc_clients
+
+  org_id     = data.zitadel_org.default.id
+  project_id = zitadel_project_v2.homelab.id
+  name       = each.key
+
+  oidc {
+    redirect_uris  = each.value.redirect_uris
+    response_types = ["OIDC_RESPONSE_TYPE_CODE"]
+    grant_types = [
+      "OIDC_GRANT_TYPE_AUTHORIZATION_CODE",
+      "OIDC_GRANT_TYPE_REFRESH_TOKEN",
+    ]
+    app_type         = "OIDC_APP_TYPE_${each.value.app_type}"
+    auth_method_type = "OIDC_AUTH_METHOD_TYPE_${each.value.auth_method}"
+    version          = "OIDC_VERSION_1_0"
+
+    # JWT access tokens: Kubernetes RBAC, MinIO policy mapping and the Go
+    # gateway all read claims, and an opaque bearer token carries none.
+    access_token_type = "OIDC_TOKEN_TYPE_JWT"
+
+    # Assert roles without the client having to request the scope. Every
+    # consumer here maps roles or groups to local permissions.
+    access_token_role_assertion = true
+    id_token_role_assertion     = true
+    id_token_userinfo_assertion = true
+
+    dev_mode = each.value.dev_mode
+  }
+}
@@ -0,0 +1,139 @@
+# Machine users and their project grants (plan §1.4, §5.3).
+#
+# Authentik gave each agent an app password or a client_credentials grant built
+# on a user password. ZITADEL has no user password on a machine user: it issues
+# a JWT profile key (`KEY_TYPE_JSON`) that the SDK signs its token request with.
+# That removes the password path entirely, which is the point — see §5.3.
+#
+# Per-agent claim values are copied from `SERVICE_ACCOUNTS` in
+# scripts/iam/provision-rbac.py. They live in user metadata as JSON strings,
+# because the groups-claim action (actions.tf) is the only thing that turns them
+# into token claims. Without that action a consumer sees roles but not
+# `minio_buckets`, `authorized_models`, `sqs_queues`, `memory_projects`,
+# `memory_visibility` or `paperless_doctypes`.
+#
+# Keys are generated per machine user and land in `terraform output -json
+# machine_keys` (sensitive). Feed each one to the owning workload as a mounted
+# file, never as an environment variable dumps-able through `ps`.
+
+locals {
+  machine_users = {
+    "memory-agent" = {
+      description      = "Memory service agent"
+      project_roles    = ["poimen-admins"]
+      authenticated_by = "poimen memory (Rust)"
+      metadata = {
+        roles             = jsonencode(["llm:inference", "memory:read", "memory:write", "notification:write", "sessions:read", "sessions:write"])
+        memory_projects   = jsonencode(["*"])
+        memory_visibility = "private"
+        authorized_models = jsonencode(["reasoning", "ornith:35b", "qwen2.5:3b"])
+      }
+    }
+    "paperless-ai-agent" = {
+      description      = "Paperless AI plugin (auto-tagging, entity extraction)"
+      project_roles    = ["poimen-devs"]
+      authenticated_by = "paperless-ai"
+      metadata = {
+        roles              = jsonencode(["llm:inference", "memory:write", "paperless:admin", "notification:write"])
+        minio_buckets      = jsonencode(["paperless"])
+        paperless_doctypes = jsonencode(["*"])
+        memory_projects    = jsonencode(["*"])
+        authorized_models  = jsonencode(["reasoning", "qwen2.5:3b"])
+      }
+    }
+    "portfolio-agent" = {
+      description      = "Portfolio service agent"
+      project_roles    = ["poimen-prod-readonly"]
+      authenticated_by = "portfolio service"
+      metadata = {
+        roles             = jsonencode(["llm:inference", "memory:read"])
+        memory_projects   = jsonencode(["homelab", "portfolio"])
+        memory_visibility = "public"
+        authorized_models = jsonencode(["ornith:35b"])
+        minio_buckets     = jsonencode(["backups"])
+      }
+    }
+    "temporal-worker-agent" = {
+      description      = "Temporal workflow worker"
+      project_roles    = ["poimen-admins"]
+      authenticated_by = "Temporal worker"
+      metadata = {
+        roles             = jsonencode(["llm:inference", "workflow:execute", "memory:read", "memory:write", "queue:send", "notification:write"])
+        memory_projects   = jsonencode(["*"])
+        sqs_queues        = jsonencode(["*"])
+        authorized_models = jsonencode(["reasoning", "ornith:35b", "qwen2.5:3b"])
+      }
+    }
+    # The gateway is the one consumer that used client_credentials on a machine
+    # token, so it needs both the OIDC app (clients.tf) and a machine user
+    # (here) plus its API application (api_apps.tf). Its capability check reads
+    # the `roles` claim, which arrives as metadata.
+    "api-gw" = {
+      description      = "Go LLM gateway machine identity"
+      project_roles    = []
+      authenticated_by = "api-gw"
+      metadata = {
+        roles = jsonencode(["llm:inference"])
+      }
+    }
+    # pi-worker has no anchor in git (plan §1.1: "not in git") and no group in
+    # the Authentik provisioning script. It gets a machine user and no project
+    # role until the owner names the consumer configuration that holds its old
+    # credentials. Creating a role grant blind would widen access, not preserve it.
+    "pi-worker" = {
+      description      = "pi sandbox worker (no git anchor for its consumer config)"
+      project_roles    = []
+      authenticated_by = "pi sandbox worker"
+      metadata = {
+        roles = jsonencode([])
+      }
+    }
+  }
+
+  machine_metadata = merge([
+    for user, spec in local.machine_users : {
+      for key, value in spec.metadata : "${user}/${key}" => {
+        user  = user
+        key   = key
+        value = value
+      }
+    }
+  ]...)
+}
+
+resource "zitadel_machine_user" "agent" {
+  for_each = local.machine_users
+
+  org_id      = data.zitadel_org.default.id
+  user_name   = each.key
+  name        = each.key
+  description = "${each.value.description} — consumer: ${each.value.authenticated_by}"
+  with_secret = false
+}
+
+resource "zitadel_machine_key" "agent" {
+  for_each = local.machine_users
+
+  org_id          = data.zitadel_org.default.id
+  user_id         = zitadel_machine_user.agent[each.key].id
+  key_type        = "KEY_TYPE_JSON"
+  expiration_date = var.machine_key_expiry
+}
+
+resource "zitadel_project_member" "agent" {
+  for_each = { for user, spec in local.machine_users : user => spec if length(spec.project_roles) > 0 }
+
+  org_id     = data.zitadel_org.default.id
+  project_id = zitadel_project_v2.homelab.id
+  user_id    = zitadel_machine_user.agent[each.key].id
+  roles      = each.value.project_roles
+}
+
+resource "zitadel_user_metadata" "agent_claim" {
+  for_each = local.machine_metadata
+
+  org_id  = data.zitadel_org.default.id
+  user_id = zitadel_machine_user.agent[each.value.user].id
+  key     = each.value.key
+  value   = each.value.value
+}
@@ -0,0 +1,46 @@
+# Outputs feed two things: the SOPS re-point of consumer secrets (§3.2, §6.9)
+# and the machine-key handoff to each agent workload (§5.3). Every value that can
+# authenticate a client is marked sensitive, so `terraform output` prints a
+# placeholder until you ask for it explicitly.
+
+output "org_id" {
+  description = "Organization that owns the project, roles and applications."
+  value       = data.zitadel_org.default.id
+}
+
+output "project_id" {
+  description = "Project ID, which ZITADEL puts in the access-token `aud` claim."
+  value       = zitadel_project_v2.homelab.id
+}
+
+output "client_ids" {
+  description = "client_id per consumer. Compare against the Authentik values: any difference means the consumer config must change too."
+  value       = { for name, app in zitadel_application_v2.oidc : name => app.oidc[0].client_id }
+}
+
+output "client_secrets" {
+  description = "Generated client secret per consumer. Write these into the existing SOPS files; never into git."
+  value       = { for name, app in zitadel_application_v2.oidc : name => app.oidc[0].client_secret }
+  sensitive   = true
+}
+
+output "api_app_client_ids" {
+  description = "client_id of each API application (machine-token surface)."
+  value       = { for name, app in zitadel_application_api.machine : name => app.client_id }
+}
+
+output "machine_user_ids" {
+  description = "ZITADEL user ID per machine user."
+  value       = { for name, user in zitadel_machine_user.agent : name => user.id }
+}
+
+output "machine_keys" {
+  description = "JWT profile key JSON per machine user. Mount as a file; do not pass through the environment."
+  value       = { for name, key in zitadel_machine_key.agent : name => key.key_details }
+  sensitive   = true
+}
+
+output "groups_claim_enabled" {
+  description = "Whether the groups-claim action and its triggers exist. False means Option A is not yet in place and no consumer may move."
+  value       = var.enable_groups_claim_action
+}
@@ -0,0 +1,62 @@
+# Project, roles and default-org lookup.
+#
+# ZITADEL has no group object. The plan §1.3 maps every Authentik group to a
+# project role, and the groups-claim action (actions.tf) reconstructs the
+# `groups` claim from those role keys. Role keys therefore EQUAL the existing
+# Authentik group names, including the hyphens, so that the resulting claim is
+# byte-identical and no consumer or RBAC file changes. The v4.19.2 API allows
+# role keys of 1-200 characters with no character-set restriction (proto
+# zitadel/project/v2/project_service.proto, AddProjectRole.role_key).
+
+data "zitadel_org" "default" {
+  id = var.org_id
+}
+
+locals {
+  # Single source: the 19 groups of scripts/iam/provision-rbac.py GROUPS.
+  # Descriptions are carried over so the console reads the same as before.
+  roles = {
+    "homelab-admins"       = "Cluster administrators with full access"
+    "minio-admins"         = "MinIO administrators"
+    "minio-photos"         = "Photos bucket (Immich) access"
+    "minio-documents"      = "Documents bucket (Paperless) access"
+    "minio-backups"        = "Backups bucket read-only access"
+    "poimen-admins"        = "Poimen memory administrators"
+    "poimen-devs"          = "Dev and staging projects access"
+    "poimen-prod-readonly" = "Production projects read-only access"
+    "paperless-admins"     = "Paperless administrators"
+    "paperless-finance"    = "Finance documents"
+    "paperless-legal"      = "Legal documents"
+    "paperless-hr"         = "HR documents"
+    "grafana-admins"       = "Grafana administrators"
+    "grafana-editors"      = "Grafana dashboard editors"
+    "grafana-viewers"      = "Grafana dashboard viewers"
+    "sqs-users"            = "SQS/Temporal queue read access"
+    "sqs-writers"          = "SQS/Temporal queue read/write access"
+    "s3-users"             = "S3 read access"
+    "s3-writers"           = "S3 read/write access"
+  }
+}
+
+resource "zitadel_project_v2" "homelab" {
+  org_id             = data.zitadel_org.default.id
+  name               = var.project_name
+  project_id         = var.project_id
+  project_role_check = false
+
+  # Roles must reach the tokens. Every consumer authorizes on them:
+  # kube-apiserver RBAC reads `groups`, MinIO reads bucket policies, Grafana
+  # reads grafana_org_role. Role assertion on the ID token is what most OIDC
+  # clients inspect.
+  project_role_assertion = true
+}
+
+resource "zitadel_project_role" "groups" {
+  for_each = local.roles
+
+  org_id       = data.zitadel_org.default.id
+  project_id   = zitadel_project_v2.homelab.id
+  role_key     = each.key
+  display_name = each.key
+  group        = "homelab"
+}
@@ -0,0 +1,53 @@
+# terraform/zitadel — identity configuration for the ZITADEL instance
+#
+# Scope: this root module declares the objects INSIDE ZITADEL (org, project,
+# roles, OIDC applications, machine users, grants). It does NOT deploy ZITADEL
+# itself — that is ArgoCD's job (`k8s/argocd/apps/30-security.yaml`, values in
+# `k8s/infra/iam/zitadel-values.yaml`).
+#
+# Migration plan: ~/workplace/homelab-doc/ZITADEL_MIGRATION.html (outside git).
+# This file implements §1.3 (roles), §1.4 (machine users), §2.4 (grants) and
+# §6.2 (Terraform provider route). The claim-injection action of §1.3 lives in
+# `actions.tf` and stays disabled until the live go/no-go gate proves it.
+#
+# Authentication to ZITADEL uses the `iam-admin` machine key that the chart's
+# setup job writes to the `iam-admin` Secret in namespace iam:
+#
+#   kubectl -n iam get secret iam-admin -o jsonpath='{.data.iam-admin\.json}' \
+#     | base64 -d > ~/.config/zitadel/iam-admin.json
+#   chmod 600 ~/.config/zitadel/iam-admin.json
+#
+#   export ZITADEL_JWT_PROFILE_FILE=$HOME/.config/zitadel/iam-admin.json
+#   terraform -chdir=terraform/zitadel init
+#   terraform -chdir=terraform/zitadel plan
+#
+# State is LOCAL and git-ignored (`terraform/*.tfstate` in .gitignore). That is
+# a deliberate first step, not the final answer: §8 risk 8 of the plan names
+# state drift as a blast-radius item. Move the state to MinIO before a second
+# operator ever runs `apply`. No secret is written to state except the generated
+# client secrets and machine keys, which are marked sensitive — they still land
+# in the state file, so treat that file as a secret.
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    zitadel = {
+      source  = "zitadel/zitadel"
+      version = "3.8.7"
+    }
+  }
+}
+
+provider "zitadel" {
+  domain = var.zitadel_domain
+  port   = var.zitadel_port
+
+  # JWT profile of the iam-admin machine user. Preferred over a PAT: it is
+  # short-lived per call and never sits in the environment as a bearer token.
+  # Set the path through ZITADEL_JWT_PROFILE_FILE.
+  jwt_profile_file = var.jwt_profile_file
+
+  # The instance serves plain HTTP behind ingress-nginx, which terminates TLS.
+  # Turn this on only if the provider must call the pod directly.
+  insecure = var.zitadel_insecure
+}
@@ -0,0 +1,103 @@
+variable "zitadel_domain" {
+  description = "External domain of the ZITADEL instance, without scheme."
+  type        = string
+  default     = "zitadel.riotpiao.com"
+}
+
+variable "zitadel_port" {
+  description = "Port for the ZITADEL API. 443 behind ingress, 8080 for a direct pod port-forward."
+  type        = string
+  default     = "443"
+}
+
+variable "zitadel_insecure" {
+  description = "Call the API over plain HTTP (set true only when port-forwarding the pod)."
+  type        = bool
+  default     = false
+}
+
+variable "jwt_profile_file" {
+  description = "Path to the iam-admin machine key JSON extracted from the iam-admin Secret."
+  type        = string
+  default     = null
+}
+
+variable "org_id" {
+  description = <<-EOT
+    ID of the ZITADEL organization that holds the project. Read it from the
+    console: https://zitadel.riotpiao.com -> Organization settings -> Resource
+    ID. The chart's setup job creates the default org; this module never creates
+    one, because creating a second org would split the role and grant space.
+  EOT
+  type        = string
+}
+
+variable "project_id" {
+  description = <<-EOT
+    Stable project ID. ZITADEL puts the project ID in the access token `aud`
+    claim, so pinning it keeps audience checks readable across a rebuild.
+    Every consumer that validates `aud` against a client ID must be re-pointed
+    to this value — see the plan §5.2.
+  EOT
+  type        = string
+  default     = "homelab"
+}
+
+variable "project_name" {
+  description = "Display name of the project that carries every role and application."
+  type        = string
+  default     = "homelab"
+}
+
+variable "machine_key_expiry" {
+  description = "Expiry of every machine key, RFC3339. Matches the chart's default iam-admin key window; rotate by changing this value and re-applying."
+  type        = string
+  default     = "2029-01-01T00:00:00Z"
+}
+
+variable "api_app_consumers" {
+  description = <<-EOT
+    Machine users that also need an API application (the client_credentials
+    replacement). Only the gateway does: every other agent signs a JWT with its
+    machine key and does not need a client_credentials grant.
+  EOT
+  type        = map(string)
+  default     = { api-gw = "Go LLM gateway" }
+}
+
+variable "enable_groups_claim_action" {
+  description = <<-EOT
+    Create the Actions v1 script and its trigger that synthesizes the `groups`
+    claim. Keep this false until the live go/no-go gate passes: the claim is
+    load-bearing for kube-apiserver RBAC, MinIO policies, Grafana role mapping
+    and every service that reads `groups`. Turning it on with an unverified
+    script can break every consumer at once.
+  EOT
+  type        = bool
+  default     = false
+}
+
+variable "groups_claim_script" {
+  description = <<-EOT
+    Path to the JavaScript body for the groups-claim action, read only when
+    enable_groups_claim_action is true. The body is deliberately not shipped
+    blind: the ZITADEL v1 Actions runtime API must be confirmed against the live
+    instance before a script is committed, because a wrong call silently emits
+    no claim instead of failing loudly.
+  EOT
+  type        = string
+  default     = null
+
+  validation {
+    # try() is required: HCL does not short-circuit `||`, so fileexists(null)
+    # would fail the whole plan even with the action disabled.
+    condition     = try(fileexists(var.groups_claim_script), true)
+    error_message = "groups_claim_script must name an existing file when it is set."
+  }
+
+  # Enabling the action without a script would reach file(null) inside actions.tf.
+  validation {
+    condition     = !var.enable_groups_claim_action || var.groups_claim_script != null
+    error_message = "enable_groups_claim_action requires groups_claim_script."
+  }
+}