merge: harness/zitadel-iam-wiring

The tunnel was remotely managed from the Zero Trust dashboard: the
Deployment ran 'run --token' and ignored the ConfigMap that already
existed in the tree. Switch to a locally-managed tunnel so the ingress
routing table lives in git.

- deployment: mount cloudflared-config (config.yml) and the
  cloudflared-credentials Secret; drop TUNNEL_TOKEN env
- configmap.yaml: per-hostname ingress rules to ingress-nginx via the
  CoreDNS rewrite, plus a 404 catch-all
- secret-generator: register the SOPS-encrypted credentials file
- homelab-ca: add the cloudflared namespace copy so the connector trusts
  the ingress certificate (force-ssl-redirect makes plaintext impossible)

Credentials were extracted from the live cloudflared-token Secret and
re-encrypted with SOPS; no plaintext secret is committed.

.gitignore

@@ -52,6 +52,10 @@ terraform/*.tfstate.*
 terraform/zitadel/*.tfstate
 terraform/zitadel/*.tfstate.*
 terraform/zitadel/*.tfvars
+terraform/edge/*.tfstate
+terraform/edge/*.tfstate.*
+terraform/edge/*.tfvars
+terraform/edge/terraform.tfvars.local
 terraform.tfvars.local
 skills-lock.json
 secrets-plaintext.yaml
@@ -81,3 +85,7 @@ scripts/iam/*.py
 # Runtime credential export from the old Authentik provisioning run. Holds live
 # M2M tokens; never commit it.
 /authentik-m2m-tokens.env
+
+# Never commit backup copies of plaintext tfvars (they carry cluster PKI)
+terraform/*.bak
+terraform/edge/.secrets/

.pi/loops/loops-01a16cb7-780d-72af-bac3-487935898ff3.json

@@ -0,0 +1,22 @@
+{
+  "nextId": 2,
+  "loops": [
+    {
+      "id": "1",
+      "prompt": "Check Longhorn and CNPG recovery progress in the homelab cluster and report terse. Run: kubectl -n longhorn-system get volumes.longhorn.io -o json (count by state/robustness), kubectl -n longhorn-system get replicas.longhorn.io -o json (count running/stopped/failed), kubectl get clusters.postgresql.cnpg.io -A (ready instances per cluster), kubectl -n cicd get pods,pvc | grep forgejo-db. Report: volumes healthy vs degraded, replicas running vs stopped, and each CNPG cluster ready count. If every volume is attached/healthy and every CNPG cluster shows 3/3 ready, say RECOVERED, list any leftover anomalies, and stop this loop with LoopDelete. Otherwise name what is still rebuilding and keep going.",
+      "trigger": {
+        "type": "cron",
+        "schedule": "*/5 * * * *"
+      },
+      "status": "expired",
+      "recurring": true,
+      "createdAt": 1792898370946,
+      "updatedAt": 1792901371403,
+      "expiresAt": 1793503170946,
+      "source": "tool",
+      "readOnly": true,
+      "maxFires": 6,
+      "fireCount": 6
+    }
+  ]
+}

k8s/apps/cloudflared/configmap.yaml

@@ -0,0 +1,58 @@
+# cloudflared ingress configuration.
+#
+# This list is the tunnel's routing table: for each incoming hostname the
+# connector dials the matching origin. The final rule is a catch-all, so a
+# hostname absent from this list gets 404 from the connector — DNS alone does
+# not route a request into a service.
+#
+# Management model: this file makes the tunnel LOCALLY managed. cloudflared is
+# given a credentials file instead of a token, so Cloudflare stops pushing the
+# remotely-managed ingess list and this ConfigMap wins. Revert the Deployment to
+# `run --token` to return to the dashboard-managed list.
+#
+# Origin services self-reference the public hostname. That resolves because
+# CoreDNS rewrites each of these names to ingress-nginx
+# (terraform/files/coredns/Corefile), so the connector reaches the in-cluster
+# ingress instead of looping back out through Cloudflare.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared-config
+  namespace: cloudflared
+data:
+  config.yml: |
+    tunnel: 32d0bf93-e1bd-45b8-93a9-71ff9f83de80
+    credentials-file: /etc/cloudflared/creds/credentials.json
+
+    # WARP-to-Tunnel routing. The remotely-managed config had this enabled;
+    # dropping the key would break WARP access to private addresses. cloudflared
+    # 2026.x has no `enabled` field — the key's presence is what enables it, and
+    # `enabled: true` here makes the binary warn about an unused key.
+    warp-routing: {}
+
+    ingress:
+      - hostname: paperless.riotpiao.com
+        service: https://paperless.riotpiao.com
+      - hostname: authentik.riotpiao.com
+        service: https://authentik.riotpiao.com
+      - hostname: img.riotpiao.com
+        service: https://img.riotpiao.com
+      - hostname: riotpiao.com
+        service: https://riotpiao.com
+      - hostname: grafana.riotpiao.com
+        service: https://grafana.riotpiao.com
+      - hostname: gotify.riotpiao.com
+        service: https://gotify.riotpiao.com
+      - hostname: igotify.riotpiao.com
+        service: https://igotify.riotpiao.com
+      - hostname: poimen.riotpiao.com
+        service: https://poimen.riotpiao.com
+      - hostname: comfyui.riotpiao.com
+        service: https://comfyui.riotpiao.com
+      - hostname: forgejo.riotpiao.com
+        service: https://forgejo.riotpiao.com
+      # ZITADEL. Without this rule the connector answers 404 for the hostname
+      # even though the DNS CNAME points at the tunnel.
+      - hostname: zitadel.riotpiao.com
+        service: https://zitadel.riotpiao.com
+      - service: http_status:404

k8s/apps/cloudflared/deployment.yaml

@@ -13,21 +13,63 @@ spec:
       labels:
         app: cloudflared
     spec:
+      # Ingress-nginx forces HTTPS (force-ssl-redirect), so cloudflared must
+      # dial https://<hostname> and trust homelab-ca or the TLS handshake fails
+      # with "certificate signed by unknown authority". Same pattern as
+      # authentik-values.yaml: merge the system bundle with homelab-ca.crt.
+      initContainers:
+        - name: merge-ca-certs
+          image: debian:bookworm
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            runAsUser: 0
+          command:
+            - sh
+            - -c
+            - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
+          volumeMounts:
+            - name: homelab-ca
+              mountPath: /homelab-ca
+              readOnly: true
+            - name: merged-ca
+              mountPath: /merged
+      volumes:
+        - name: config
+          configMap:
+            name: cloudflared-config
+        - name: credentials
+          secret:
+            secretName: cloudflared-credentials
+        - name: homelab-ca
+          configMap:
+            name: homelab-ca
+        - name: merged-ca
+          emptyDir: {}
       containers:
         - name: cloudflared
           image: cloudflare/cloudflared:latest
           args:
             - tunnel
             - --no-autoupdate
+            - --protocol
+            - http2
+            - --config
+            - /etc/cloudflared/config.yml
             - run
-            - --token
-            - $(TUNNEL_TOKEN)
           env:
-            - name: TUNNEL_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: cloudflared-token
-                  key: token
+            - name: SSL_CERT_FILE
+              value: /merged/ca-bundle.crt
+          volumeMounts:
+            - name: config
+              mountPath: /etc/cloudflared/config.yml
+              subPath: config.yml
+              readOnly: true
+            - name: credentials
+              mountPath: /etc/cloudflared/creds
+              readOnly: true
+            - name: merged-ca
+              mountPath: /merged
+              readOnly: true
           resources:
             requests:
               cpu: 10m

k8s/apps/cloudflared/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Kustomization
 namespace: cloudflared
 resources:
   - deployment.yaml
+  - configmap.yaml

k8s/apps/paperless/backup-cronjob.yaml

@@ -48,6 +48,7 @@ spec:
                   mountPath: /backup
           containers:
             - name: mc-mirror
+              # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
               image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
               env:
                 - name: ACCESS_KEY

k8s/apps/sms/deployment-macos.yaml

@@ -1,120 +0,0 @@
-# macOS VM (Docker-OSX) hosting the BlueBubbles server.
-#
-# ── Why a VM and not a container ────────────────────────────────────────────
-# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
-# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
-# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
-# container — a VM in a box, not a macOS container.
-#
-# ── Why this works on worker-2 ──────────────────────────────────────────────
-# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
-# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
-# module). Bare metal, so no nested virtualisation needed.
-#
-# ── Read this before relying on it ──────────────────────────────────────────
-# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
-#    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
-#    into iMessage, THEN install BlueBubbles inside the guest. This manifest
-#    only provides the machine; it does not provision macOS.
-# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
-#    own guidance: "test sending an iMessage to yourself. If it does not
-#    succeed, it's likely best to restart from the beginning."
-# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
-#    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
-# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
-#    warranty".
-#
-# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
-# inside the guest and is NOT required for plain send/receive. Skip it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: macos-bluebubbles
-  labels:
-    app.kubernetes.io/name: macos-bluebubbles
-    app.kubernetes.io/part-of: sms
-spec:
-  # Scaled to 0 on 2026-10-24. The pod pinned itself to a node labelled
-  # workload=imessage, and no such node exists any more. The PVC macos-disk also
-  # uses StorageClass longhorn-imessage-local, which pins the same dead node and
-  # stayed unbound for 71 days. The macOS guest therefore cannot start. Raise
-  # this to 1 again only after a KVM-capable node carries that label and taint.
-  replicas: 0
-  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
-  # concurrently — two QEMU processes on one image corrupts it.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: macos-bluebubbles
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: macos-bluebubbles
-        app.kubernetes.io/part-of: sms
-    spec:
-      # Dedicated node. The taint keeps everything else off worker-2; this
-      # toleration is what lets the VM on. Both halves are required.
-      nodeSelector:
-        workload: imessage
-      tolerations:
-        - key: workload
-          operator: Equal
-          value: imessage
-          effect: NoSchedule
-      containers:
-        - name: macos
-          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
-          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
-          # only schedules because the sms namespace is labelled privileged.
-          securityContext:
-            privileged: true
-          env:
-            # Generates a unique serial / board-serial / UUID / MAC and persists
-            # them to bootdisk.qcow2. This synthetic identity is what iMessage
-            # activates against — it must stay stable across restarts, which is
-            # why the PVC matters.
-            - name: GENERATE_UNIQUE
-              value: "true"
-            # Identity is only plausible if it matches a real product line.
-            - name: DEVICE_MODEL
-              value: "iMacPro1,1"
-            - name: RAM
-              value: "12"
-            - name: CORES
-              value: "6"
-            - name: EXTRA
-              # Expose the BlueBubbles server port from the guest to the pod.
-              # Guest :1234 (BlueBubbles default) -> pod :1234.
-              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
-          ports:
-            - name: vnc
-              containerPort: 5999
-              protocol: TCP
-            - name: bluebubbles
-              containerPort: 1234
-              protocol: TCP
-          resources:
-            requests:
-              cpu: "6"
-              memory: 14Gi
-            limits:
-              cpu: "12"
-              memory: 20Gi
-          volumeMounts:
-            - name: macos-disk
-              mountPath: /home/arch/OSX-KVM/disk
-            - name: kvm
-              mountPath: /dev/kvm
-          # No readiness probe on purpose. The guest takes many minutes to boot,
-          # and until macOS + BlueBubbles are installed BY HAND there is nothing
-          # listening on 1234. A probe here would crash-loop the pod through the
-          # entire interactive install.
-      volumes:
-        - name: macos-disk
-          persistentVolumeClaim:
-            claimName: macos-disk
-        - name: kvm
-          hostPath:
-            path: /dev/kvm
-            type: CharDevice

k8s/apps/sms/kustomization.yaml

@@ -1,10 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: sms
-resources:
-  - namespace.yaml
-  - storageclass.yaml
-  - pvc-macos.yaml
-  - deployment-macos.yaml
-  - service.yaml
-  - networkpolicy.yaml

k8s/apps/sms/namespace.yaml

@@ -1,19 +0,0 @@
-# iMessage delivery for the cluster.
-#
-# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
-# that hooks Messages.app private APIs — it cannot be containerised on Linux,
-# because containers share the host kernel and macOS needs XNU + Cocoa. The only
-# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
-# needs a privileged pod with /dev/kvm.
-#
-# Hence privileged PodSecurity: the cluster default from the Talos controlplane
-# is `enforce: baseline`, which forbids privileged containers and host devices.
-# Scope is limited to this namespace.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: sms
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged

k8s/apps/sms/networkpolicy.yaml

@@ -1,22 +0,0 @@
-# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
-# ID and exposes an unauthenticated VNC console; nothing should reach it except
-# opted-in clients.
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: sms-default-deny
-spec:
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/part-of: sms
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        - namespaceSelector: {}
-          podSelector:
-            matchLabels:
-              sms-client: "true"
-      ports:
-        - protocol: TCP
-          port: 1234

k8s/apps/sms/pvc-macos.yaml

@@ -1,25 +0,0 @@
-# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
-#
-# This volume is NOT disposable: it holds the VM's serial number, board serial,
-# UUID and MAC, which together form the identity iMessage was activated against.
-# Losing it means re-running activation, which is the least reliable step of the
-# whole setup.
-#
-# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
-# the installer, the base system, and qcow2 growth.
-#
-# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
-# Losing worker-2's disk therefore means losing the activated identity and
-# redoing iMessage activation. Once the guest is installed and activated, take
-# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: macos-disk
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-imessage-local
-  resources:
-    requests:
-      storage: 200Gi

k8s/apps/sms/service.yaml

@@ -1,31 +0,0 @@
-# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
-# it is an unauthenticated console onto a machine holding a live Apple ID
-# session. Reach it with `kubectl port-forward`, never an Ingress.
-apiVersion: v1
-kind: Service
-metadata:
-  name: macos-vnc
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: macos-bluebubbles
-  ports:
-    - name: vnc
-      port: 5999
-      targetPort: vnc
----
-# The BlueBubbles REST API, once installed inside the guest. This is the stable
-# name cluster services use, so callers never depend on the pod IP or on whether
-# the backend is this VM or a real Mac mini later.
-apiVersion: v1
-kind: Service
-metadata:
-  name: bluebubbles
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: macos-bluebubbles
-  ports:
-    - name: http
-      port: 1234
-      targetPort: bluebubbles

k8s/apps/sms/storageclass.yaml

@@ -1,32 +0,0 @@
-# Dedicated StorageClass for the macOS VM disk.
-#
-# The default `longhorn` class does not work here, for two independent reasons:
-#
-# 1. Replica count. Default is 3, and Longhorn schedules against
-#    storageMaximum - storageScheduled with over-provisioning at 100%. Free
-#    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
-#    volume has only one node that can hold even a single replica — a 3-replica
-#    volume fails outright with ReplicaSchedulingFailure.
-# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
-#    created, before any pod is scheduled. Combined with strict-local that
-#    pins the data to an arbitrary node, not the one the VM runs on.
-#
-# So: one replica, kept local to the VM, bound only once the pod has a node.
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: longhorn-imessage-local
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-reclaimPolicy: Delete
-# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
-# the replica is placed on that node and not somewhere else.
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
-  # Serving it over the network from another node's disk would be the single
-  # worst thing for guest responsiveness, so force it local.
-  numberOfReplicas: "1"
-  dataLocality: "strict-local"
-  staleReplicaTimeout: "30"
-  fsType: "ext4"

k8s/argocd/apps/60-applications.yaml

@@ -110,37 +110,6 @@ spec:
     syncOptions:
       - CreateNamespace=true
 ---
-# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
-# running the BlueBubbles server, plus its dedicated local StorageClass.
-#
-# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching
-# toleration for that node's taint. Until worker-2 is provisioned this app
-# syncs everything except the pod, which stays Pending — that is expected.
-#
-# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which
-# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would
-# not have those labels.
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: sms
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/sms
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: sms
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
----
 # Document management. Raw manifests (no Helm): postgres is the dedicated
 # paperless-db CNPG cluster in k8s/infra/databases (wave 2), redis is
 # in-cluster only (no PVC), media lives on the cp-3 USB HDD (see

k8s/argocd/secrets/cloudflared-credentials.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: cloudflared-credentials
+    namespace: cloudflared
+type: Opaque
+stringData:
+    credentials.json: ENC[AES256_GCM,data:LnJVGuxJAOwDYbLgX00OHICDDldZgwgvGafOIwKoJKnjg3FjFMQYThNCOL/H+6tctuKixJNcAckjg70tmCKWxNq9I35XX/YSqob6Qv4fywv32hsNIAe44BU035MGZ5zmc6bZZ8wAgTJZw6Th4TzY7taFBCRdj2fW4T+qVfq7strn68G4ufVMK6UXlOkEDLjR2Vxj32lqSLdbdMma96XB+265hwL6IVfaugG662mcGhPwgHk=,iv:NFmyHN5m2mJUBgn1qgmJZ2YgkFxBZyp7+gU1Ng6B/7M=,tag:/66JtHK5Yu7pRxB0b0eifQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaDVycUE4RFpoRjdpTlpX
+            dGpzN3hwRFQ2cmtXVXVzWnZJOXJEZXZRWEhNCkJwbm9QV0lGcGtNRTJqbStrc3l1
+            Q2M4bE1WbzJpTXZQWjJyTGpFS0JYcVUKLS0tIDBvU2U5aEtVTTZUaE85eEkyYitG
+            bGtSVkhKNnRuNlZWMWh6Z1M4VXM1WU0KdusvtM85V0xrO+uJFUY+UH0tuwdF0GiW
+            79J4Rlxk8ne25psKsvwjLYLxZ53xjFHCC4FCHTNkYwTKCDEKjw46Lg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-26T02:45:04Z"
+    mac: ENC[AES256_GCM,data:KF7KwXCnNLAg1YI3u0lLI/ho4sITHAq09ac2WO4TK3TCthkuivb89YjhuuJKs0NFhjn6aRLqmZavUqcXH2HujiryqbLNVexOShQM0wSzgCzmWdbi0UzpgJ4KcQA3PufDtsQz4HHLXx396EbZrM3Ie3WSu/PvRm5iWOxzTGCGc9Q=,iv:Eo920oXXD1FLZVJXywZ6C+LRL+qZjmnZXv0+Oa/GMb8=,tag:6LuPVSet4WjvrvGUAxaJlw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -11,6 +11,7 @@ files:
     - agent-pod-ssh-key.enc.yaml
     - authentik-secrets.enc.yaml
     - cloudflare-secrets.enc.yaml
+    - cloudflared-credentials.enc.yaml
     - forgejo-registry-pat-api.enc.yaml
     - forgejo-registry-pull.enc.yaml
     - forgejo-runner-token.enc.yaml
@@ -38,3 +39,8 @@ files:
     - pi-sandbox-secrets.enc.yaml
     - etcd-defrag-creds.enc.yaml
     - zitadel-masterkey.enc.yaml
+    # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
+    # owns it. Without this, the chart re-generates the keypair on every render,
+    # which changes checksum/secret-login-service-key and leaves both Zitadel
+    # Deployments permanently OutOfSync.
+    - zitadel-login-service-key.enc.yaml

k8s/argocd/secrets/zitadel-login-service-key.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: zitadel-login-service-key
+    namespace: iam
+type: kubernetes.io/tls
+data:
+    tls.crt: ENC[AES256_GCM,data:Ad4omL7pD4QhixW6wSVsVor5FtHbWyOHeYbxOQsEBAaSuvb7JIkvnKqo+Xy6zKuMmRViUIbqhlLQDXCaseAVGgpu3PdtShwJ7LRXJmr0XtQV1M6YtRbRexN6UpFA3XdD+JfifMhtvd29rM3WzLPX69uk4k7voYN5m0pNXSN23jVeIeqm+xB1CpxDW/guZ2hA5+b5XBzVOkMsSOSdWFWH2lK7nInv6ZatCa0/fuIKmB5nZpSV9tHpCTiTnBuNc0rW1X4HOWa8zMiSbjQAqtDitm7lrVHJyImXwBjmuS0wM35VwrwxVUUEjS8kgqnyb02JvNRdNjuwOmIcyKjrdgO1B1ft0CrAbRNkk2dYR8Y7SNhR4F4tYYF5L2szQgj+5tcYrnRGEI2QkXbcmTa0Y9V99+D2bKjt5t59Pi/1r8/jt0CNy0GeeZDvsN5scy0/A4WqtJ2sZDyTnTPwtUpAAwVC2WSqsD11gkzhyRJAgmAmEgfPIL7kTZsZBoSQeuuCJ82Ghqx9kLTpui6kLULxxUpP9oefsKw9PjTwExRbNNcPV5p7PPi9dN00TNe9iYuFqoZFP/lp5SMjDqbsV4Ie1ffcfyZsIHMuOv5Ths7n8ExYdR0zIn+aQePXASt+dE6njhFiblxyBZsn0xON3EIUaE9e3WsdcQKjlPOaAW/DvU6PwBqCeaB2nYzM2NFk6W3lN8BahTE9k7xXrILHIrXuIXezotCK6jclb5yZmdm/fTCX1iejGVDktvvcmX6bbv3YXTRZmsviRhLFHA8oClU0alTsY5Q3nn2Xl0UfXU+rW+10AIpY5WFMzP6A8hXvHx0ISC1S4AIqjmlsDmq2yEZKpbiL+Bs52ymW7t2rQ+KGhWH/bcC3VrJwUWnVYP9b80xyHZdKqubwLc5UcDzCs/Lan23VpLxSJkqb8vZP7SHQ1nc+rlwrqFDkzQ+yMNLZmutqbaAmMyXbrR51Rz3VqofraPXjS+lmLAuEcuTEFbrImsct5Zb57MtKvhHi6CUWt/Vj/NOUJlwy09zPy4JAMMJ81ooyBj6CBHY2f1vfR/UUDN5VZdfg4dUWXS0CQdOb3hx19COr4lR4RcpA46qJEjum+4icm5ckziYT5Nm7NB+ydXJ+Eys2Ba0dGRM6CIeQVhghhCDOZ6uoQ2A6DNql6wjrfUml8UwuJNO0nXVqNS28pLwddsOjlbsysJAUEma6EHXb2Vjb90JZErVswGuhQ2+UdhtpuIkgBYX/MMCZV942C+K41A9478BaCiWuC9KbnAcx81ceR+T3syARctZvQaYaiH7WfF8qyo0VOGzZRYjFbqeyHefiOmPmWTS/XbIti3evyhmxHwjUcidK2njrPB0O/3w6R69UA1p7ph9EgFsj18gPYnJthikgX8vcfVaVnPIAwCqY5DJJmIFTNXVkNDBX+2orrW2qj6PEH+go+uc+aUmu646niqaMAcrUl7gGNGd2OWFHdIRKdtogOvoVpQMDB4eWHPRsL29EVt3rnknHfpq7cIjQXuk9XtlB8zjllYdrh50Q8ZNBOvzWgCiMvAyanfIA+kW9KHQlmb1qJGYp7it2Lrg1K7sDEekrlbuoNsf+2rl8YRQz8pVBuCSzG9DPp4+Rj+ddiqX/Rrs3ufWDKFV9zJoN1EpJ8+z6n8rsQJBe/lSFLW1mLVkaPSNNdIQfdJ6MbsHpMlqqFYP8dOKckxv82uOkHoCLHrgaXiiMd1dBsX1fBiv2NvzDiuLyN81d7R2wNAQbkG9b+/+C6wVr6kDg0n6seseoCyeSI++PJIkCrkTXzd7rLs8Of9CHZQEB15wjemlmYGlJnCiFWPFa6h0jJPatPrfTYv+GpqzWV8u1Uh3aNjd+g1wVQok/ojNOZJahUSI4B67HkJI+I7WOF9C+rRCwNutqshIxYzW7CpyVey5P6Kyv+/cucDEgb0ggH032MBtzTxazpJcE4Y6fTrOiXHp9j2KjzyUt2YeCCL8UMwbwJvW9p1I5gx3ArsOuyHm1kA==,iv:PLsjmrTwjM7IUS7ZBSyVvJfAJzCSUtU+UAvHbSwoaVk=,tag:pUQwbHoZgfHW4CpPKoULAw==,type:str]
+    tls.key: ENC[AES256_GCM,data:rO1NVJZP9TP96C48it4+2lGyRutwqdlX6qiM9R/+0hgYc5ceeeopaCUG/i/LOgltxRQK8qkyed/W3zU2VI6alBpt75XkT3o9ZRy3uxe8mB3ixS7+xQnMc0ci8Mf2bCtGzqwUQj8XPRP4K7Mxw8r4XO5rQLnSmNme0+BxS+XiE6hqifS16UUKfLuFeUtTEoI850sg1GQt0uaHf9kOrpmvsbg3sflO2dQXk8uSFi3B1ghG5DdWImVoc4f5kFK86XG65WVVfYjLaUvm7ZBPT0mwvq3SgPdEXeWq1fp7tMQqCRKuluRZon1e+J5SlWOPUTxwyTq336oK2Nfk3HsiQ3uCG76oCmI5yTe2tCN1f2IXFXBvLNM9QTxc/KFhI4uI6bvHjimOSpXYuRdroH0hc4ohLP44WG0E8Nz7sAb3Ai3PbDTKnqYQemvGEUTdcdbWuMK5/JE4f1IE9DgOOO9RF+s7uXTdzZAmoMcRtVsckQJEJl7AKgqSn7BbK0cjRL/QXclNySQ/qcgSDFiV/pPbGC9JFweJon5F0Qn/AIAnGGzqmVbS6x45UAJieI5oI/30OxjzF+Dov6EGMoE0mys+6l4j8v1EFCtHhhopS7zM4jZRTnNzlZWPtEISU2kopLroGpKESE145VQnB7X66F40KXcOIiXmU1H7Yxw5rJ0SqLAFWQw7gyNdzbnbfCmGASByxnYYwZRy3QDZYrQlthLd5tlGe0cg7j02ZSHmmJNQOx5yOmSQI7z4njDvJlOPncC14T59yDZUIYYLTAYjL8nmCNk2yD55ImVGRIb/f0JGAc2vHMlDN6fkuiSLtbASz11PwWzdln9tS9BTpRx7sfv5+kMhFkzomufftAJbs5XdOxcvGRHkRkdyCn3t8jcRXaNDBpq3o8zWLNh+411I65DMDN1B0c4+7Uw4N17I+KfrVh5Fb8ix0LW9aEL/SUxeIqdEu0ZMTncxGS0QVzZmtJUTdWuT0JjtgfsD65jqpg9l8RdIKX4i0ox2+GeQuXKCQiCaXS47yDxUcICYZB7UDnfwBcXlnGWw1frvkBBqwvwUVa7G7Al1A1a35ob1qb9gpyJZGqmEyV4Qe3PmfUHkp+/bAWarLXnk8P4l2faFjCEnPDIejGMQgDdtkodq7lfqwtHwg02bXNl9X3FbXjyp51/IJshE9dhHLkniWyHwZvHZJ4jjqndf08682gxTnW0bscfv7d6lKznE7rgd/el5ygdlL0PSr0pbxJtcd7MYrACMfZZxRuY3PWZayr1n4iaqh161S8ldeEcLCiHFvDeGHslUK+Sl383AHhs46NHEVgMbhuKJ1thHckQSgpqm9vGE1gBIo0d7z40EulgD+KS4Qm/vdydCgqtzyNmFDg17L+SVCQiyckGxgl/j1YzQKz0o1EWXNssk121m7Nh/e/bYNN8kC6doJPBQBkdhU/+DVZa4+/je+qpqvqnjpheX42k49ubk638QhOkcMmx6ZNnEbL9Ra0mNJHZO27t6++LgN2XkPg6t8VO94yx1c/VFyoF9Qg21O1+BadK3RF9l9s1eUB7n93YwUDhxZpimr8j8oLCn+PvksKwzWEIiAFXUWvuROwh6yC31GghO6veGx7UqXGPM2O8+mkOxXZObbnYAAhteRhXO/k15HL3qf94pudZ6+tETmeej4FKdhZzF9iOsTu/69izQFhl57E+n5hDjRzeVANKQOdq2x7qDG7pGfpJzrDVR95Z1q7Y/d+7FJjBML+YvzGvbxZno+PbPmLGrgxSebAWgO73fUDkBHvosMDjeHjg0NKBXYfR3kImceo4iJwF0bOeFQA09j+6OKW1EId9r1EzYwPGR0DUDt+A9f4DkU/27dbKUmJovBPIAXtYHErwZ7EW0ITd8ebNXyOiArz8JJietDsR++lwetArzrDszsPsiumoSu1/FajtbBfUQqllxJBeoIqNEPHQED2ysSWzqTlHe8tfmY3Vp/PlZh8GbpA/EV5uikWzDSaZbADqXGny/xd+g+8mm4J3DxgPOEEgFqeg+q4C1U4yEy3gYkknT6AcIO8MGOa3daRowEv2zfhVlrNhv2ezLjzsPBQJSK/ph9KZbYiREZVCBQioJJlvfG/dkkycjkVqvTNtn3swhipxUqYc0vKmXFG+R9+fUOP5u9ctqmdIf7Jclew4OBO1ML+sH+l6NojgfX+T+OqQ8EyNMY2Aa7i24ppwS8ySDwyYEk9kjopVZ8L3MSw3CQnWBNMg734qUawCUoFMtGPZ0jC+o6DiV18xpkB1Ffrmjs+72AoKk6wR9zIYyVQG7wr7O4PLWeXEX2NgBDeKaWi0gHInVAb2puJKgGgaZbImuecSaKaqA0y1fXu3+4teozv9eLkBxhYUJRwIBQbA7E3lQ4ocziE3ZMxdAIGAOEJHgDs6UreB4ZMLgdKxqhYzR3GxrmZmspOw/TVBGGjClrefGZh0/gLkXSDZ0uugW1rNM3TkA9t2U0eBCNyRh+8sScUKzja4Ei2ukkfoCNEf0alJkG3udwBtjDBcTRhQJARFD+UWMut5Xqw/dAxs777/VmY00/wD9FaCq0aVc/CJI++oGYgunPDd7ACSTyuY0UST1AEBrCrFBB+T+eKrFcU/4gBXEGiAY4x8lrp7fNZ/nsGFvBvs5sxq9NvLFXGA8Tv3LHU8x8m8oGHVnT0AaFJIKM8vGf2bkdD89EmQkKG9pfnKPCTbyFszqOJlgORDLSn9bNvDYlVC3dS5HQbqdv1a1fn1WrLDEXTNuD1Nj6dJXUBLAZfqzRHaPRi04YNpw9MY+c6f3XmAvNmwrFgdnxHja47ObX4XGigltlCWUSImRboBinCeZ9k9GdQ82Pw36dO433Z3IjtjuFLTWFsUurpAvmgiemzJ3xnhra6pOEr2jy2dw6fUL4gMQtPpSwqB4Qd9TY6MOUpwBqK0MpuRMBisa3zOBphJyvnXOIG9RCYzivzOQM+PKzLmznjTyeql1Ipj/9LwEgQ==,iv:PbRgjgm5ERWlAkkWc7As8eIx8e0rjTUkQ7XlDBrEUlU=,tag:yh2/JuHu5I+dWlxVAmkkXA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByOGNaakZoV3lhemhMSVVF
+            RklKbEsxeVFMWWdwQ3ZqR0lhRHlPM1dWT3h3Cnk2RnZUNnplVEwvNW9qMExsSUlt
+            VVVXejMrcDRRTEdtT1krdThqWThLWWcKLS0tIEVZZEVBWFZyVi84S1puRHdtMWR1
+            cDBHc1J5aHFpQkxSSEx2VDRjZ1RsdXcKG5MgnUBpFmlNGSkxBijXp7m492ujyqsC
+            TbTfU8SWJwXhio7V+zDbXG+JW4R51wr9F9Sj47DCw7b64nbsD3N5TQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-26T02:15:51Z"
+    mac: ENC[AES256_GCM,data:+2eGERwwzeK5JE9H6qhsu+tskR3Y9c7DecVGopfG8Ggq5SHQBfYyvz//IU+MGCOb6nIyP3JPf9+9IDS2DthVcw+RLtviQkN1TeWuUTDbxBxHdCoUv9zepwU7+d0qUMrdUK0N9QQSRZ4TpiwceViDeMuhPF/vTmADP9QWwid/qwk=,iv:nuJo37Dh8MAf+qI/Ty2jsW0IheNhgmZ4l0Va8pJG6+4=,tag:vCpifXvYF1/QU7XExc6raQ==,type:str]
+    version: 3.13.2

k8s/bootstrap/cert-manager/homelab-ca-configmap.yaml

@@ -271,3 +271,40 @@ data:
     NDKi7/Gxha790QN3TD/U7p9GcEtfXKv8KISKfxnfD9o9Yx6pWOXcgvGefQkpXuRm
     kGzDM+aY/VKs5Et5/RyKavBDihuOcumLsmoRg/bFoB/xpeURTyiehUdu
     -----END CERTIFICATE-----
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: homelab-ca
+  namespace: cloudflared
+data:
+  ca.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIFBjCCAu6gAwIBAgIJAKoBPdKPDL5AMA0GCSqGSIb3DQEBCwUAMDAxEzARBgNV
+    BAMMCmhvbWVsYWItY2ExGTAXBgNVBAoMEHJpb3RwaWFvLmhvbWVsYWIwHhcNMjYw
+    ODEyMjA1NDQzWhcNMzYwODA5MjA1NDQzWjAwMRMwEQYDVQQDDApob21lbGFiLWNh
+    MRkwFwYDVQQKDBByaW90cGlhby5ob21lbGFiMIICIjANBgkqhkiG9w0BAQEFAAOC
+    Ag8AMIICCgKCAgEAzw796vZRFOBc/orUu9J6rv91A3SNLJsqz/VuetwAXCaVSlK5
+    kL/RaBKKkcXToCC5avwix6W21FnwLUW4R8QlQdin8+XEHpx2LY5R4GmSvaCbU7wm
+    YwYgVGQVv8KlgcMbz/pF09vXU29cHUHwlkN+BWVGAxVGWxam94j1pBzV0dOhhRw2
+    q5sZYUUrPO5l775od2FMMnMzoRFugxDvJof0p3fAChNAIiFhP95lrcb2YsTwK+a2
+    6mk3AZc0gMtmgVsH+zryGFcAmWLixq41ye/H23ktj2bjEHCr9mIDpuSOKLyLBzrg
+    js/HqdxvwkZkkxwXft0Vi7GN0/LGHMdPrDE499pJelAHbxUpBoOk8sfXSNnfESI8
+    v20LybATGrKXFlOhMKshY1a3JLAoZTaW0xkG4PM4HT7lK/ovr/NLF6kdP7HxJFFQ
+    +4WjySiwbjusNobpTaQWVwCjD3Imf+2feAqJeX3z+P3EhxfrmvEojLOdUiiPcjMn
+    CFl9u2EpSlax53umSkcBQJGfFKM17fZzhtakjqhCTzAfXXlxZy4hsHDtfzRcfaoB
+    Zz/PiJ0TtJSC7TvxZZoPpGexQdoeT7kJEksTvGi5OSfJD7HaKe5RQSZGgpz7ATB+
+    YMBKbjhMXVu+TAdcO1qZRN0UCncrliN61ki9F0tE8kNgpS9biZ6kheoJMZUCAwEA
+    AaMjMCEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcN
+    AQELBQADggIBAFN1At11d/+Ue6cEjCklnRQpLuD5agXaE+pi5oHNfnggSlO8UjUj
+    lDzfnmxi3IAI6HruzDARM9J95qJboUxWOAEmK9SNnZNS/1b0GyikP3YcF1gIFqtW
+    6SgmVfC1cVUFKGt/+QQzZxhOLuoydv9Ozu12nxsTW/WY2t+cpeL+1wrsM22RA6W/
+    xV/bzYbwzCUT9v4tMZLZHk5CkXztVFXSJyW/8w9rVZdUGZM+mwwn81PLSE0v3s/2
+    wJH8eUqGl73i3Xgn7CVanN4cJYMGYVVlRD6Qr6muBN4Vh0ETY1PrjzsZ6XsV6vk9
+    glMbQnErYMZHZIDtnVdaBt60MecUD9O4eLeip0SE6bVRAucKBXA9xNMilMQEL8/X
+    FBMAxXl10rowBlbX2oaFSOM80eLTla4a3gSL7+0vua3C4Dtv2bLo8ZjCyqXwOJeb
+    N85cqcVZStz3qlgQN1nG2qGE3BS6scOs7+mUGe9CmS8acJV++PKm8CtmLH6/ibjV
+    tAc/6+Vg/keo0kty7H7L6oQd+gKpuaJ2I+wKYe4l9UwHi9zEb8YjzGB4n2AtsoWI
+    NDKi7/Gxha790QN3TD/U7p9GcEtfXKv8KISKfxnfD9o9Yx6pWOXcgvGefQkpXuRm
+    kGzDM+aY/VKs5Et5/RyKavBDihuOcumLsmoRg/bFoB/xpeURTyiehUdu
+    -----END CERTIFICATE-----

k8s/infra/iam/zitadel-values.yaml

@@ -32,6 +32,15 @@ zitadel:
     TLS:
       Enabled: false
 
+    # First boot is COMPLETE (2026-10-26): the instance, the system org and the
+    # `iam-admin` machine key all exist. Skip stops the setup hook from running
+    # the first-instance step again. Without it, every sync re-runs the
+    # machinekeyWriter, which silently rotates the iam-admin key Terraform
+    # reads. ZITADEL objects are declared by scripts/iam/zitadel-provision.py.
+    # Never set this back to false.
+    FirstInstance:
+      Skip: true
+
   # Rendered into a Secret by the chart; read by pods and init jobs alike.
   masterkeySecretName: zitadel-masterkey
 
@@ -39,6 +48,20 @@ zitadel:
     requests: { memory: "512Mi", cpu: "250m" }
     limits: { memory: "2Gi", cpu: "1" }
 
+  # Best practice: run as non-root, drop all capabilities, and refuse
+  # privilege escalation. The chart's image already supports a non-root UID.
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+
   # FirstInstance defaults stay for the FIRST sync: the chart creates org
   # `zitadel` plus the `iam-admin` machine user and its machine-key Secret.
   # Terraform consumes that key to declare projects, roles, OIDC apps and the
@@ -84,16 +107,64 @@ postgresql:
 # CNPG already created the database, the `app` user and its grants, so the init
 # job runs only the ZITADEL-internal step. Running the full default command
 # would try to create a database and a superuser it does not own.
+#
+# enabled: false after the first successful boot (2026-10-26). The schema is in
+# place; the init hook has nothing left to do and re-running it on every sync
+# only adds a PreSync failure surface. Re-enable it for a deliberate schema
+# upgrade, then set it back to false.
 initJob:
+  enabled: false
   command: "zitadel"
   activeDeadlineSeconds: 600
 
+# The setup job runs the full first-instance creation: schema migrations, the
+# system org, the iam-admin machine key, and 13 projection replays. 20 minutes
+# leaves room for a cold database without masking a genuine hang.
+#
+# DONE, 2026-10-26 — first boot completed. The annotations are moved from
+# pre-install/pre-upgrade to post-delete because a PreSync setup run is not
+# harmless after boot:
+#  - it re-runs every repeatable migration on every sync;
+#  - interrupting one (a sync aborted, or a human deleting the Job) leaves a
+#    `system.migration.started` event with no matching `done`, and ZITADEL then
+#    waits forever on `migration already started`. That is exactly how this
+#    instance deadlocked on projections.projects4 and later on
+#    repeatable_delete_stale_org_fields.
+# A post-delete hook is never executed by ArgoCD sync, so setup runs only on
+# first install. Re-enable it deliberately (and only with initJob too) for a
+# schema upgrade, then revert.
+setupJob:
+  activeDeadlineSeconds: 1200
+  annotations:
+    helm.sh/hook: post-delete
+    helm.sh/hook-delete-policy: hook-succeeded
+    helm.sh/hook-weight: "2"
+
 # Two API replicas: the read path survives one node drain, and both share the
 # same masterkey and database. The login UI stays at one replica — a session
 # cookie is not shared, so a second pod buys nothing.
 replicaCount: 2
 revisionHistoryLimit: 3
 
+# Best practice: a PDB guarantees at least one API pod stays available during
+# voluntary disruptions (node drain, rolling upgrade). Without it a drain can
+# take both replicas down at once. minAvailable 1 is correct for 2 replicas;
+# minAvailable 2 would block every drain forever.
+pdb:
+  enabled: true
+  minAvailable: 1
+
+# Best practice: spread replicas across nodes so a single node failure cannot
+# take both API pods. Anti-affinity alone only prefers separation; topology
+# spread enforces the failure-domain budget.
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: ScheduleAnyway
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: zitadel
+
 ingress:
   enabled: true
   className: nginx
@@ -101,10 +172,31 @@ ingress:
 
 login:
   enabled: true
+  # ksops owns this Secret (k8s/argocd/secrets/zitadel-login-service-key.enc.yaml,
+  # rendered by the sops-secrets Application). Pointing the chart at it stops the
+  # chart from auto-generating an RSA keypair every render, which changed
+  # checksum/secret-login-service-key and kept both Deployments permanently
+  # OutOfSync. The keypair must stay RSA: the login container signs JWTs (RS256).
+  loginServiceKeySecretName: zitadel-login-service-key
   replicaCount: 1
+  revisionHistoryLimit: 3
   ingress:
     enabled: true
     className: nginx
+  # Best practice: the login UI is the public attack surface. Harden the pod
+  # even at one replica. The chart has no login PDB key; a single replica has
+  # nothing to protect with minAvailable.
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
 
 # Chart-owned ServiceMonitor. The cluster's Prometheus runs with
 # serviceMonitorSelectorNilUsesHelmValues=false, so the release label is not

k8s/infra/minio/minio-provision-paperless-job.yaml

@@ -59,6 +59,7 @@ spec:
               mountPath: /shared
       containers:
         - name: provision
+          # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
           image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
           volumeMounts:
             - name: shared

k8s/infra/monitoring/prometheus-values.yaml

@@ -55,7 +55,10 @@ alertmanager:
 prometheus:
   prometheusSpec:
     retention: 30d
-    retentionSize: "95GB"
+    # Must stay under the PVC size below (25Gi). At 95GB the limit never
+    # triggered, the volume filled, and Prometheus crashlooped with
+    # "no space left on device" writing /prometheus/queries.active.
+    retentionSize: "18GB"
 
     # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
     # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no

scripts/iam/zitadel-provision.py

@@ -6,7 +6,7 @@ One command replaces the manual sequence the Authentik script
 fixed order and every phase is safe to re-run:
 
   keys         extract the iam-admin machine key and PAT from namespace iam
-  apply        terraform apply terraform/zitadel (project, roles, apps, machines)
+  apply        create project, roles and OIDC apps through the ZITADEL API
   secrets      write generated client ids/secrets back into the SOPS files
   users        import one human user, converting the Django password hash
   verify       check discovery, client count and the groups-claim gate
@@ -43,13 +43,85 @@ from pathlib import Path
 import yaml
 
 REPO = Path(__file__).resolve().parents[2]
-TF_DIR = REPO / "terraform" / "zitadel"
 SOPS_DIR = REPO / "k8s" / "argocd" / "secrets"
 SOPS_CONFIG = REPO / ".sops.yaml"
 CONF_DIR = Path(os.environ.get("ZITADEL_CONF_DIR", Path.home() / ".config" / "zitadel"))
+STATE_FILE = CONF_DIR / "state.json"
 NAMESPACE = os.environ.get("ZITADEL_NAMESPACE", "iam")
 BASE_URL = os.environ.get("ZITADEL_BASE_URL", "https://zitadel.riotpiao.com")
 
+# ── desired ZITADEL state ──────────────────────────────────────────────────
+# Declared here, applied through the API by the `apply` phase. This replaces the
+# retired terraform/zitadel root module: same objects, no state file to drift and
+# no second writer.
+PROJECT_NAME = os.environ.get("ZITADEL_PROJECT_NAME", "homelab")
+PROJECT_GROUP = "homelab"
+
+# Role keys EQUAL the Authentik group names, so the `groups` claim keeps working
+# for every consumer that maps on it (kube-apiserver RBAC, MinIO policies,
+# Grafana roles).
+ROLES = {
+    "homelab-admins": "Cluster administrators with full access",
+    "minio-admins": "MinIO administrators",
+    "minio-photos": "Photos bucket (Immich) access",
+    "minio-documents": "Documents bucket (Paperless) access",
+    "minio-backups": "Backups bucket read-only access",
+    "poimen-admins": "Poimen memory administrators",
+    "poimen-devs": "Dev and staging projects access",
+    "poimen-prod-readonly": "Production projects read-only access",
+    "paperless-admins": "Paperless administrators",
+    "paperless-finance": "Finance documents",
+    "paperless-legal": "Legal documents",
+    "paperless-hr": "HR documents",
+    "grafana-admins": "Grafana administrators",
+    "grafana-editors": "Grafana dashboard editors",
+    "grafana-viewers": "Grafana dashboard viewers",
+    "sqs-users": "SQS/Temporal queue read access",
+    "sqs-writers": "SQS/Temporal queue read/write access",
+    "s3-users": "S3 read access",
+    "s3-writers": "S3 read/write access",
+}
+
+_WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"}
+_NATIVE = {"app_type_id": "OIDC_APP_TYPE_NATIVE", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+_PKCE = {"app_type_id": "OIDC_APP_TYPE_USER_AGENT", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+
+OIDC_APPS = {
+    "api-gw": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://api.riotpiao.com/callback"]},
+    "argocd": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/auth/callback",
+        "https://argocd.riotpiao.com/auth/callback"]},
+    "forgejo": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/user/oauth2/authentik/callback",
+        "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
+    "grafana": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/login/generic_oauth",
+        "https://grafana.riotpiao.com/login/generic_oauth"]},
+    "homarr": {**_WEB, "redirect_uris": [
+        "http://localhost:7575/auth/callback",
+        "https://homarr.riotpiao.com/auth/callback"]},
+    "immich": {**_WEB, "app_type_id": "OIDC_APP_TYPE_NATIVE", "redirect_uris": [
+        "app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login",
+        "https://img.riotpiao.com/user/oauth2/callback"]},
+    "kubernetes": {**_PKCE, "dev_mode": True, "redirect_uris": [
+        "http://localhost:8000", "http://localhost:18000"]},
+    "minio": {**_WEB, "redirect_uris": [
+        "http://localhost:9000/auth/sso/oauth2/code",
+        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+    "paperless": {**_WEB, "redirect_uris": [
+        "http://localhost:8000/accounts/oidc/authentik/login/callback/",
+        "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    "poimen": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+    "queue-sqs": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
+    "vault": {**_WEB, "redirect_uris": [
+        "http://localhost:8200/ui/vault/auth/oidc/oidc/callback",
+        "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"]},
+}
+
+
 _TEMP_FILES: set[str] = set()
 
 
@@ -155,6 +227,13 @@ def curl_json(method: str, url: str, token: str | None = None, body: dict | None
     cfg = [f'url = "{esc(url)}"', f'request = "{method}"', "silent", "show-error",
            f"max-time = {timeout}", 'write-out = "\\n%{http_code}"',
            'header = "Content-Type: application/json"']
+    # Reaching the instance by name needs either public DNS through the tunnel
+    # or a pinned address. ZITADEL picks the instance from the Host header, so a
+    # plain port-forward fails with "Instance not found". Set this to bypass DNS:
+    #   ZITADEL_CURL_RESOLVE=zitadel.riotpiao.com:443:192.168.1.160
+    pinned = os.environ.get("ZITADEL_CURL_RESOLVE", "").strip()
+    if pinned:
+        cfg.append(f'resolve = "{esc(pinned)}"')
     if token:
         cfg.append(f'header = "Authorization: Bearer {esc(token)}"')
     if body is not None:
@@ -179,27 +258,99 @@ def phase_keys(_args: argparse.Namespace) -> int:
 
 # ── phase: apply ───────────────────────────────────────────────────────────
 
-def phase_apply(args: argparse.Namespace) -> int:
-    need("terraform")
-    if not args.org_id:
-        die("apply requires --org-id (ZITADEL console -> Organization settings -> Resource ID)")
-    key = CONF_DIR / "iam-admin.json"
-    if not key.exists():
-        die(f"{key} missing — run the keys phase first")
-    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
+def _api(method: str, path: str, token: str, body: dict | None = None,
+         timeout: int = 30) -> tuple[int, dict]:
+    code, payload = curl_json(method, f"{BASE_URL}{path}", token=token, body=body, timeout=timeout)
+    try:
+        return code, (json.loads(payload) if payload.strip() else {})
+    except json.JSONDecodeError:
+        return code, {"_raw": payload}
 
-    for cmd in (["init", "-input=false"], ["fmt", "-check", "-recursive"]):
-        cp = run(["terraform", f"-chdir={TF_DIR}", *cmd], env=env)
-        if cp.returncode != 0:
-            die(f"terraform {' '.join(cmd)} failed: {cp.stderr.strip() or cp.stdout.strip()}")
 
-    cp = run(["terraform", f"-chdir={TF_DIR}", "apply", "-input=false", "-auto-approve",
-              f"-var=org_id={args.org_id}"], env=env)
-    sys.stdout.write(cp.stdout)
-    sys.stderr.write(cp.stderr)
-    if cp.returncode != 0:
-        die("terraform apply failed — no SOPS value was written. Objects created before the "
-            "failure stay in ZITADEL; re-run apply once the cause is fixed")
+def _ensure_project(token: str, name: str) -> str:
+    code, d = _api("POST", "/management/v1/projects/_search", token, {})
+    for row in d.get("result") or []:
+        if row.get("name") == name:
+            return row["id"]
+    code, d = _api("POST", "/management/v1/projects", token, {"name": name})
+    if code not in (200, 201) or not d.get("id"):
+        die(f"create project {name} failed: {code} {d}")
+    return d["id"]
+
+
+def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
+    have = {r.get("roleKey") for r in (d.get("result") or [])}
+    for key, display in roles.items():
+        if key in have:
+            continue
+        code, d = _api("POST", f"/management/v1/projects/{project_id}/roles", token,
+                       {"roleKey": key, "displayName": display, "group": PROJECT_GROUP})
+        if code not in (200, 201):
+            die(f"create role {key} failed: {code} {d}")
+
+
+def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {})
+    existing = {a.get("name"): a for a in (d.get("result") or [])}
+    ids: dict[str, str] = {}
+    secrets: dict[str, str] = {}
+    for name, spec in apps.items():
+        row = existing.get(name)
+        if row:
+            ids[name] = (row.get("oidcConfig") or {}).get("clientId", "")
+            app_id = row.get("id")
+        else:
+            body = {
+                "name": name,
+                "redirectUris": spec["redirect_uris"],
+                "responseTypes": ["OIDC_RESPONSE_TYPE_CODE"],
+                "grantTypes": ["OIDC_GRANT_TYPE_AUTHORIZATION_CODE",
+                               "OIDC_GRANT_TYPE_REFRESH_TOKEN"],
+                "appType": spec.get("app_type_id", "OIDC_APP_TYPE_WEB"),
+                "authMethodType": spec.get("auth_method_id", "OIDC_AUTH_METHOD_TYPE_BASIC"),
+                "devMode": bool(spec.get("dev_mode")),
+                "accessTokenType": "OIDC_TOKEN_TYPE_BEARER",
+                "version": "OIDC_VERSION_1_0",
+                "idTokenRoleAssertion": True,
+            }
+            code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/oidc", token, body)
+            if code not in (200, 201) or not d.get("appId"):
+                die(f"create app {name} failed: {code} {d}")
+            app_id = d["appId"]
+            ids[name] = d.get("clientId", "")
+        if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE":
+            continue
+        code, d = _api("POST",
+                       f"/management/v1/projects/{project_id}/apps/{app_id}"
+                       "/oidc_config/_generate_client_secret", token, {})
+        if code in (200, 201) and d.get("clientSecret"):
+            secrets[name] = d["clientSecret"]
+    return ids, secrets
+
+
+def phase_apply(_args: argparse.Namespace) -> int:
+    """Declare the ZITADEL objects through the API. No Terraform, no state drift."""
+    token = bootstrap_token()
+    if not token:
+        die("no iam-admin PAT — run the keys phase first")
+
+    project_id = _ensure_project(token, PROJECT_NAME)
+    print(f"project   : {PROJECT_NAME} ({project_id})")
+    _ensure_roles(token, project_id, ROLES)
+    print(f"roles     : {len(ROLES)} ensured")
+    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS)
+    print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued")
+
+    state = {
+        "project_id": {"value": project_id},
+        "client_ids": {"value": ids},
+        "client_secrets": {"value": secrets},
+        "machine_keys": {"value": {}},
+        "groups_claim_enabled": {"value": False},
+    }
+    write_private(STATE_FILE, json.dumps(state, indent=2, sort_keys=True).encode())
+    print(f"state     -> {STATE_FILE} (0600)")
     return 0
 
 
@@ -238,16 +389,11 @@ MANUAL_ONLY = {
 }
 
 
-def terraform_outputs() -> dict:
-    need("terraform")
-    key = CONF_DIR / "iam-admin.json"
-    if not key.exists():
-        die(f"{key} missing — run the keys phase first")
-    env = dict(os.environ, ZITADEL_JWT_PROFILE_FILE=str(key))
-    cp = run(["terraform", f"-chdir={TF_DIR}", "output", "-json"], env=env)
-    if cp.returncode != 0:
-        die(f"terraform output failed: {cp.stderr.strip()}")
-    return json.loads(cp.stdout or "{}")
+def state_outputs() -> dict:
+    """Read the object map written by the apply phase."""
+    if not STATE_FILE.exists():
+        die(f"{STATE_FILE} missing — run the apply phase first")
+    return json.loads(STATE_FILE.read_text())
 
 
 def sops_env() -> dict:
@@ -345,13 +491,13 @@ def update_client_id(client: str, value: str, dry_run: bool, apply_config: bool)
 
 def phase_secrets(args: argparse.Namespace) -> int:
     need("sops")
-    out = terraform_outputs()
+    out = state_outputs()
     ids = out.get("client_ids", {}).get("value", {})
     secrets = out.get("client_secrets", {}).get("value", {})
     machine_keys = out.get("machine_keys", {}).get("value", {})
 
     if not ids and not machine_keys:
-        die("terraform has no ZITADEL outputs yet — run the apply phase first")
+        die("no ZITADEL objects recorded yet — run the apply phase first")
 
     print("[1/3] client secrets into SOPS")
     for client, entries in SECRET_MAP.items():
@@ -438,7 +584,7 @@ def phase_verify(_args: argparse.Namespace) -> int:
     if code != 200:
         die("discovery endpoint is not 200 — the instance or the ingress is not ready")
 
-    out = terraform_outputs()
+    out = state_outputs()
     ids = out.get("client_ids", {}).get("value", {})
     print(f"client apps    : {len(ids)}")
     enabled = bool(out.get("groups_claim_enabled", {}).get("value"))

terraform/edge/cloudflare.tf

@@ -0,0 +1,51 @@
+# Cloudflare side of the edge.
+#
+# The tunnel itself is remotely managed today: its token lives in
+# k8s/apps/cloudflared and the public hostnames are configured in the Zero
+# Trust dashboard. Terraform can own the DNS records without owning the tunnel,
+# which is the safe half: a record change is reversible, a tunnel config rewrite
+# removes every hostname the plan does not list.
+
+locals {
+  # CNAME target of every remotely-managed tunnel.
+  tunnel_target = "${var.tunnel_id}.cfargotunnel.com"
+}
+
+resource "cloudflare_dns_record" "tunnel" {
+  for_each = toset(var.cloudflare_tunnel_hostnames)
+
+  zone_id = var.cloudflare_zone_id
+  name    = each.value
+  type    = "CNAME"
+  content = local.tunnel_target
+  proxied = var.cloudflare_dns_proxied
+  ttl     = var.cloudflare_dns_ttl
+  comment = "cloudflared tunnel; managed by terraform/edge"
+
+  # A proxied CNAME must point at the tunnel target. A CNAME to any other
+  # hostname breaks the tunnel mapping.
+  lifecycle {
+    precondition {
+      condition     = var.cloudflare_dns_proxied
+      error_message = "Tunnel CNAMEs must be proxied; an unproxied record bypasses the tunnel."
+    }
+  }
+}
+
+# Optional: take the tunnel configuration into Terraform. Off by default.
+# Applying it REPLACES the whole ingress list. Import first:
+#   terraform import cloudflare_zero_trust_tunnel_cloudflared_config.this \
+#     <account_id>/<tunnel_id>
+resource "cloudflare_zero_trust_tunnel_cloudflared_config" "this" {
+  count = var.manage_tunnel_config ? 1 : 0
+
+  account_id = var.cloudflare_account_id
+  tunnel_id  = var.tunnel_id
+
+  config = {
+    ingress = concat(
+      [for host, service in var.tunnel_ingress : { hostname = host, service = service }],
+      [{ service = "http_status:404" }],
+    )
+  }
+}

terraform/edge/outputs.tf

@@ -0,0 +1,40 @@
+output "cloudflare_records" {
+  description = "Hostnames now managed in the Cloudflare zone."
+  value       = sort(keys(cloudflare_dns_record.tunnel))
+}
+
+output "tunnel_config_managed" {
+  description = "Whether Terraform owns the tunnel ingress list. False means the dashboard still does."
+  value       = var.manage_tunnel_config
+}
+
+output "relay_public_ip" {
+  description = "Public address of the Tencent relay. Point the .cn DNS at it, or let the module do it."
+  value       = tencentcloud_instance.relay.public_ip
+}
+
+output "china_hostnames" {
+  description = "Fully qualified China hostnames served by the relay."
+  value       = [for host in var.china_hostnames : "${host}.${var.china_domain}"]
+}
+
+output "wireguard_peer" {
+  description = "Peer stanza for the homelab side. Add it to the cluster's wg0.conf; the relay already trusts this public key."
+  value       = <<-EOT
+    [Peer]
+    PublicKey = <relay public key; see /etc/wireguard/wg0.conf on the relay>
+    Endpoint = ${tencentcloud_instance.relay.public_ip}:${var.wireguard_port}
+    AllowedIPs = ${var.wireguard_relay_address}
+    PersistentKeepalive = 25
+  EOT
+}
+
+output "next_manual_steps" {
+  description = "What this module cannot do for you."
+  value = [
+    "Confirm the ICP filing for ${var.china_domain} if the relay is in a mainland region.",
+    "Issue a certificate for the China hostnames on the relay (Caddy ACME needs the DNS-01 challenge for a .cn domain).",
+    "Run a WireGuard peer in the cluster that dials ${tencentcloud_instance.relay.public_ip}:${var.wireguard_port}.",
+    "Add the .cn origins to the OIDC client redirect URIs before any China user logs in.",
+  ]
+}

terraform/edge/providers.tf

@@ -0,0 +1,47 @@
+# terraform/edge — the China-facing edge, plus DNS ownership for the existing zone
+#
+# Purpose: give users in mainland China a short, stable path to the homelab
+# without touching the Cloudflare path that serves everyone else.
+#
+#   China user -> <service>.<china_domain> -> Tencent CVM (this module)
+#              -> WireGuard tunnel -> ingress-nginx (192.168.1.160) -> service
+#
+#   Everyone else -> Cloudflare -> cloudflared tunnel -> same ingress (unchanged)
+#
+# The homelab still opens no inbound port: the tunnel is established outbound
+# from the cluster to the CVM.
+#
+# Credentials come from the environment, never from a file in git:
+#   export CLOUDFLARE_API_TOKEN=...
+#   export TENCENTCLOUD_SECRET_ID=...
+#   export TENCENTCLOUD_SECRET_KEY=...
+#   terraform -chdir=terraform/edge init
+#   terraform -chdir=terraform/edge plan -var china_domain=example.cn ...
+#
+# State is local and git-ignored. It carries no
+# credential today, but it will once InstanceData or certificates are added, so
+# treat it as a secret and move it to MinIO before a second operator applies.
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "5.28.1"
+    }
+    tencentcloud = {
+      source  = "tencentcloudstack/tencentcloud"
+      version = "1.83.38"
+    }
+  }
+}
+
+provider "cloudflare" {
+  api_token = var.cloudflare_api_token
+}
+
+provider "tencentcloud" {
+  secret_id  = var.tencent_secret_id
+  secret_key = var.tencent_secret_key
+  region     = var.tencent_region
+}

terraform/edge/templates/Caddyfile.tpl

@@ -0,0 +1,22 @@
+# Rendered by terraform/edge. Do not edit on the instance.
+#
+# Caddy terminates TLS for the .cn hostnames and forwards over the WireGuard
+# tunnel to ingress-nginx. flush_interval -1 disables response buffering so
+# Server-Sent Events reach the client chunk-by-chunk.
+{
+	email ${acme_email}
+	admin off
+}
+
+${hosts} {
+	reverse_proxy ${ingress_upstream} {
+		flush_interval -1
+	}
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "strict-origin-when-cross-origin"
+	}
+}

terraform/edge/tencent.tf

@@ -0,0 +1,169 @@
+# The relay: one small CVM in Tencent Cloud that terminates TLS for the .cn
+# domain and forwards over WireGuard to the homelab ingress.
+#
+# Why a relay and not Cloudflare: mainland clients reach a Tencent edge far
+# faster than a Cloudflare edge, and Cloudflare's China network is an
+# Enterprise product. The homelab keeps zero inbound exposure because the
+# tunnel is dialled outbound from the cluster.
+#
+# ICP FILING: a mainland resource serving HTTP/HTTPS for a domain requires an
+# ICP filing naming that domain. Without it this instance accepts the
+# connection and the mainland network resets the HTTP layer. Put the relay in a
+# non-mainland region (ap-hongkong, ap-singapore) when no filing exists.
+
+resource "tencentcloud_vpc" "edge" {
+  name       = "${replace(var.china_domain, ".", "_")}_edge"
+  cidr_block = var.edge_vpc_cidr
+}
+
+resource "tencentcloud_subnet" "edge" {
+  name              = "${replace(var.china_domain, ".", "_")}_edge"
+  vpc_id            = tencentcloud_vpc.edge.id
+  availability_zone = var.tencent_availability_zone
+  cidr_block        = var.edge_subnet_cidr
+}
+
+resource "tencentcloud_key_pair" "edge" {
+  key_name   = "${replace(var.china_domain, ".", "_")}_edge"
+  public_key = var.relay_ssh_public_key
+}
+
+resource "tencentcloud_security_group" "edge" {
+  name        = "${replace(var.china_domain, ".", "_")}_edge"
+  description = "TLS + WireGuard for the ${var.china_domain} relay; SSH from the operator only"
+}
+
+resource "tencentcloud_security_group_rule" "ssh" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  port_range        = "22"
+  cidr_ip           = var.relay_admin_cidr
+  policy            = "accept"
+}
+
+resource "tencentcloud_security_group_rule" "tls" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  port_range        = "80,443"
+  cidr_ip           = "0.0.0.0/0"
+  policy            = "accept"
+}
+
+resource "tencentcloud_security_group_rule" "wireguard" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "udp"
+  port_range        = tostring(var.wireguard_port)
+  # The homelab dials outbound and has no static public address, so this is
+  # 0.0.0.0/0 by default. WireGuard authenticates every packet by public key,
+  # and an unauthenticated packet is dropped before it reaches the tunnel.
+  # Narrow this to the homelab egress CIDR when that address is stable.
+  cidr_ip = var.wireguard_allowed_cidr
+  policy  = "accept"
+}
+
+# cloud-init: WireGuard listener, forwarding sysctls, and Caddy for TLS.
+# The relay never holds application credentials; it forwards bytes.
+#
+# NOTE ON THE PRIVATE KEY: user_data is visible in the instance metadata and in
+# Terraform state. That is a known limitation of bootstrapping WireGuard this
+# way. The argument is wrapped in sensitive() to keep it out of plan/apply
+# output, but anyone with console or state access can read it. Rotate the key
+# if the state or the instance metadata is ever exposed.
+locals {
+  china_fqdns = [for host in var.china_hostnames : "${host}.${var.china_domain}"]
+
+  caddyfile = templatefile("${path.module}/templates/Caddyfile.tpl", {
+    acme_email       = var.acme_email
+    hosts            = join(", ", local.china_fqdns)
+    ingress_upstream = var.ingress_upstream
+  })
+
+  relay_user_data = <<-EOT
+    #cloud-config
+    package_update: true
+    package_upgrade: false
+    manage_etc_hosts: true
+    timezone: Asia/Shanghai
+    packages: [wireguard, wireguard-tools, caddy]
+    write_files:
+      - path: /etc/wireguard/wg0.conf
+        permissions: "0600"
+        content: |
+          [Interface]
+          Address = ${var.wireguard_relay_address}
+          ListenPort = ${var.wireguard_port}
+          PrivateKey = ${var.wireguard_relay_private_key}
+          # Masquerade forwarded traffic so the homelab peer only has to accept
+          # packets sourced from the relay's tunnel address. Without this the
+          # homelab peer's AllowedIPs would drop the original client source IP.
+          PostUp = iptables -t nat -A POSTROUTING -o wg0 -j MASQUERADE
+          PostDown = iptables -t nat -D POSTROUTING -o wg0 -j MASQUERADE
+          [Peer]
+          PublicKey = ${var.wireguard_homelab_public_key}
+          AllowedIPs = ${var.wireguard_homelab_allowed_ips}
+          PersistentKeepalive = 25
+      - path: /etc/sysctl.d/99-wireguard.conf
+        content: |
+          net.ipv4.ip_forward = 1
+      - path: /etc/caddy/Caddyfile
+        permissions: "0644"
+        content: ${jsonencode(local.caddyfile)}
+    runcmd:
+      - sysctl --system
+      - systemctl enable --now wg-quick@wg0
+      - caddy validate --config /etc/caddy/Caddyfile
+      - systemctl enable --now caddy
+      - systemctl restart caddy
+  EOT
+}
+
+resource "tencentcloud_instance" "relay" {
+  instance_name     = "${replace(var.china_domain, ".", "_")}_edge"
+  availability_zone = var.tencent_availability_zone
+  image_id          = var.relay_image_id
+  instance_type     = var.relay_instance_type
+
+  vpc_id             = tencentcloud_vpc.edge.id
+  subnet_id          = tencentcloud_subnet.edge.id
+  security_groups    = [tencentcloud_security_group.edge.id]
+  key_ids            = [tencentcloud_key_pair.edge.id]
+  allocate_public_ip = true
+
+  # Outbound only needs a few Mbps; inbound TLS is the same order.
+  internet_max_bandwidth_out = var.relay_internet_max_bandwidth_out
+  system_disk_type           = var.relay_system_disk_type
+  system_disk_size           = var.relay_system_disk_size
+
+  # The private key is embedded in this bootstrap script. sensitive() keeps it
+  # out of CLI output; it does not remove it from state or instance metadata.
+  user_data = sensitive(local.relay_user_data)
+
+  tags = {
+    role   = "china-edge"
+    domain = var.china_domain
+  }
+
+  # Accidental `terraform destroy` would take the only China-facing edge down.
+  # Remove this block deliberately when a replacement path exists.
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+# DNS for the .cn domain lives at DNSPod (Tencent). Each subdomain resolves to
+# the relay's public address, and the relay proxies to the homelab over the
+# tunnel. The "中国境内" line is the default line; no separate line records.
+resource "tencentcloud_dnspod_record" "china" {
+  for_each = toset(var.china_hostnames)
+
+  domain      = var.china_domain
+  sub_domain  = each.value
+  record_type = "A"
+  record_line = var.china_dns_record_line
+  value       = tencentcloud_instance.relay.public_ip
+  ttl         = var.china_dns_ttl
+  remark      = "China edge relay; origin is the homelab ingress over WireGuard"
+}

terraform/edge/terraform.tfvars.example

@@ -0,0 +1,45 @@
+# Copy to terraform.tfvars.local (git-ignored) and fill in.
+# Secrets never belong in a committed file: pass them as environment variables
+# instead — TF_VAR_wireguard_relay_private_key, CLOUDFLARE_API_TOKEN,
+# TENCENTCLOUD_SECRET_ID, TENCENTCLOUD_SECRET_KEY.
+
+# ── Cloudflare (existing zone) ───────────────────────────────────────────────
+cloudflare_zone_id = "REPLACE_WITH_ZONE_ID"
+tunnel_id          = "REPLACE_WITH_TUNNEL_UUID"
+
+# Adopt existing records with `terraform import` before the first apply, or the
+# create call collides with the hand-made record of the same name:
+#   terraform import 'cloudflare_dns_record.tunnel["zitadel"]' \
+#     <zone_id>/<record_id>
+#
+# A tunnel CNAME is only half of the path. The tunnel's Public Hostname list
+# (Zero Trust -> Networks -> Tunnels -> Public Hostname) must also map the name
+# to https://<name>.riotpiao.com, or Cloudflare answers 404 from the tunnel even
+# though DNS resolves.
+cloudflare_tunnel_hostnames = ["paperless", "img", "poimen", "zitadel"]
+
+# Leave false until the tunnel config import is understood. True makes
+# Terraform replace the entire ingress list with tunnel_ingress.
+manage_tunnel_config = false
+tunnel_ingress       = {}
+
+# ── China edge (Tencent Cloud) ───────────────────────────────────────────────
+china_domain   = "REPLACE.example.cn"
+china_hostnames = ["paperless", "img"]
+
+# Mainland regions require an ICP filing for the domain. Use ap-hongkong or
+# ap-singapore when no filing exists.
+tencent_region            = "ap-guangzhou"
+tencent_availability_zone = "ap-guangzhou-6"
+relay_instance_type       = "S5.SMALL2"
+relay_image_id            = "REPLACE_WITH_IMAGE_ID"
+relay_ssh_public_key      = "ssh-ed25519 AAAA... operator@laptop"
+relay_admin_cidr          = "REPLACE_WITH_OPERATOR_CIDR"
+
+# ── WireGuard between the relay and the cluster ──────────────────────────────
+# The relay private key comes from `wg genkey`. Keep it in the environment, or
+# in a SOPS file mounted for the run. It is not written by this module.
+wireguard_port                 = 51820
+wireguard_homelab_public_key   = "REPLACE_WITH_HOMELAB_WG_PUBKEY"
+wireguard_relay_address        = "10.88.0.1/24"
+wireguard_homelab_address      = "10.88.0.2/24"

terraform/edge/variables.tf

@@ -0,0 +1,227 @@
+variable "cloudflare_api_token" {
+  description = "Cloudflare API token with Zone:DNS:Edit and, for tunnel config, Account:Cloudflare Tunnel:Edit."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "cloudflare_zone_id" {
+  description = "Zone ID of the existing public zone (riotpiao.com)."
+  type        = string
+}
+
+variable "cloudflare_account_id" {
+  description = "Account ID. Required only when manage_tunnel_config is true."
+  type        = string
+  default     = null
+}
+
+variable "tunnel_id" {
+  description = "UUID of the cloudflared tunnel that the DNS records point at."
+  type        = string
+}
+
+# The dashboard remains the source of truth for the remotely-managed tunnel
+# today. Setting this true hands that config to Terraform, which then rewrites
+# the whole ingress list: every hostname not listed in tunnel_ingress is
+# removed from the tunnel. Import first and read the plan before applying.
+variable "manage_tunnel_config" {
+  description = "Take ownership of the remotely-managed tunnel configuration."
+  type        = bool
+  default     = false
+}
+
+variable "tunnel_ingress" {
+  description = "hostname => origin service, used only when manage_tunnel_config is true."
+  type        = map(string)
+  default     = {}
+}
+
+# ── DNS records managed in the Cloudflare zone ───────────────────────────────
+# These names already exist, created by hand in the dashboard. Adopt them with
+# `terraform import` before the first apply, or Terraform will try to create a
+# second record with the same name and Cloudflare will reject it.
+variable "cloudflare_tunnel_hostnames" {
+  description = "Hostnames in the Cloudflare zone that must CNAME to the tunnel."
+  type        = list(string)
+  default     = []
+
+  validation {
+    condition     = alltrue([for h in var.cloudflare_tunnel_hostnames : can(regex("^[a-zA-Z0-9._-]+$", h))])
+    error_message = "cloudflare_tunnel_hostnames entries must be DNS labels or relative names (letters, digits, dot, underscore, hyphen)."
+  }
+}
+
+variable "cloudflare_dns_ttl" {
+  description = "TTL for tunnel CNAME records. 1 means automatic, which is required when proxied is true."
+  type        = number
+  default     = 1
+}
+
+variable "cloudflare_dns_proxied" {
+  description = "Whether tunnel CNAME records are proxied through Cloudflare. Must stay true for a tunnel origin."
+  type        = bool
+  default     = true
+}
+
+# ── China edge ───────────────────────────────────────────────────────────────
+variable "china_domain" {
+  description = "The .cn domain served from Tencent Cloud, without a trailing dot."
+  type        = string
+}
+
+variable "china_hostnames" {
+  description = "Subdomains of china_domain that point at the Tencent relay."
+  type        = list(string)
+  default     = ["paperless", "img"]
+}
+
+variable "tencent_secret_id" {
+  description = "Tencent Cloud SecretId. Prefer TENCENTCLOUD_SECRET_ID in the environment."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "tencent_secret_key" {
+  description = "Tencent Cloud SecretKey. Prefer TENCENTCLOUD_SECRET_KEY in the environment."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "tencent_region" {
+  description = "Tencent Cloud region for the relay. Mainland regions need an ICP filing for the domain."
+  type        = string
+  default     = "ap-guangzhou"
+}
+
+variable "tencent_availability_zone" {
+  description = "Availability zone inside tencent_region."
+  type        = string
+  default     = "ap-guangzhou-6"
+}
+
+variable "relay_instance_type" {
+  description = "CVM spec for the relay. A relay only terminates TLS and forwards bytes."
+  type        = string
+  default     = "S5.SMALL2"
+}
+
+variable "relay_image_id" {
+  description = "CVM image ID. Find one with: tencentcloud images --image-name 'Ubuntu 24.04' (or the console)."
+  type        = string
+}
+
+variable "relay_ssh_public_key" {
+  description = "Public key injected into the relay for break-glass SSH. Never the private key."
+  type        = string
+}
+
+variable "relay_admin_cidr" {
+  description = "CIDR allowed to reach SSH and WireGuard on the relay. Keep it to the operator, not 0.0.0.0/0."
+  type        = string
+
+  validation {
+    condition     = var.relay_admin_cidr != "0.0.0.0/0" && var.relay_admin_cidr != "::/0"
+    error_message = "relay_admin_cidr must not be the whole internet. Set the operator's egress CIDR."
+  }
+}
+
+variable "wireguard_port" {
+  description = "UDP port of the WireGuard listener on the relay."
+  type        = number
+  default     = 51820
+}
+
+variable "wireguard_allowed_cidr" {
+  description = "CIDR allowed to reach the WireGuard UDP port. The homelab dials outbound, so this is often 0.0.0.0/0; WireGuard authenticates by key. Restrict it to the homelab egress CIDR when that address is static."
+  type        = string
+  default     = "0.0.0.0/0"
+}
+
+variable "wireguard_relay_private_key" {
+  description = "WireGuard private key of the relay side. Generate with `wg genkey`; pass via TF_VAR_wireguard_relay_private_key."
+  type        = string
+  sensitive   = true
+}
+
+variable "wireguard_homelab_public_key" {
+  description = "WireGuard public key of the homelab peer that dials the relay."
+  type        = string
+}
+
+variable "wireguard_relay_address" {
+  description = "Tunnel address of the relay side."
+  type        = string
+  default     = "10.88.0.1/24"
+}
+
+variable "wireguard_homelab_address" {
+  description = "Tunnel address of the homelab side."
+  type        = string
+  default     = "10.88.0.2/24"
+}
+
+# ── Relay networking / sizing ───────────────────────────────────────────────
+variable "edge_vpc_cidr" {
+  description = "CIDR of the relay VPC."
+  type        = string
+  default     = "10.66.0.0/16"
+}
+
+variable "edge_subnet_cidr" {
+  description = "CIDR of the relay subnet."
+  type        = string
+  default     = "10.66.1.0/24"
+}
+
+variable "relay_internet_max_bandwidth_out" {
+  description = "Peak outbound bandwidth in Mbps. The relay only terminates TLS and forwards bytes."
+  type        = number
+  default     = 20
+}
+
+variable "relay_system_disk_type" {
+  description = "System disk type for the relay."
+  type        = string
+  default     = "CLOUD_PREMIUM"
+}
+
+variable "relay_system_disk_size" {
+  description = "System disk size in GB for the relay."
+  type        = number
+  default     = 50
+}
+
+# ── DNS records ──────────────────────────────────────────────────────────────
+variable "china_dns_ttl" {
+  description = "TTL in seconds for the .cn A records."
+  type        = number
+  default     = 600
+}
+
+variable "china_dns_record_line" {
+  description = "DNSPod line for the .cn A records. Use 默认 for the default line."
+  type        = string
+  default     = "默认"
+}
+
+# ── Caddy reverse proxy ──────────────────────────────────────────────────────
+variable "wireguard_homelab_allowed_ips" {
+  description = "AllowedIPs for the homelab peer. Must include the homelab LAN so Caddy can reach ingress-nginx over the tunnel."
+  type        = string
+  default     = "10.88.0.2/32,192.168.1.0/24"
+}
+
+variable "ingress_upstream" {
+  description = "Upstream URL Caddy proxies to over WireGuard (the homelab ingress-nginx)."
+  type        = string
+  default     = "http://192.168.1.160:80"
+}
+
+variable "acme_email" {
+  description = "Contact email for Caddy ACME (Let's Encrypt)."
+  type        = string
+  default     = "[email protected]"
+}

terraform/terraform.tfvars.local.bak

@@ -1,97 +0,0 @@
-# RECOVERED from live cluster state via talosctl get machineconfig.
-# Regenerated after the original tfvars.local was lost/corrupted.
-
-cluster_id  = "Rtc4g2av9EP0mOdxA4M0-QQitzHLWICz-rLBNfrOjgw="
-cluster_secret = "8E0CAeylAPKmmUEQmIGmHQAhQf+8c7NUf43CdrQZ+vg="
-bootstrap_token = "b2q7lh.9w7hwgrulrd65gr3"
-machine_token = "hq9wlf.96l9z46efd79jtr2"
-machine_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBMkUwRWZqbG41L1J3eTVjMVJmNkNSakFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qWXdOekE1TURVd056VTRXaGNOTXpZd056QTJNRFV3TnpVNFdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUNwR3NQZkVHdEU4anVmNG9JTkNHNnlCQmN6aURFaEpLbDV3CnJib0hSR0tUbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkdmaExZUUdaOGYzd3lZZAo0SFI3KzlONnZKQXJNQVVHQXl0bGNBTkJBQ1ZQVlAwcGYxMkdUakYvSHJMZmcwZHBLemdBMlE1OGR5Y0paY0ZvClQvNDdPQk8ra29VZm11dXNjVVNNQnBXS0VuWHNYMFNocmNab3hQd1FHM3diblFFPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-machine_ca_key = "LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM0Q0FRQXdCUVlESzJWd0JDSUVJSlR4MXRqZEVOTTg1cGNRRFR0WWRtMFd0QXNBd0tzL1VESlZLN0ZqYU5uUgotLS0tLUVORCBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0K"
-kubernetes_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVRDZ0F3SUJBZ0lSQUtwT1ZaK29CRzljNHFycEpwaUc4TkV3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOakEzTURrd05UQTNOVGhhRncwek5qQTNNRFl3TlRBMwpOVGhhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVQ1VjJvNkliMUpRZkcza3lCTTBCeTRkd2FLbnlGY2tVdjNVem9yOG1Ba3RaN2JrT2ZKZDlmL2VmcVkKYXBzUFpLV1RHQnYxM3JZbFdvOGtuU3ZnNm83Um8yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGQ1dPZVJUVHdnN2tnNVNWMG9raFQwY0VDNGwzTUFvR0NDcUdTTTQ5QkFNQ0EwY0FNRVFDSURURCtNYXUKb2JXdkp6UkNMVEdJaHJHc1daalFnalVmRWl2MnhCTXB6RnNVQWlCQzNNWkYwMjVqVHk4Uno2YjI5czVJQmpkNgpZQTVWd0kvNVFieXlwSGFXQlE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-kubernetes_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUUrSEdPcUJJYXpJMzdqNmJJUlA1emVxeXEwZzhBU2xZeGplZUlJcEpIcXBvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFK1ZkcU9pRzlTVUh4dDVNZ1ROQWN1SGNHaXA4aFhKRkw5MU02Sy9KZ0pMV2UyNURueVhmWAovM242bUdxYkQyU2xreGdiOWQ2MkpWcVBKSjByNE9xTzBRPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-etcd_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJmVENDQVNTZ0F3SUJBZ0lSQVAyaHkwdUhvTm5vSzQyZE9LTk1nU2t3Q2dZSUtvWkl6ajBFQXdJd0R6RU4KTUFzR0ExVUVDaE1FWlhSalpEQWVGdzB5TmpBM01Ea3dOVEEzTlRoYUZ3MHpOakEzTURZd05UQTNOVGhhTUE4eApEVEFMQmdOVkJBb1RCR1YwWTJRd1dUQVRCZ2NxaGtqT1BRSUJCZ2dxaGtqT1BRTUJCd05DQUFRVGlKYUJpSEJPCmJha3JuL0dVdkUxVFd5czRVUkVzVGtVNEM4OG1EdWZYQURjV0NnN2RTRzc0QjkzOGFwSWgybGc4UDhKRDFFRUoKN0RkU1lQVG5zYWh1bzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSApBd0VHQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkliYTBLaEhmM3hhClBQNk1hb3dESUNWVXBLdDVNQW9HQ0NxR1NNNDlCQU1DQTBjQU1FUUNJRzZMYUJGeGgvcys2WXpGdVlwaUxUWlYKS2prOGh5UVNvMmVTZTJTUy81MG5BaUI0a0RNWnR4b1UzTitHc3BEOHVEbXFrNlhxbzZoeE0vbG0yU21OMzlPNAovdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
-etcd_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUUwZ3JiMk0yblA4c1hxWjVhd3NzNThwbUdvb1FlSWg3a3RoWVlxNzhZZThvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFRTRpV2dZaHdUbTJwSzUveGxMeE5VMXNyT0ZFUkxFNUZPQXZQSmc3bjF3QTNGZ29PM1VodQorQWZkL0dxU0lkcFlQRC9DUTlSQkNldzNVbUQwNTdHb2JnPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-aggregator_ca_crt = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJZVENDQVFhZ0F3SUJBZ0lSQUpxeFI3alBzcmhzRUp2cmtEWndYR3N3Q2dZSUtvWkl6ajBFQXdJd0FEQWUKRncweU5qQTNNRGt3TlRBM05UaGFGdzB6TmpBM01EWXdOVEEzTlRoYU1BQXdXVEFUQmdjcWhrak9QUUlCQmdncQpoa2pPUFFNQkJ3TkNBQVFBWWlieHY1ZDVFRGdTbWhlRHlhYjJLZGh4ZFZnZ3lQc2pNcjBET0JMVmxtQmpMcXFqClpNSkk2d1ZmV2hkUjBRVGxVdEFLZDJvREJZLy9TeDBzQi9qY28yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXcKSFFZRFZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4dwpIUVlEVlIwT0JCWUVGSTkrWm1EYVBybDhrTnhXUUxOT3ErTmVzeEh0TUFvR0NDcUdTTTQ5QkFNQ0Ewa0FNRVlDCklRRC9DRUZ2SUVHMW9qcmRZRVUzUldmTklLV1FwVXlZQWJ1ZGE0T0ZWQS9yOUFJaEFPS1VOZFF6bUk2WVZOdzgKeklDejlLblRxazQrT3dvV3RjNVl5SmlKR29zUgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="
-aggregator_ca_key = "LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUFoOEwycXFLbUxoYkpmczJ2M3ZRWXBFZHF6Ri9hTGZhSmoraEZRazdPY2NvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFQUdJbThiK1hlUkE0RXBvWGc4bW05aW5ZY1hWWUlNajdJeks5QXpnUzFaWmdZeTZxbzJUQwpTT3NGWDFvWFVkRUU1VkxRQ25kcUF3V1AvMHNkTEFmNDNBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo="
-service_account_key = "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKSndJQkFBS0NBZ0VBd3NNZU40eE1YQkZ0VVE2dVZ2NEZFU1dNT2ZLc1RKdmxHa0ptVG1URjJIazg1SlJvCkhuQU1nMEkvMEJiMGFUQTJ3MjlkSEUrdUN6dFd5eVVqTzV5eWdJSDQ0Q2RtVnNHVzVua1Z0TmFGNzNpcFllbjYKaG11bmRoWDY1VndqOHpVUmFYZUxtUEc0Y2d4dk5SOXdGYUZXS0ZZdzQzdGtlWjg5S0F5QTNibjFXN3JHTloyOQovcXRYWEcrQnFSSlVuUzhXQmt4dkErUktyRE1OVHF4bjVGOEl1VkpTWkw5bDVEd3lSUnNqMjVMZVdRZUFYenl6CllWcmFKZmVubSt6L3hLZjhLdHFZMSs5U1pwbkZsS2N6TGlWVzg4WkJ5TnR0KytYZjB4N3FSL2lQOHhRQXV0ckIKV1pTbytoa01FRjV5YU0wQWdxcG5mbDBsUmhsM0I0UUx3NHkyTUpoTEpyQ1ltQjl1QllLWU5oRE5NOXk0Z1lrZgpacmdEdm9NdGpsSUJUUHRyVjMyemxYN05Od3d5UjdXOXp5ZlBSVjEvbjhpMk9KR0szeittdGNCR2Y0TS8veDFsClJRNEhzWWhDUVJRaE8vQWp2U2QvQlhFRnB3dEhsb1VYRU5nQ2lPSW00MnFrZTNWb3lLbjJ2Q3g2QlhodlJVZDAKUHgvK0JpM0d4RmRoa2Uxa0doelJLQTdySGhEMVBkVGsyM04wUXV3WHNNVDZRWXVrRDdPZG5KdFZtZVN4TkxqcwptdVpybmpwRzZsaHRDSmdvMGdVeXBYQ2RlWGVnZ0dIc0JCMks4NjIrdThVa0dwMk9IQUhGeldsdDF5L2VXTW9PCmhneWx1SEVLcjkzUU5uNDZsTGtZTDViWjlVTUZ0NXBMRFVrSFQxNkV5dTh0V21ET2ZoQ3Z3M1lmRWY4Q0F3RUEKQVFLQ0FnQUdSbUlSV1JoV3VRc0VHd3g3NmdoQXdxeHZhNFdvbkRjMzd0Njc5Tnc0K3NMKy9GY1ViL2kvTytHeApjeVBoeGJkbCtZOE82L1JJRVZ2ZEJLL0xhbU9INTJnYzFMZ2o0RzNidEJnQ2NRejBwN2NSWEFnQno3TWdCMXBECmpJSHVBbzR5anpMMHRRa0R4Nm5IbE9FNEdUQWM4WlgycGxHWTU0d0JYOUhCRXc0NEs5N1orR0NZTlc0Rm9PUVYKRGUyaStOTGxWZzRYbW9IYlpYT3V6cmcwTCttb2l1SHp0QVQwNHdtZGwxL0M0Y3IvSkZJNi8wb3FQMUthK1kweApaV1BpTXFWWnZoeEJqTWpqWEYzMHlhUkkvdFA3MjYzZjZrM3pXVGNxWnFzV3NZZjF4WFcyajNpK1NaOWVHM043CmpZZHpIL085d2Y2K29BS2s3UW9jT0dGbXBnQnlwdm9JbytlSFdjZlpZNFNXNloyeTRacUl0M2xTSWFHMEtmQjEKUnVrSVBtMlYzNDNlc3Azdy9TV2liRVU2elhvd01sWjlZc0dPalY1MTZDQmJZK0lQcmY2RXY0UFhDWjlnUERxcgpnUFFIZ0lFS294aUdYK2dENG9pYWdUMWVVSk1iYWZkaEkzSHJMWk9YbUpBZU40RHpOZXR1SGRsTk13YnpZZHN5CnpyMllSMkhqNTZydk9hcWZZUFJKV1NmS3ozRlhQdDNDMHJRVmNRekVqV1ZZbm9MZG9yR3FkaU1uMyt1NzF2RW4KSnluZksvN3VRSDY5VTU3NGNKK3FjOFNtNTlPeHBnN1RoODljMUZTeSszTk03bTE2czlJcjQvcG94Q2pWaHNWeQpKRmN2cHE4UTMwek9CQk0waUxyblNKcVRXaXJDRU8xNHlRY0VLbFBVd2VGUWdJK05pUUtDQVFFQTMwMWNrTnZuCmpaRHduRXFQaWp4SlBDSUNEeHcxYWtnVTZUNUpMVkt5SUFnK3BYNHlPZDZtendvV2hpWE5oVUQ5eWxPL21lTEsKSDRPUXVzeWdJUDdkOVNKZi9ZVTV5a1RZaVZLMUdzM1loZXovRmhRQXdBQUg4UmdBdW8rZU9UTUcyT0IxM2loLworQlFYL1lrOG9VR1M0YlpsZWNGWG5pSTZ5S0g1MUJUQ3g0ZTZZcStvdGMrMFc2RzBRVHVBT3JWcjdXWklGY1htClVDdElReXRJN2s2UVd6SXU0K3dnVnU3OUdrUEJTMHNBaXhzU0ppYXRTNzk1TjlhYjYwRzNDeVFWZDZWajJaVmEKWGpCR1oyQXREalpEWS9MMDlZQTlRVUxDbWVqT2hYVFlGNkJuNnRkYmVjeVlZTHdNS2MzWEhqNEt2U2ZaQ01HTwpvUXROdnFoUFFlSWpxUUtDQVFFQTMwZnFCSThDK2QwdXJlM1JhNFFRdUlzUC9xVTMyejIzMUdzbTZOOXlQd0hUClM4ZzlrS0ZDYzhMVGlmWFdnajhIRWxTVjVLSXJseERwZEVWQ1pEM05qdy9GVDdHcHV5SVplN1E3VlhiNld3MnYKY1I0M1FkdUQvOEc4ZkdlZGkrZ1BnKzlzeURCQXFIVzFGSHl1RmVaWGNsVTF1cndOV3V3TlFKUnZHczFoK1NuQQpXVmJxd1ZUL01GMmYxTjlBaFN0alhkM0Nscm1rb0o4ek05YW0zVWg5VUprQ2xSZi9mZVRONXZndE1odS9NNGtGCmVQYUlBMTVqT3h1cEFMN3ptVVVxZ0h1NE9yMm5mYTVNNHMzWTR6TVRYYm9VZGNVT09oRTRzU0J2bGlaak5KL3QKV2pSTmJmUWhxbWRBTTNZZjFGUC8vRW9CYTBPejBxMHNXci92cEhDUlp3S0NBUUFxMlkySnZxa1FZVi9LbmdRdApZcVFyQmR1ZlNxcDFXcCtvb21zb1oxWUhENDMxOCtGdmVXcEpFSWFCOTM4WXN3QUFjMUd4RmZQeldDdk5yTGFOCm5scTVUMzljQnRTd0c4WHhsQTFzdDFOMVg2VVRkNE10Vk5ReFQ0blVRdnI1dnZEeGJTRXhJRlJ1Sm16MEdnR28KY0F6ZmcwQzF2SVF6dEIzVG9rRnVrUTFQZkp3bms4MnNGYzltUmdGeEF4bjRLaGdyMWhTL0dOcTVSNVQyVHJnUQpBc053dkpDQzdDeklnZFBQMW5DaElpTllqamxOV042b1NuWFlZVFpLVHJIeFVWdE5PaytPMFRvbUdOMXB1T3JzCmJ6MC9VTC93M0VyazJ3cTh2Zy9qVENpclgveVE5QUo1dk9rQXB4VXVjSEYzUERDVFc3SXFHL3Bpck9pZVRXM28KRnAwQkFvSUJBRjBxa3JsSU8wT3JTUmtHRE1aQ0d3QUY5cXlZb0EvNVZzVnAySmgrOUJyYVZpSmU4V0Z5Q0ZwcApSdjlmOXh2dDFMT1BXK1JFenMrQUhRbUpCTVR6RE56UEJkUFZIQytiY09xdkw3cmZwR050K0hESTNPRzhDUDRsCkJ0TWFJU0VKdWIraG5kQ0NZZGhwRlIveFRtcVE3SmdtZWY3ckROK05jNUlvM1p0ZmE2d2VBY2JGZjdzZ0RrTk8KTGEwVFlzYXViZzN5eElsRCtTK1VmamI1TUROUlZnalZiOEJxZlE4NDg3bVdnTFZSNHB4TVpsNHM4R0FIZUh4bgpkRU45YWdQZ1duVzJLZzlJcDZUSG9BbGJQMDYrTnl4NndxTEprTUFtQTNQVlJ2cHVGaU1WUUdMTlJDbkhIbTBPCkhEbmM1amNndmNXMTA1WEFjRDVPU0IydHpQN2VnYTBDZ2dFQUZhTHJxMDJ6M2tXaW1iVzdoNi9Pby9YRHdjZ2YKSmdXMDYyYWdWUlpEMjM3Mm4valZSL25kMnFybmwxTFdWaXpzWW91b3hvY2N6NUwvQ2h3MktCOC9Rc1Zxai9KOApOUWh1ZDJ2ZUxjQlUvVzNseVhENnJpajJZMythNzRvM1A4b1psOGdDQmJEck1jajNsMHRZMXRWbm9nOWo0eHQ3Ckp3UXA3djNXb3ArSFVuRWVjbGpscTdlN1VuTGNwZlRDOE5PUmdxbjBGSUtCbXdFNlpJQnQxUzB6NmxSKytJdVMKQk1oTTZTSkZmeUFOVVdjTCt2cHJoRzBjWUZmcUYzajlPZ1dXRnBGTHFWQmdSTlZBVDRoM0U1Ymh4L0lsMTVHMAo2USs2aWlkRElqMEhNNFAzcjhja2lRUlVPdFI1R2NVS1RYZGh1TjkwMjZGRTdEckRvSURVZ0F5dGVnPT0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K"
-secretbox_encryption_secret = "RLkR4RmeaLmH/abyK3eYf6q22umJ/byLhheVCnh/yrA="
-
-controlplane_configs = {
-  "talos-cp-1" = {
-    hostname     = "talos-cp-1"
-    lan_ip       = "192.168.1.166"
-    public_ip    = "https://k8s-api.riotpiao.com"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/nvme0n1"
-    longhorn_disks = []
-    zone             = "az-a"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-  "talos-cp-2" = {
-    hostname     = "talos-cp-2"
-    lan_ip       = "192.168.1.214"
-    public_ip    = "https://k8s-api.riotpiao.com"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
-    longhorn_disks = [
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
-        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
-        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
-        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
-        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
-        # re-add it.
-      ]
-    zone             = "az-b"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-  "talos-cp-3" = {
-    hostname     = "talos-cp-3"
-    lan_ip       = "192.168.1.162"
-    public_ip    = "https://k8s-api.riotpiao.com"
-    lan_subnet   = "192.168.1.0/24"
-    lan_gateway  = "192.168.1.254"
-    install_disk = "/dev/nvme0n1"
-    longhorn_disks = [
-        { device = "/dev/disk/by-id/usb-Seagate_One_Touch_w_PW_00000000NABV3H34-0:0", mountpoint = "/var/lib/longhorn-paperless-media" },
-      ]
-    zone             = "az-c"
-    allow_scheduling = true
-    cloudflare_talos_sans     = []
-    cloudflare_apiserver_sans = []
-  },
-}
-
-worker_configs = {
-  "worker-1" = {
-    hostname          = "worker-1"
-    lan_ip            = "192.168.1.223"
-    lan_subnet        = "192.168.1.0/24"
-    lan_gateway       = "192.168.1.254"
-    install_disk      = "/dev/nvme0n1"
-    network_interface = "enp28s0f0np0"
-    zone              = "az-a"
-    gpu_count         = 4
-    node_labels       = {}
-    node_taints       = []
-    factory_image     = "factory.talos.dev/metal-installer/0a2153a6dc099a371bf2f63d6c3c22d275c876bf6302dd154c5813072924cb3f:v1.13.3"
-    swap_size         = "64GiB"
-    ephemeral_max_size = "700GiB"
-    extra_disks       = []
-  }
-}
-
-cluster_config = {
-  controlplane_ip = "192.168.1.166"
-  pod_subnets     = ["10.244.0.0/16"]
-  service_subnets = ["10.96.0.0/12"]
-  dns_servers     = ["8.8.8.8", "1.1.1.1"]
-  dns_domain      = "cluster.local"
-}

terraform/zitadel/actions.tf

@@ -1,58 +0,0 @@
-# The groups claim — the single load-bearing decision of the migration.
-#
-# ZITADEL does not issue a `groups` claim. Authentik did, and these consumers
-# read it: kube-apiserver RBAC (`--oidc-groups-claim=groups` plus 11 RoleBindings
-# on oidc:<group>), MinIO (`MINIO_IDENTITY_OPENID_CLAIM_NAME=groups`), Grafana
-# role mapping, and every service that filters on a group name.
-#
-# Plan §1.3 Option A keeps those consumers unchanged by synthesizing the claim.
-# Option B refactors all of them. This module takes responsibility only for the
-# ZITADEL side of Option A: the action resource and its trigger.
-#
-# The script body is NOT shipped by default. The go/no-go gate (§1.3, §1.5 task
-# 1.3) is: prove on the live instance that the action runtime can read project
-# roles plus user metadata and write both `groups` and the per-agent claims,
-# for a human token AND a machine-user token. Until that proof exists, enabling
-# this resource would move every consumer onto an unverified claim source:
-#
-#   terraform apply -var enable_groups_claim_action=true \
-#                   -var groups_claim_script=./groups-claim.js
-#
-# After it applies, verify with a real token exchange before moving a consumer:
-#
-#   curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
-#     -d grant_type=refresh_token -d refresh_token=<t> \
-#     -d client_id=kubernetes | jq -R 'split(".")[1] | @base64d | fromjson'
-#
-# The decoded access token must contain `"groups":["homelab-admins", ...]`.
-
-resource "zitadel_action" "groups_claim" {
-  count = var.enable_groups_claim_action ? 1 : 0
-
-  org_id          = data.zitadel_org.default.id
-  name            = "homelab-groups-claim"
-  script          = file(var.groups_claim_script)
-  timeout         = "5s"
-  allowed_to_fail = false
-}
-
-# Both triggers are needed. The access-token trigger fills the JWT that
-# Kubernetes and MinIO read. The userinfo trigger fills the response that
-# Grafana, paperless and Forgejo read.
-resource "zitadel_trigger_actions" "groups_claim_access_token" {
-  count = var.enable_groups_claim_action ? 1 : 0
-
-  org_id       = data.zitadel_org.default.id
-  flow_type    = "FLOW_TYPE_CUSTOMISE_TOKEN"
-  trigger_type = "TRIGGER_TYPE_PRE_ACCESS_TOKEN_CREATION"
-  action_ids   = [zitadel_action.groups_claim[0].id]
-}
-
-resource "zitadel_trigger_actions" "groups_claim_userinfo" {
-  count = var.enable_groups_claim_action ? 1 : 0
-
-  org_id       = data.zitadel_org.default.id
-  flow_type    = "FLOW_TYPE_CUSTOMISE_TOKEN"
-  trigger_type = "TRIGGER_TYPE_PRE_USERINFO_CREATION"
-  action_ids   = [zitadel_action.groups_claim[0].id]
-}

terraform/zitadel/api_apps.tf

@@ -1,18 +0,0 @@
-# API applications — the machine-token surface.
-#
-# ZITADEL keeps two different objects apart: an OIDC application is for a
-# browser flow, and an API application is what a client_credentials / JWT-profile
-# caller presents. Authentik modelled both with one provider, so the plan §1.1
-# row 1 ("OIDC app + machine user") needs one of each here.
-#
-# The gateway signs its token request with the machine key from machines.tf, so
-# the auth method is PRIVATE_KEY_JWT. No secret is stored for it.
-
-resource "zitadel_application_api" "machine" {
-  for_each = { for name, spec in local.machine_users : name => spec if contains(keys(var.api_app_consumers), name) }
-
-  org_id           = data.zitadel_org.default.id
-  project_id       = zitadel_project_v2.homelab.id
-  name             = "${each.key}-machine"
-  auth_method_type = "API_AUTH_METHOD_TYPE_PRIVATE_KEY_JWT"
-}

terraform/zitadel/clients.tf

@@ -1,136 +0,0 @@
-# OIDC applications — one per consumer that logs a browser in.
-#
-# Client IDs: the plan §3.2 wants every client_id byte-identical after the move,
-# so that a consumer change is only the issuer URL plus the secret.
-#
-# THAT GOAL IS NOT ACHIEVABLE HERE, verified twice on 2026-10-24:
-#   - ZITADEL v4.19.2 CreateOIDCApplicationRequest has no client_id field, and
-#     CreateOIDCApplicationResponse documents the value as "generated"
-#     (proto/zitadel/app/v2beta/app_service.proto:540-650).
-#   - zitadel provider 3.8.7 exposes no settable application id: `id` on
-#     `zitadel_application_v2` is not a valid argument (terraform validate
-#     rejects it), and `client_id` on `zitadel_application_oidc` is computed.
-# Consequence: every consumer takes a NEW client_id as well as a new secret.
-# Budget the extra config change per consumer — this is not the one-line
-# re-point the plan assumed.
-#
-# The only way to keep the old value is to create each app through the v2 REST
-# API with an explicit body id, then import it:
-#   POST /v2/projects/<project_id>/apps
-#   {"id":"<old-client-id>","name":"<old-client-id>","oidcRequest":{...}}
-# Not implemented here: it would put app creation outside Terraform state.
-#
-# Secrets are generated by ZITADEL, never written here. `terraform output -json
-# client_ids` and `client_secrets` feed the SOPS re-point (§3.2, §6.9). Do not
-# commit the tfstate: it carries those values.
-
-locals {
-  oidc_clients = {
-    # Gateway. Browser sessions for the console, plus a machine path
-    # (machines.tf) for callers that need client_credentials.
-    "api-gw" = {
-      redirect_uris = ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "argocd" = {
-      redirect_uris = ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "forgejo" = {
-      redirect_uris = ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "grafana" = {
-      redirect_uris = ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "homarr" = {
-      redirect_uris = ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "immich" = {
-      redirect_uris = ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"]
-      app_type      = "NATIVE"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "kubernetes" = {
-      redirect_uris = ["http://localhost:8000", "http://localhost:18000"]
-      app_type      = "WEB"
-      auth_method   = "NONE"
-      dev_mode      = true
-    }
-    "minio" = {
-      redirect_uris = ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "paperless" = {
-      redirect_uris = ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "poimen" = {
-      redirect_uris = ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "queue-sqs" = {
-      redirect_uris = ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-    "vault" = {
-      redirect_uris = ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"]
-      app_type      = "WEB"
-      auth_method   = "BASIC"
-      dev_mode      = false
-    }
-  }
-}
-
-resource "zitadel_application_v2" "oidc" {
-  for_each = local.oidc_clients
-
-  org_id     = data.zitadel_org.default.id
-  project_id = zitadel_project_v2.homelab.id
-  name       = each.key
-
-  oidc {
-    redirect_uris  = each.value.redirect_uris
-    response_types = ["OIDC_RESPONSE_TYPE_CODE"]
-    grant_types = [
-      "OIDC_GRANT_TYPE_AUTHORIZATION_CODE",
-      "OIDC_GRANT_TYPE_REFRESH_TOKEN",
-    ]
-    app_type         = "OIDC_APP_TYPE_${each.value.app_type}"
-    auth_method_type = "OIDC_AUTH_METHOD_TYPE_${each.value.auth_method}"
-    version          = "OIDC_VERSION_1_0"
-
-    # JWT access tokens: Kubernetes RBAC, MinIO policy mapping and the Go
-    # gateway all read claims, and an opaque bearer token carries none.
-    access_token_type = "OIDC_TOKEN_TYPE_JWT"
-
-    # Assert roles without the client having to request the scope. Every
-    # consumer here maps roles or groups to local permissions.
-    access_token_role_assertion = true
-    id_token_role_assertion     = true
-    id_token_userinfo_assertion = true
-
-    dev_mode = each.value.dev_mode
-  }
-}

terraform/zitadel/machines.tf

@@ -1,139 +0,0 @@
-# Machine users and their project grants (plan §1.4, §5.3).
-#
-# Authentik gave each agent an app password or a client_credentials grant built
-# on a user password. ZITADEL has no user password on a machine user: it issues
-# a JWT profile key (`KEY_TYPE_JSON`) that the SDK signs its token request with.
-# That removes the password path entirely, which is the point — see §5.3.
-#
-# Per-agent claim values are copied from `SERVICE_ACCOUNTS` in
-# scripts/iam/provision-rbac.py. They live in user metadata as JSON strings,
-# because the groups-claim action (actions.tf) is the only thing that turns them
-# into token claims. Without that action a consumer sees roles but not
-# `minio_buckets`, `authorized_models`, `sqs_queues`, `memory_projects`,
-# `memory_visibility` or `paperless_doctypes`.
-#
-# Keys are generated per machine user and land in `terraform output -json
-# machine_keys` (sensitive). Feed each one to the owning workload as a mounted
-# file, never as an environment variable dumps-able through `ps`.
-
-locals {
-  machine_users = {
-    "memory-agent" = {
-      description      = "Memory service agent"
-      project_roles    = ["poimen-admins"]
-      authenticated_by = "poimen memory (Rust)"
-      metadata = {
-        roles             = jsonencode(["llm:inference", "memory:read", "memory:write", "notification:write", "sessions:read", "sessions:write"])
-        memory_projects   = jsonencode(["*"])
-        memory_visibility = "private"
-        authorized_models = jsonencode(["reasoning", "ornith:35b", "qwen2.5:3b"])
-      }
-    }
-    "paperless-ai-agent" = {
-      description      = "Paperless AI plugin (auto-tagging, entity extraction)"
-      project_roles    = ["poimen-devs"]
-      authenticated_by = "paperless-ai"
-      metadata = {
-        roles              = jsonencode(["llm:inference", "memory:write", "paperless:admin", "notification:write"])
-        minio_buckets      = jsonencode(["paperless"])
-        paperless_doctypes = jsonencode(["*"])
-        memory_projects    = jsonencode(["*"])
-        authorized_models  = jsonencode(["reasoning", "qwen2.5:3b"])
-      }
-    }
-    "portfolio-agent" = {
-      description      = "Portfolio service agent"
-      project_roles    = ["poimen-prod-readonly"]
-      authenticated_by = "portfolio service"
-      metadata = {
-        roles             = jsonencode(["llm:inference", "memory:read"])
-        memory_projects   = jsonencode(["homelab", "portfolio"])
-        memory_visibility = "public"
-        authorized_models = jsonencode(["ornith:35b"])
-        minio_buckets     = jsonencode(["backups"])
-      }
-    }
-    "temporal-worker-agent" = {
-      description      = "Temporal workflow worker"
-      project_roles    = ["poimen-admins"]
-      authenticated_by = "Temporal worker"
-      metadata = {
-        roles             = jsonencode(["llm:inference", "workflow:execute", "memory:read", "memory:write", "queue:send", "notification:write"])
-        memory_projects   = jsonencode(["*"])
-        sqs_queues        = jsonencode(["*"])
-        authorized_models = jsonencode(["reasoning", "ornith:35b", "qwen2.5:3b"])
-      }
-    }
-    # The gateway is the one consumer that used client_credentials on a machine
-    # token, so it needs both the OIDC app (clients.tf) and a machine user
-    # (here) plus its API application (api_apps.tf). Its capability check reads
-    # the `roles` claim, which arrives as metadata.
-    "api-gw" = {
-      description      = "Go LLM gateway machine identity"
-      project_roles    = []
-      authenticated_by = "api-gw"
-      metadata = {
-        roles = jsonencode(["llm:inference"])
-      }
-    }
-    # pi-worker has no anchor in git (plan §1.1: "not in git") and no group in
-    # the Authentik provisioning script. It gets a machine user and no project
-    # role until the owner names the consumer configuration that holds its old
-    # credentials. Creating a role grant blind would widen access, not preserve it.
-    "pi-worker" = {
-      description      = "pi sandbox worker (no git anchor for its consumer config)"
-      project_roles    = []
-      authenticated_by = "pi sandbox worker"
-      metadata = {
-        roles = jsonencode([])
-      }
-    }
-  }
-
-  machine_metadata = merge([
-    for user, spec in local.machine_users : {
-      for key, value in spec.metadata : "${user}/${key}" => {
-        user  = user
-        key   = key
-        value = value
-      }
-    }
-  ]...)
-}
-
-resource "zitadel_machine_user" "agent" {
-  for_each = local.machine_users
-
-  org_id      = data.zitadel_org.default.id
-  user_name   = each.key
-  name        = each.key
-  description = "${each.value.description} — consumer: ${each.value.authenticated_by}"
-  with_secret = false
-}
-
-resource "zitadel_machine_key" "agent" {
-  for_each = local.machine_users
-
-  org_id          = data.zitadel_org.default.id
-  user_id         = zitadel_machine_user.agent[each.key].id
-  key_type        = "KEY_TYPE_JSON"
-  expiration_date = var.machine_key_expiry
-}
-
-resource "zitadel_project_member" "agent" {
-  for_each = { for user, spec in local.machine_users : user => spec if length(spec.project_roles) > 0 }
-
-  org_id     = data.zitadel_org.default.id
-  project_id = zitadel_project_v2.homelab.id
-  user_id    = zitadel_machine_user.agent[each.key].id
-  roles      = each.value.project_roles
-}
-
-resource "zitadel_user_metadata" "agent_claim" {
-  for_each = local.machine_metadata
-
-  org_id  = data.zitadel_org.default.id
-  user_id = zitadel_machine_user.agent[each.value.user].id
-  key     = each.value.key
-  value   = each.value.value
-}

terraform/zitadel/outputs.tf

@@ -1,46 +0,0 @@
-# Outputs feed two things: the SOPS re-point of consumer secrets (§3.2, §6.9)
-# and the machine-key handoff to each agent workload (§5.3). Every value that can
-# authenticate a client is marked sensitive, so `terraform output` prints a
-# placeholder until you ask for it explicitly.
-
-output "org_id" {
-  description = "Organization that owns the project, roles and applications."
-  value       = data.zitadel_org.default.id
-}
-
-output "project_id" {
-  description = "Project ID, which ZITADEL puts in the access-token `aud` claim."
-  value       = zitadel_project_v2.homelab.id
-}
-
-output "client_ids" {
-  description = "client_id per consumer. Compare against the Authentik values: any difference means the consumer config must change too."
-  value       = { for name, app in zitadel_application_v2.oidc : name => app.oidc[0].client_id }
-}
-
-output "client_secrets" {
-  description = "Generated client secret per consumer. Write these into the existing SOPS files; never into git."
-  value       = { for name, app in zitadel_application_v2.oidc : name => app.oidc[0].client_secret }
-  sensitive   = true
-}
-
-output "api_app_client_ids" {
-  description = "client_id of each API application (machine-token surface)."
-  value       = { for name, app in zitadel_application_api.machine : name => app.client_id }
-}
-
-output "machine_user_ids" {
-  description = "ZITADEL user ID per machine user."
-  value       = { for name, user in zitadel_machine_user.agent : name => user.id }
-}
-
-output "machine_keys" {
-  description = "JWT profile key JSON per machine user. Mount as a file; do not pass through the environment."
-  value       = { for name, key in zitadel_machine_key.agent : name => key.key_details }
-  sensitive   = true
-}
-
-output "groups_claim_enabled" {
-  description = "Whether the groups-claim action and its triggers exist. False means Option A is not yet in place and no consumer may move."
-  value       = var.enable_groups_claim_action
-}

terraform/zitadel/project.tf

@@ -1,62 +0,0 @@
-# Project, roles and default-org lookup.
-#
-# ZITADEL has no group object. The plan §1.3 maps every Authentik group to a
-# project role, and the groups-claim action (actions.tf) reconstructs the
-# `groups` claim from those role keys. Role keys therefore EQUAL the existing
-# Authentik group names, including the hyphens, so that the resulting claim is
-# byte-identical and no consumer or RBAC file changes. The v4.19.2 API allows
-# role keys of 1-200 characters with no character-set restriction (proto
-# zitadel/project/v2/project_service.proto, AddProjectRole.role_key).
-
-data "zitadel_org" "default" {
-  id = var.org_id
-}
-
-locals {
-  # Single source: the 19 groups of scripts/iam/provision-rbac.py GROUPS.
-  # Descriptions are carried over so the console reads the same as before.
-  roles = {
-    "homelab-admins"       = "Cluster administrators with full access"
-    "minio-admins"         = "MinIO administrators"
-    "minio-photos"         = "Photos bucket (Immich) access"
-    "minio-documents"      = "Documents bucket (Paperless) access"
-    "minio-backups"        = "Backups bucket read-only access"
-    "poimen-admins"        = "Poimen memory administrators"
-    "poimen-devs"          = "Dev and staging projects access"
-    "poimen-prod-readonly" = "Production projects read-only access"
-    "paperless-admins"     = "Paperless administrators"
-    "paperless-finance"    = "Finance documents"
-    "paperless-legal"      = "Legal documents"
-    "paperless-hr"         = "HR documents"
-    "grafana-admins"       = "Grafana administrators"
-    "grafana-editors"      = "Grafana dashboard editors"
-    "grafana-viewers"      = "Grafana dashboard viewers"
-    "sqs-users"            = "SQS/Temporal queue read access"
-    "sqs-writers"          = "SQS/Temporal queue read/write access"
-    "s3-users"             = "S3 read access"
-    "s3-writers"           = "S3 read/write access"
-  }
-}
-
-resource "zitadel_project_v2" "homelab" {
-  org_id             = data.zitadel_org.default.id
-  name               = var.project_name
-  project_id         = var.project_id
-  project_role_check = false
-
-  # Roles must reach the tokens. Every consumer authorizes on them:
-  # kube-apiserver RBAC reads `groups`, MinIO reads bucket policies, Grafana
-  # reads grafana_org_role. Role assertion on the ID token is what most OIDC
-  # clients inspect.
-  project_role_assertion = true
-}
-
-resource "zitadel_project_role" "groups" {
-  for_each = local.roles
-
-  org_id       = data.zitadel_org.default.id
-  project_id   = zitadel_project_v2.homelab.id
-  role_key     = each.key
-  display_name = each.key
-  group        = "homelab"
-}

terraform/zitadel/providers.tf

@@ -1,53 +0,0 @@
-# terraform/zitadel — identity configuration for the ZITADEL instance
-#
-# Scope: this root module declares the objects INSIDE ZITADEL (org, project,
-# roles, OIDC applications, machine users, grants). It does NOT deploy ZITADEL
-# itself — that is ArgoCD's job (`k8s/argocd/apps/30-security.yaml`, values in
-# `k8s/infra/iam/zitadel-values.yaml`).
-#
-# Migration plan: ~/workplace/homelab-doc/ZITADEL_MIGRATION.html (outside git).
-# This file implements §1.3 (roles), §1.4 (machine users), §2.4 (grants) and
-# §6.2 (Terraform provider route). The claim-injection action of §1.3 lives in
-# `actions.tf` and stays disabled until the live go/no-go gate proves it.
-#
-# Authentication to ZITADEL uses the `iam-admin` machine key that the chart's
-# setup job writes to the `iam-admin` Secret in namespace iam:
-#
-#   kubectl -n iam get secret iam-admin -o jsonpath='{.data.iam-admin\.json}' \
-#     | base64 -d > ~/.config/zitadel/iam-admin.json
-#   chmod 600 ~/.config/zitadel/iam-admin.json
-#
-#   export ZITADEL_JWT_PROFILE_FILE=$HOME/.config/zitadel/iam-admin.json
-#   terraform -chdir=terraform/zitadel init
-#   terraform -chdir=terraform/zitadel plan
-#
-# State is LOCAL and git-ignored (`terraform/*.tfstate` in .gitignore). That is
-# a deliberate first step, not the final answer: §8 risk 8 of the plan names
-# state drift as a blast-radius item. Move the state to MinIO before a second
-# operator ever runs `apply`. No secret is written to state except the generated
-# client secrets and machine keys, which are marked sensitive — they still land
-# in the state file, so treat that file as a secret.
-terraform {
-  required_version = ">= 1.6.0"
-
-  required_providers {
-    zitadel = {
-      source  = "zitadel/zitadel"
-      version = "3.8.7"
-    }
-  }
-}
-
-provider "zitadel" {
-  domain = var.zitadel_domain
-  port   = var.zitadel_port
-
-  # JWT profile of the iam-admin machine user. Preferred over a PAT: it is
-  # short-lived per call and never sits in the environment as a bearer token.
-  # Set the path through ZITADEL_JWT_PROFILE_FILE.
-  jwt_profile_file = var.jwt_profile_file
-
-  # The instance serves plain HTTP behind ingress-nginx, which terminates TLS.
-  # Turn this on only if the provider must call the pod directly.
-  insecure = var.zitadel_insecure
-}

terraform/zitadel/variables.tf

@@ -1,103 +0,0 @@
-variable "zitadel_domain" {
-  description = "External domain of the ZITADEL instance, without scheme."
-  type        = string
-  default     = "zitadel.riotpiao.com"
-}
-
-variable "zitadel_port" {
-  description = "Port for the ZITADEL API. 443 behind ingress, 8080 for a direct pod port-forward."
-  type        = string
-  default     = "443"
-}
-
-variable "zitadel_insecure" {
-  description = "Call the API over plain HTTP (set true only when port-forwarding the pod)."
-  type        = bool
-  default     = false
-}
-
-variable "jwt_profile_file" {
-  description = "Path to the iam-admin machine key JSON extracted from the iam-admin Secret."
-  type        = string
-  default     = null
-}
-
-variable "org_id" {
-  description = <<-EOT
-    ID of the ZITADEL organization that holds the project. Read it from the
-    console: https://zitadel.riotpiao.com -> Organization settings -> Resource
-    ID. The chart's setup job creates the default org; this module never creates
-    one, because creating a second org would split the role and grant space.
-  EOT
-  type        = string
-}
-
-variable "project_id" {
-  description = <<-EOT
-    Stable project ID. ZITADEL puts the project ID in the access token `aud`
-    claim, so pinning it keeps audience checks readable across a rebuild.
-    Every consumer that validates `aud` against a client ID must be re-pointed
-    to this value — see the plan §5.2.
-  EOT
-  type        = string
-  default     = "homelab"
-}
-
-variable "project_name" {
-  description = "Display name of the project that carries every role and application."
-  type        = string
-  default     = "homelab"
-}
-
-variable "machine_key_expiry" {
-  description = "Expiry of every machine key, RFC3339. Matches the chart's default iam-admin key window; rotate by changing this value and re-applying."
-  type        = string
-  default     = "2029-01-01T00:00:00Z"
-}
-
-variable "api_app_consumers" {
-  description = <<-EOT
-    Machine users that also need an API application (the client_credentials
-    replacement). Only the gateway does: every other agent signs a JWT with its
-    machine key and does not need a client_credentials grant.
-  EOT
-  type        = map(string)
-  default     = { api-gw = "Go LLM gateway" }
-}
-
-variable "enable_groups_claim_action" {
-  description = <<-EOT
-    Create the Actions v1 script and its trigger that synthesizes the `groups`
-    claim. Keep this false until the live go/no-go gate passes: the claim is
-    load-bearing for kube-apiserver RBAC, MinIO policies, Grafana role mapping
-    and every service that reads `groups`. Turning it on with an unverified
-    script can break every consumer at once.
-  EOT
-  type        = bool
-  default     = false
-}
-
-variable "groups_claim_script" {
-  description = <<-EOT
-    Path to the JavaScript body for the groups-claim action, read only when
-    enable_groups_claim_action is true. The body is deliberately not shipped
-    blind: the ZITADEL v1 Actions runtime API must be confirmed against the live
-    instance before a script is committed, because a wrong call silently emits
-    no claim instead of failing loudly.
-  EOT
-  type        = string
-  default     = null
-
-  validation {
-    # try() is required: HCL does not short-circuit `||`, so fileexists(null)
-    # would fail the whole plan even with the action disabled.
-    condition     = try(fileexists(var.groups_claim_script), true)
-    error_message = "groups_claim_script must name an existing file when it is set."
-  }
-
-  # Enabling the action without a script would reach file(null) inside actions.tf.
-  validation {
-    condition     = !var.enable_groups_claim_action || var.groups_claim_script != null
-    error_message = "enable_groups_claim_action requires groups_claim_script."
-  }
-}