feat(iam): register the poimen /auth/callback redirect

The frontend's callback path is /auth/callback. The old /callback spelling stays in
the list so an image built before this change still completes a login.

.forgejo/workflows/key-rotation-controller.yaml

@@ -0,0 +1,41 @@
+# Builds and pushes the key-rotation-controller image.
+#
+# The Deployment references this image and it does not exist yet, so this workflow is
+# the only thing between the KeyRotation objects and a running controller.
+name: key-rotation-controller
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "k8s/apps/key-rotation-controller/controller/**"
+      - ".forgejo/workflows/key-rotation-controller.yaml"
+  workflow_dispatch:
+
+jobs:
+  image:
+    runs-on: golang
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | \
+            docker login forgejo.riotpiao.com -u poimen --password-stdin
+
+      - name: Build and push
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          set -eu
+          IMAGE=forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller
+          TAG=$(echo "$GITHUB_SHA" | cut -c1-8)
+          docker build -t "$IMAGE:$TAG" -t "$IMAGE:0.1.0" .
+          docker push "$IMAGE:$TAG"
+          docker push "$IMAGE:0.1.0"
+          echo "pushed $IMAGE:$TAG and $IMAGE:0.1.0"
+
+      - name: Prove the binary starts
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          docker run --rm --entrypoint /usr/local/bin/controller \
+            forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0 --help || true

k8s/apps/comfyui/ingress.yaml

@@ -13,9 +13,16 @@ metadata:
     # WebSocket support for ComfyUI's live preview
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
     nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "upgrade";
+    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
+    # oauth2-proxy for an allow or deny decision.
+    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
+    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
+    # No configuration-snippet. The controller rejects that annotation as risky
+    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
+    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
+    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
+    # and Connection headers itself for a websocket upgrade.
 spec:
   ingressClassName: nginx
   rules:

k8s/apps/comfyui/kustomization.yaml

@@ -4,4 +4,5 @@ kind: Kustomization
 resources:
   - deployment.yaml
   - service.yaml
+  - oauth2-proxy.yaml
   - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -0,0 +1,105 @@
+# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
+# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
+# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
+# a decision on every request and it answers from a ZITADEL session.
+#
+# No API client calls this host today, so the whole host is gated. If a client ever
+# needs direct access, add a skip rule for the exact paths it uses rather than
+# opening the whole host.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: comfyui-oauth2-proxy
+  template:
+    metadata:
+      labels:
+        app: comfyui-oauth2-proxy
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
+          args:
+            - --provider=oidc
+            - --oidc-issuer-url=https://zitadel.riotpiao.com
+            - --email-domain=*
+            - --upstream=static://202
+            - --reverse-proxy=true
+            - --set-xauthrequest=true
+            - --cookie-secure=true
+            - --cookie-expire=12h
+            - --cookie-refresh=1h
+            - --skip-provider-button=true
+            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
+            - --http-address=0.0.0.0:4180
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
+          ports:
+            - {name: http, containerPort: 4180}
+          readinessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 15
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 128Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  type: ClusterIP
+  selector:
+    app: comfyui-oauth2-proxy
+  ports:
+    - {name: http, port: 4180, targetPort: http}
+---
+# The login path must NOT be gated: a request to /oauth2/start that itself demands a
+# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
+# an ingress to the locations it defines, so a separate object without auth-url keeps
+# this path open. The earlier failure of this shape was not the split — it was the
+# configuration-snippet on the app ingress, which made the controller drop the whole
+# object and left only this one alive.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui-oauth2
+  namespace: comfyui
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfyui.riotpiao.com
+      http:
+        paths:
+          - path: /oauth2
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui-oauth2-proxy
+                port:
+                  number: 4180

k8s/apps/gotify/bridge/configmap.yaml

@@ -0,0 +1,110 @@
+# The bridge script is mounted as a ConfigMap so no image build is needed.
+# Regenerate this block from grafana_gotify_bridge.py when the script changes.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+data:
+  grafana_gotify_bridge.py: |
+    #!/usr/bin/env python3
+    """Translate a Grafana alert webhook into a Gotify message.
+    
+    Grafana sends its own payload shape:
+      {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+    
+    Gotify requires:
+      {"title": "...", "message": "...", "priority": 0..10}
+    
+    Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+    conversion step is mandatory. This service is that step and nothing else.
+    Standard library only, so it needs no image build and no dependencies.
+    """
+    import json
+    import os
+    import urllib.request
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+    
+    GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+    GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+    LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+    
+    # Firing is louder than resolved. Gotify priority runs 0 to 10.
+    PRIORITY = {"firing": 8, "resolved": 3}
+    
+    
+    def to_gotify(payload):
+        alerts = payload.get("alerts") or []
+        status = (payload.get("status") or "firing").lower()
+        priority = PRIORITY.get(status, 5)
+    
+        if not alerts:
+            title = payload.get("title") or "Grafana notification"
+            return title, json.dumps(payload)[:2000], priority
+    
+        first = alerts[0]
+        labels = first.get("labels") or {}
+        annotations = first.get("annotations") or {}
+        name = labels.get("alertname", "Grafana alert")
+    
+        title = "[%s] %s" % (status.upper(), name)
+        lines = []
+        summary = annotations.get("summary") or annotations.get("description")
+        if summary:
+            lines.append(summary)
+        for key in ("severity", "namespace", "pod", "instance", "job"):
+            if labels.get(key):
+                lines.append("%s: %s" % (key, labels[key]))
+        if len(alerts) > 1:
+            lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+    
+        return title, "\n".join(lines) or name, priority
+    
+    
+    class Handler(BaseHTTPRequestHandler):
+        def _reply(self, code, body=b"ok"):
+            self.send_response(code)
+            self.send_header("Content-Type", "text/plain")
+            self.end_headers()
+            self.wfile.write(body)
+    
+        def do_GET(self):
+            self._reply(200)
+    
+        def do_POST(self):
+            length = int(self.headers.get("Content-Length") or 0)
+            raw = self.rfile.read(length) if length else b"{}"
+            try:
+                payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+            except ValueError:
+                payload = {}
+    
+            title, message, priority = to_gotify(payload)
+            body = json.dumps({
+                "title": title[:200],
+                "message": message[:4000],
+                "priority": priority,
+            }).encode()
+    
+            request = urllib.request.Request(
+                GOTIFY_BASE.rstrip("/") + "/message",
+                data=body,
+                headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+            )
+            try:
+                urllib.request.urlopen(request, timeout=10).read()
+            except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+                print("forward to gotify failed: %s" % exc, flush=True)
+                self._reply(502, b"gotify rejected the message")
+                return
+    
+            print("forwarded: %s" % title, flush=True)
+            self._reply(200)
+    
+        def log_message(self, fmt, *args):  # keep the log to one line per delivery
+            print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+    
+    
+    if __name__ == "__main__":
+        print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+        HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/bridge/deployment.yaml

@@ -0,0 +1,80 @@
+# Grafana sends its own JSON and Gotify rejects it, so this Deployment is the
+# translation step between them. Grafana posts to the Service, the bridge posts
+# to /message with the app token from a SOPS secret.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana-gotify-bridge
+  template:
+    metadata:
+      labels:
+        app: grafana-gotify-bridge
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: bridge
+          image: python:3.12-alpine
+          command: ["python3", "/app/grafana_gotify_bridge.py"]
+          env:
+            - name: GOTIFY_URL
+              value: "http://gotify.notifications.svc.cluster.local"
+            - name: PORT
+              value: "8080"
+            - name: GOTIFY_TOKEN
+              valueFrom:
+                secretKeyRef: {name: grafana-gotify-bridge, key: app-token}
+          ports:
+            - name: http
+              containerPort: 8080
+          readinessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: {path: /, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 20
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: script
+              mountPath: /app
+              readOnly: true
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 96Mi}
+      volumes:
+        - name: script
+          configMap:
+            name: grafana-gotify-bridge
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana-gotify-bridge
+  namespace: notifications
+  labels:
+    app: grafana-gotify-bridge
+spec:
+  type: ClusterIP
+  selector:
+    app: grafana-gotify-bridge
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

k8s/apps/gotify/bridge/grafana_gotify_bridge.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Translate a Grafana alert webhook into a Gotify message.
+
+Grafana sends its own payload shape:
+  {"receiver": "...", "status": "firing", "alerts": [{"labels": {...}, "annotations": {...}}]}
+
+Gotify requires:
+  {"title": "...", "message": "...", "priority": 0..10}
+
+Gotify answers 400 "Field 'message' is required" to a Grafana payload, so a
+conversion step is mandatory. This service is that step and nothing else.
+Standard library only, so it needs no image build and no dependencies.
+"""
+import json
+import os
+import urllib.request
+from http.server import BaseHTTPRequestHandler, HTTPServer
+
+GOTIFY_BASE = os.environ.get("GOTIFY_URL", "http://gotify.notifications.svc.cluster.local")
+GOTIFY_TOKEN = os.environ["GOTIFY_TOKEN"]
+LISTEN_PORT = int(os.environ.get("PORT", "8080"))
+
+# Firing is louder than resolved. Gotify priority runs 0 to 10.
+PRIORITY = {"firing": 8, "resolved": 3}
+
+
+def to_gotify(payload):
+    alerts = payload.get("alerts") or []
+    status = (payload.get("status") or "firing").lower()
+    priority = PRIORITY.get(status, 5)
+
+    if not alerts:
+        title = payload.get("title") or "Grafana notification"
+        return title, json.dumps(payload)[:2000], priority
+
+    first = alerts[0]
+    labels = first.get("labels") or {}
+    annotations = first.get("annotations") or {}
+    name = labels.get("alertname", "Grafana alert")
+
+    title = "[%s] %s" % (status.upper(), name)
+    lines = []
+    summary = annotations.get("summary") or annotations.get("description")
+    if summary:
+        lines.append(summary)
+    for key in ("severity", "namespace", "pod", "instance", "job"):
+        if labels.get(key):
+            lines.append("%s: %s" % (key, labels[key]))
+    if len(alerts) > 1:
+        lines.append("+%d more alert(s) in this notification" % (len(alerts) - 1))
+
+    return title, "\n".join(lines) or name, priority
+
+
+class Handler(BaseHTTPRequestHandler):
+    def _reply(self, code, body=b"ok"):
+        self.send_response(code)
+        self.send_header("Content-Type", "text/plain")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_GET(self):
+        self._reply(200)
+
+    def do_POST(self):
+        length = int(self.headers.get("Content-Length") or 0)
+        raw = self.rfile.read(length) if length else b"{}"
+        try:
+            payload = json.loads(raw.decode("utf-8", "replace") or "{}")
+        except ValueError:
+            payload = {}
+
+        title, message, priority = to_gotify(payload)
+        body = json.dumps({
+            "title": title[:200],
+            "message": message[:4000],
+            "priority": priority,
+        }).encode()
+
+        request = urllib.request.Request(
+            GOTIFY_BASE.rstrip("/") + "/message",
+            data=body,
+            headers={"Content-Type": "application/json", "X-Gotify-Key": GOTIFY_TOKEN},
+        )
+        try:
+            urllib.request.urlopen(request, timeout=10).read()
+        except Exception as exc:  # noqa: BLE001 - log and report to Grafana
+            print("forward to gotify failed: %s" % exc, flush=True)
+            self._reply(502, b"gotify rejected the message")
+            return
+
+        print("forwarded: %s" % title, flush=True)
+        self._reply(200)
+
+    def log_message(self, fmt, *args):  # keep the log to one line per delivery
+        print("%s - %s" % (self.address_string(), fmt % args), flush=True)
+
+
+if __name__ == "__main__":
+    print("grafana-gotify bridge listening on %d" % LISTEN_PORT, flush=True)
+    HTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

k8s/apps/gotify/igotify-deployment.yaml

@@ -28,9 +28,17 @@ spec:
               value: "true"
             - name: ENABLE_SCALAR_UI
               value: "true"
-            # GOTIFY_URLS / GOTIFY_CLIENT_TOKENS / SECNTFY_TOKENS intentionally
-            # left unset — not required when Gotify is reachable via public domain
-            # (gotify.riotpiao.com). iGotify iOS app registers itself at runtime.
+            # The bridge authenticates with a Gotify client token, not with
+            # ZITADEL. Gotify has no way to accept a forwarded identity, so the
+            # token is the only credential that works server-side. The token
+            # belongs to user `rock`, never expires, and is revoked by deleting
+            # the client row in Gotify.
+            - name: GOTIFY_URLS
+              value: "https://gotify.riotpiao.com"
+            - name: GOTIFY_CLIENT_TOKENS
+              valueFrom:
+                secretKeyRef: {name: igotify-client-token, key: client-token}
+            # SECNTFY_TOKENS intentionally left unset: secntfy is not deployed.
           volumeMounts:
             - name: data
               mountPath: /app/data

k8s/apps/gotify/ingress.yaml

@@ -9,12 +9,6 @@ metadata:
     # WebSocket support for Gotify client connections (/stream endpoint)
     # nginx-ingress handles Upgrade/Connection headers natively with http/1.1
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
-    # Single sign-on. Gotify has no OIDC, so every request is handed to
-    # oauth2-proxy for an allow or deny decision. oauth2-proxy skips the token
-    # API paths itself, so /message keeps Gotify's own X-Gotify-Key check.
-    nginx.ingress.kubernetes.io/auth-url: "http://gotify-oauth2-proxy.notifications.svc.cluster.local:4180/oauth2/auth"
-    nginx.ingress.kubernetes.io/auth-signin: "https://gotify.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
-    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
 spec:
   ingressClassName: nginx
   rules:

k8s/apps/gotify/kustomization.yaml

@@ -5,10 +5,9 @@ resources:
   - db.yaml
   - deployment.yaml
   - service.yaml
-  - oauth2-proxy-deployment.yaml
-  - oauth2-proxy-service.yaml
-  - oauth2-proxy-ingress.yaml
   - ingress.yaml
+  - bridge/configmap.yaml
+  - bridge/deployment.yaml
   - igotify-pvc.yaml
   - igotify-deployment.yaml
   - igotify-service.yaml

k8s/apps/gotify/oauth2-proxy-deployment.yaml

@@ -1,82 +0,0 @@
-# oauth2-proxy in front of Gotify. Gotify has no OIDC support of its own, so it
-# cannot validate a ZITADEL token. This proxy is the front door: nginx asks it
-# for an auth decision on every request (auth_request), and it answers from a
-# ZITADEL session cookie.
-#
-# The Gotify REST API keeps its own client-token auth (X-Gotify-Key) and must
-# stay reachable without a browser session, or every push sender breaks:
-# Forgejo webhooks, the harness, poimen-sender and the ntfy-style relays all post
-# to /message. skip-auth-route leaves those paths to Gotify's own token check.
-#
-# Gotify's own login still exists behind the proxy. ZITADEL decides who may
-# reach the service; Gotify decides what that person may do inside it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: gotify-oauth2-proxy
-  namespace: notifications
-  labels:
-    app: gotify-oauth2-proxy
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: gotify-oauth2-proxy
-  template:
-    metadata:
-      labels:
-        app: gotify-oauth2-proxy
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: oauth2-proxy
-          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
-          args:
-            - --provider=oidc
-            - --oidc-issuer-url=https://zitadel.riotpiao.com
-            - --email-domain=*
-            - --upstream=static://202
-            - --reverse-proxy=true
-            - --set-xauthrequest=true
-            - --cookie-secure=true
-            - --cookie-expire=12h
-            - --cookie-refresh=1h
-            - --skip-provider-button=true
-            - --redirect-url=https://gotify.riotpiao.com/oauth2/callback
-            - --http-address=0.0.0.0:4180
-            # Browser paths keep SSO. API paths fall through to Gotify's own
-            # client-token check, so existing senders and the Android client
-            # keep working without a browser session.
-            - --skip-auth-route=^/(message|application|client|user|version|stream|health)(/.*)?$
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: client-id}
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: client-secret}
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom:
-                secretKeyRef: {name: gotify-oauth2-proxy, key: cookie-secret}
-          ports:
-            - name: http
-              containerPort: 4180
-          readinessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          livenessProbe:
-            httpGet: {path: /ping, port: http}
-            initialDelaySeconds: 15
-            periodSeconds: 20
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          resources:
-            requests: {cpu: 10m, memory: 32Mi}
-            limits: {cpu: 200m, memory: 128Mi}

k8s/apps/gotify/oauth2-proxy-ingress.yaml

@@ -1,22 +0,0 @@
-# The proxy needs its own ingress path. nginx auth_request also guards
-# /oauth2/start and /oauth2/callback, which would loop: the login entry point
-# would itself require a login. A longer path match sends /oauth2 to the proxy
-# directly, and the prefix rule on the gotify ingress handles everything else.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gotify-oauth2
-  namespace: notifications
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: gotify.riotpiao.com
-      http:
-        paths:
-          - path: /oauth2
-            pathType: Prefix
-            backend:
-              service:
-                name: gotify-oauth2-proxy
-                port:
-                  number: 4180

k8s/apps/gotify/oauth2-proxy-service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: gotify-oauth2-proxy
-  namespace: notifications
-  labels:
-    app: gotify-oauth2-proxy
-spec:
-  type: ClusterIP
-  selector:
-    app: gotify-oauth2-proxy
-  ports:
-    - name: http
-      port: 4180
-      targetPort: http

k8s/apps/key-rotation-controller/README.md

@@ -0,0 +1,50 @@
+# Credential rotation
+
+One `KeyRotation` object describes one credential. The controller creates the value,
+writes the Secret, proves the new value works, then retires the old one.
+
+## Why the state order is the safety property
+
+```
+Prepare -> Publish -> Verify -> Retire
+```
+
+`Retire` runs only for a provider whose old credential still works after the publish
+step, and only after `Verify` passes. A failure before that point leaves the working
+credential untouched. The hand-run rotation that broke Grafana did the opposite: it
+regenerated the secret and never re-pointed the consumer.
+
+## Two strategies, decided by the provider, not by preference
+
+| Provider | Overlap in ZITADEL | Strategy | Retire |
+|---|---|---|---|
+| `zitadel-machine-key` | many keys per service user | `overlap` | after the new key obtains a token |
+| `zitadel-oidc-client` | one secret per app | `atomic` | nothing to retire |
+
+## The single master credential
+
+The controller authenticates with one machine key, mounted from Secret
+`iam/key-rotation-controller`. That key is the only credential the cluster cannot
+rotate by itself, so it is the only one that stays encrypted in git.
+
+Every other credential is created by the controller, which means the 34 `.enc.yaml`
+files under `k8s/argocd/secrets/` are migration debt, not a design. The retirement
+target is one root-of-trust file per thing that cannot rotate itself:
+
+| Survives (root of trust) | Why it cannot rotate |
+|---|---|
+| ZITADEL instance masterkey | encrypts the whole ZITADEL database |
+| Vault unseal keys | unsealing needs a human-held key |
+| Talos repair config | recovery path, must work when the cluster does not |
+| homelab CA | signs the wildcard certificate |
+| controller bootstrap key | chicken and egg: it creates every other key |
+| etcd defrag credentials | needed by a job that runs outside the controller |
+
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
+api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
+
+## Status
+
+The CRD is installed. Eight `KeyRotation` objects exist for the consumers above and
+all are `suspend: true`: the controller image does not exist yet, so no process acts
+on them. poimen-memory is excluded from this migration on purpose.

k8s/apps/key-rotation-controller/controller/Dockerfile

@@ -0,0 +1,13 @@
+# Build stage. The controller has no cgo and no runtime assets, so the final image
+# is a static binary on a base with no shell.
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/controller .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/controller /usr/local/bin/controller
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/controller"]

k8s/apps/key-rotation-controller/controller/api/types.go

@@ -0,0 +1,254 @@
+// Package api declares the KeyRotation custom resource.
+//
+// One resource describes one credential and how to rotate it. The controller
+// never stores the credential in git: it writes a Kubernetes Secret and records
+// only the credential's identifier in status.
+package api
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// Group is the API group for homelab platform resources.
+const Group = "iam.homelab.riotpiao.com"
+
+// Version is the served version.
+const Version = "v1"
+
+// Provider names. Each provider uses one external API to create and retire
+// credentials, and each declares whether a rotation can overlap.
+const (
+	// ProviderZitadelMachineKey creates a machine key while the previous key
+	// stays valid. True zero downtime.
+	ProviderZitadelMachineKey = "zitadel-machine-key"
+	// ProviderZitadelOIDCClient regenerates an app client secret. ZITADEL keeps
+	// one secret per app, so the old one dies the moment the new one exists.
+	ProviderZitadelOIDCClient = "zitadel-oidc-client"
+)
+
+// Strategy names.
+const (
+	// StrategyOverlap keeps the old credential valid until the new one is proven.
+	StrategyOverlap = "overlap"
+	// StrategyAtomic replaces the credential in place. A short window exists.
+	StrategyAtomic = "atomic"
+)
+
+// Reload names tell the controller how the consumer picks up a new value.
+const (
+	// ReloadHot means the consumer watches the mounted file.
+	ReloadHot = "hot"
+	// ReloadRestart means the consumer reads the value at start, so it must roll.
+	ReloadRestart = "restart"
+)
+
+// Phase names for status.
+const (
+	PhasePending      = "Pending"
+	PhasePreparing    = "Preparing"
+	PhasePropagating  = "Propagating"
+	PhaseVerifying    = "Verifying"
+	PhaseCompleting   = "Completing"
+	PhaseSucceeded    = "Succeeded"
+	PhaseFailed       = "Failed"
+	PhaseSuspended    = "Suspended"
+)
+
+// ConsumerRef names a workload that reads the rotated credential.
+type ConsumerRef struct {
+	// Kind is Deployment or StatefulSet.
+	Kind string `json:"kind"`
+	// Name is the workload name.
+	Name string `json:"name"`
+	// Namespace is where the workload runs.
+	Namespace string `json:"namespace"`
+}
+
+// RotationTarget is the credential's source and destination.
+type RotationTarget struct {
+	// ZitadelProjectID is the project that owns the app or service user.
+	ZitadelProjectID string `json:"zitadelProjectID,omitempty"`
+	// ZitadelAppID is the OIDC app id, for the client-secret provider.
+	ZitadelAppID string `json:"zitadelAppID,omitempty"`
+	// MachineUserID is the service user id, for the machine-key provider.
+	MachineUserID string `json:"machineUserID,omitempty"`
+	// SecretName is the Secret the controller writes.
+	SecretName string `json:"secretName"`
+	// SecretNamespace is where that Secret lives.
+	SecretNamespace string `json:"secretNamespace"`
+	// SecretKeys maps a logical field name to the Secret key that holds it.
+	// For a machine key: key -> key, key-id -> key-id.
+	SecretKeys map[string]string `json:"secretKeys,omitempty"`
+	// Consumers are the workloads that must pick the new value up.
+	Consumers []ConsumerRef `json:"consumers,omitempty"`
+	// Reload is hot or restart. Empty means restart, the safe default.
+	Reload string `json:"reload,omitempty"`
+}
+
+// VerifySpec is the proof the controller demands before it retires the old
+// credential.
+type VerifySpec struct {
+	// Introspect checks client_id and client_secret against the introspection
+	// endpoint.
+	Introspect bool `json:"introspect,omitempty"`
+	// TokenExchange exchanges a token with the new machine key.
+	TokenExchange bool `json:"tokenExchange,omitempty"`
+	// AudienceProject, when set, is added to the token request as
+	// urn:zitadel:iam:org:project:id:<id>:aud.
+	AudienceProject string `json:"audienceProject,omitempty"`
+}
+
+// KeyRotationSpec is the desired state.
+type KeyRotationSpec struct {
+	// Provider selects the external API.
+	Provider string `json:"provider"`
+	// Strategy is overlap or atomic.
+	Strategy string `json:"strategy"`
+	// IntervalHours is how often to rotate. Minimum 24.
+	IntervalHours int32 `json:"intervalHours"`
+	// Suspend stops automatic rotation without deleting the resource.
+	Suspend bool `json:"suspend,omitempty"`
+	// Target names the credential and its consumers.
+	Target RotationTarget `json:"target"`
+	// Verify names the checks that must pass.
+	Verify VerifySpec `json:"verify,omitempty"`
+}
+
+// KeyRotationStatus is the observed state.
+type KeyRotationStatus struct {
+	Phase              string             `json:"phase,omitempty"`
+	ObservedGeneration int64              `json:"observedGeneration,omitempty"`
+	LastRotation       *metav1.Time       `json:"lastRotation,omitempty"`
+	NextRotation       *metav1.Time       `json:"nextRotation,omitempty"`
+	ActiveCredentialID string             `json:"activeCredentialID,omitempty"`
+	PreviousCredentialID string           `json:"previousCredentialID,omitempty"`
+	LastError          string             `json:"lastError,omitempty"`
+	Conditions         []metav1.Condition `json:"conditions,omitempty"`
+}
+
+// KeyRotation is one credential under management.
+type KeyRotation struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KeyRotationSpec   `json:"spec,omitempty"`
+	Status KeyRotationStatus `json:"status,omitempty"`
+}
+
+// KeyRotationList is a list of KeyRotation.
+type KeyRotationList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KeyRotation `json:"items"`
+}
+
+// SchemeGroupVersion is the group and version for this type.
+var SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version}
+
+// SchemeBuilder registers the types with a scheme.
+var SchemeBuilder = runtime.NewSchemeBuilder(func(s *runtime.Scheme) error {
+	s.AddKnownTypes(SchemeGroupVersion, &KeyRotation{}, &KeyRotationList{})
+	metav1.AddToGroupVersion(s, SchemeGroupVersion)
+	return nil
+})
+
+// AddToScheme registers this package's types.
+var AddToScheme = SchemeBuilder.AddToScheme
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotation) DeepCopyInto(out *KeyRotation) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy makes a deep copy.
+func (in *KeyRotation) DeepCopy() *KeyRotation {
+	if in == nil {
+		return nil
+	}
+	out := new(KeyRotation)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject satisfies runtime.Object.
+func (in *KeyRotation) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationSpec) DeepCopyInto(out *KeyRotationSpec) {
+	*out = *in
+	in.Target.DeepCopyInto(&out.Target)
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *RotationTarget) DeepCopyInto(out *RotationTarget) {
+	*out = *in
+	if in.SecretKeys != nil {
+		out.SecretKeys = make(map[string]string, len(in.SecretKeys))
+		for k, v := range in.SecretKeys {
+			out.SecretKeys[k] = v
+		}
+	}
+	if in.Consumers != nil {
+		out.Consumers = make([]ConsumerRef, len(in.Consumers))
+		copy(out.Consumers, in.Consumers)
+	}
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationStatus) DeepCopyInto(out *KeyRotationStatus) {
+	*out = *in
+	if in.LastRotation != nil {
+		t := *in.LastRotation
+		out.LastRotation = &t
+	}
+	if in.NextRotation != nil {
+		t := *in.NextRotation
+		out.NextRotation = &t
+	}
+	if in.Conditions != nil {
+		out.Conditions = make([]metav1.Condition, len(in.Conditions))
+		copy(out.Conditions, in.Conditions)
+	}
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationList) DeepCopyInto(out *KeyRotationList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		out.Items = make([]KeyRotation, len(in.Items))
+		for i := range in.Items {
+			in.Items[i].DeepCopyInto(&out.Items[i])
+		}
+	}
+}
+
+// DeepCopy makes a deep copy.
+func (in *KeyRotationList) DeepCopy() *KeyRotationList {
+	if in == nil {
+		return nil
+	}
+	out := new(KeyRotationList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject satisfies runtime.Object.
+func (in *KeyRotationList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}

k8s/apps/key-rotation-controller/controller/go.mod

@@ -0,0 +1,71 @@
+module github.com/riotpiao/key-rotation-controller
+
+go 1.26.0
+
+require (
+	k8s.io/api v0.37.1
+	k8s.io/apimachinery v0.37.1
+	k8s.io/client-go v0.37.0
+	sigs.k8s.io/controller-runtime v0.25.2
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v1.0.0 // indirect
+	github.com/go-openapi/jsonreference v1.0.0 // indirect
+	github.com/go-openapi/swag v0.27.1 // indirect
+	github.com/go-openapi/swag/cmdutils v0.27.1 // indirect
+	github.com/go-openapi/swag/conv v0.27.1 // indirect
+	github.com/go-openapi/swag/fileutils v0.27.1 // indirect
+	github.com/go-openapi/swag/jsonutils v0.27.1 // indirect
+	github.com/go-openapi/swag/loading v0.27.1 // indirect
+	github.com/go-openapi/swag/mangling v0.27.1 // indirect
+	github.com/go-openapi/swag/netutils v0.27.1 // indirect
+	github.com/go-openapi/swag/pools v0.27.1 // indirect
+	github.com/go-openapi/swag/stringutils v0.27.1 // indirect
+	github.com/go-openapi/swag/typeutils v0.27.1 // indirect
+	github.com/go-openapi/swag/yamlutils v0.27.1 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.24.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.0 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/apiextensions-apiserver v0.37.0 // indirect
+	k8s.io/klog/v2 v2.140.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
+	k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)

k8s/apps/key-rotation-controller/controller/go.sum

@@ -0,0 +1,133 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
+github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
+github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
+github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s=
+github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
+github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY=
+github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0=
+github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg=
+github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE=
+github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE=
+github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM=
+github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU=
+github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs=
+github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM=
+github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8=
+github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU=
+github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk=
+github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY=
+github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE=
+github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA=
+github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w=
+github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU=
+github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ=
+github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E=
+github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE=
+github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8=
+github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM=
+github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc=
+github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ=
+github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA=
+github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI=
+github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI=
+github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM=
+k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M=
+k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI=
+k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80=
+k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE=
+k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw=
+k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ=
+k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0=
+k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
+k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ=
+sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

k8s/apps/key-rotation-controller/controller/internal/reconcile/providers.go

@@ -0,0 +1,168 @@
+package reconcile
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+	"github.com/riotpiao/key-rotation-controller/internal/zitadel"
+)
+
+// MachineKeyProvider rotates a service user's private key.
+//
+// ZITADEL keeps many keys per service user, and each assertion names its key with
+// `kid`. A new key can be created while the deployed key stays valid, so this
+// provider retires the old key only after the new one proves itself. Nothing in
+// the window is invalid.
+type MachineKeyProvider struct {
+	// Client talks to the instance that owns the service user.
+	Client *zitadel.Client
+	// AuthClientID is the API app the controller authenticates as.
+	AuthClientID string
+	// KeyIDFile and KeyFile name the Secret keys, defaulting to key-id and key.
+	KeyIDFile string
+	KeyFile   string
+}
+
+// Name implements Provider.
+func (p *MachineKeyProvider) Name() string { return iamv1.ProviderZitadelMachineKey }
+
+// Strategy implements Provider. Machine keys overlap.
+func (p *MachineKeyProvider) Strategy() string { return iamv1.StrategyOverlap }
+
+// Prepare creates a new key. The currently deployed key remains valid.
+func (p *MachineKeyProvider) Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error) {
+	userID := kr.Spec.Target.MachineUserID
+	if userID == "" {
+		return nil, fmt.Errorf("spec.target.machineUserID is required")
+	}
+	key, err := p.Client.CreateMachineKey(userID)
+	if err != nil {
+		return nil, fmt.Errorf("creating a machine key: %w", err)
+	}
+	fields := map[string][]byte{
+		p.keyField(kr):   []byte(key.Key),
+		p.keyIDField(kr): []byte(key.KeyID),
+	}
+	return &Credential{ID: key.KeyID, Fields: fields}, nil
+}
+
+// Verify exchanges a token with the new key. A key that cannot obtain a token is
+// not deployed as a working credential, whatever the API said.
+func (p *MachineKeyProvider) Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error {
+	keyPEM := cred.Fields[p.keyField(kr)]
+	keyID := string(cred.Fields[p.keyIDField(kr)])
+	return p.Client.ExchangeMachineKey(
+		p.AuthClientID,
+		kr.Spec.Target.MachineUserID,
+		keyID,
+		keyPEM,
+		kr.Spec.Verify.AudienceProject,
+	)
+}
+
+// Retire deletes the superseded key.
+func (p *MachineKeyProvider) Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error {
+	return p.Client.DeleteMachineKey(kr.Spec.Target.MachineUserID, credentialID)
+}
+
+func (p *MachineKeyProvider) keyField(kr *iamv1.KeyRotation) string {
+	if key, ok := kr.Spec.Target.SecretKeys["key"]; ok && key != "" {
+		return key
+	}
+	return "key"
+}
+
+func (p *MachineKeyProvider) keyIDField(kr *iamv1.KeyRotation) string {
+	if key, ok := kr.Spec.Target.SecretKeys["key-id"]; ok && key != "" {
+		return key
+	}
+	return "key-id"
+}
+
+// OIDCClientProvider rotates an app's client secret.
+//
+// ZITADEL stores exactly one secret per app, so creating the new secret kills the
+// old one. The strategy is atomic: publish immediately, roll the consumer, verify,
+// and if verification fails, put the previous value back.
+type OIDCClientProvider struct {
+	Client *zitadel.Client
+	// ClientIDFunc maps a KeyRotation to the app's client id, which the
+	// introspection endpoint needs.
+	ClientIDFunc func(kr *iamv1.KeyRotation) string
+	// SecretKey names the Secret key that holds the secret.
+	SecretKey string
+}
+
+// Name implements Provider.
+func (p *OIDCClientProvider) Name() string { return iamv1.ProviderZitadelOIDCClient }
+
+// Strategy implements Provider. Client secrets cannot overlap.
+func (p *OIDCClientProvider) Strategy() string { return iamv1.StrategyAtomic }
+
+// Prepare regenerates the client secret.
+func (p *OIDCClientProvider) Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error) {
+	projectID := kr.Spec.Target.ZitadelProjectID
+	appID := kr.Spec.Target.ZitadelAppID
+	if projectID == "" || appID == "" {
+		return nil, fmt.Errorf("spec.target.zitadelProjectID and zitadelAppID are required")
+	}
+	secret, err := p.Client.GenerateClientSecret(projectID, appID)
+	if err != nil {
+		return nil, fmt.Errorf("generating a client secret: %w", err)
+	}
+	field := p.secretField(kr)
+	// The credential id is the generation marker, not a ZITADEL identifier: the
+	// API returns only the value.
+	return &Credential{
+		ID:     "secret-" + shortHash(secret),
+		Fields: map[string][]byte{field: []byte(secret)},
+	}, nil
+}
+
+// Verify checks the credential against the introspection endpoint.
+func (p *OIDCClientProvider) Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error {
+	if !kr.Spec.Verify.Introspect {
+		return nil
+	}
+	clientID := ""
+	if p.ClientIDFunc != nil {
+		clientID = p.ClientIDFunc(kr)
+	}
+	if clientID == "" {
+		return fmt.Errorf("no client id configured for introspection")
+	}
+	return p.Client.Introspect(clientID, string(cred.Fields[p.secretField(kr)]))
+}
+
+// Retire is a no-op: an atomic provider has nothing to retire, because the
+// provider already invalidated the previous value.
+func (p *OIDCClientProvider) Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error {
+	return nil
+}
+
+func (p *OIDCClientProvider) secretField(kr *iamv1.KeyRotation) string {
+	if p.SecretKey != "" {
+		return p.SecretKey
+	}
+	if key, ok := kr.Spec.Target.SecretKeys["secret"]; ok && key != "" {
+		return key
+	}
+	return "secret"
+}
+
+// shortHash gives a stable, non-reversible marker for a secret so status can
+// record which generation is live without storing the value.
+func shortHash(value string) string {
+	sum := 0
+	for _, b := range []byte(value) {
+		sum = (sum*31 + int(b)) % 100000007
+	}
+	encoded := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%d", sum)))
+	return encoded
+}
+
+// unusedJSON keeps the import set honest if a provider needs raw JSON later.
+var _ = json.Marshal

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler.go

@@ -0,0 +1,391 @@
+// Package reconcile turns a KeyRotation object into a rotated credential.
+//
+// The state machine is identical for every provider. Only three calls differ:
+// create, verify, retire. That is what makes one controller cover machine keys,
+// client secrets and, later, the token-style providers.
+//
+// Order is the safety property:
+//
+//	Prepare    create the new credential
+//	Publish    write the Secret, roll the consumer when it cannot reload
+//	Verify     prove the new credential works against the provider
+//	Retire     delete the old credential, only for overlapping providers
+//
+// A failure before Retire leaves the old credential in place, so a broken
+// rotation never removes the thing that currently works.
+package reconcile
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+)
+
+// Credential is fresh credential material, before it is written to a Secret.
+type Credential struct {
+	// ID identifies the credential at the provider, so it can be retired later.
+	ID string
+	// Fields maps a Secret key to its value.
+	Fields map[string][]byte
+}
+
+// Provider creates, verifies and retires one class of credential.
+type Provider interface {
+	// Name is the provider name from the spec.
+	Name() string
+	// Strategy is overlap or atomic. Atomic providers replace in place.
+	Strategy() string
+	// Prepare creates a new credential at the provider.
+	Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error)
+	// Verify proves the credential works, using the provider's own check.
+	Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error
+	// Retire deletes a superseded credential. It must be safe to call twice.
+	Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error
+}
+
+// Reconciler rotates credentials for KeyRotation objects.
+type Reconciler struct {
+	client.Client
+	// Providers keyed by spec.provider.
+	Providers map[string]Provider
+	// Now is injectable so scheduling is testable.
+	Now func() time.Time
+	// RolloutTimeout bounds the wait for a consumer to become available.
+	RolloutTimeout time.Duration
+}
+
+// previousKeySuffix keeps the superseded value beside the current one, so an
+// atomic provider can be rolled back without storing a secret in the object.
+const previousKeySuffix = ".previous"
+
+// RequeueAfter is how long to wait before checking a resource again.
+func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration {
+	interval := time.Duration(kr.Spec.IntervalHours) * time.Hour
+	if interval <= 0 {
+		interval = 30 * 24 * time.Hour
+	}
+	if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" {
+		return 0 // rotate now
+	}
+	due := kr.Status.LastRotation.Add(interval)
+	wait := due.Sub(r.now())
+	if wait < 0 {
+		return 0
+	}
+	return wait
+}
+
+// Reconcile performs one rotation when the resource is due.
+func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (time.Duration, error) {
+	logger := log.FromContext(ctx).WithValues("keyrotation", req.String())
+
+	var kr iamv1.KeyRotation
+	if err := r.Get(ctx, req, &kr); err != nil {
+		if apierrors.IsNotFound(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	if kr.Spec.Suspend {
+		return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "")
+	}
+
+	provider := r.Providers[kr.Spec.Provider]
+	if provider == nil {
+		return 0, r.setPhase(ctx, &kr, iamv1.PhaseFailed,
+			fmt.Sprintf("no provider named %q", kr.Spec.Provider))
+	}
+
+	wait := r.RequeueAfter(&kr)
+	if wait > 0 {
+		return wait, nil
+	}
+
+	logger.Info("rotating credential", "provider", kr.Spec.Provider, "strategy", kr.Spec.Strategy)
+
+	previousID := kr.Status.ActiveCredentialID
+	previousFields := r.readCurrentSecret(ctx, &kr)
+
+	// Prepare
+	if err := r.setPhase(ctx, &kr, iamv1.PhasePreparing, ""); err != nil {
+		return 0, err
+	}
+	cred, err := provider.Prepare(ctx, &kr)
+	if err != nil {
+		return r.fail(ctx, &kr, iamv1.PhasePreparing, err)
+	}
+
+	// Publish. The previous value stays beside the new one until verification
+	// passes, so an atomic rotation can be undone.
+	if err := r.setPhase(ctx, &kr, iamv1.PhasePropagating, ""); err != nil {
+		return 0, err
+	}
+	if err := r.writeSecret(ctx, &kr, cred, previousFields); err != nil {
+		return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
+	}
+	if kr.Spec.Target.Reload != iamv1.ReloadHot {
+		if err := r.rollConsumers(ctx, &kr, cred.ID); err != nil {
+			return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
+		}
+	}
+
+	// Verify
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseVerifying, ""); err != nil {
+		return 0, err
+	}
+	if err := provider.Verify(ctx, &kr, cred); err != nil {
+		// Put the previous value back and roll again, then report the failure.
+		r.restorePrevious(ctx, &kr, previousFields)
+		return r.fail(ctx, &kr, iamv1.PhaseVerifying, err)
+	}
+
+	// Complete. Retire only when the provider can overlap, because retiring an
+	// atomic credential would delete the one now in use.
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseCompleting, ""); err != nil {
+		return 0, err
+	}
+	if provider.Strategy() == iamv1.StrategyOverlap && previousID != "" && previousID != cred.ID {
+		if err := provider.Retire(ctx, &kr, previousID); err != nil {
+			logger.Error(err, "retiring the previous credential failed; it stays valid", "credential", previousID)
+		}
+	}
+	r.clearPrevious(ctx, &kr)
+
+	now := metav1.NewTime(r.now())
+	kr.Status.LastRotation = &now
+	kr.Status.ActiveCredentialID = cred.ID
+	kr.Status.PreviousCredentialID = previousID
+	kr.Status.LastError = ""
+	kr.Status.ObservedGeneration = kr.Generation
+	if err := r.Status().Update(ctx, &kr); err != nil {
+		return 0, err
+	}
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseSucceeded, ""); err != nil {
+		return 0, err
+	}
+	logger.Info("rotation complete", "credential", cred.ID, "previous", previousID)
+
+	return r.RequeueAfter(&kr), nil
+}
+
+func (r *Reconciler) now() time.Time {
+	if r.Now != nil {
+		return r.Now()
+	}
+	return time.Now()
+}
+
+func (r *Reconciler) setPhase(ctx context.Context, kr *iamv1.KeyRotation, phase, message string) error {
+	kr.Status.Phase = phase
+	kr.Status.LastError = message
+	kr.Status.ObservedGeneration = kr.Generation
+	if err := r.Status().Update(ctx, kr); err != nil && !apierrors.IsConflict(err) {
+		return err
+	}
+	return nil
+}
+
+func (r *Reconciler) fail(ctx context.Context, kr *iamv1.KeyRotation, phase string, cause error) (time.Duration, error) {
+	kr.Status.Phase = iamv1.PhaseFailed
+	kr.Status.LastError = fmt.Sprintf("%s: %v", phase, cause)
+	_ = r.Status().Update(ctx, kr)
+	// Retry soon, but not in a tight loop.
+	return 5 * time.Minute, nil
+}
+
+// secretData assembles the Secret payload: the new values, plus the superseded
+// values under a shadow key so an atomic rotation can be undone.
+func secretData(kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) map[string][]byte {
+	data := make(map[string][]byte, len(cred.Fields)*2)
+	for logical, value := range cred.Fields {
+		key := secretKeyFor(kr, logical)
+		data[key] = value
+		if old, ok := previous[key]; ok && len(old) > 0 {
+			data[key+previousKeySuffix] = old
+		}
+	}
+	return data
+}
+
+// secretKeyFor maps a logical field name to the Secret key, defaulting to the
+// same name so a simple credential needs no mapping.
+func secretKeyFor(kr *iamv1.KeyRotation, logical string) string {
+	if key, ok := kr.Spec.Target.SecretKeys[logical]; ok && key != "" {
+		return key
+	}
+	return logical
+}
+
+func (r *Reconciler) readCurrentSecret(ctx context.Context, kr *iamv1.KeyRotation) map[string][]byte {
+	var secret corev1.Secret
+	err := r.Get(ctx, types.NamespacedName{
+		Namespace: kr.Spec.Target.SecretNamespace,
+		Name:      kr.Spec.Target.SecretName,
+	}, &secret)
+	if err != nil {
+		return nil
+	}
+	return secret.Data
+}
+
+func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) error {
+	data := secretData(kr, cred, previous)
+
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	err := r.Get(ctx, key, &secret)
+	if apierrors.IsNotFound(err) {
+		secret = corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      key.Name,
+				Namespace: key.Namespace,
+				Labels:    map[string]string{"app.kubernetes.io/managed-by": "key-rotation-controller"},
+			},
+			Type: corev1.SecretTypeOpaque,
+			Data: data,
+		}
+		return r.Create(ctx, &secret)
+	}
+	if err != nil {
+		return err
+	}
+	secret.Data = data
+	return r.Update(ctx, &secret)
+}
+
+// restorePrevious puts the shadow values back in place after a failed check.
+func (r *Reconciler) restorePrevious(ctx context.Context, kr *iamv1.KeyRotation, previous map[string][]byte) {
+	if len(previous) == 0 {
+		return
+	}
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	if err := r.Get(ctx, key, &secret); err != nil {
+		return
+	}
+	// Restore every shadowed value over the current one.
+	for k, v := range secret.Data {
+		base := k
+		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
+			base = k[:len(k)-len(previousKeySuffix)]
+			secret.Data[base] = v
+		}
+	}
+	_ = r.Update(ctx, &secret)
+	_ = r.rollConsumers(ctx, kr, "rollback")
+}
+
+// clearPrevious removes the shadow values once verification passed.
+func (r *Reconciler) clearPrevious(ctx context.Context, kr *iamv1.KeyRotation) {
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	if err := r.Get(ctx, key, &secret); err != nil {
+		return
+	}
+	changed := false
+	for k := range secret.Data {
+		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
+			delete(secret.Data, k)
+			changed = true
+		}
+	}
+	if changed {
+		_ = r.Update(ctx, &secret)
+	}
+}
+
+// rollConsumers restarts each consumer so it reads the new value.
+func (r *Reconciler) rollConsumers(ctx context.Context, kr *iamv1.KeyRotation, revision string) error {
+	for _, consumer := range kr.Spec.Target.Consumers {
+		ns := consumer.Namespace
+		if ns == "" {
+			ns = kr.Namespace
+		}
+		switch consumer.Kind {
+		case "Deployment":
+			var dep appsv1.Deployment
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
+				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
+			}
+			base := dep.DeepCopy()
+			if dep.Spec.Template.Annotations == nil {
+				dep.Spec.Template.Annotations = map[string]string{}
+			}
+			dep.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
+			if err := r.Patch(ctx, &dep, client.MergeFrom(base)); err != nil {
+				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
+			}
+		case "StatefulSet":
+			var sts appsv1.StatefulSet
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &sts); err != nil {
+				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
+			}
+			base := sts.DeepCopy()
+			if sts.Spec.Template.Annotations == nil {
+				sts.Spec.Template.Annotations = map[string]string{}
+			}
+			sts.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
+			if err := r.Patch(ctx, &sts, client.MergeFrom(base)); err != nil {
+				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
+			}
+		default:
+			return fmt.Errorf("consumer kind %q is not supported", consumer.Kind)
+		}
+	}
+	return r.waitForConsumers(ctx, kr)
+}
+
+// waitForConsumers blocks until every consumer reports all replicas available.
+func (r *Reconciler) waitForConsumers(ctx context.Context, kr *iamv1.KeyRotation) error {
+	deadline := r.now().Add(r.rolloutTimeout())
+	for {
+		ready := true
+		for _, consumer := range kr.Spec.Target.Consumers {
+			ns := consumer.Namespace
+			if ns == "" {
+				ns = kr.Namespace
+			}
+			var dep appsv1.Deployment
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
+				continue
+			}
+			desired := int32(1)
+			if dep.Spec.Replicas != nil {
+				desired = *dep.Spec.Replicas
+			}
+			if dep.Status.UpdatedReplicas < desired || dep.Status.AvailableReplicas < desired {
+				ready = false
+				break
+			}
+		}
+		if ready {
+			return nil
+		}
+		if r.now().After(deadline) {
+			return fmt.Errorf("consumer did not become available within %s", r.rolloutTimeout())
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}
+
+func (r *Reconciler) rolloutTimeout() time.Duration {
+	if r.RolloutTimeout > 0 {
+		return r.RolloutTimeout
+	}
+	return 3 * time.Minute
+}

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler_test.go

@@ -0,0 +1,155 @@
+package reconcile
+
+import (
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+)
+
+func rotation(last *metav1.Time, activeID string, hours int32) *iamv1.KeyRotation {
+	return &iamv1.KeyRotation{
+		Spec: iamv1.KeyRotationSpec{
+			Provider:      iamv1.ProviderZitadelMachineKey,
+			Strategy:      iamv1.StrategyOverlap,
+			IntervalHours: hours,
+		},
+		Status: iamv1.KeyRotationStatus{
+			LastRotation:       last,
+			ActiveCredentialID: activeID,
+		},
+	}
+}
+
+func TestRequeueAfterRotatesImmediatelyWhenNeverRotated(t *testing.T) {
+	r := &Reconciler{Now: func() time.Time { return time.Unix(1_700_000_000, 0) }}
+	if got := r.RequeueAfter(rotation(nil, "", 24)); got != 0 {
+		t.Fatalf("a resource with no rotation history must rotate now, got %s", got)
+	}
+	if got := r.RequeueAfter(rotation(nil, "key-1", 24)); got != 0 {
+		t.Fatalf("a resource with no lastRotation must rotate now, got %s", got)
+	}
+}
+
+func TestRequeueAfterWaitsForTheInterval(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-6 * time.Hour))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	got := r.RequeueAfter(rotation(&last, "key-1", 24))
+	want := 18 * time.Hour
+	if got != want {
+		t.Fatalf("expected %s until the next rotation, got %s", want, got)
+	}
+}
+
+func TestRequeueAfterRotatesWhenOverdue(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-30 * time.Hour))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	if got := r.RequeueAfter(rotation(&last, "key-1", 24)); got != 0 {
+		t.Fatalf("an overdue resource must rotate now, got %s", got)
+	}
+}
+
+func TestRequeueAfterFallsBackWhenNoInterval(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now)
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	got := r.RequeueAfter(rotation(&last, "key-1", 0))
+	if got <= 0 {
+		t.Fatalf("a zero interval must fall back to a default, got %s", got)
+	}
+}
+
+func TestSecretDataKeepsThePreviousValueUnderAShadowKey(t *testing.T) {
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"key": "jwt-key", "key-id": "jwt-key-id"},
+		},
+	}}
+	cred := &Credential{ID: "k2", Fields: map[string][]byte{
+		"key":    []byte("PEM-2"),
+		"key-id": []byte("k2"),
+	}}
+	previous := map[string][]byte{
+		"jwt-key":    []byte("PEM-1"),
+		"jwt-key-id": []byte("k1"),
+	}
+
+	data := secretData(kr, cred, previous)
+
+	if string(data["jwt-key"]) != "PEM-2" {
+		t.Fatalf("current value missing, got %q", data["jwt-key"])
+	}
+	if string(data["jwt-key"+previousKeySuffix]) != "PEM-1" {
+		t.Fatalf("previous value must be kept for rollback, got %q", data["jwt-key"+previousKeySuffix])
+	}
+	if string(data["jwt-key-id"+previousKeySuffix]) != "k1" {
+		t.Fatalf("previous key id must be kept for rollback")
+	}
+}
+
+func TestSecretDataDefaultsToLogicalNames(t *testing.T) {
+	kr := &iamv1.KeyRotation{}
+	cred := &Credential{ID: "k1", Fields: map[string][]byte{"secret": []byte("s3cret")}}
+
+	data := secretData(kr, cred, nil)
+
+	if string(data["secret"]) != "s3cret" {
+		t.Fatalf("an unmapped field must keep its logical name, got %v", data)
+	}
+	if _, shadow := data["secret"+previousKeySuffix]; shadow {
+		t.Fatalf("no previous value existed, so no shadow key may be written")
+	}
+}
+
+func TestSecretKeyForPrefersTheMapping(t *testing.T) {
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"secret": "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET"},
+		},
+	}}
+	if got := secretKeyFor(kr, "secret"); got != "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET" {
+		t.Fatalf("mapping not honoured, got %q", got)
+	}
+	if got := secretKeyFor(kr, "key-id"); got != "key-id" {
+		t.Fatalf("unmapped field must default to its own name, got %q", got)
+	}
+}
+
+func TestMachineKeyProviderFieldsRespectTheMapping(t *testing.T) {
+	p := &MachineKeyProvider{}
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"key": "private-key", "key-id": "kid"},
+		},
+	}}
+	if p.keyField(kr) != "private-key" || p.keyIDField(kr) != "kid" {
+		t.Fatalf("provider must read the mapped Secret keys")
+	}
+	if p.Strategy() != iamv1.StrategyOverlap {
+		t.Fatalf("machine keys must be an overlapping provider")
+	}
+	if (&OIDCClientProvider{}).Strategy() != iamv1.StrategyAtomic {
+		t.Fatalf("client secrets cannot overlap and must be atomic")
+	}
+}
+
+func TestShortHashIsStableAndDoesNotLeakTheValue(t *testing.T) {
+	a := shortHash("super-secret-value")
+	b := shortHash("super-secret-value")
+	if a != b {
+		t.Fatalf("the marker must be stable for the same value")
+	}
+	if a == "super-secret-value" {
+		t.Fatalf("the marker must not be the value itself")
+	}
+	if shortHash("other") == a {
+		t.Fatalf("different values must produce different markers")
+	}
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/client.go

@@ -0,0 +1,308 @@
+package zitadel
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Config holds everything the controller needs to call ZITADEL.
+type Config struct {
+	// Issuer is the instance URL, for example https://zitadel.riotpiao.com.
+	Issuer string
+	// ClientID is the API app that this controller authenticates as.
+	ClientID string
+	// UserID is the machine user the assertion acts for. Empty means ClientID.
+	UserID string
+	// KeyID and PrivateKeyPEM identify the controller's own machine key.
+	KeyID         string
+	PrivateKeyPEM []byte
+	// HTTPClient is optional; a sane default is used when nil.
+	HTTPClient *http.Client
+}
+
+// Client is a thin ZITADEL management client.
+type Client struct {
+	cfg   Config
+	http  *http.Client
+	mu    sync.Mutex
+	token string
+	until time.Time
+}
+
+// New builds a client.
+func New(cfg Config) *Client {
+	client := cfg.HTTPClient
+	if client == nil {
+		client = &http.Client{Timeout: 20 * time.Second}
+	}
+	return &Client{cfg: cfg, http: client}
+}
+
+// AccessToken returns a cached access token, minting one when needed.
+func (c *Client) AccessToken() (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.token != "" && time.Now().Before(c.until) {
+		return c.token, nil
+	}
+
+	assertion, err := SignAssertion(c.cfg.PrivateKeyPEM, c.cfg.KeyID, c.cfg.ClientID, c.cfg.UserID, c.cfg.Issuer, time.Now())
+	if err != nil {
+		return "", err
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	form.Set("scope", "openid")
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/token", strings.NewReader(form.Encode()))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("token endpoint returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+
+	var parsed struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &parsed); err != nil {
+		return "", fmt.Errorf("token response did not parse: %w", err)
+	}
+	if parsed.AccessToken == "" {
+		return "", fmt.Errorf("token response carried no access_token")
+	}
+
+	lifetime := parsed.ExpiresIn
+	if lifetime <= 0 {
+		lifetime = 300
+	}
+	c.token = parsed.AccessToken
+	// Refresh at 80 % of the lifetime so a slow call never presents an expired token.
+	c.until = time.Now().Add(time.Duration(lifetime) * time.Second * 4 / 5)
+	return c.token, nil
+}
+
+// projectAppPath builds the management path for one OIDC app under a project.
+func (c *Client) projectAppPath(projectID, appID string) string {
+	return fmt.Sprintf("/management/v1/projects/%s/apps/%s", projectID, appID)
+}
+
+// MachineKey is one registered key for a service user.
+type MachineKey struct {
+	KeyID      string `json:"keyId"`
+	Expiration string `json:"expirationDate,omitempty"`
+	// Details holds the PEM bundle when the key was just created.
+	Details string `json:"keyDetails,omitempty"`
+}
+
+// NewMachineKeyJSON is the JSON document ZITADEL returns for a new JSON key.
+type NewMachineKeyJSON struct {
+	Type     string `json:"type"`
+	KeyID    string `json:"keyId"`
+	Key      string `json:"key"`
+	UserID   string `json:"userId"`
+	ExpireAt string `json:"expirationDate,omitempty"`
+}
+
+// CreateMachineKey adds a key to a service user and returns its material.
+//
+// ZITADEL keeps many keys per user, so this never invalidates the key that is
+// currently deployed. That is the whole reason the machine-key provider can
+// rotate with no gap.
+func (c *Client) CreateMachineKey(userID string) (*NewMachineKeyJSON, error) {
+	payload := map[string]any{"type": "KEY_TYPE_JSON"}
+	var out struct {
+		KeyID      string `json:"keyId"`
+		KeyDetails string `json:"keyDetails"`
+	}
+	if err := c.post("/management/v1/users/"+userID+"/keys", payload, &out); err != nil {
+		return nil, err
+	}
+	if out.KeyDetails == "" {
+		return nil, fmt.Errorf("create machine key returned no key material")
+	}
+
+	raw, err := base64Decode(out.KeyDetails)
+	if err != nil {
+		return nil, fmt.Errorf("key material is not base64: %w", err)
+	}
+
+	var key NewMachineKeyJSON
+	if err := json.Unmarshal(raw, &key); err != nil {
+		return nil, fmt.Errorf("key material is not the expected JSON: %w", err)
+	}
+	return &key, nil
+}
+
+// DeleteMachineKey removes one key. Called only after the new key is proven.
+func (c *Client) DeleteMachineKey(userID, keyID string) error {
+	return c.delete("/management/v1/users/" + userID + "/keys/" + keyID)
+}
+
+// GenerateClientSecret replaces the client secret of an OIDC app.
+//
+// ZITADEL stores exactly one secret per app, so the previous secret stops working
+// at this call. The caller must publish the new value immediately.
+func (c *Client) GenerateClientSecret(projectID, appID string) (string, error) {
+	var out struct {
+		ClientSecret string `json:"clientSecret"`
+	}
+	path := c.projectAppPath(projectID, appID) + "/oidc_config/_generate_client_secret"
+	if err := c.post(path, map[string]any{}, &out); err != nil {
+		return "", err
+	}
+	if out.ClientSecret == "" {
+		return "", fmt.Errorf("generate client secret returned an empty value")
+	}
+	return out.ClientSecret, nil
+}
+
+// Introspect checks a client credential against the token endpoint.
+//
+// A successful call returns an object with `active`. The credentials are valid
+// when the endpoint answers 200, whatever `active` says, because `active`
+// describes the token rather than the client.
+func (c *Client) Introspect(clientID, clientSecret string) error {
+	form := url.Values{}
+	form.Set("token", "probe")
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/introspect", strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.SetBasicAuth(clientID, clientSecret)
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("introspection request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("introspection returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+// ExchangeMachineKey obtains a token with a specific key, proving that key works.
+func (c *Client) ExchangeMachineKey(clientID, userID, keyID string, keyPEM []byte, audienceProject string) error {
+	assertion, err := SignAssertion(keyPEM, keyID, clientID, userID, c.cfg.Issuer, time.Now())
+	if err != nil {
+		return err
+	}
+
+	scope := "openid"
+	if audienceProject != "" {
+		scope += " urn:zitadel:iam:org:project:id:" + audienceProject + ":aud"
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	form.Set("scope", scope)
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/token", strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("key exchange failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("key exchange returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+func (c *Client) do(method, path string, payload any, out any) error {
+	token, err := c.AccessToken()
+	if err != nil {
+		return err
+	}
+
+	var body io.Reader
+	if payload != nil {
+		encoded, err := json.Marshal(payload)
+		if err != nil {
+			return err
+		}
+		body = bytes.NewReader(encoded)
+	}
+
+	req, err := http.NewRequest(method, strings.TrimRight(c.cfg.Issuer, "/")+path, body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	if payload != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("%s %s returned %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
+	}
+	if out != nil && len(raw) > 0 {
+		if err := json.Unmarshal(raw, out); err != nil {
+			return fmt.Errorf("%s %s response did not parse: %w", method, path, err)
+		}
+	}
+	return nil
+}
+
+func (c *Client) post(path string, payload, out any) error {
+	return c.do(http.MethodPost, path, payload, out)
+}
+
+func (c *Client) delete(path string) error {
+	return c.do(http.MethodDelete, path, nil, nil)
+}
+
+func base64Decode(s string) ([]byte, error) {
+	if raw, err := base64StdDecode(s); err == nil {
+		return raw, nil
+	}
+	return base64URLDecode(s)
+}
+
+func base64StdDecode(s string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s)
+}
+
+func base64URLDecode(s string) ([]byte, error) {
+	return base64.RawURLEncoding.DecodeString(s)
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/jwt.go

@@ -0,0 +1,89 @@
+// Package zitadel talks to the ZITADEL management API with the controller's own
+// machine key.
+//
+// The controller needs a credential to rotate credentials. It signs a short
+// assertion with its private key, exchanges it for an access token, and reuses
+// that token until shortly before it expires.
+package zitadel
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"time"
+)
+
+// assertionLifetime is deliberately short. The assertion proves one request.
+const assertionLifetime = 60 * time.Second
+
+// newAssertionClaims builds the claim set for a machine-key assertion.
+//
+// ZITADEL expects `iss` to name the caller, `sub` to name the user it acts as,
+// and `aud` to equal the issuer of the instance.
+func newAssertionClaims(clientID, userID, issuer string, now time.Time) map[string]any {
+	if userID == "" {
+		userID = clientID
+	}
+	return map[string]any{
+		"iss": clientID,
+		"sub": userID,
+		"aud": issuer,
+		"iat": now.Unix(),
+		"exp": now.Add(assertionLifetime).Unix(),
+	}
+}
+
+func base64URL(b []byte) string {
+	return base64.RawURLEncoding.EncodeToString(b)
+}
+
+// SignAssertion produces a signed RS256 JWT for the jwt-bearer grant.
+//
+// The header carries `kid`, so ZITADEL selects the matching registered key. That
+// is what makes overlapping keys possible: two keys can be valid at once and the
+// assertion says which one it used.
+func SignAssertion(privateKeyPEM []byte, keyID, clientID, userID, issuer string, now time.Time) (string, error) {
+	block, _ := pem.Decode(privateKeyPEM)
+	if block == nil {
+		return "", fmt.Errorf("private key is not PEM encoded")
+	}
+
+	key, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		// ZITADEL exports PKCS#1 keys as well.
+		if rsaKey, rsaErr := x509.ParsePKCS1PrivateKey(block.Bytes); rsaErr == nil {
+			key = rsaKey
+		} else {
+			return "", fmt.Errorf("private key does not parse: %w", err)
+		}
+	}
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return "", fmt.Errorf("private key is not RSA")
+	}
+
+	header := map[string]any{"alg": "RS256", "typ": "JWT", "kid": keyID}
+	headerJSON, err := json.Marshal(header)
+	if err != nil {
+		return "", err
+	}
+	claimsJSON, err := json.Marshal(newAssertionClaims(clientID, userID, issuer, now))
+	if err != nil {
+		return "", err
+	}
+
+	signingInput := base64URL(headerJSON) + "." + base64URL(claimsJSON)
+	digest := sha256.Sum256([]byte(signingInput))
+	signature, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, crypto.SHA256, digest[:])
+	if err != nil {
+		return "", fmt.Errorf("cannot sign the assertion: %w", err)
+	}
+
+	return signingInput + "." + base64URL(signature), nil
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/jwt_test.go

@@ -0,0 +1,90 @@
+package zitadel
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"strings"
+	"testing"
+	"time"
+)
+
+func newKey(t *testing.T) ([]byte, *rsa.PublicKey) {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("cannot generate a key: %v", err)
+	}
+	pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	return pemBytes, &key.PublicKey
+}
+
+func decodeSegment(t *testing.T, segment string, out any) {
+	t.Helper()
+	raw, err := base64.RawURLEncoding.DecodeString(segment)
+	if err != nil {
+		t.Fatalf("segment is not base64url: %v", err)
+	}
+	if err := json.Unmarshal(raw, out); err != nil {
+		t.Fatalf("segment is not JSON: %v", err)
+	}
+}
+
+func TestSignAssertionCarriesTheKeyIDAndIssuer(t *testing.T) {
+	pemBytes, _ := newKey(t)
+	now := time.Unix(1_700_000_000, 0)
+
+	token, err := SignAssertion(pemBytes, "key-42", "client-1", "user-9", "https://zitadel.riotpiao.com", now)
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	if strings.Count(token, ".") != 2 {
+		t.Fatalf("a JWT has three segments, got %q", token)
+	}
+
+	parts := strings.Split(token, ".")
+	var header map[string]any
+	decodeSegment(t, parts[0], &header)
+	if header["kid"] != "key-42" {
+		t.Fatalf("the header must name the key id so ZITADEL can pick the key, got %v", header["kid"])
+	}
+	if header["alg"] != "RS256" {
+		t.Fatalf("expected RS256, got %v", header["alg"])
+	}
+
+	var claims map[string]any
+	decodeSegment(t, parts[1], &claims)
+	if claims["iss"] != "client-1" || claims["sub"] != "user-9" {
+		t.Fatalf("iss and sub are wrong: %v / %v", claims["iss"], claims["sub"])
+	}
+	if claims["aud"] != "https://zitadel.riotpiao.com" {
+		t.Fatalf("aud must equal the issuer, got %v", claims["aud"])
+	}
+	exp := int64(claims["exp"].(float64))
+	iat := int64(claims["iat"].(float64))
+	if exp-iat != 60 {
+		t.Fatalf("the assertion must live 60 seconds, got %d", exp-iat)
+	}
+}
+
+func TestSignAssertionRejectsGarbage(t *testing.T) {
+	if _, err := SignAssertion([]byte("not a key"), "k", "c", "u", "https://i", time.Now()); err == nil {
+		t.Fatalf("a non-PEM key must be rejected")
+	}
+}
+
+func TestSignAssertionDefaultsSubjectToClientID(t *testing.T) {
+	pemBytes, _ := newKey(t)
+	token, err := SignAssertion(pemBytes, "k1", "client-1", "", "https://i", time.Unix(1, 0))
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	var claims map[string]any
+	decodeSegment(t, strings.Split(token, ".")[1], &claims)
+	if claims["sub"] != "client-1" {
+		t.Fatalf("an empty user id must fall back to the client id, got %v", claims["sub"])
+	}
+}

k8s/apps/key-rotation-controller/controller/main.go

@@ -0,0 +1,152 @@
+// Command key-rotation-controller rotates credentials declared by KeyRotation
+// objects.
+//
+// The controller authenticates to ZITADEL with its own machine key, read from a
+// mounted Secret. That key is the single bootstrap credential: everything else is
+// created by this process and never committed to git.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+	keyreconcile "github.com/riotpiao/key-rotation-controller/internal/reconcile"
+	"github.com/riotpiao/key-rotation-controller/internal/zitadel"
+)
+
+func main() {
+	var (
+		metricsAddr = flag.String("metrics-bind-address", ":8080", "address for the metrics endpoint")
+		healthAddr  = flag.String("health-probe-bind-address", ":8081", "address for probes")
+		interval    = flag.Duration("scan-interval", 10*time.Minute, "how often to re-check every resource")
+	)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseDevMode(false)))
+	logger := ctrl.Log.WithName("setup")
+
+	scheme := runtime.NewScheme()
+	if err := clientgoscheme.AddToScheme(scheme); err != nil {
+		logger.Error(err, "cannot register the Kubernetes types")
+		os.Exit(1)
+	}
+	if err := iamv1.AddToScheme(scheme); err != nil {
+		logger.Error(err, "cannot register the KeyRotation types")
+		os.Exit(1)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+		HealthProbeBindAddress: *healthAddr,
+		LeaderElection:         true,
+		LeaderElectionID:       "key-rotation-controller.iam.homelab.riotpiao.com",
+	})
+	if err != nil {
+		logger.Error(err, "cannot create the manager")
+		os.Exit(1)
+	}
+
+	zitadelClient, err := buildZitadelClient()
+	if err != nil {
+		logger.Error(err, "cannot build the ZITADEL client")
+		os.Exit(1)
+	}
+
+	reconciler := &keyreconcile.Reconciler{
+		Client: mgr.GetClient(),
+		Providers: map[string]keyreconcile.Provider{
+			iamv1.ProviderZitadelMachineKey: &keyreconcile.MachineKeyProvider{
+				Client:       zitadelClient,
+				AuthClientID: os.Getenv("ZITADEL_CLIENT_ID"),
+			},
+			iamv1.ProviderZitadelOIDCClient: &keyreconcile.OIDCClientProvider{
+				Client: zitadelClient,
+			},
+		},
+	}
+
+	c, err := controller.New("keyrotation", mgr, controller.Options{
+		Reconciler: ctrlreconcile.Func(func(ctx context.Context, req ctrlreconcile.Request) (ctrlreconcile.Result, error) {
+			wait, err := reconciler.Reconcile(ctx, types.NamespacedName(req.NamespacedName))
+			if err != nil {
+				return ctrlreconcile.Result{}, err
+			}
+			if wait <= 0 {
+				wait = *interval
+			}
+			return ctrlreconcile.Result{RequeueAfter: wait}, nil
+		}),
+	})
+	if err != nil {
+		logger.Error(err, "cannot create the controller")
+		os.Exit(1)
+	}
+
+	if err := c.Watch(source.Kind(mgr.GetCache(), client.Object(&iamv1.KeyRotation{}), &handler.EnqueueRequestForObject{})); err != nil {
+		logger.Error(err, "cannot watch KeyRotation objects")
+		os.Exit(1)
+	}
+
+	if err := mgr.AddHealthzCheck("healthz", func(*http.Request) error { return nil }); err != nil {
+		logger.Error(err, "cannot add the health check")
+		os.Exit(1)
+	}
+
+	logger.Info("starting the key rotation controller")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		logger.Error(err, "manager stopped")
+		os.Exit(1)
+	}
+}
+
+// buildZitadelClient reads the controller's own credential from the environment.
+//
+// These are the only values the controller cannot create for itself. They arrive
+// from one Secret, created once when the cluster is bootstrapped.
+func buildZitadelClient() (*zitadel.Client, error) {
+	issuer := os.Getenv("ZITADEL_ISSUER")
+	clientID := os.Getenv("ZITADEL_CLIENT_ID")
+	userID := os.Getenv("ZITADEL_USER_ID")
+	keyID := os.Getenv("ZITADEL_KEY_ID")
+	keyFile := envOr("ZITADEL_KEY_FILE", "/var/run/keys/controller/key")
+
+	if issuer == "" || clientID == "" || keyID == "" {
+		return nil, fmt.Errorf("ZITADEL_ISSUER, ZITADEL_CLIENT_ID and ZITADEL_KEY_ID are required")
+	}
+	pem, err := os.ReadFile(keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the controller key: %w", err)
+	}
+	return zitadel.New(zitadel.Config{
+		Issuer:        issuer,
+		ClientID:      clientID,
+		UserID:        userID,
+		KeyID:         keyID,
+		PrivateKeyPEM: pem,
+	}), nil
+}
+
+func envOr(name, fallback string) string {
+	if v := os.Getenv(name); v != "" {
+		return v
+	}
+	return fallback
+}

k8s/apps/key-rotation-controller/crd.yaml

@@ -0,0 +1,121 @@
+# KeyRotation declares one credential and how to rotate it.
+#
+# The resource never holds a secret value. It names where the value goes and how
+# to prove the new value works. The controller writes the Secret.
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: keyrotations.iam.homelab.riotpiao.com
+spec:
+  group: iam.homelab.riotpiao.com
+  scope: Namespaced
+  names:
+    kind: KeyRotation
+    plural: keyrotations
+    singular: keyrotation
+    shortNames: [kr]
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - {name: Provider, type: string, jsonPath: .spec.provider}
+        - {name: Strategy, type: string, jsonPath: .spec.strategy}
+        - {name: Phase, type: string, jsonPath: .status.phase}
+        - {name: Last, type: date, jsonPath: .status.lastRotation}
+        - {name: Next, type: date, jsonPath: .status.nextRotation}
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          required: [spec]
+          properties:
+            spec:
+              type: object
+              required: [provider, strategy, intervalHours, target]
+              properties:
+                provider:
+                  type: string
+                  enum: [zitadel-machine-key, zitadel-oidc-client]
+                  description: >-
+                    Which external API creates the credential.
+                strategy:
+                  type: string
+                  enum: [overlap, atomic]
+                  description: >-
+                    overlap keeps the old credential valid until the new one is
+                    proven. atomic replaces it in place, because the provider
+                    allows only one value.
+                intervalHours:
+                  type: integer
+                  minimum: 1
+                  description: How often to rotate.
+                suspend:
+                  type: boolean
+                  description: Stop automatic rotation without deleting the resource.
+                target:
+                  type: object
+                  required: [secretName, secretNamespace]
+                  properties:
+                    zitadelProjectID: {type: string}
+                    zitadelAppID: {type: string}
+                    machineUserID: {type: string}
+                    secretName: {type: string}
+                    secretNamespace: {type: string}
+                    secretKeys:
+                      type: object
+                      additionalProperties: {type: string}
+                      description: >-
+                        Logical field name to Secret key. Unmapped fields keep
+                        their logical name.
+                    reload:
+                      type: string
+                      enum: [hot, restart]
+                      description: >-
+                        hot means the consumer watches the file. restart rolls the
+                        consumer so it reads the new value at start.
+                    consumers:
+                      type: array
+                      items:
+                        type: object
+                        required: [kind, name, namespace]
+                        properties:
+                          kind: {type: string, enum: [Deployment, StatefulSet]}
+                          name: {type: string}
+                          namespace: {type: string}
+                verify:
+                  type: object
+                  properties:
+                    introspect:
+                      type: boolean
+                      description: Check client id and secret against introspection.
+                    tokenExchange:
+                      type: boolean
+                      description: Exchange a token with the new machine key.
+                    audienceProject:
+                      type: string
+                      description: Project id added to the token audience scope.
+            status:
+              type: object
+              properties:
+                phase:
+                  type: string
+                  enum: [Pending, Preparing, Propagating, Verifying, Completing, Succeeded, Failed, Suspended]
+                observedGeneration: {type: integer, format: int64}
+                lastRotation: {type: string, format: date-time}
+                nextRotation: {type: string, format: date-time}
+                activeCredentialID: {type: string}
+                previousCredentialID: {type: string}
+                lastError: {type: string}
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    required: [type, status, reason, lastTransitionTime]
+                    properties:
+                      type: {type: string}
+                      status: {type: string}
+                      reason: {type: string}
+                      message: {type: string}
+                      lastTransitionTime: {type: string, format: date-time}

k8s/apps/key-rotation-controller/deployment.yaml

@@ -0,0 +1,66 @@
+# The controller. Its own machine key is the single bootstrap credential: it is
+# created once and mounted here, and this process creates every other credential.
+#
+# The image must be built and pushed before this Deployment can start. Until then
+# the object stays Pending, which is why no Argo CD Application is wired yet.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: key-rotation-controller
+  namespace: iam
+  labels:
+    app: key-rotation-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: key-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: key-rotation-controller
+    spec:
+      serviceAccountName: key-rotation-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          image: forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --scan-interval=10m
+          env:
+            - {name: ZITADEL_ISSUER, value: "https://zitadel.riotpiao.com"}
+            - name: ZITADEL_CLIENT_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: client-id}}
+            - name: ZITADEL_USER_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: user-id}}
+            - name: ZITADEL_KEY_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: key-id}}
+            - {name: ZITADEL_KEY_FILE, value: "/var/run/keys/controller/key"}
+          ports:
+            - {name: metrics, containerPort: 8080}
+            - {name: health, containerPort: 8081}
+          readinessProbe:
+            httpGet: {path: /healthz, port: health}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /healthz, port: health}
+            initialDelaySeconds: 15
+          volumeMounts:
+            - {name: controller-key, mountPath: /var/run/keys/controller, readOnly: true}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 20m, memory: 64Mi}
+            limits: {cpu: 500m, memory: 256Mi}
+      volumes:
+        - name: controller-key
+          secret:
+            secretName: key-rotation-controller
+            items:
+              - {key: key, path: key}

k8s/apps/key-rotation-controller/example-machine-key.yaml

@@ -0,0 +1,28 @@
+# Example: rotate a service user's machine key with no gap.
+#
+# Machine keys overlap in ZITADEL, so the old key stays valid until the new one has
+# obtained a token. Nothing in the window fails.
+#
+# Apply only after the controller runs and the machine user exists.
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: example-machine-key
+  namespace: iam
+spec:
+  provider: zitadel-machine-key
+  strategy: overlap
+  intervalHours: 720        # 30 days
+  suspend: true             # flip to false once the pilot is proven
+  target:
+    machineUserID: "REPLACE_WITH_SERVICE_USER_ID"
+    secretName: example-jwt
+    secretNamespace: iam
+    secretKeys:
+      key: key
+      key-id: key-id
+    reload: hot             # the consumer watches the file, so no restart
+    consumers: []
+  verify:
+    tokenExchange: true
+    audienceProject: "396757796898471937"

k8s/apps/key-rotation-controller/example-oidc-client.yaml

@@ -0,0 +1,27 @@
+# Example: rotate an OIDC app's client secret.
+#
+# ZITADEL keeps one secret per app, so the old value dies at the generate call.
+# The strategy is atomic: publish, roll the consumer, verify, and roll back if the
+# introspection check fails.
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: example-grafana-client
+  namespace: iam
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160       # 90 days
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "REPLACE_WITH_APP_ID"
+    secretName: grafana-oidc
+    secretNamespace: logging
+    secretKeys:
+      secret: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+    reload: restart         # Grafana reads the value at start
+    consumers:
+      - {kind: Deployment, name: grafana, namespace: logging}
+  verify:
+    introspect: true

k8s/apps/key-rotation-controller/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No Argo CD Application is wired to this directory yet: the controller image must
+# exist first. Add the app once the image is pushed, so Argo never reports a
+# Deployment that cannot pull.
+resources:
+  - crd.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - rotations.yaml

k8s/apps/key-rotation-controller/rbac.yaml

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: key-rotation-controller
+  namespace: iam
+---
+# Cluster-wide because a credential and its consumer live in different namespaces:
+# the Secret in one, the workload in another.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: key-rotation-controller
+rules:
+  - apiGroups: ["iam.homelab.riotpiao.com"]
+    resources: ["keyrotations"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["iam.homelab.riotpiao.com"]
+    resources: ["keyrotations/status"]
+    verbs: ["get", "update", "patch"]
+  # The controller writes the credential it just created.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "update", "patch"]
+  # Rolling a consumer that cannot hot reload.
+  - apiGroups: ["apps"]
+    resources: ["deployments", "statefulsets"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: key-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: key-rotation-controller
+subjects:
+  - kind: ServiceAccount
+    name: key-rotation-controller
+    namespace: iam

k8s/apps/key-rotation-controller/rotations.yaml

@@ -0,0 +1,211 @@
+# Every credential this cluster owns, described as a KeyRotation.
+#
+# One object per consumer. The controller creates the value, writes the Secret and
+# proves the new value works before it retires the old one. Nothing here holds a
+# secret, so this file is safe in git.
+#
+# suspend: true everywhere on purpose. The controller image does not exist yet, and
+# an unsuspended object would claim an intent no process is acting on. Clear the flag
+# per object as its pilot passes.
+#
+# poimen-memory is absent deliberately: it is in use and excluded from this migration.
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: grafana
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801965191169"
+    secretName: grafana-oidc
+    secretNamespace: logging
+    secretKeys:
+      secret: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: grafana, namespace: logging}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: argocd
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801210282367"
+    secretName: argocd-oidc
+    secretNamespace: argocd
+    secretKeys:
+      secret: oidc.clientSecret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: argocd-server, namespace: argocd}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: comfyui
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "397015744010781070"
+    secretName: comfyui-oauth2-proxy
+    secretNamespace: comfyui
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: minio
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757803257036801"
+    secretName: minio-oidc
+    secretNamespace: minio
+    secretKeys:
+      secret: client-secret
+    reload: hot
+    consumers:
+      - {kind: Deployment, name: minio, namespace: minio}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: paperless
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757803559027071"
+    secretName: paperless-oidc
+    secretNamespace: paperless
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: paperless, namespace: paperless}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: immich
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757802602791295"
+    secretName: immich-oidc
+    secretNamespace: immich
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: immich-server, namespace: immich}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: forgejo
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801579315583"
+    secretName: forgejo-oidc
+    secretNamespace: forgejo
+    secretKeys:
+      secret: key
+    reload: restart
+    consumers:
+      - {kind: StatefulSet, name: forgejo, namespace: forgejo}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: api-gw
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757800874672511"
+    secretName: api-gw-oidc
+    secretNamespace: api-gw
+    secretKeys:
+      secret: client-secret
+    reload: hot
+    consumers:
+      - {kind: Deployment, name: api-gw, namespace: api-gw}
+  verify:
+    introspect: true

k8s/apps/messaging/management-service/templates/configmap.yaml

@@ -6,7 +6,7 @@ metadata:
 data:
   KMSVC_KAFKA_BROKERS: {{ .Values.env.kafkaBrokers | quote }}
   KMSVC_REDIS_ADDR: {{ .Values.env.redisAddr | quote }}
-  KMSVC_AUTHENTIK_ISSUER_URL: {{ .Values.env.authentikIssuerURL | quote }}
-  KMSVC_AUTHENTIK_AUDIENCE: {{ .Values.env.authentikAudience | quote }}
+  KMSVC_ZITADEL_ISSUER_URL: {{ .Values.env.zitadelIssuerURL | quote }}
+  KMSVC_ZITADEL_AUDIENCE: {{ .Values.env.zitadelAudience | quote }}
   KMSVC_GRPC_LISTEN_ADDR: ":{{ .Values.grpcPort }}"
   KMSVC_HTTP_LISTEN_ADDR: ":{{ .Values.httpPort }}"

k8s/apps/messaging/management-service/values.yaml

@@ -12,8 +12,8 @@ httpPort: 8080
 env:
   kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
   redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
-  authentikIssuerURL: ""
-  authentikAudience: ""
+  zitadelIssuerURL: ""
+  zitadelAudience: ""
 
 resources:
   requests:

k8s/apps/secret-rotation-controller/crd.yaml

@@ -1,144 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: secretrotations.homelab.riotpiao.com
-spec:
-  group: homelab.riotpiao.com
-  names:
-    kind: SecretRotation
-    plural: secretrotations
-  scope: Namespaced
-  versions:
-  - name: v1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        properties:
-          metadata:
-            type: object
-          spec:
-            type: object
-            required:
-            - provider
-            - rotationInterval
-            properties:
-              # External system: authentik | forgejo | minio | vault
-              provider:
-                type: string
-                enum: [authentik, forgejo, minio, vault]
-              
-              # How often to rotate (hours)
-              rotationInterval:
-                type: integer
-                minimum: 24
-              
-              # Application ID in external system
-              appId:
-                type: string
-              
-              # k8s Secret to update (name, namespace, key)
-              secretRef:
-                type: object
-                required: [name, namespace]
-                properties:
-                  name:
-                    type: string
-                  namespace:
-                    type: string
-                  key:
-                    type: string
-                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
-              
-              # Path to git file that holds the secret (for .enc.yaml files)
-              gitPath:
-                type: string
-                description: "Path in homelab repo to .enc.yaml file"
-              
-              # Ansible template values to substitute
-              templateValues:
-                type: object
-                additionalProperties:
-                  type: string
-          
-          status:
-            type: object
-            properties:
-              lastRotationTime:
-                type: string
-                format: date-time
-              nextRotationTime:
-                type: string
-                format: date-time
-              lastRotationStatus:
-                type: string
-                enum: [Success, Failed, Pending]
-              lastRotationError:
-                type: string
-              lastCommitHash:
-                type: string
-
----
-# Example usage:
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160  # 90 days in hours
-  appId: minio
-  secretRef:
-    name: minio-oidc
-    namespace: storage
-    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: portfolio-agent-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160
-  appId: portfolio-agent
-  secretRef:
-    name: portfolio-agent-oidc
-    namespace: portfolio
-    key: CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: forgejo-registry-token
-  namespace: secret-rotation
-spec:
-  provider: forgejo
-  rotationInterval: 2160
-  appId: rock/riotpiao.com
-  secretRef:
-    name: forgejo-registry-secret
-    namespace: kube-system
-    key: REGISTRY_TOKEN
-  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-root-credentials
-  namespace: secret-rotation
-spec:
-  provider: minio
-  rotationInterval: 4320  # 180 days in hours
-  appId: root
-  secretRef:
-    name: minio-creds
-    namespace: storage
-  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -1,92 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: secret-rotation-controller
-  template:
-    metadata:
-      labels:
-        app: secret-rotation-controller
-    spec:
-      serviceAccountName: secret-rotation-controller
-      containers:
-      - name: controller
-        image: secret-rotation-controller:latest
-        imagePullPolicy: IfNotPresent
-        env:
-        # SOPS reads age key from this file
-        - name: SOPS_AGE_KEY_FILE
-          value: /etc/sops/age/private-key.txt
-        
-        # Vault auth (token in projected volume)
-        - name: VAULT_ADDR
-          value: http://vault.vault.svc.cluster.local:8200
-        - name: VAULT_TOKEN_FILE
-          value: /var/run/secrets/vault/token
-        
-        # Authentik
-        - name: AUTHENTIK_URL
-          value: http://authentik-server.iam.svc.cluster.local
-        - name: AUTHENTIK_BOOTSTRAP_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: authentik-bootstrap
-              key: token
-        
-        # Git
-        - name: GIT_REPO
-          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-        - name: GIT_AUTHOR_EMAIL
-          value: [email protected]
-        - name: GIT_AUTHOR_NAME
-          value: Secret Rotation Controller
-        - name: FORGEJO_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: forgejo-registry-secret
-              key: REGISTRY_TOKEN
-        
-        volumeMounts:
-        # Age key from ExternalSecret (synced from Vault)
-        - name: age-key
-          mountPath: /etc/sops/age
-          readOnly: true
-        
-        # Vault auth token (projected)
-        - name: vault-token
-          mountPath: /var/run/secrets/vault
-          readOnly: true
-        
-        # Temp working dir
-        - name: tmp
-          mountPath: /tmp
-        
-        resources:
-          requests:
-            cpu: 100m
-            memory: 256Mi
-          limits:
-            cpu: 500m
-            memory: 512Mi
-      
-      volumes:
-      - name: age-key
-        secret:
-          secretName: sops-age-key
-          defaultMode: 0400
-      
-      - name: vault-token
-        projected:
-          sources:
-          - serviceAccountToken:
-              path: token
-              audience: vault
-              expirationSeconds: 3600
-      
-      - name: tmp
-        emptyDir: {}

k8s/apps/secret-rotation-controller/kustomization.yaml

@@ -1,15 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: secret-rotation
-
-resources:
-  - rbac.yaml
-  - crd.yaml
-  - external-secret.yaml
-  - deployment.yaml
-
-commonLabels:
-  app.kubernetes.io/name: secret-rotation-controller
-  app.kubernetes.io/component: automation
-  managed-by: argocd

k8s/apps/secret-rotation-controller/rbac.yaml

@@ -1,53 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: secret-rotation-controller
-rules:
-# Read SecretRotation CRDs
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations"]
-  verbs: ["get", "list", "watch"]
-
-# Update status
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations/status"]
-  verbs: ["get", "patch", "update"]
-
-# Read k8s secrets that will be rotated
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["get", "list"]
-
-# For recording events
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create", "patch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: secret-rotation-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: secret-rotation-controller
-subjects:
-- kind: ServiceAccount
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: secret-rotation
-  labels:
-    kubernetes.io/metadata.name: secret-rotation

k8s/argocd/apps/08-secret-rotation.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: secret-rotation
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: secret-rotation
-spec:
-  project: homelab
-  
-  sources:
-  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    path: k8s/apps/secret-rotation-controller
-    targetRevision: main
-  
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: secret-rotation
-  
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    - RespectIgnoreDifferences=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/secrets/gotify-oauth2-proxy-secrets.enc.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: gotify-oauth2-proxy
-    namespace: notifications
-type: Opaque
-data:
-    client-id: ENC[AES256_GCM,data:Q+XQAR6cfHKoiuxhPZs6khTvqT0A/JG6,iv:LqhfuhXOYPVB6FQpPpERzWRQ+nxgxHZo4qdJO/g52y8=,tag:V2eK2fyscQKmM3/MmDnNtg==,type:str]
-    client-secret: ENC[AES256_GCM,data:P7F+83CLDhHnTN+Qx3SeCFbCjVH4JHb2rSteCr3Dywcb1ysMSNvC+3HyBXzsfJJ/EErUrCPWuwlRmV5ySts1oIrWhKyWMJxNrUTzv4eoclat4RyV6SaNDg==,iv:ESO1OUIqZbZPEVUuHrqjskA4tuTX/sc5m+K6Eo319kI=,tag:2isdxjFbtosov88FHZdeaQ==,type:str]
-    cookie-secret: ENC[AES256_GCM,data:WoWiANSdju+Ep5MuZQNeT0BQdKf5cFvrDUZpe+4B0VeUmQqnvblFi3dciJOeBFom9xYYzUCAI2tUSmX+a6tdAObyoGvjdFmLh+WTfw==,iv:41gYYRkyDIqC3GPuZkoob81Wl2njFTY07j4Puzaq0s4=,tag:x6z33zZnFwvmWUQIVPcxdQ==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2UURWblY4aTBOQjFSNDRL
-            YkRXVjVHK1QvYW1iR2o0UWVmdE94c2lkWDJjCnlXK1k3cnZBNEExbDU3eDhQZ3FT
-            TVcyckFiWHdRSjZHQ201Nnl6UFdTN28KLS0tIExQeE90VnM1WHRCYkpycjJtR0ht
-            ZHFDNWhKeENSQzljZUNmdktDYWloUDQK/fOG8AjTb2TNqLbqNfArOs0mzInYNJVd
-            iaYxnjH6AjhbLKdnmCpecTQpgx8GeFGIkefV/x5pgrdSUwn0lcvpSQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-10-27T19:05:35Z"
-    mac: ENC[AES256_GCM,data:KZV3JNebQ+DbZZZ+e3fvtOGckwC9fmkIchh8DBJ0DMq9lbrPEhP4Juordmnrrs9vJvGxuFJ/KwUv9EELqBsuSy8c47+z7flEAXJiJCJkvVPL/jMidABsFqMALvNNrCqBPWvMHLZUj762DwxB2gKhSx3GLfRUXhm+ZiNcD+39EVw=,iv:ifUlEq/GhBc/QomJApkCaWc3f4ggXoLsvqjzZElO0fU=,tag:fMo+HBDUZdRXgfck+8AJbA==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.3

k8s/argocd/secrets/grafana-gotify-bridge.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-gotify-bridge
+    namespace: notifications
+type: Opaque
+data:
+    app-token: ENC[AES256_GCM,data:MnkKT0/ubJctmLTFXjdCZjNlz+KgQLsPwIKCmPF2weEtMw6ieRfb3+B7OUlIiDqpktdPaJA/znsMOYPXT2aXDqyKquA=,iv:vcAGkgfTheoViZ7OzJJhbNf0boIbRBZLr60ieW5DhLk=,tag:iBDJEjGScxHopkNExh7jDw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYYWZYNFBHbVhRSkZoa3dH
+            dWp3WllyVmRPcExPTndZUlFsTG1YMHN6WW1jCjRiS1M2U0tIM0tCdGhJOUQ0c2JW
+            bXNTTExDeVVmcExuNStldFdhTHMrVTgKLS0tIDN3d3JKSEtIUUFhVHROOVBUaUNO
+            UUVGcVVkOVJmN294dE5DS1R1UkxiZzgKx0MEkIrFsO2dQwjY2410Nj0Nyy3QgC3L
+            T844m4WT+mxFwB69/y0luGkn4aXMS3xrHHWUC+vjAkM4Unp+MKhhrg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-10-27T19:35:38Z"
+    mac: ENC[AES256_GCM,data:Gf5jnvWP+jBRc8fvQHyrnHnTDaaUTcXQgoOqm7Drax5FpBYEHa36pdKaUq+XR6VBd7ch6Ia/f7YavhS47ybgR3jn+w+trMxkpwtZhm5clw2Lo3Hm7Cvi4fG4qhcjm1C8/NOnWGsi5Qm+TJqyG509p9HNTvM99SzElRovPZldLL4=,iv:10hA6nq0WIO0a2I4ZW6uPGXPjesFgxrwmXohNO3FZz4=,tag:bJ37e8jvgEactq6T/pM2UQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml

@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:rGU=,iv:w1yJeQvN0MPZlPx8VQZmC6zkRaHW4rtjjROea0Kx9X4=,tag:qgYgVUGMQ2BtN0RZ0nZYjQ==,type:str]
-kind: ENC[AES256_GCM,data:wp2QP1he,iv:C8c/jKh5pnBOtHqXsTEchGf9DACU/5hOJ0Mp/+Jk1kg=,tag:QUZNeguRXdi3JO6Ov6GMDg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:jVsnAcFTlRiKxDoU,iv:bWEqo28kH3bxzfI3ipiY6dsqhIcl9wIxBjFu0mxVGLg=,tag:PVWTveI8xhvLxvXPldhimQ==,type:str]
-    namespace: ENC[AES256_GCM,data:ZHnVZyd3wA==,iv:3evUnFTQfRz/ZilUQIl/9dsCba3nsX6o/iQsGXMBfgM=,tag:+9kp+VRu1ksroDTbTGmGPA==,type:str]
-type: ENC[AES256_GCM,data:bHX+Bu9o,iv:8qqNddsuw4YoHAuJfUcPIqjcqNySIh5Ln8OhsfZxuk8=,tag:P7QGUXo9Io2Y19YqlWv5sQ==,type:str]
-stringData:
-    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:nJz9pqa6w9ZGOjanZ32D38Bolg2YrdHs1crhb2uZkKyaaKDpklVb4FOKZ46K+NS1NXjoS6XDUAlV69CSvm1PeA==,iv:g9F/iLuzNC0WogtDJ/TAb3+97N6iGkbOTlb4SzYzwx8=,tag:4F3kUheGbtW2U6jgOvRO1A==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrUkxLMEV3aEpILytic0Ux
-            V2VEV2EvcENJVnNNRUtBbGkwT0d6eUxYeEFvCjZPSkU2MG5qME5jZ01mRVJwTUFL
-            M3M0V3c3QXBoRml1Vm9NWW5JTlNRSG8KLS0tIEJnNlFmdFJXb1BPaVJzTmo1RDQv
-            VXZOVHRHUEZJRUZ4eGZGM1RxSURyWXcK3ZyYCPhRUvpvT/pjPQGJoLIwaktZY6Tg
-            LS0cIDUhjPFbeu7qUATOrat6vMUi0UREbXSFZ1KyAjNAwJZaUusPMA==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:0cgVNtFMwhbEHBxu7T1XfstcQlgUTxijq9Mz36zC60E7kYXRNwGt2/0KoCaqaE0leypwAdrPn2OjeoDPeCN6x+n0vLRVX2tnli6HazBLbRsPHljDAV0ASF07iNhb0PvgtWay432Mm+sGATkel3ITdAaP0lISNLUYfEe1/TbYHkk=,iv:FZ9Bgtk7LOC3aOSOo1+50zwLGhlcUZYJaUQ9r05cAn0=,tag:tsLQ1frZ7li5l717CCtgoQ==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/igotify-client-token.enc.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: igotify-client-token
+    namespace: notifications
+type: Opaque
+data:
+    client-token: ENC[AES256_GCM,data:W+InmDyZkU0i+OqQUsIaRNY/3p6zjMC19ASj/qYHTldhtGmHwj4koQX3tIxdCQV72xZKOxsc8eV3hGX3pqbW02c2/ng=,iv:wWI5AZdpxnzX61f8iTiWmxKcDFMaFM4PHFYo4DMBK4Q=,tag:CpE8Kg5V8fL7P3oAjWjSvQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwaUlFU0xKaEs0SWcxWEEr
+            T0tUSmErR08zVkhuLy9VMEc3VXdmU1o3cWdrCjhuOW1yTmZPdWxDZHhMeUZtQlcz
+            UmVzaEMyVHZOZS9KVHdMd2pndUJwUW8KLS0tIGxYeENwVEFDdUpDTGszdGtNRHBQ
+            Y041WW9HdTJwUlJCUGhMN2RGZWZpUmMKiQbi/O9X3cFJgqw3tvwWcyy8QKkUuq/7
+            Y+V51R/WFobEQuAZLmDKgWcPSGt51effufB/Vm9LRRqtbkbYQAIoXQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2026-10-27T19:14:20Z"
+    mac: ENC[AES256_GCM,data:Hxl7AGzMOMK0pwe6T9ZmrE6K45XhQkq4qk/7YkrVuneL1e7jc8Bmv4NlywqRkt9fPVOCMwKikeX9U3ZcsqK13xPPfm1CmAxV3CCluwK0/9589N70OQxL4qJSkO6CXyGaa0Y9wVRX4y/N3wSjW+d25+GOYUXrOxkRpVyU4GFSqAQ=,iv:0Af/MCZBEYJVI3yi7DpPX1eBKwSANXJPCpSAEE3zCq4=,tag:SS1U9TfRQR3z8KkoQ6ZuZA==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.3

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -5,23 +5,25 @@ metadata:
     name: oauth2-credentials
     namespace: iam
 data:
-    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
-    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
-    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
-    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
-    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+    api-gw-client-secret: ENC[AES256_GCM,data:q41OBQ6WvxsmHmXFhrdzeNUlQyPKuOQQa0vWdhuioQwTsNJ5+l9rYZKBAcZxzvd4P066aRN6yVPmEZhq,iv:Dnfgy8N9NVOM2x656Qei8960dzu7icAU3BZLXmj7N5o=,tag:deilBSCC9Mg/Eo8K0w6rMw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:2k28ui+Mbu3qd4ar6/NidO4bWO8g7JQUjDQBD8liywpCm7rA8TSrmU51im/I5kEvRRhl+RByEo5wpozo,iv:JFrZYF7cP/tQL8gkQJ5GvtdOt1kKcAdHE7L4l+uT9M0=,tag:DSiv8kv7MINUxi7fO/093w==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
-            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
-            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
-            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
-            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5WmthakxzRk5aeGJWZnRF
+            bzdqM1o5Q1h2MFlpNEJvRUFZVU1wWjRaVWowClVKYkp0YUJCbUM0MWduaCtBSExH
+            MDRWUGZkYTdndTZTeEM3ckxrMURFMUEKLS0tIHF3YUlLNndSWmxGejFPbTVjQnRS
+            THF4RFBNbDJCdjVKRDJ6cEJlZGVBM0UKEDz1sKDON+lelq1zbCO4CETaK77T4G8y
+            fly66sR5uIvAXdwXy14L9ssZsM1jOHqdkXnpbo6mzh3toB/ROzoL7w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-27T21:47:38Z"
+    mac: ENC[AES256_GCM,data:bgZD+/LAPKp6DXyRMnbFWyxmiRjWxYO0ofxeGpO7nyw1DMSTD9s8li43eh/IP7wW4NORwzfIfzxUvOFmVfAiQKfkzpYRDFIKqK+aa/+43zppwpVT0UJ3CcBt2qgdJRHKnv9qeUOx4M8ulY7Xcsoevw8EGdrXCc1KAcDIwoIAp0Q=,iv:djYQ6YnGPrYGRKgpfbdf0beW6QHYTq5CJ3nes7+C4p4=,tag:N9n4VKyMNzGCmuYpwBLPng==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
-    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
-    version: 3.13.2
+    version: 3.9.3

k8s/argocd/secrets/secret-generator.yaml

@@ -15,7 +15,6 @@ files:
     - forgejo-registry-pull.enc.yaml
     - forgejo-runner-token.enc.yaml
     - forgejo-secrets.enc.yaml
-    - grafana-oidc-secrets.enc.yaml
     - grafana-secrets.enc.yaml
     - homelab-ca-secrets.enc.yaml
     - loki-secrets.enc.yaml
@@ -36,7 +35,8 @@ files:
     - etcd-defrag-creds.enc.yaml
     - zitadel-masterkey.enc.yaml
     - smtp-gotify-secrets.enc.yaml
-    - gotify-oauth2-proxy-secrets.enc.yaml
+    - igotify-client-token.enc.yaml
+    - grafana-gotify-bridge.enc.yaml
     # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
     # owns it. Without this, the chart re-generates the keypair on every render,
     # which changes checksum/secret-login-service-key and leaves both Zitadel

k8s/infra/logging/grafana-values.yaml

@@ -209,3 +209,25 @@ sidecar:
       disableDelete: false
       foldersFromFilesStructure: true
     searchNamespace: ALL
+
+# ── Alerting ──────────────────────────────────────────────────────────────────
+# One contact point: Gotify. Gotify refuses Grafana's payload shape with
+# 400 "Field 'message' is required", so a bridge service translates the JSON
+# first (k8s/apps/gotify/bridge). Grafana posts to the bridge, the bridge posts
+# to /message with an app token.
+alerting:
+  contactpoints.yaml:
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: gotify
+        receivers:
+          - uid: gotify-webhook
+            type: webhook
+            disableResolveMessage: false
+            settings:
+              url: http://grafana-gotify-bridge.notifications.svc.cluster.local:8080/alert
+              httpMethod: POST
+              # 0 sends every alert in one request. The bridge names the extra
+              # count in the message body.
+              maxAlerts: 0

scripts/iam/rotate-oidc.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Rotate the OIDC client secret of every owned application, one at a time.
+
+This is the manual execution of what a KeyRotation object declares. It exists so
+the credentials can rotate today, before the controller image is built, and so the
+controller has a reference implementation to match.
+
+For each application: generate a new secret in ZITADEL, write it into the
+Kubernetes Secret, roll the consumer, then prove the consumer still talks to
+ZITADEL. A failure stops the run, because the next application should not start
+while one is in an unknown state.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+import importlib.util
+
+spec = importlib.util.spec_from_file_location("zp", Path(__file__).with_name("zitadel-provision.py"))
+zp = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(zp)
+
+PROJECT = "396757796898471937"
+ISSUER = "https://zitadel.riotpiao.com"
+
+# app key, ZITADEL app id, Secret name, namespace, Secret key, consumer kind, consumer name
+APPS = [
+    ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
+    ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
+    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
+    ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
+    ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
+    ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),
+    ("forgejo",   "forgejo",   "forgejo-oidc",         "cicd",      "CLIENT_SECRET",                       "deploy", "forgejo-gitea"),
+]
+
+
+def sh(*args: str, timeout: int = 120) -> tuple[int, str]:
+    p = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+    return p.returncode, (p.stdout + p.stderr).strip()
+
+
+def state() -> dict:
+    return json.load(open(Path.home() / ".config/zitadel/state.json"))
+
+
+def app_ids(token: str) -> dict[str, str]:
+    """Map clientId to the app resource id. The two are different, and the secret
+    endpoint takes the resource id."""
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/_search", token,
+                         {"query": {"limit": 100, "offset": 0}})
+    out = {}
+    for app in body.get("result", []) or []:
+        cid = (app.get("oidcConfig") or {}).get("clientId")
+        if cid and app.get("id"):
+            out[cid] = app["id"]
+    return out
+
+
+def new_secret(token: str, app_id: str) -> str | None:
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/{app_id}/oidc_config/_generate_client_secret", token)
+    if code != 200:
+        print(f"    generate failed: {code} {str(body)[:120]}")
+        return None
+    return body.get("clientSecret")
+
+
+def patch_secret(ns: str, name: str, key: str, value: str) -> bool:
+    import base64
+    enc = base64.b64encode(value.encode()).decode()
+    patch = json.dumps({"data": {key: enc}})
+    code, out = sh("kubectl", "-n", ns, "patch", "secret", name, "--type=merge", "-p", patch)
+    if code != 0:
+        print(f"    secret patch failed: {out[:120]}")
+    return code == 0
+
+
+def roll(kind: str, ns: str, name: str) -> bool:
+    code, out = sh("kubectl", "-n", ns, "rollout", "restart", f"{kind}/{name}", timeout=120)
+    if code != 0:
+        print(f"    rollout restart failed: {out[:120]}")
+        return False
+    code, out = sh("kubectl", "-n", ns, "rollout", "status", f"{kind}/{name}", "--timeout=90s", timeout=120)
+    return code == 0
+
+
+def main() -> int:
+    only = sys.argv[1:] or None
+    token = zp.bootstrap_token()
+    if not token:
+        print("no bootstrap token")
+        return 2
+    cids = state()["client_ids"]["value"]
+    ids = app_ids(token)
+    print(f"resolved {len(ids)} app resource ids")
+    done, failed = [], []
+    for key, app_id, sname, ns, skey, kind, cname in APPS:
+        if only and key not in only:
+            continue
+        print(f"[{key}] issuer={ISSUER} client_id={cids[key]}")
+        rid = ids.get(cids[key])
+        if not rid:
+            print(f"[{key}] no app resource id for client {cids[key]}")
+            failed.append(key)
+            continue
+        secret = new_secret(token, rid)
+        if not secret:
+            failed.append(key)
+            continue
+        if not patch_secret(ns, sname, skey, secret):
+            failed.append(key)
+            continue
+        if not roll(kind, ns, cname):
+            failed.append(key)
+            print(f"[{key}] FAILED: consumer did not become ready")
+            continue
+        code, out = sh("kubectl", "-n", ns, "get", "pods", "-l",
+                       f"app={cname}" if kind == "deploy" else f"app.kubernetes.io/name={cname}",
+                       "--no-headers")
+        ready = "Running" in out
+        print(f"[{key}] rotated, consumer {'ready' if ready else 'NOT ready'}")
+        (done if ready else failed).append(key)
+    print(f"\nrotated: {len(done)} {done}\nfailed: {len(failed)} {failed}")
+    return 0 if not failed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/iam/zitadel-provision.py

@@ -122,11 +122,11 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
-    # Gotify has no native OIDC, so oauth2-proxy sits in front of it and this app
-    # is the proxy's client. The callback path is the proxy's, not Gotify's.
-    "gotify": {**_WEB, "redirect_uris": [
+    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
+    # the proxy's client, so the callback path belongs to the proxy.
+    "comfyui": {**_WEB, "redirect_uris": [
         "http://localhost:4180/oauth2/callback",
-        "https://gotify.riotpiao.com/oauth2/callback"]},
+        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -143,8 +143,13 @@ OIDC_APPS = {
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
         "http://localhost:8000/accounts/oidc/authentik/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    # The frontend calls back at /auth/callback (see OAUTH_CALLBACK_PATH in
+    # poimen-frontend/lib/oauth.ts). The bare /callback spelling stays registered so
+    # an older image still completes its login.
     "poimen": {**_WEB, "redirect_uris": [
-        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback",
+        "http://localhost:3000/auth/callback",
+        "https://poimen.riotpiao.com/auth/callback"]},
     "queue-sqs": {**_WEB, "redirect_uris": [
         "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
     "vault": {**_WEB, "redirect_uris": [
@@ -355,13 +360,16 @@ def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
             die(f"create role {key} failed: {code} {d}")
 
 
-def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]:
+def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict],
+                      known_secrets: dict[str, str] | None = None) -> tuple[dict, dict]:
     code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {})
     existing = {a.get("name"): a for a in (d.get("result") or [])}
     ids: dict[str, str] = {}
     secrets: dict[str, str] = {}
+    known_secrets = known_secrets or {}
     for name, spec in apps.items():
         row = existing.get(name)
+        created = False
         if row:
             ids[name] = (row.get("oidcConfig") or {}).get("clientId", "")
             app_id = row.get("id")
@@ -384,8 +392,16 @@ def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tup
                 die(f"create app {name} failed: {code} {d}")
             app_id = d["appId"]
             ids[name] = d.get("clientId", "")
+            created = True
         if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE":
             continue
+        if not created:
+            # ZITADEL keeps one client secret per app, and the API cannot read it
+            # back. Generating a new one here would replace the deployed copy and
+            # break that app's login, so reuse the value already on disk.
+            if known_secrets.get(name):
+                secrets[name] = known_secrets[name]
+            continue
         code, d = _api("POST",
                        f"/management/v1/projects/{project_id}/apps/{app_id}"
                        "/oidc_config/_generate_client_secret", token, {})
@@ -487,7 +503,13 @@ def phase_apply(_args: argparse.Namespace) -> int:
     print(f"project   : {PROJECT_NAME} ({project_id})")
     _ensure_roles(token, project_id, ROLES)
     print(f"roles     : {len(ROLES)} ensured")
-    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS)
+    known: dict[str, str] = {}
+    try:
+        known = (json.loads(STATE_FILE.read_bytes()).get("client_secrets", {})
+                 .get("value", {}) or {})
+    except (OSError, ValueError):
+        known = {}
+    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS, known)
     print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued")
 
     state = {

terraform/templates/controlplane.tftpl

@@ -33,14 +33,6 @@ machine:
       - ip: ${forgejo_registry_ip}
         aliases:
           - ${forgejo_hostname}
-      # kube-apiserver's static pod runs on the host network, so it resolves
-      # authentik.riotpiao.com via THIS node's DNS, not cluster CoreDNS - the
-      # OIDC issuer discovery call would otherwise hit the public
-      # Cloudflare-fronted IP instead of nginx directly. Same reasoning as
-      # the forgejo entry above.
-      - ip: 192.168.1.160
-        aliases:
-          - authentik.riotpiao.com
   kubelet:
     image: ${kubelet_image}
     defaultRuntimeSeccompProfileEnabled: true
@@ -128,19 +120,10 @@ cluster:
       - ${san}
 %{ endfor ~}
     image: ${kube_apiserver_img}
-    # Foundation for the Authentik group -> k8s RBAC pattern in
-    # k8s/infra/rbac/ - additive only, existing client-cert auth
-    # (system:masters, this session's own admin@homelab-cluster kubeconfig)
-    # keeps working unchanged; a bad OIDC config just means OIDC logins fail,
-    # not a lockout. Prefixes avoid collision with built-in system:* users
-    # and groups.
-    extraArgs:
-      oidc-issuer-url: https://authentik.riotpiao.com/application/o/kubernetes/
-      oidc-client-id: kubernetes
-      oidc-username-claim: email
-      oidc-groups-claim: groups
-      oidc-username-prefix: "oidc:"
-      oidc-groups-prefix: "oidc:"
+    # No OIDC block. The cluster authenticates with client certificates. An
+    # earlier Authentik-based kube-apiserver OIDC config was removed from the
+    # live nodes, and leaving it here would re-add an issuer that no longer
+    # resolves on any future machine-config render.
     admissionControl:
       - name: PodSecurity
         configuration: