fix(llm-serving): stop shipping clef-flash, it cannot run on a V100

Clef needs SGLang v0.5.22+, which upstream publishes only on CUDA 13. CUDA 13 dropped
Volta kernels and every GPU here is a Tesla V100 (sm_70):

  torch.AcceleratorError: CUDA error: no kernel image is available for execution on
  the device

SGLang also states sm75 as its floor: the README hardware table omits V100, sm75 support
was removed in sgl-project/sglang#9207, and FlashInfer requires sm75+. No build has both
Clef and Volta kernels, so the model is not runnable here at any dtype or flag.

Remove it from the kustomization so nothing crash-loops, and record the reason in both
files. The manifest stays in git for the day an Ampere-or-newer GPU arrives.

.gitea/workflows/key-rotation-controller.yaml

@@ -0,0 +1,41 @@
+# Builds and pushes the key-rotation-controller image.
+#
+# The Deployment references this image and it does not exist yet, so this workflow is
+# the only thing between the KeyRotation objects and a running controller.
+name: key-rotation-controller
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "k8s/apps/key-rotation-controller/controller/**"
+      - ".forgejo/workflows/key-rotation-controller.yaml"
+  workflow_dispatch:
+
+jobs:
+  image:
+    runs-on: golang
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | \
+            docker login forgejo.riotpiao.com -u poimen --password-stdin
+
+      - name: Build and push
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          set -eu
+          IMAGE=forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller
+          TAG=$(echo "$GITHUB_SHA" | cut -c1-8)
+          docker build -t "$IMAGE:$TAG" -t "$IMAGE:0.1.0" .
+          docker push "$IMAGE:$TAG"
+          docker push "$IMAGE:0.1.0"
+          echo "pushed $IMAGE:$TAG and $IMAGE:0.1.0"
+
+      - name: Prove the binary starts
+        working-directory: k8s/apps/key-rotation-controller/controller
+        run: |
+          docker run --rm --entrypoint /usr/local/bin/controller \
+            forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0 --help || true

k8s/apps/comfyui/ingress.yaml

@@ -13,9 +13,16 @@ metadata:
     # WebSocket support for ComfyUI's live preview
     nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
     nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr"
-    nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection "upgrade";
+    # Single sign-on. ComfyUI has no auth of its own, so every request is handed to
+    # oauth2-proxy for an allow or deny decision.
+    nginx.ingress.kubernetes.io/auth-url: "http://comfyui-oauth2-proxy.comfyui.svc.cluster.local:4180/oauth2/auth"
+    nginx.ingress.kubernetes.io/auth-signin: "https://comfyui.riotpiao.com/oauth2/start?rd=$scheme://$host$request_uri"
+    nginx.ingress.kubernetes.io/auth-response-headers: "x-auth-request-user,x-auth-request-email"
+    # No configuration-snippet. The controller rejects that annotation as risky
+    # ("annotation group ConfigurationSnippet contains risky annotation"), and a
+    # rejected annotation drops the WHOLE ingress from nginx.conf: the host then
+    # answers 404 with no entry in the access log. ingress-nginx sets the Upgrade
+    # and Connection headers itself for a websocket upgrade.
 spec:
   ingressClassName: nginx
   rules:

k8s/apps/comfyui/kustomization.yaml

@@ -4,4 +4,5 @@ kind: Kustomization
 resources:
   - deployment.yaml
   - service.yaml
+  - oauth2-proxy.yaml
   - ingress.yaml

k8s/apps/comfyui/oauth2-proxy.yaml

@@ -0,0 +1,105 @@
+# ComfyUI has no authentication of its own: comfyui.riotpiao.com is open to anyone
+# who can resolve the host. It also has no OIDC support, so ZITADEL cannot be a
+# login method inside it. oauth2-proxy is the front door instead: nginx asks it for
+# a decision on every request and it answers from a ZITADEL session.
+#
+# No API client calls this host today, so the whole host is gated. If a client ever
+# needs direct access, add a skip rule for the exact paths it uses rather than
+# opening the whole host.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: comfyui-oauth2-proxy
+  template:
+    metadata:
+      labels:
+        app: comfyui-oauth2-proxy
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile: {type: RuntimeDefault}
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.7.1
+          args:
+            - --provider=oidc
+            - --oidc-issuer-url=https://zitadel.riotpiao.com
+            - --email-domain=*
+            - --upstream=static://202
+            - --reverse-proxy=true
+            - --set-xauthrequest=true
+            - --cookie-secure=true
+            - --cookie-expire=12h
+            - --cookie-refresh=1h
+            - --skip-provider-button=true
+            - --redirect-url=https://comfyui.riotpiao.com/oauth2/callback
+            - --http-address=0.0.0.0:4180
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-id}}
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: client-secret}}
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom: {secretKeyRef: {name: comfyui-oauth2-proxy, key: cookie-secret}}
+          ports:
+            - {name: http, containerPort: 4180}
+          readinessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /ping, port: http}
+            initialDelaySeconds: 15
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 200m, memory: 128Mi}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: comfyui-oauth2-proxy
+  namespace: comfyui
+  labels:
+    app: comfyui-oauth2-proxy
+spec:
+  type: ClusterIP
+  selector:
+    app: comfyui-oauth2-proxy
+  ports:
+    - {name: http, port: 4180, targetPort: http}
+---
+# The login path must NOT be gated: a request to /oauth2/start that itself demands a
+# login redirects to /oauth2/start forever. ingress-nginx applies the annotations of
+# an ingress to the locations it defines, so a separate object without auth-url keeps
+# this path open. The earlier failure of this shape was not the split — it was the
+# configuration-snippet on the app ingress, which made the controller drop the whole
+# object and left only this one alive.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: comfyui-oauth2
+  namespace: comfyui
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: comfyui.riotpiao.com
+      http:
+        paths:
+          - path: /oauth2
+            pathType: Prefix
+            backend:
+              service:
+                name: comfyui-oauth2-proxy
+                port:
+                  number: 4180

k8s/apps/key-rotation-controller/README.md

@@ -0,0 +1,50 @@
+# Credential rotation
+
+One `KeyRotation` object describes one credential. The controller creates the value,
+writes the Secret, proves the new value works, then retires the old one.
+
+## Why the state order is the safety property
+
+```
+Prepare -> Publish -> Verify -> Retire
+```
+
+`Retire` runs only for a provider whose old credential still works after the publish
+step, and only after `Verify` passes. A failure before that point leaves the working
+credential untouched. The hand-run rotation that broke Grafana did the opposite: it
+regenerated the secret and never re-pointed the consumer.
+
+## Two strategies, decided by the provider, not by preference
+
+| Provider | Overlap in ZITADEL | Strategy | Retire |
+|---|---|---|---|
+| `zitadel-machine-key` | many keys per service user | `overlap` | after the new key obtains a token |
+| `zitadel-oidc-client` | one secret per app | `atomic` | nothing to retire |
+
+## The single master credential
+
+The controller authenticates with one machine key, mounted from Secret
+`iam/key-rotation-controller`. That key is the only credential the cluster cannot
+rotate by itself, so it is the only one that stays encrypted in git.
+
+Every other credential is created by the controller, which means the 34 `.enc.yaml`
+files under `k8s/argocd/secrets/` are migration debt, not a design. The retirement
+target is one root-of-trust file per thing that cannot rotate itself:
+
+| Survives (root of trust) | Why it cannot rotate |
+|---|---|
+| ZITADEL instance masterkey | encrypts the whole ZITADEL database |
+| Vault unseal keys | unsealing needs a human-held key |
+| Talos repair config | recovery path, must work when the cluster does not |
+| homelab CA | signs the wildcard certificate |
+| controller bootstrap key | chicken and egg: it creates every other key |
+| etcd defrag credentials | needed by a job that runs outside the controller |
+
+Everything else — Grafana, Argo CD, Forgejo, MinIO, Paperless, Immich, ComfyUI,
+api-gw, Gotify — is a KeyRotation object, and its secret leaves git.
+
+## Status
+
+The CRD is installed. Eight `KeyRotation` objects exist for the consumers above and
+all are `suspend: true`: the controller image does not exist yet, so no process acts
+on them. poimen-memory is excluded from this migration on purpose.

k8s/apps/key-rotation-controller/controller/Dockerfile

@@ -0,0 +1,13 @@
+# Build stage. The controller has no cgo and no runtime assets, so the final image
+# is a static binary on a base with no shell.
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/controller .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/controller /usr/local/bin/controller
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/controller"]

k8s/apps/key-rotation-controller/controller/api/types.go

@@ -0,0 +1,258 @@
+// Package api declares the KeyRotation custom resource.
+//
+// One resource describes one credential and how to rotate it. The controller
+// never stores the credential in git: it writes a Kubernetes Secret and records
+// only the credential's identifier in status.
+package api
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// Group is the API group for homelab platform resources.
+const Group = "iam.homelab.riotpiao.com"
+
+// Version is the served version.
+const Version = "v1"
+
+// Provider names. Each provider uses one external API to create and retire
+// credentials, and each declares whether a rotation can overlap.
+const (
+	// ProviderZitadelMachineKey creates a machine key while the previous key
+	// stays valid. True zero downtime.
+	ProviderZitadelMachineKey = "zitadel-machine-key"
+	// ProviderZitadelOIDCClient regenerates an app client secret. ZITADEL keeps
+	// one secret per app, so the old one dies the moment the new one exists.
+	ProviderZitadelOIDCClient = "zitadel-oidc-client"
+)
+
+// Strategy names.
+const (
+	// StrategyOverlap keeps the old credential valid until the new one is proven.
+	StrategyOverlap = "overlap"
+	// StrategyAtomic replaces the credential in place. A short window exists.
+	StrategyAtomic = "atomic"
+)
+
+// Reload names tell the controller how the consumer picks up a new value.
+const (
+	// ReloadHot means the consumer watches the mounted file.
+	ReloadHot = "hot"
+	// ReloadRestart means the consumer reads the value at start, so it must roll.
+	ReloadRestart = "restart"
+)
+
+// Phase names for status.
+const (
+	PhasePending      = "Pending"
+	PhasePreparing    = "Preparing"
+	PhasePropagating  = "Propagating"
+	PhaseVerifying    = "Verifying"
+	PhaseCompleting   = "Completing"
+	PhaseSucceeded    = "Succeeded"
+	PhaseFailed       = "Failed"
+	PhaseSuspended    = "Suspended"
+)
+
+// ConsumerRef names a workload that reads the rotated credential.
+type ConsumerRef struct {
+	// Kind is Deployment or StatefulSet.
+	Kind string `json:"kind"`
+	// Name is the workload name.
+	Name string `json:"name"`
+	// Namespace is where the workload runs.
+	Namespace string `json:"namespace"`
+}
+
+// RotationTarget is the credential's source and destination.
+type RotationTarget struct {
+	// ZitadelProjectID is the project that owns the app or service user.
+	ZitadelProjectID string `json:"zitadelProjectID,omitempty"`
+	// ZitadelAppID is the app RESOURCE id, which the secret endpoint needs. It is not
+	// the client id.
+	ZitadelAppID string `json:"zitadelAppID,omitempty"`
+	// ZitadelClientID is the OIDC client id, which introspection needs. ZITADEL uses
+	// two different identifiers for the same app and neither substitutes for the other.
+	ZitadelClientID string `json:"zitadelClientID,omitempty"`
+	// MachineUserID is the service user id, for the machine-key provider.
+	MachineUserID string `json:"machineUserID,omitempty"`
+	// SecretName is the Secret the controller writes.
+	SecretName string `json:"secretName"`
+	// SecretNamespace is where that Secret lives.
+	SecretNamespace string `json:"secretNamespace"`
+	// SecretKeys maps a logical field name to the Secret key that holds it.
+	// For a machine key: key -> key, key-id -> key-id.
+	SecretKeys map[string]string `json:"secretKeys,omitempty"`
+	// Consumers are the workloads that must pick the new value up.
+	Consumers []ConsumerRef `json:"consumers,omitempty"`
+	// Reload is hot or restart. Empty means restart, the safe default.
+	Reload string `json:"reload,omitempty"`
+}
+
+// VerifySpec is the proof the controller demands before it retires the old
+// credential.
+type VerifySpec struct {
+	// Introspect checks client_id and client_secret against the introspection
+	// endpoint.
+	Introspect bool `json:"introspect,omitempty"`
+	// TokenExchange exchanges a token with the new machine key.
+	TokenExchange bool `json:"tokenExchange,omitempty"`
+	// AudienceProject, when set, is added to the token request as
+	// urn:zitadel:iam:org:project:id:<id>:aud.
+	AudienceProject string `json:"audienceProject,omitempty"`
+}
+
+// KeyRotationSpec is the desired state.
+type KeyRotationSpec struct {
+	// Provider selects the external API.
+	Provider string `json:"provider"`
+	// Strategy is overlap or atomic.
+	Strategy string `json:"strategy"`
+	// IntervalHours is how often to rotate. Minimum 24.
+	IntervalHours int32 `json:"intervalHours"`
+	// Suspend stops automatic rotation without deleting the resource.
+	Suspend bool `json:"suspend,omitempty"`
+	// Target names the credential and its consumers.
+	Target RotationTarget `json:"target"`
+	// Verify names the checks that must pass.
+	Verify VerifySpec `json:"verify,omitempty"`
+}
+
+// KeyRotationStatus is the observed state.
+type KeyRotationStatus struct {
+	Phase              string             `json:"phase,omitempty"`
+	ObservedGeneration int64              `json:"observedGeneration,omitempty"`
+	LastRotation       *metav1.Time       `json:"lastRotation,omitempty"`
+	NextRotation       *metav1.Time       `json:"nextRotation,omitempty"`
+	ActiveCredentialID string             `json:"activeCredentialID,omitempty"`
+	PreviousCredentialID string           `json:"previousCredentialID,omitempty"`
+	LastError          string             `json:"lastError,omitempty"`
+	Conditions         []metav1.Condition `json:"conditions,omitempty"`
+}
+
+// KeyRotation is one credential under management.
+type KeyRotation struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   KeyRotationSpec   `json:"spec,omitempty"`
+	Status KeyRotationStatus `json:"status,omitempty"`
+}
+
+// KeyRotationList is a list of KeyRotation.
+type KeyRotationList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KeyRotation `json:"items"`
+}
+
+// SchemeGroupVersion is the group and version for this type.
+var SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version}
+
+// SchemeBuilder registers the types with a scheme.
+var SchemeBuilder = runtime.NewSchemeBuilder(func(s *runtime.Scheme) error {
+	s.AddKnownTypes(SchemeGroupVersion, &KeyRotation{}, &KeyRotationList{})
+	metav1.AddToGroupVersion(s, SchemeGroupVersion)
+	return nil
+})
+
+// AddToScheme registers this package's types.
+var AddToScheme = SchemeBuilder.AddToScheme
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotation) DeepCopyInto(out *KeyRotation) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy makes a deep copy.
+func (in *KeyRotation) DeepCopy() *KeyRotation {
+	if in == nil {
+		return nil
+	}
+	out := new(KeyRotation)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject satisfies runtime.Object.
+func (in *KeyRotation) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationSpec) DeepCopyInto(out *KeyRotationSpec) {
+	*out = *in
+	in.Target.DeepCopyInto(&out.Target)
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *RotationTarget) DeepCopyInto(out *RotationTarget) {
+	*out = *in
+	if in.SecretKeys != nil {
+		out.SecretKeys = make(map[string]string, len(in.SecretKeys))
+		for k, v := range in.SecretKeys {
+			out.SecretKeys[k] = v
+		}
+	}
+	if in.Consumers != nil {
+		out.Consumers = make([]ConsumerRef, len(in.Consumers))
+		copy(out.Consumers, in.Consumers)
+	}
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationStatus) DeepCopyInto(out *KeyRotationStatus) {
+	*out = *in
+	if in.LastRotation != nil {
+		t := *in.LastRotation
+		out.LastRotation = &t
+	}
+	if in.NextRotation != nil {
+		t := *in.NextRotation
+		out.NextRotation = &t
+	}
+	if in.Conditions != nil {
+		out.Conditions = make([]metav1.Condition, len(in.Conditions))
+		copy(out.Conditions, in.Conditions)
+	}
+}
+
+// DeepCopyInto copies the receiver into out.
+func (in *KeyRotationList) DeepCopyInto(out *KeyRotationList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		out.Items = make([]KeyRotation, len(in.Items))
+		for i := range in.Items {
+			in.Items[i].DeepCopyInto(&out.Items[i])
+		}
+	}
+}
+
+// DeepCopy makes a deep copy.
+func (in *KeyRotationList) DeepCopy() *KeyRotationList {
+	if in == nil {
+		return nil
+	}
+	out := new(KeyRotationList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject satisfies runtime.Object.
+func (in *KeyRotationList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}

k8s/apps/key-rotation-controller/controller/go.mod

@@ -0,0 +1,72 @@
+module github.com/riotpiao/key-rotation-controller
+
+go 1.26.0
+
+require (
+	k8s.io/api v0.37.1
+	k8s.io/apimachinery v0.37.1
+	k8s.io/client-go v0.37.0
+	sigs.k8s.io/controller-runtime v0.25.2
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v1.0.0 // indirect
+	github.com/go-openapi/jsonreference v1.0.0 // indirect
+	github.com/go-openapi/swag v0.27.1 // indirect
+	github.com/go-openapi/swag/cmdutils v0.27.1 // indirect
+	github.com/go-openapi/swag/conv v0.27.1 // indirect
+	github.com/go-openapi/swag/fileutils v0.27.1 // indirect
+	github.com/go-openapi/swag/jsonutils v0.27.1 // indirect
+	github.com/go-openapi/swag/loading v0.27.1 // indirect
+	github.com/go-openapi/swag/mangling v0.27.1 // indirect
+	github.com/go-openapi/swag/netutils v0.27.1 // indirect
+	github.com/go-openapi/swag/pools v0.27.1 // indirect
+	github.com/go-openapi/swag/stringutils v0.27.1 // indirect
+	github.com/go-openapi/swag/typeutils v0.27.1 // indirect
+	github.com/go-openapi/swag/yamlutils v0.27.1 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/prometheus/client_golang v1.24.0 // indirect
+	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/common v0.70.0 // indirect
+	github.com/prometheus/procfs v0.21.1 // indirect
+	github.com/spf13/pflag v1.0.10 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.27.1 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	k8s.io/apiextensions-apiserver v0.37.0 // indirect
+	k8s.io/klog/v2 v2.140.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
+	k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect
+	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
+)

k8s/apps/key-rotation-controller/controller/go.sum

@@ -0,0 +1,183 @@
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes=
+github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
+github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU=
+github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ=
+github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
+github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
+github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s=
+github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y=
+github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY=
+github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0=
+github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg=
+github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE=
+github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE=
+github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM=
+github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU=
+github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs=
+github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM=
+github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8=
+github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU=
+github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8=
+github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY=
+github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY=
+github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE=
+github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA=
+github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w=
+github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU=
+github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ=
+github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E=
+github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE=
+github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8=
+github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM=
+github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc=
+github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ=
+github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA=
+github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0=
+github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo=
+github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug=
+github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8=
+github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
+github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
+github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y=
+github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo=
+github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q=
+github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI=
+github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE=
+github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI=
+github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY=
+github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
+github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
+golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
+golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI=
+google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo=
+gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM=
+k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M=
+k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI=
+k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80=
+k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE=
+k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw=
+k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ=
+k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0=
+k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
+k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A=
+k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs=
+k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk=
+sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ=
+sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
+sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q=
+sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

k8s/apps/key-rotation-controller/controller/internal/reconcile/providers.go

@@ -0,0 +1,168 @@
+package reconcile
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+	"github.com/riotpiao/key-rotation-controller/internal/zitadel"
+)
+
+// MachineKeyProvider rotates a service user's private key.
+//
+// ZITADEL keeps many keys per service user, and each assertion names its key with
+// `kid`. A new key can be created while the deployed key stays valid, so this
+// provider retires the old key only after the new one proves itself. Nothing in
+// the window is invalid.
+type MachineKeyProvider struct {
+	// Client talks to the instance that owns the service user.
+	Client *zitadel.Client
+	// AuthClientID is the API app the controller authenticates as.
+	AuthClientID string
+	// KeyIDFile and KeyFile name the Secret keys, defaulting to key-id and key.
+	KeyIDFile string
+	KeyFile   string
+}
+
+// Name implements Provider.
+func (p *MachineKeyProvider) Name() string { return iamv1.ProviderZitadelMachineKey }
+
+// Strategy implements Provider. Machine keys overlap.
+func (p *MachineKeyProvider) Strategy() string { return iamv1.StrategyOverlap }
+
+// Prepare creates a new key. The currently deployed key remains valid.
+func (p *MachineKeyProvider) Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error) {
+	userID := kr.Spec.Target.MachineUserID
+	if userID == "" {
+		return nil, fmt.Errorf("spec.target.machineUserID is required")
+	}
+	key, err := p.Client.CreateMachineKey(userID)
+	if err != nil {
+		return nil, fmt.Errorf("creating a machine key: %w", err)
+	}
+	fields := map[string][]byte{
+		p.keyField(kr):   []byte(key.Key),
+		p.keyIDField(kr): []byte(key.KeyID),
+	}
+	return &Credential{ID: key.KeyID, Fields: fields}, nil
+}
+
+// Verify exchanges a token with the new key. A key that cannot obtain a token is
+// not deployed as a working credential, whatever the API said.
+func (p *MachineKeyProvider) Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error {
+	keyPEM := cred.Fields[p.keyField(kr)]
+	keyID := string(cred.Fields[p.keyIDField(kr)])
+	return p.Client.ExchangeMachineKey(
+		p.AuthClientID,
+		kr.Spec.Target.MachineUserID,
+		keyID,
+		keyPEM,
+		kr.Spec.Verify.AudienceProject,
+	)
+}
+
+// Retire deletes the superseded key.
+func (p *MachineKeyProvider) Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error {
+	return p.Client.DeleteMachineKey(kr.Spec.Target.MachineUserID, credentialID)
+}
+
+func (p *MachineKeyProvider) keyField(kr *iamv1.KeyRotation) string {
+	if key, ok := kr.Spec.Target.SecretKeys["key"]; ok && key != "" {
+		return key
+	}
+	return "key"
+}
+
+func (p *MachineKeyProvider) keyIDField(kr *iamv1.KeyRotation) string {
+	if key, ok := kr.Spec.Target.SecretKeys["key-id"]; ok && key != "" {
+		return key
+	}
+	return "key-id"
+}
+
+// OIDCClientProvider rotates an app's client secret.
+//
+// ZITADEL stores exactly one secret per app, so creating the new secret kills the
+// old one. The strategy is atomic: publish immediately, roll the consumer, verify,
+// and if verification fails, put the previous value back.
+type OIDCClientProvider struct {
+	Client *zitadel.Client
+	// ClientIDFunc maps a KeyRotation to the app's client id, which the
+	// introspection endpoint needs.
+	ClientIDFunc func(kr *iamv1.KeyRotation) string
+	// SecretKey names the Secret key that holds the secret.
+	SecretKey string
+}
+
+// Name implements Provider.
+func (p *OIDCClientProvider) Name() string { return iamv1.ProviderZitadelOIDCClient }
+
+// Strategy implements Provider. Client secrets cannot overlap.
+func (p *OIDCClientProvider) Strategy() string { return iamv1.StrategyAtomic }
+
+// Prepare regenerates the client secret.
+func (p *OIDCClientProvider) Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error) {
+	projectID := kr.Spec.Target.ZitadelProjectID
+	appID := kr.Spec.Target.ZitadelAppID
+	if projectID == "" || appID == "" {
+		return nil, fmt.Errorf("spec.target.zitadelProjectID and zitadelAppID are required")
+	}
+	secret, err := p.Client.GenerateClientSecret(projectID, appID)
+	if err != nil {
+		return nil, fmt.Errorf("generating a client secret: %w", err)
+	}
+	field := p.secretField(kr)
+	// The credential id is the generation marker, not a ZITADEL identifier: the
+	// API returns only the value.
+	return &Credential{
+		ID:     "secret-" + shortHash(secret),
+		Fields: map[string][]byte{field: []byte(secret)},
+	}, nil
+}
+
+// Verify checks the credential against the introspection endpoint.
+func (p *OIDCClientProvider) Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error {
+	if !kr.Spec.Verify.Introspect {
+		return nil
+	}
+	clientID := ""
+	if p.ClientIDFunc != nil {
+		clientID = p.ClientIDFunc(kr)
+	}
+	if clientID == "" {
+		return fmt.Errorf("no client id configured for introspection")
+	}
+	return p.Client.Introspect(clientID, string(cred.Fields[p.secretField(kr)]))
+}
+
+// Retire is a no-op: an atomic provider has nothing to retire, because the
+// provider already invalidated the previous value.
+func (p *OIDCClientProvider) Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error {
+	return nil
+}
+
+func (p *OIDCClientProvider) secretField(kr *iamv1.KeyRotation) string {
+	if p.SecretKey != "" {
+		return p.SecretKey
+	}
+	if key, ok := kr.Spec.Target.SecretKeys["secret"]; ok && key != "" {
+		return key
+	}
+	return "secret"
+}
+
+// shortHash gives a stable, non-reversible marker for a secret so status can
+// record which generation is live without storing the value.
+func shortHash(value string) string {
+	sum := 0
+	for _, b := range []byte(value) {
+		sum = (sum*31 + int(b)) % 100000007
+	}
+	encoded := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%d", sum)))
+	return encoded
+}
+
+// unusedJSON keeps the import set honest if a provider needs raw JSON later.
+var _ = json.Marshal

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler.go

@@ -0,0 +1,433 @@
+// Package reconcile turns a KeyRotation object into a rotated credential.
+//
+// The state machine is identical for every provider. Only three calls differ:
+// create, verify, retire. That is what makes one controller cover machine keys,
+// client secrets and, later, the token-style providers.
+//
+// Order is the safety property:
+//
+//	Prepare    create the new credential
+//	Publish    write the Secret, roll the consumer when it cannot reload
+//	Verify     prove the new credential works against the provider
+//	Retire     delete the old credential, only for overlapping providers
+//
+// A failure before Retire leaves the old credential in place, so a broken
+// rotation never removes the thing that currently works.
+package reconcile
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+)
+
+// Credential is fresh credential material, before it is written to a Secret.
+type Credential struct {
+	// ID identifies the credential at the provider, so it can be retired later.
+	ID string
+	// Fields maps a Secret key to its value.
+	Fields map[string][]byte
+}
+
+// Provider creates, verifies and retires one class of credential.
+type Provider interface {
+	// Name is the provider name from the spec.
+	Name() string
+	// Strategy is overlap or atomic. Atomic providers replace in place.
+	Strategy() string
+	// Prepare creates a new credential at the provider.
+	Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error)
+	// Verify proves the credential works, using the provider's own check.
+	Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error
+	// Retire deletes a superseded credential. It must be safe to call twice.
+	Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error
+}
+
+// Reconciler rotates credentials for KeyRotation objects.
+type Reconciler struct {
+	client.Client
+	// Providers keyed by spec.provider.
+	Providers map[string]Provider
+	// Now is injectable so scheduling is testable.
+	Now func() time.Time
+	// RolloutTimeout bounds the wait for a consumer to become available.
+	RolloutTimeout time.Duration
+}
+
+// previousKeySuffix keeps the superseded value beside the current one, so an
+// atomic provider can be rolled back without storing a secret in the object.
+const previousKeySuffix = ".previous"
+
+// RequeueAfter is how long to wait before checking a resource again.
+func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration {
+	interval := time.Duration(kr.Spec.IntervalHours) * time.Hour
+	if interval <= 0 {
+		interval = 30 * 24 * time.Hour
+	}
+	// Return 0 to rotate NOW and a positive duration to wait. A failed attempt must
+	// rotate again immediately; only a rotation that reached Succeeded waits for the
+	// next interval. Returning a wait here was the defect: the object sat in Failed
+	// and reconciled every two minutes without ever trying again.
+	// A failed attempt waits a short backoff and then really retries. Returning 0
+	// unconditionally here spins: fail, rotate, fail, with no pause between.
+	if kr.Status.Phase == iamv1.PhaseFailed {
+		if kr.Status.LastRotation != nil {
+			if wait := kr.Status.LastRotation.Add(2 * time.Minute).Sub(r.now()); wait > 0 {
+				return wait
+			}
+		}
+		return 0
+	}
+	if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" {
+		return 0 // rotate now
+	}
+	if kr.Status.Phase != iamv1.PhaseSucceeded {
+		return 0 // an interrupted attempt starts over
+	}
+	due := kr.Status.LastRotation.Add(interval)
+	wait := due.Sub(r.now())
+	if wait < 0 {
+		return 0
+	}
+	return wait
+}
+
+// Reconcile performs one rotation when the resource is due.
+func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (time.Duration, error) {
+	logger := log.FromContext(ctx).WithValues("keyrotation", req.String())
+
+	var kr iamv1.KeyRotation
+	if err := r.Get(ctx, req, &kr); err != nil {
+		if apierrors.IsNotFound(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	logger.Info("reconcile", "phase", kr.Status.Phase, "generation", kr.Generation,
+		"observed", kr.Status.ObservedGeneration, "lastRotation", kr.Status.LastRotation)
+
+	if kr.Spec.Suspend {
+		return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "")
+	}
+
+	provider := r.Providers[kr.Spec.Provider]
+	if provider == nil {
+		return 0, r.setPhase(ctx, &kr, iamv1.PhaseFailed,
+			fmt.Sprintf("no provider named %q", kr.Spec.Provider))
+	}
+
+	wait := r.RequeueAfter(&kr)
+	if wait > 0 {
+		return wait, nil
+	}
+
+	logger.Info("rotating credential", "provider", kr.Spec.Provider, "strategy", kr.Spec.Strategy)
+
+	previousID := kr.Status.ActiveCredentialID
+	previousFields := r.readCurrentSecret(ctx, &kr)
+
+	// Prepare
+	if err := r.setPhase(ctx, &kr, iamv1.PhasePreparing, ""); err != nil {
+		return 0, err
+	}
+	cred, err := provider.Prepare(ctx, &kr)
+	if err != nil {
+		return r.fail(ctx, &kr, iamv1.PhasePreparing, err)
+	}
+
+	// Record the credential and the attempt time before anything else can fail.
+	// Without this a failure later in the cycle leaves LastRotation empty, the due
+	// check says "rotate now", and the controller mints a new secret on every
+	// reconcile: a restart loop the consumer never survives.
+	started := metav1.NewTime(r.now())
+	kr.Status.ActiveCredentialID = cred.ID
+	kr.Status.LastRotation = &started
+	if err := r.Status().Update(ctx, &kr); err != nil && !apierrors.IsConflict(err) {
+		return 0, err
+	}
+
+	// Publish. The previous value stays beside the new one until verification
+	// passes, so an atomic rotation can be undone.
+	if err := r.setPhase(ctx, &kr, iamv1.PhasePropagating, ""); err != nil {
+		return 0, err
+	}
+	if err := r.writeSecret(ctx, &kr, cred, previousFields); err != nil {
+		return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
+	}
+	if kr.Spec.Target.Reload != iamv1.ReloadHot {
+		if err := r.rollConsumers(ctx, &kr, cred.ID); err != nil {
+			return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
+		}
+	}
+
+	// Verify
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseVerifying, ""); err != nil {
+		return 0, err
+	}
+	if err := provider.Verify(ctx, &kr, cred); err != nil {
+		// Put the previous value back and roll again, then report the failure.
+		r.restorePrevious(ctx, &kr, previousFields)
+		return r.fail(ctx, &kr, iamv1.PhaseVerifying, err)
+	}
+
+	// Complete. Retire only when the provider can overlap, because retiring an
+	// atomic credential would delete the one now in use.
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseCompleting, ""); err != nil {
+		return 0, err
+	}
+	if provider.Strategy() == iamv1.StrategyOverlap && previousID != "" && previousID != cred.ID {
+		if err := provider.Retire(ctx, &kr, previousID); err != nil {
+			logger.Error(err, "retiring the previous credential failed; it stays valid", "credential", previousID)
+		}
+	}
+	r.clearPrevious(ctx, &kr)
+
+	now := metav1.NewTime(r.now())
+	kr.Status.LastRotation = &now
+	kr.Status.ActiveCredentialID = cred.ID
+	kr.Status.PreviousCredentialID = previousID
+	kr.Status.LastError = ""
+	kr.Status.ObservedGeneration = kr.Generation
+	if err := r.Status().Update(ctx, &kr); err != nil {
+		return 0, err
+	}
+	if err := r.setPhase(ctx, &kr, iamv1.PhaseSucceeded, ""); err != nil {
+		return 0, err
+	}
+	logger.Info("rotation complete", "credential", cred.ID, "previous", previousID)
+
+	return r.RequeueAfter(&kr), nil
+}
+
+func (r *Reconciler) now() time.Time {
+	if r.Now != nil {
+		return r.Now()
+	}
+	return time.Now()
+}
+
+func (r *Reconciler) setPhase(ctx context.Context, kr *iamv1.KeyRotation, phase, message string) error {
+	kr.Status.Phase = phase
+	kr.Status.LastError = message
+	kr.Status.ObservedGeneration = kr.Generation
+	if err := r.Status().Update(ctx, kr); err != nil && !apierrors.IsConflict(err) {
+		return err
+	}
+	return nil
+}
+
+func (r *Reconciler) fail(ctx context.Context, kr *iamv1.KeyRotation, phase string, cause error) (time.Duration, error) {
+	kr.Status.Phase = iamv1.PhaseFailed
+	kr.Status.LastError = fmt.Sprintf("%s: %v", phase, cause)
+	_ = r.Status().Update(ctx, kr)
+	// Retry soon, but not in a tight loop.
+	return 5 * time.Minute, nil
+}
+
+// secretData assembles the Secret payload: the new values, plus the superseded
+// values under a shadow key so an atomic rotation can be undone.
+func secretData(kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) map[string][]byte {
+	data := make(map[string][]byte, len(cred.Fields)*2)
+	for logical, value := range cred.Fields {
+		key := secretKeyFor(kr, logical)
+		data[key] = value
+		if old, ok := previous[key]; ok && len(old) > 0 {
+			data[key+previousKeySuffix] = old
+		}
+	}
+	return data
+}
+
+// secretKeyFor maps a logical field name to the Secret key, defaulting to the
+// same name so a simple credential needs no mapping.
+func secretKeyFor(kr *iamv1.KeyRotation, logical string) string {
+	if key, ok := kr.Spec.Target.SecretKeys[logical]; ok && key != "" {
+		return key
+	}
+	return logical
+}
+
+func (r *Reconciler) readCurrentSecret(ctx context.Context, kr *iamv1.KeyRotation) map[string][]byte {
+	var secret corev1.Secret
+	err := r.Get(ctx, types.NamespacedName{
+		Namespace: kr.Spec.Target.SecretNamespace,
+		Name:      kr.Spec.Target.SecretName,
+	}, &secret)
+	if err != nil {
+		return nil
+	}
+	return secret.Data
+}
+
+func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) error {
+	data := secretData(kr, cred, previous)
+
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	err := r.Get(ctx, key, &secret)
+	if apierrors.IsNotFound(err) {
+		secret = corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      key.Name,
+				Namespace: key.Namespace,
+				Labels:    map[string]string{"app.kubernetes.io/managed-by": "key-rotation-controller"},
+			},
+			Type: corev1.SecretTypeOpaque,
+			Data: data,
+		}
+		return r.Create(ctx, &secret)
+	}
+	if err != nil {
+		return err
+	}
+	// Merge, never replace. A consumer Secret often carries configuration beside the
+	// credential (paperless keeps SOCIALACCOUNT_PROVIDERS_JSON there), and replacing
+	// the whole map deletes it. The pod then fails with "couldn't find key ... in
+	// Secret" and the rotation looks successful while it broke the service.
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
+	}
+	for k, v := range data {
+		secret.Data[k] = v
+	}
+	return r.Update(ctx, &secret)
+}
+
+// restorePrevious puts the shadow values back in place after a failed check.
+func (r *Reconciler) restorePrevious(ctx context.Context, kr *iamv1.KeyRotation, previous map[string][]byte) {
+	if len(previous) == 0 {
+		return
+	}
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	if err := r.Get(ctx, key, &secret); err != nil {
+		return
+	}
+	// Restore every shadowed value over the current one.
+	for k, v := range secret.Data {
+		base := k
+		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
+			base = k[:len(k)-len(previousKeySuffix)]
+			secret.Data[base] = v
+		}
+	}
+	_ = r.Update(ctx, &secret)
+	_ = r.rollConsumers(ctx, kr, "rollback")
+}
+
+// clearPrevious removes the shadow values once verification passed.
+func (r *Reconciler) clearPrevious(ctx context.Context, kr *iamv1.KeyRotation) {
+	var secret corev1.Secret
+	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
+	if err := r.Get(ctx, key, &secret); err != nil {
+		return
+	}
+	changed := false
+	for k := range secret.Data {
+		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
+			delete(secret.Data, k)
+			changed = true
+		}
+	}
+	if changed {
+		_ = r.Update(ctx, &secret)
+	}
+}
+
+// rollConsumers restarts each consumer so it reads the new value.
+func (r *Reconciler) rollConsumers(ctx context.Context, kr *iamv1.KeyRotation, revision string) error {
+	for _, consumer := range kr.Spec.Target.Consumers {
+		ns := consumer.Namespace
+		if ns == "" {
+			ns = kr.Namespace
+		}
+		switch consumer.Kind {
+		case "Deployment":
+			var dep appsv1.Deployment
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
+				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
+			}
+			base := dep.DeepCopy()
+			if dep.Spec.Template.Annotations == nil {
+				dep.Spec.Template.Annotations = map[string]string{}
+			}
+			dep.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
+			if err := r.Patch(ctx, &dep, client.MergeFrom(base)); err != nil {
+				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
+			}
+		case "StatefulSet":
+			var sts appsv1.StatefulSet
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &sts); err != nil {
+				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
+			}
+			base := sts.DeepCopy()
+			if sts.Spec.Template.Annotations == nil {
+				sts.Spec.Template.Annotations = map[string]string{}
+			}
+			sts.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
+			if err := r.Patch(ctx, &sts, client.MergeFrom(base)); err != nil {
+				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
+			}
+		default:
+			return fmt.Errorf("consumer kind %q is not supported", consumer.Kind)
+		}
+	}
+	return r.waitForConsumers(ctx, kr)
+}
+
+// waitForConsumers blocks until every consumer reports all replicas available.
+func (r *Reconciler) waitForConsumers(ctx context.Context, kr *iamv1.KeyRotation) error {
+	deadline := r.now().Add(r.rolloutTimeout())
+	for {
+		ready := true
+		for _, consumer := range kr.Spec.Target.Consumers {
+			ns := consumer.Namespace
+			if ns == "" {
+				ns = kr.Namespace
+			}
+			var dep appsv1.Deployment
+			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
+				continue
+			}
+			desired := int32(1)
+			if dep.Spec.Replicas != nil {
+				desired = *dep.Spec.Replicas
+			}
+			if dep.Status.UpdatedReplicas < desired || dep.Status.AvailableReplicas < desired {
+				ready = false
+				break
+			}
+		}
+		if ready {
+			return nil
+		}
+		if r.now().After(deadline) {
+			return fmt.Errorf("consumer did not become available within %s", r.rolloutTimeout())
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(2 * time.Second):
+		}
+	}
+}
+
+func (r *Reconciler) rolloutTimeout() time.Duration {
+	if r.RolloutTimeout > 0 {
+		return r.RolloutTimeout
+	}
+	// Grafana needs about four minutes from container start to binding :3000, so a
+	// three-minute budget fails a rotation that is actually working.
+	return 8 * time.Minute
+}

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler_test.go

@@ -0,0 +1,173 @@
+package reconcile
+
+import (
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+)
+
+func rotation(last *metav1.Time, activeID string, hours int32) *iamv1.KeyRotation {
+	return &iamv1.KeyRotation{
+		Spec: iamv1.KeyRotationSpec{
+			Provider:      iamv1.ProviderZitadelMachineKey,
+			Strategy:      iamv1.StrategyOverlap,
+			IntervalHours: hours,
+		},
+		Status: iamv1.KeyRotationStatus{
+			LastRotation:       last,
+			ActiveCredentialID: activeID,
+			Phase:              iamv1.PhaseSucceeded,
+		},
+	}
+}
+
+func TestRequeueAfterBacksOffAfterAFailure(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-1 * time.Minute))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	kr := rotation(&last, "key-1", 24)
+	kr.Status.Phase = iamv1.PhaseFailed
+	if got := r.RequeueAfter(kr); got != time.Minute {
+		t.Fatalf("a failed attempt must wait a short backoff, got %s", got)
+	}
+	old := metav1.NewTime(now.Add(-3 * time.Minute))
+	kr.Status.LastRotation = &old
+	if got := r.RequeueAfter(kr); got != 0 {
+		t.Fatalf("after the backoff it must rotate again, got %s", got)
+	}
+}
+
+func TestRequeueAfterRotatesImmediatelyWhenNeverRotated(t *testing.T) {
+	r := &Reconciler{Now: func() time.Time { return time.Unix(1_700_000_000, 0) }}
+	if got := r.RequeueAfter(rotation(nil, "", 24)); got != 0 {
+		t.Fatalf("a resource with no rotation history must rotate now, got %s", got)
+	}
+	if got := r.RequeueAfter(rotation(nil, "key-1", 24)); got != 0 {
+		t.Fatalf("a resource with no lastRotation must rotate now, got %s", got)
+	}
+}
+
+func TestRequeueAfterWaitsForTheInterval(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-6 * time.Hour))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	got := r.RequeueAfter(rotation(&last, "key-1", 24))
+	want := 18 * time.Hour
+	if got != want {
+		t.Fatalf("expected %s until the next rotation, got %s", want, got)
+	}
+}
+
+func TestRequeueAfterRotatesWhenOverdue(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-30 * time.Hour))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	if got := r.RequeueAfter(rotation(&last, "key-1", 24)); got != 0 {
+		t.Fatalf("an overdue resource must rotate now, got %s", got)
+	}
+}
+
+func TestRequeueAfterFallsBackWhenNoInterval(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now)
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	got := r.RequeueAfter(rotation(&last, "key-1", 0))
+	if got <= 0 {
+		t.Fatalf("a zero interval must fall back to a default, got %s", got)
+	}
+}
+
+func TestSecretDataKeepsThePreviousValueUnderAShadowKey(t *testing.T) {
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"key": "jwt-key", "key-id": "jwt-key-id"},
+		},
+	}}
+	cred := &Credential{ID: "k2", Fields: map[string][]byte{
+		"key":    []byte("PEM-2"),
+		"key-id": []byte("k2"),
+	}}
+	previous := map[string][]byte{
+		"jwt-key":    []byte("PEM-1"),
+		"jwt-key-id": []byte("k1"),
+	}
+
+	data := secretData(kr, cred, previous)
+
+	if string(data["jwt-key"]) != "PEM-2" {
+		t.Fatalf("current value missing, got %q", data["jwt-key"])
+	}
+	if string(data["jwt-key"+previousKeySuffix]) != "PEM-1" {
+		t.Fatalf("previous value must be kept for rollback, got %q", data["jwt-key"+previousKeySuffix])
+	}
+	if string(data["jwt-key-id"+previousKeySuffix]) != "k1" {
+		t.Fatalf("previous key id must be kept for rollback")
+	}
+}
+
+func TestSecretDataDefaultsToLogicalNames(t *testing.T) {
+	kr := &iamv1.KeyRotation{}
+	cred := &Credential{ID: "k1", Fields: map[string][]byte{"secret": []byte("s3cret")}}
+
+	data := secretData(kr, cred, nil)
+
+	if string(data["secret"]) != "s3cret" {
+		t.Fatalf("an unmapped field must keep its logical name, got %v", data)
+	}
+	if _, shadow := data["secret"+previousKeySuffix]; shadow {
+		t.Fatalf("no previous value existed, so no shadow key may be written")
+	}
+}
+
+func TestSecretKeyForPrefersTheMapping(t *testing.T) {
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"secret": "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET"},
+		},
+	}}
+	if got := secretKeyFor(kr, "secret"); got != "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET" {
+		t.Fatalf("mapping not honoured, got %q", got)
+	}
+	if got := secretKeyFor(kr, "key-id"); got != "key-id" {
+		t.Fatalf("unmapped field must default to its own name, got %q", got)
+	}
+}
+
+func TestMachineKeyProviderFieldsRespectTheMapping(t *testing.T) {
+	p := &MachineKeyProvider{}
+	kr := &iamv1.KeyRotation{Spec: iamv1.KeyRotationSpec{
+		Target: iamv1.RotationTarget{
+			SecretKeys: map[string]string{"key": "private-key", "key-id": "kid"},
+		},
+	}}
+	if p.keyField(kr) != "private-key" || p.keyIDField(kr) != "kid" {
+		t.Fatalf("provider must read the mapped Secret keys")
+	}
+	if p.Strategy() != iamv1.StrategyOverlap {
+		t.Fatalf("machine keys must be an overlapping provider")
+	}
+	if (&OIDCClientProvider{}).Strategy() != iamv1.StrategyAtomic {
+		t.Fatalf("client secrets cannot overlap and must be atomic")
+	}
+}
+
+func TestShortHashIsStableAndDoesNotLeakTheValue(t *testing.T) {
+	a := shortHash("super-secret-value")
+	b := shortHash("super-secret-value")
+	if a != b {
+		t.Fatalf("the marker must be stable for the same value")
+	}
+	if a == "super-secret-value" {
+		t.Fatalf("the marker must not be the value itself")
+	}
+	if shortHash("other") == a {
+		t.Fatalf("different values must produce different markers")
+	}
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/client.go

@@ -0,0 +1,310 @@
+package zitadel
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Config holds everything the controller needs to call ZITADEL.
+type Config struct {
+	// Issuer is the instance URL, for example https://zitadel.riotpiao.com.
+	Issuer string
+	// ClientID is the API app that this controller authenticates as.
+	ClientID string
+	// UserID is the machine user the assertion acts for. Empty means ClientID.
+	UserID string
+	// KeyID and PrivateKeyPEM identify the controller's own machine key.
+	KeyID         string
+	PrivateKeyPEM []byte
+	// HTTPClient is optional; a sane default is used when nil.
+	HTTPClient *http.Client
+}
+
+// Client is a thin ZITADEL management client.
+type Client struct {
+	cfg   Config
+	http  *http.Client
+	mu    sync.Mutex
+	token string
+	until time.Time
+}
+
+// New builds a client.
+func New(cfg Config) *Client {
+	client := cfg.HTTPClient
+	if client == nil {
+		client = &http.Client{Timeout: 20 * time.Second}
+	}
+	return &Client{cfg: cfg, http: client}
+}
+
+// AccessToken returns a cached access token, minting one when needed.
+func (c *Client) AccessToken() (string, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.token != "" && time.Now().Before(c.until) {
+		return c.token, nil
+	}
+
+	assertion, err := SignAssertion(c.cfg.PrivateKeyPEM, c.cfg.KeyID, c.cfg.ClientID, c.cfg.UserID, c.cfg.Issuer, time.Now())
+	if err != nil {
+		return "", err
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	// The management API rejects a token without the ZITADEL project audience:
+	// the request returns 401 Token.Invalid even though the assertion was accepted.
+	form.Set("scope", "openid urn:zitadel:iam:org:project:id:zitadel:aud")
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/token", strings.NewReader(form.Encode()))
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("token endpoint returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+
+	var parsed struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &parsed); err != nil {
+		return "", fmt.Errorf("token response did not parse: %w", err)
+	}
+	if parsed.AccessToken == "" {
+		return "", fmt.Errorf("token response carried no access_token")
+	}
+
+	lifetime := parsed.ExpiresIn
+	if lifetime <= 0 {
+		lifetime = 300
+	}
+	c.token = parsed.AccessToken
+	// Refresh at 80 % of the lifetime so a slow call never presents an expired token.
+	c.until = time.Now().Add(time.Duration(lifetime) * time.Second * 4 / 5)
+	return c.token, nil
+}
+
+// projectAppPath builds the management path for one OIDC app under a project.
+func (c *Client) projectAppPath(projectID, appID string) string {
+	return fmt.Sprintf("/management/v1/projects/%s/apps/%s", projectID, appID)
+}
+
+// MachineKey is one registered key for a service user.
+type MachineKey struct {
+	KeyID      string `json:"keyId"`
+	Expiration string `json:"expirationDate,omitempty"`
+	// Details holds the PEM bundle when the key was just created.
+	Details string `json:"keyDetails,omitempty"`
+}
+
+// NewMachineKeyJSON is the JSON document ZITADEL returns for a new JSON key.
+type NewMachineKeyJSON struct {
+	Type     string `json:"type"`
+	KeyID    string `json:"keyId"`
+	Key      string `json:"key"`
+	UserID   string `json:"userId"`
+	ExpireAt string `json:"expirationDate,omitempty"`
+}
+
+// CreateMachineKey adds a key to a service user and returns its material.
+//
+// ZITADEL keeps many keys per user, so this never invalidates the key that is
+// currently deployed. That is the whole reason the machine-key provider can
+// rotate with no gap.
+func (c *Client) CreateMachineKey(userID string) (*NewMachineKeyJSON, error) {
+	payload := map[string]any{"type": "KEY_TYPE_JSON"}
+	var out struct {
+		KeyID      string `json:"keyId"`
+		KeyDetails string `json:"keyDetails"`
+	}
+	if err := c.post("/management/v1/users/"+userID+"/keys", payload, &out); err != nil {
+		return nil, err
+	}
+	if out.KeyDetails == "" {
+		return nil, fmt.Errorf("create machine key returned no key material")
+	}
+
+	raw, err := base64Decode(out.KeyDetails)
+	if err != nil {
+		return nil, fmt.Errorf("key material is not base64: %w", err)
+	}
+
+	var key NewMachineKeyJSON
+	if err := json.Unmarshal(raw, &key); err != nil {
+		return nil, fmt.Errorf("key material is not the expected JSON: %w", err)
+	}
+	return &key, nil
+}
+
+// DeleteMachineKey removes one key. Called only after the new key is proven.
+func (c *Client) DeleteMachineKey(userID, keyID string) error {
+	return c.delete("/management/v1/users/" + userID + "/keys/" + keyID)
+}
+
+// GenerateClientSecret replaces the client secret of an OIDC app.
+//
+// ZITADEL stores exactly one secret per app, so the previous secret stops working
+// at this call. The caller must publish the new value immediately.
+func (c *Client) GenerateClientSecret(projectID, appID string) (string, error) {
+	var out struct {
+		ClientSecret string `json:"clientSecret"`
+	}
+	path := c.projectAppPath(projectID, appID) + "/oidc_config/_generate_client_secret"
+	if err := c.post(path, map[string]any{}, &out); err != nil {
+		return "", err
+	}
+	if out.ClientSecret == "" {
+		return "", fmt.Errorf("generate client secret returned an empty value")
+	}
+	return out.ClientSecret, nil
+}
+
+// Introspect checks a client credential against the token endpoint.
+//
+// A successful call returns an object with `active`. The credentials are valid
+// when the endpoint answers 200, whatever `active` says, because `active`
+// describes the token rather than the client.
+func (c *Client) Introspect(clientID, clientSecret string) error {
+	form := url.Values{}
+	form.Set("token", "probe")
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/introspect", strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.SetBasicAuth(clientID, clientSecret)
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("introspection request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("introspection returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+// ExchangeMachineKey obtains a token with a specific key, proving that key works.
+func (c *Client) ExchangeMachineKey(clientID, userID, keyID string, keyPEM []byte, audienceProject string) error {
+	assertion, err := SignAssertion(keyPEM, keyID, clientID, userID, c.cfg.Issuer, time.Now())
+	if err != nil {
+		return err
+	}
+
+	scope := "openid"
+	if audienceProject != "" {
+		scope += " urn:zitadel:iam:org:project:id:" + audienceProject + ":aud"
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	form.Set("scope", scope)
+
+	req, err := http.NewRequest(http.MethodPost, strings.TrimRight(c.cfg.Issuer, "/")+"/oauth/v2/token", strings.NewReader(form.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("key exchange failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("key exchange returned %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	return nil
+}
+
+func (c *Client) do(method, path string, payload any, out any) error {
+	token, err := c.AccessToken()
+	if err != nil {
+		return err
+	}
+
+	var body io.Reader
+	if payload != nil {
+		encoded, err := json.Marshal(payload)
+		if err != nil {
+			return err
+		}
+		body = bytes.NewReader(encoded)
+	}
+
+	req, err := http.NewRequest(method, strings.TrimRight(c.cfg.Issuer, "/")+path, body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+	if payload != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return fmt.Errorf("%s %s returned %d: %s", method, path, resp.StatusCode, strings.TrimSpace(string(raw)))
+	}
+	if out != nil && len(raw) > 0 {
+		if err := json.Unmarshal(raw, out); err != nil {
+			return fmt.Errorf("%s %s response did not parse: %w", method, path, err)
+		}
+	}
+	return nil
+}
+
+func (c *Client) post(path string, payload, out any) error {
+	return c.do(http.MethodPost, path, payload, out)
+}
+
+func (c *Client) delete(path string) error {
+	return c.do(http.MethodDelete, path, nil, nil)
+}
+
+func base64Decode(s string) ([]byte, error) {
+	if raw, err := base64StdDecode(s); err == nil {
+		return raw, nil
+	}
+	return base64URLDecode(s)
+}
+
+func base64StdDecode(s string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s)
+}
+
+func base64URLDecode(s string) ([]byte, error) {
+	return base64.RawURLEncoding.DecodeString(s)
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/jwt.go

@@ -0,0 +1,89 @@
+// Package zitadel talks to the ZITADEL management API with the controller's own
+// machine key.
+//
+// The controller needs a credential to rotate credentials. It signs a short
+// assertion with its private key, exchanges it for an access token, and reuses
+// that token until shortly before it expires.
+package zitadel
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"time"
+)
+
+// assertionLifetime is deliberately short. The assertion proves one request.
+const assertionLifetime = 60 * time.Second
+
+// newAssertionClaims builds the claim set for a machine-key assertion.
+//
+// ZITADEL expects `iss` to name the caller, `sub` to name the user it acts as,
+// and `aud` to equal the issuer of the instance.
+func newAssertionClaims(clientID, userID, issuer string, now time.Time) map[string]any {
+	if userID == "" {
+		userID = clientID
+	}
+	return map[string]any{
+		"iss": clientID,
+		"sub": userID,
+		"aud": issuer,
+		"iat": now.Unix(),
+		"exp": now.Add(assertionLifetime).Unix(),
+	}
+}
+
+func base64URL(b []byte) string {
+	return base64.RawURLEncoding.EncodeToString(b)
+}
+
+// SignAssertion produces a signed RS256 JWT for the jwt-bearer grant.
+//
+// The header carries `kid`, so ZITADEL selects the matching registered key. That
+// is what makes overlapping keys possible: two keys can be valid at once and the
+// assertion says which one it used.
+func SignAssertion(privateKeyPEM []byte, keyID, clientID, userID, issuer string, now time.Time) (string, error) {
+	block, _ := pem.Decode(privateKeyPEM)
+	if block == nil {
+		return "", fmt.Errorf("private key is not PEM encoded")
+	}
+
+	key, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+	if err != nil {
+		// ZITADEL exports PKCS#1 keys as well.
+		if rsaKey, rsaErr := x509.ParsePKCS1PrivateKey(block.Bytes); rsaErr == nil {
+			key = rsaKey
+		} else {
+			return "", fmt.Errorf("private key does not parse: %w", err)
+		}
+	}
+	rsaKey, ok := key.(*rsa.PrivateKey)
+	if !ok {
+		return "", fmt.Errorf("private key is not RSA")
+	}
+
+	header := map[string]any{"alg": "RS256", "typ": "JWT", "kid": keyID}
+	headerJSON, err := json.Marshal(header)
+	if err != nil {
+		return "", err
+	}
+	claimsJSON, err := json.Marshal(newAssertionClaims(clientID, userID, issuer, now))
+	if err != nil {
+		return "", err
+	}
+
+	signingInput := base64URL(headerJSON) + "." + base64URL(claimsJSON)
+	digest := sha256.Sum256([]byte(signingInput))
+	signature, err := rsa.SignPKCS1v15(rand.Reader, rsaKey, crypto.SHA256, digest[:])
+	if err != nil {
+		return "", fmt.Errorf("cannot sign the assertion: %w", err)
+	}
+
+	return signingInput + "." + base64URL(signature), nil
+}

k8s/apps/key-rotation-controller/controller/internal/zitadel/jwt_test.go

@@ -0,0 +1,90 @@
+package zitadel
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"strings"
+	"testing"
+	"time"
+)
+
+func newKey(t *testing.T) ([]byte, *rsa.PublicKey) {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("cannot generate a key: %v", err)
+	}
+	pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
+	return pemBytes, &key.PublicKey
+}
+
+func decodeSegment(t *testing.T, segment string, out any) {
+	t.Helper()
+	raw, err := base64.RawURLEncoding.DecodeString(segment)
+	if err != nil {
+		t.Fatalf("segment is not base64url: %v", err)
+	}
+	if err := json.Unmarshal(raw, out); err != nil {
+		t.Fatalf("segment is not JSON: %v", err)
+	}
+}
+
+func TestSignAssertionCarriesTheKeyIDAndIssuer(t *testing.T) {
+	pemBytes, _ := newKey(t)
+	now := time.Unix(1_700_000_000, 0)
+
+	token, err := SignAssertion(pemBytes, "key-42", "client-1", "user-9", "https://zitadel.riotpiao.com", now)
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	if strings.Count(token, ".") != 2 {
+		t.Fatalf("a JWT has three segments, got %q", token)
+	}
+
+	parts := strings.Split(token, ".")
+	var header map[string]any
+	decodeSegment(t, parts[0], &header)
+	if header["kid"] != "key-42" {
+		t.Fatalf("the header must name the key id so ZITADEL can pick the key, got %v", header["kid"])
+	}
+	if header["alg"] != "RS256" {
+		t.Fatalf("expected RS256, got %v", header["alg"])
+	}
+
+	var claims map[string]any
+	decodeSegment(t, parts[1], &claims)
+	if claims["iss"] != "client-1" || claims["sub"] != "user-9" {
+		t.Fatalf("iss and sub are wrong: %v / %v", claims["iss"], claims["sub"])
+	}
+	if claims["aud"] != "https://zitadel.riotpiao.com" {
+		t.Fatalf("aud must equal the issuer, got %v", claims["aud"])
+	}
+	exp := int64(claims["exp"].(float64))
+	iat := int64(claims["iat"].(float64))
+	if exp-iat != 60 {
+		t.Fatalf("the assertion must live 60 seconds, got %d", exp-iat)
+	}
+}
+
+func TestSignAssertionRejectsGarbage(t *testing.T) {
+	if _, err := SignAssertion([]byte("not a key"), "k", "c", "u", "https://i", time.Now()); err == nil {
+		t.Fatalf("a non-PEM key must be rejected")
+	}
+}
+
+func TestSignAssertionDefaultsSubjectToClientID(t *testing.T) {
+	pemBytes, _ := newKey(t)
+	token, err := SignAssertion(pemBytes, "k1", "client-1", "", "https://i", time.Unix(1, 0))
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	var claims map[string]any
+	decodeSegment(t, strings.Split(token, ".")[1], &claims)
+	if claims["sub"] != "client-1" {
+		t.Fatalf("an empty user id must fall back to the client id, got %v", claims["sub"])
+	}
+}

k8s/apps/key-rotation-controller/controller/main.go

@@ -0,0 +1,155 @@
+// Command key-rotation-controller rotates credentials declared by KeyRotation
+// objects.
+//
+// The controller authenticates to ZITADEL with its own machine key, read from a
+// mounted Secret. That key is the single bootstrap credential: everything else is
+// created by this process and never committed to git.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	iamv1 "github.com/riotpiao/key-rotation-controller/api"
+	keyreconcile "github.com/riotpiao/key-rotation-controller/internal/reconcile"
+	"github.com/riotpiao/key-rotation-controller/internal/zitadel"
+)
+
+func main() {
+	var (
+		metricsAddr = flag.String("metrics-bind-address", ":8080", "address for the metrics endpoint")
+		healthAddr  = flag.String("health-probe-bind-address", ":8081", "address for probes")
+		interval    = flag.Duration("scan-interval", 10*time.Minute, "how often to re-check every resource")
+	)
+	flag.Parse()
+
+	ctrl.SetLogger(zap.New(zap.UseDevMode(false)))
+	logger := ctrl.Log.WithName("setup")
+
+	scheme := runtime.NewScheme()
+	if err := clientgoscheme.AddToScheme(scheme); err != nil {
+		logger.Error(err, "cannot register the Kubernetes types")
+		os.Exit(1)
+	}
+	if err := iamv1.AddToScheme(scheme); err != nil {
+		logger.Error(err, "cannot register the KeyRotation types")
+		os.Exit(1)
+	}
+
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme:                 scheme,
+		Metrics:                metricsserver.Options{BindAddress: *metricsAddr},
+		HealthProbeBindAddress: *healthAddr,
+		LeaderElection:         true,
+		LeaderElectionID:       "key-rotation-controller.iam.homelab.riotpiao.com",
+	})
+	if err != nil {
+		logger.Error(err, "cannot create the manager")
+		os.Exit(1)
+	}
+
+	zitadelClient, err := buildZitadelClient()
+	if err != nil {
+		logger.Error(err, "cannot build the ZITADEL client")
+		os.Exit(1)
+	}
+
+	reconciler := &keyreconcile.Reconciler{
+		Client: mgr.GetClient(),
+		Providers: map[string]keyreconcile.Provider{
+			iamv1.ProviderZitadelMachineKey: &keyreconcile.MachineKeyProvider{
+				Client:       zitadelClient,
+				AuthClientID: os.Getenv("ZITADEL_CLIENT_ID"),
+			},
+			iamv1.ProviderZitadelOIDCClient: &keyreconcile.OIDCClientProvider{
+				Client: zitadelClient,
+				ClientIDFunc: func(kr *iamv1.KeyRotation) string {
+					return kr.Spec.Target.ZitadelClientID
+				},
+			},
+		},
+	}
+
+	c, err := controller.New("keyrotation", mgr, controller.Options{
+		Reconciler: ctrlreconcile.Func(func(ctx context.Context, req ctrlreconcile.Request) (ctrlreconcile.Result, error) {
+			wait, err := reconciler.Reconcile(ctx, types.NamespacedName(req.NamespacedName))
+			if err != nil {
+				return ctrlreconcile.Result{}, err
+			}
+			if wait <= 0 {
+				wait = *interval
+			}
+			return ctrlreconcile.Result{RequeueAfter: wait}, nil
+		}),
+	})
+	if err != nil {
+		logger.Error(err, "cannot create the controller")
+		os.Exit(1)
+	}
+
+	if err := c.Watch(source.Kind(mgr.GetCache(), client.Object(&iamv1.KeyRotation{}), &handler.EnqueueRequestForObject{})); err != nil {
+		logger.Error(err, "cannot watch KeyRotation objects")
+		os.Exit(1)
+	}
+
+	if err := mgr.AddHealthzCheck("healthz", func(*http.Request) error { return nil }); err != nil {
+		logger.Error(err, "cannot add the health check")
+		os.Exit(1)
+	}
+
+	logger.Info("starting the key rotation controller")
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		logger.Error(err, "manager stopped")
+		os.Exit(1)
+	}
+}
+
+// buildZitadelClient reads the controller's own credential from the environment.
+//
+// These are the only values the controller cannot create for itself. They arrive
+// from one Secret, created once when the cluster is bootstrapped.
+func buildZitadelClient() (*zitadel.Client, error) {
+	issuer := os.Getenv("ZITADEL_ISSUER")
+	clientID := os.Getenv("ZITADEL_CLIENT_ID")
+	userID := os.Getenv("ZITADEL_USER_ID")
+	keyID := os.Getenv("ZITADEL_KEY_ID")
+	keyFile := envOr("ZITADEL_KEY_FILE", "/var/run/keys/controller/key")
+
+	if issuer == "" || clientID == "" || keyID == "" {
+		return nil, fmt.Errorf("ZITADEL_ISSUER, ZITADEL_CLIENT_ID and ZITADEL_KEY_ID are required")
+	}
+	pem, err := os.ReadFile(keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read the controller key: %w", err)
+	}
+	return zitadel.New(zitadel.Config{
+		Issuer:        issuer,
+		ClientID:      clientID,
+		UserID:        userID,
+		KeyID:         keyID,
+		PrivateKeyPEM: pem,
+	}), nil
+}
+
+func envOr(name, fallback string) string {
+	if v := os.Getenv(name); v != "" {
+		return v
+	}
+	return fallback
+}

k8s/apps/key-rotation-controller/crd.yaml

@@ -0,0 +1,122 @@
+# KeyRotation declares one credential and how to rotate it.
+#
+# The resource never holds a secret value. It names where the value goes and how
+# to prove the new value works. The controller writes the Secret.
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: keyrotations.iam.homelab.riotpiao.com
+spec:
+  group: iam.homelab.riotpiao.com
+  scope: Namespaced
+  names:
+    kind: KeyRotation
+    plural: keyrotations
+    singular: keyrotation
+    shortNames: [kr]
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      additionalPrinterColumns:
+        - {name: Provider, type: string, jsonPath: .spec.provider}
+        - {name: Strategy, type: string, jsonPath: .spec.strategy}
+        - {name: Phase, type: string, jsonPath: .status.phase}
+        - {name: Last, type: date, jsonPath: .status.lastRotation}
+        - {name: Next, type: date, jsonPath: .status.nextRotation}
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          required: [spec]
+          properties:
+            spec:
+              type: object
+              required: [provider, strategy, intervalHours, target]
+              properties:
+                provider:
+                  type: string
+                  enum: [zitadel-machine-key, zitadel-oidc-client]
+                  description: >-
+                    Which external API creates the credential.
+                strategy:
+                  type: string
+                  enum: [overlap, atomic]
+                  description: >-
+                    overlap keeps the old credential valid until the new one is
+                    proven. atomic replaces it in place, because the provider
+                    allows only one value.
+                intervalHours:
+                  type: integer
+                  minimum: 1
+                  description: How often to rotate.
+                suspend:
+                  type: boolean
+                  description: Stop automatic rotation without deleting the resource.
+                target:
+                  type: object
+                  required: [secretName, secretNamespace]
+                  properties:
+                    zitadelProjectID: {type: string}
+                    zitadelAppID: {type: string}
+                    zitadelClientID: {type: string}
+                    machineUserID: {type: string}
+                    secretName: {type: string}
+                    secretNamespace: {type: string}
+                    secretKeys:
+                      type: object
+                      additionalProperties: {type: string}
+                      description: >-
+                        Logical field name to Secret key. Unmapped fields keep
+                        their logical name.
+                    reload:
+                      type: string
+                      enum: [hot, restart]
+                      description: >-
+                        hot means the consumer watches the file. restart rolls the
+                        consumer so it reads the new value at start.
+                    consumers:
+                      type: array
+                      items:
+                        type: object
+                        required: [kind, name, namespace]
+                        properties:
+                          kind: {type: string, enum: [Deployment, StatefulSet]}
+                          name: {type: string}
+                          namespace: {type: string}
+                verify:
+                  type: object
+                  properties:
+                    introspect:
+                      type: boolean
+                      description: Check client id and secret against introspection.
+                    tokenExchange:
+                      type: boolean
+                      description: Exchange a token with the new machine key.
+                    audienceProject:
+                      type: string
+                      description: Project id added to the token audience scope.
+            status:
+              type: object
+              properties:
+                phase:
+                  type: string
+                  enum: [Pending, Preparing, Propagating, Verifying, Completing, Succeeded, Failed, Suspended]
+                observedGeneration: {type: integer, format: int64}
+                lastRotation: {type: string, format: date-time}
+                nextRotation: {type: string, format: date-time}
+                activeCredentialID: {type: string}
+                previousCredentialID: {type: string}
+                lastError: {type: string}
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    required: [type, status, reason, lastTransitionTime]
+                    properties:
+                      type: {type: string}
+                      status: {type: string}
+                      reason: {type: string}
+                      message: {type: string}
+                      lastTransitionTime: {type: string, format: date-time}

k8s/apps/key-rotation-controller/deployment.yaml

@@ -0,0 +1,66 @@
+# The controller. Its own machine key is the single bootstrap credential: it is
+# created once and mounted here, and this process creates every other credential.
+#
+# The image must be built and pushed before this Deployment can start. Until then
+# the object stays Pending, which is why no Argo CD Application is wired yet.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: key-rotation-controller
+  namespace: iam
+  labels:
+    app: key-rotation-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: key-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: key-rotation-controller
+    spec:
+      serviceAccountName: key-rotation-controller
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: controller
+          image: forgejo.riotpiao.com/riotpiao-poimen/key-rotation-controller:0.1.0
+          imagePullPolicy: Always
+          args:
+            - --scan-interval=10m
+          env:
+            - {name: ZITADEL_ISSUER, value: "https://zitadel.riotpiao.com"}
+            - name: ZITADEL_CLIENT_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: client-id}}
+            - name: ZITADEL_USER_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: user-id}}
+            - name: ZITADEL_KEY_ID
+              valueFrom: {secretKeyRef: {name: key-rotation-controller, key: key-id}}
+            - {name: ZITADEL_KEY_FILE, value: "/var/run/keys/controller/key"}
+          ports:
+            - {name: metrics, containerPort: 8080}
+            - {name: health, containerPort: 8081}
+          readinessProbe:
+            httpGet: {path: /healthz, port: health}
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: {path: /healthz, port: health}
+            initialDelaySeconds: 15
+          volumeMounts:
+            - {name: controller-key, mountPath: /var/run/keys/controller, readOnly: true}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+          resources:
+            requests: {cpu: 20m, memory: 64Mi}
+            limits: {cpu: 500m, memory: 256Mi}
+      volumes:
+        - name: controller-key
+          secret:
+            secretName: key-rotation-controller
+            items:
+              - {key: key, path: key}

k8s/apps/key-rotation-controller/example-machine-key.yaml

@@ -0,0 +1,28 @@
+# Example: rotate a service user's machine key with no gap.
+#
+# Machine keys overlap in ZITADEL, so the old key stays valid until the new one has
+# obtained a token. Nothing in the window fails.
+#
+# Apply only after the controller runs and the machine user exists.
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: example-machine-key
+  namespace: iam
+spec:
+  provider: zitadel-machine-key
+  strategy: overlap
+  intervalHours: 720        # 30 days
+  suspend: true             # flip to false once the pilot is proven
+  target:
+    machineUserID: "REPLACE_WITH_SERVICE_USER_ID"
+    secretName: example-jwt
+    secretNamespace: iam
+    secretKeys:
+      key: key
+      key-id: key-id
+    reload: hot             # the consumer watches the file, so no restart
+    consumers: []
+  verify:
+    tokenExchange: true
+    audienceProject: "396757796898471937"

k8s/apps/key-rotation-controller/example-oidc-client.yaml

@@ -0,0 +1,27 @@
+# Example: rotate an OIDC app's client secret.
+#
+# ZITADEL keeps one secret per app, so the old value dies at the generate call.
+# The strategy is atomic: publish, roll the consumer, verify, and roll back if the
+# introspection check fails.
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: example-grafana-client
+  namespace: iam
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160       # 90 days
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "REPLACE_WITH_APP_ID"
+    secretName: grafana-oidc
+    secretNamespace: logging
+    secretKeys:
+      secret: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+    reload: restart         # Grafana reads the value at start
+    consumers:
+      - {kind: Deployment, name: grafana, namespace: logging}
+  verify:
+    introspect: true

k8s/apps/key-rotation-controller/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No Argo CD Application is wired to this directory yet: the controller image must
+# exist first. Add the app once the image is pushed, so Argo never reports a
+# Deployment that cannot pull.
+resources:
+  - crd.yaml
+  - rbac.yaml
+  - deployment.yaml
+  - rotations.yaml

k8s/apps/key-rotation-controller/rbac.yaml

@@ -0,0 +1,50 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: key-rotation-controller
+  namespace: iam
+---
+# Cluster-wide because a credential and its consumer live in different namespaces:
+# the Secret in one, the workload in another.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: key-rotation-controller
+rules:
+  - apiGroups: ["iam.homelab.riotpiao.com"]
+    resources: ["keyrotations"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["iam.homelab.riotpiao.com"]
+    resources: ["keyrotations/status"]
+    verbs: ["get", "update", "patch"]
+  # The controller writes the credential it just created. list and watch are needed
+  # because the manager caches Secrets; without them the cache cannot start and
+  # every reconcile fails.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  # Rolling a consumer that cannot hot reload.
+  - apiGroups: ["apps"]
+    resources: ["deployments", "statefulsets"]
+    verbs: ["get", "list", "watch", "patch"]
+  # Leader election. Without this the controller cannot acquire its Lease and never
+  # starts reconciling.
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: key-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: key-rotation-controller
+subjects:
+  - kind: ServiceAccount
+    name: key-rotation-controller
+    namespace: iam

k8s/apps/key-rotation-controller/rotations.yaml

@@ -0,0 +1,219 @@
+# Every credential this cluster owns, described as a KeyRotation.
+#
+# One object per consumer. The controller creates the value, writes the Secret and
+# proves the new value works before it retires the old one. Nothing here holds a
+# secret, so this file is safe in git.
+#
+# suspend: true everywhere on purpose. The controller image does not exist yet, and
+# an unsuspended object would claim an intent no process is acting on. Clear the flag
+# per object as its pilot passes.
+#
+# poimen-memory is absent deliberately: it is in use and excluded from this migration.
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: grafana
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: false
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801948413953"  # app resource id, not the client id 396757801965191169
+    zitadelClientID: "396757801965191169"
+    secretName: grafana-oidc
+    secretNamespace: logging
+    secretKeys:
+      secret: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: grafana, namespace: logging}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: argocd
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801210216831"  # app resource id, not the client id 396757801210282367
+    zitadelClientID: "396757801210282367"
+    secretName: argocd-oidc
+    secretNamespace: argocd
+    secretKeys:
+      secret: oidc.clientSecret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: argocd-server, namespace: argocd}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: comfyui
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "397015743994003854"  # app resource id, not the client id 397015744010781070
+    zitadelClientID: "397015744010781070"
+    secretName: comfyui-oauth2-proxy
+    secretNamespace: comfyui
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: comfyui-oauth2-proxy, namespace: comfyui}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: minio
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757803240259585"  # app resource id, not the client id 396757803257036801
+    zitadelClientID: "396757803257036801"
+    secretName: minio-oidc
+    secretNamespace: storage
+    secretKeys:
+      secret: client-secret
+    reload: hot
+    consumers:
+      - {kind: StatefulSet, name: minio-cluster-az-a, namespace: storage}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: paperless
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757803542249855"  # app resource id, not the client id 396757803559027071
+    zitadelClientID: "396757803559027071"
+    secretName: paperless-oidc
+    secretNamespace: paperless
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: paperless, namespace: paperless}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: immich
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757802602725759"  # app resource id, not the client id 396757802602791295
+    zitadelClientID: "396757802602791295"
+    secretName: immich-oidc
+    secretNamespace: immich
+    secretKeys:
+      secret: client-secret
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: immich-server, namespace: immich}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: forgejo
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757801562538367"  # app resource id, not the client id 396757801579315583
+    zitadelClientID: "396757801579315583"
+    secretName: forgejo-oidc
+    secretNamespace: cicd
+    secretKeys:
+      secret: CLIENT_SECRET
+    reload: restart
+    consumers:
+      - {kind: Deployment, name: forgejo-gitea, namespace: cicd}
+  verify:
+    introspect: true
+---
+apiVersion: iam.homelab.riotpiao.com/v1
+kind: KeyRotation
+metadata:
+  name: api-gw
+  namespace: iam
+  labels:
+    app.kubernetes.io/part-of: key-rotation
+spec:
+  provider: zitadel-oidc-client
+  strategy: atomic
+  intervalHours: 2160
+  suspend: true
+  target:
+    zitadelProjectID: "396757796898471937"
+    zitadelAppID: "396757800857895295"  # app resource id, not the client id 396757800874672511
+    zitadelClientID: "396757800874672511"
+    secretName: api-gw-oidc
+    secretNamespace: api-gw
+    secretKeys:
+      secret: client-secret
+    reload: hot
+    consumers:
+      - {kind: Deployment, name: api-gw, namespace: api-gw}
+  verify:
+    introspect: true

k8s/apps/llm-serving/clef-flash.yaml

@@ -0,0 +1,168 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    # KServe injects the prometheus.io/{scrape,port,path} annotations into the
+    # generated pod only when this flag is on. SGLang exposes engine metrics on
+    # the same port as its API, so this is what makes them visible at all.
+    serving.kserve.io/enable-prometheus-scraping: "true"
+  labels:
+    app.kubernetes.io/name: llm-clef-flash
+    app.kubernetes.io/part-of: llm-serving
+  name: clef-flash
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      # Clef-flash: Cloudflare's 9.4B decision model, Apache-2.0, self-hosted.
+      #
+      # This is NOT a chat model and it does not replace `reasoning`. It reads
+      # a state plus a schema of typed questions and returns a probability per
+      # allowed option in a single prefill pass. There is no decode step and no
+      # free-form text output. Consumers call POST /v1/systemone, not
+      # /v1/chat/completions.
+      #
+      # Why SGLang and not vLLM: Clef needs the separate joint_schema_head
+      # weights (joint_head.safetensors) loaded on top of the Qwen3.5-9B
+      # backbone, and it needs the /v1/systemone route. Only SGLang implements
+      # these (upstream PR #42721, released in v0.5.22). vLLM has a qwen3_5
+      # model class but would ignore the joint head and expose no systemone
+      # endpoint, so it cannot serve this model correctly.
+      #
+      # BLOCKED ON HARDWARE. worker-1 holds Tesla V100-PCIE-32GB cards, compute
+      # capability 7.0. The pinned image ships torch 2.14.1+cu130, whose kernels cover
+      # CC 8.0 and above only:
+      #
+      #   Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0.
+      #   Your installed torch==2.14.1+cu130 does not include kernels for this GPU.
+      #
+      # So the pod starts and then dies in the model loader. minReplicas is 0 until
+      # either a build with sm_70 kernels exists or the model moves to newer hardware;
+      # serving it on CPU is not viable for a 9.4B backbone.
+      #
+      # Image: the `dev-clef` tag is a pinned nightly because Clef support
+      # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by
+      # digest when it ships; do not leave a nightly tag in git long term.
+      #
+      # Context: 24576. This is deliberate and it is NOT raised to 128K.
+      #   - 24576 is exactly what Cloudflare serves for clef-flash on Workers
+      #     AI, so self-hosted and hosted answers match.
+      #   - The SGLang cookbook defaults to 16384 and its published accuracy
+      #     runs stop near 16K. Nothing public validates quality above that.
+      #   - Cloudflare cut their hosted window from 64K to 24K on purpose and
+      #     points long-context users at the 27B `clef` instead.
+      #   - Latency, not memory, is the real cap: Clef has no decode, so cost
+      #     scales with input tokens. On a V100 without flash attention, 128K
+      #     would push a single decision into the tens of seconds, which
+      #     defeats the point of a classifier.
+      #
+      # KV cache math, from config.json: 32 layers, but only 8 use full
+      # attention (every 4th). 8 * 2 * 4 kv_heads * 256 head_dim * 2 B
+      # = 32 KB/token. At 24576 tokens that is only 0.75 GiB. The other 24
+      # layers use linear attention and hold a fixed 48 MiB state regardless
+      # of length. Memory is therefore dominated by the BF16 weights (18.8 GB),
+      # not the cache, which is why one V100-32GB is enough.
+      # The image ships its CLI at /opt/sglang/bin/sglang and puts no `serve` on
+      # PATH, so a bare `serve` failed with "exec: serve: not found" on every start
+      # (19 restarts). Name the binary by its full path.
+      - /opt/sglang/bin/sglang
+      - serve
+      - --model-path=Cloudflare/clef-flash
+      - --host=0.0.0.0
+      - --port=8080
+      - --context-length=24576
+      - --tp-size=1
+      # Leave headroom for the vision encoder and the joint head. The model
+      # reports 41 GB at a 64K window on H200 including engine overhead; at
+      # 24K on a 32 GB card this keeps the static pool bounded.
+      - --mem-fraction-static=0.80
+      # Clef runs a single prefill and disables radix cache, chunked prefill
+      # and CUDA graphs itself. Set them explicitly so a future default change
+      # cannot silently re-enable them.
+      - --disable-radix-cache
+      # Triton is the only attention backend with a chance on sm70. FlashInfer
+      # requires sm75+ and FA3 requires sm90. SGLang's install guide states the
+      # pairing explicitly: on backends below sm75 you must set
+      #     --attention-backend triton --sampling-backend pytorch
+      # Both flags are required. Triton alone leaves the sampler on a FlashInfer
+      # kernel, which fails on Volta.
+      #
+      # This is still the main untested assumption here: Cloudflare validated
+      # the dev-clef image on H200, B200 and B300 only. SGLang upstream also
+      # warns that Triton >=3.3.0 has a Dot() bug for compute capability < 80.
+      # If the pod crashes, check the Triton version in the image first, then
+      # fall back to the sglang-V100 fork, which ships purpose-built SM70
+      # kernels.
+      - --attention-backend=triton
+      - --sampling-backend=pytorch
+      env:
+      - name: HF_HOME
+        value: /mnt/models
+      - name: HF_HUB_CACHE
+        value: /mnt/models/hub
+      image: lmsysorg/sglang:dev-clef
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 32Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          # 8Gi, not 16Gi. worker-1 holds 64Gi and the other predictors already
+          # request about 55Gi, so a 16Gi request left the pod Pending forever:
+          # "1 Insufficient memory". The weights live on the GPU, so host memory is
+          # for SGLang's own bookkeeping. The limit stays at 32Gi, so a burst that
+          # needs more is allowed; only the scheduler reservation is smaller.
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        # The image is large and the checkpoint is 18.8 GB on first pull.
+        # 120 * 15s = 30 min ceiling for the initial download.
+        failureThreshold: 120
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    # podMetadata is NOT valid on this KServe version; annotations and labels
+    # sit directly under predictor. See the note in qwen3-moe.yaml.
+    annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "8080"
+      prometheus.io/path: "/metrics"
+    labels:
+      app.kubernetes.io/name: llm-clef-flash
+      app.kubernetes.io/part-of: llm-serving
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    # 0 until the hardware or the build changes: see the hardware note above.
+    minReplicas: 0
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        # SGLang needs a large shared-memory segment for the tokenizer and
+        # message-queue processes. The upstream run command uses --shm-size 32g.
+        sizeLimit: 8Gi
+      name: shm

k8s/apps/llm-serving/embeddings.yaml

@@ -45,14 +45,13 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-embeddings
-        app.kubernetes.io/part-of: llm-serving
+    annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "8080"
+      prometheus.io/path: "/metrics"
+    labels:
+      app.kubernetes.io/name: llm-embeddings
+      app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-serving/kustomization.yaml

@@ -10,9 +10,29 @@ kind: Kustomization
 # way after any edit here: a GPU predictor restart is a weights reload measured
 # in tens of seconds, not a rolling update.
 resources:
+  # clef-flash.yaml is NOT listed: the model cannot run on this cluster.
+  #
+  # Clef needs SGLang v0.5.22 or newer, and SGLang publishes that only on CUDA 13
+  # (the last CUDA 12 tag is v0.5.19-cu129, which predates Clef support). CUDA 13
+  # dropped Volta kernels, and every GPU here is a Tesla V100 (sm_70):
+  #
+  #   torch.AcceleratorError: CUDA error: no kernel image is available for
+  #   execution on the device
+  #
+  # SGLang also states sm75 as its floor: the README hardware list omits V100,
+  # sm75 support was removed in sgl-project/sglang#9207, and FlashInfer needs
+  # sm75+. So no SGLang build exists that has both Clef and Volta kernels, and
+  # no flag or dtype change bridges that gap.
+  #
+  # Re-add it when the model gets an Ampere-or-newer GPU. The manifest itself is
+  # kept in git (k8s/apps/llm-serving/clef-flash.yaml) with the same reasoning.
+  # ornith and reasoning are live but unmanaged: their manifests were never committed
+  # to this directory, so listing them made `kustomize build` fail and left the whole
+  # Application in ComparisonError. Remove the entries rather than the models. The
+  # Application runs with prune=false, so an object that is no longer in the desired
+  # state is left alone, not deleted. Re-export both from live state when they are
+  # next edited.
   - embeddings.yaml
-  - ornith.yaml
-  - reasoning.yaml
   - reranker.yaml
   - qwen-cpu.yaml
   - networkpolicy.yaml

k8s/apps/llm-serving/reranker.yaml

@@ -45,14 +45,13 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reranker
-        app.kubernetes.io/part-of: llm-serving
+    annotations:
+      prometheus.io/scrape: "true"
+      prometheus.io/port: "8080"
+      prometheus.io/path: "/metrics"
+    labels:
+      app.kubernetes.io/name: llm-reranker
+      app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-token-vendor/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.26 AS build
+WORKDIR /src
+COPY go.mod ./
+COPY *.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/vendor .
+
+FROM gcr.io/distroless/static:nonroot
+COPY --from=build /out/vendor /usr/local/bin/vendor
+USER 65532:65532
+ENTRYPOINT ["/usr/local/bin/vendor"]

k8s/apps/llm-token-vendor/go.mod

@@ -0,0 +1,3 @@
+module github.com/riotpiao/llm-token-vendor
+
+go 1.26.0

k8s/apps/llm-token-vendor/main.go

@@ -0,0 +1,197 @@
+// llm-token-vendor mints ZITADEL access tokens for a caller that cannot sign.
+//
+// Some callers are vendor images: paperless-ai can only read a bearer token from a
+// file. It cannot sign an assertion, so it cannot use a machine key, and a client
+// secret would rotate with a window.
+//
+// This sidecar closes the gap. It signs with a machine key, exchanges the assertion
+// for an access token, and writes the token to the file the vendor image already
+// reads. The file is replaced atomically and refreshed at 80% of the token lifetime,
+// so the vendor never reads a partial or expired value.
+//
+// The key is re-read on every mint. A rotation therefore lands without a restart:
+// publish the new key, the next mint uses it, retire the old one.
+package main
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+type config struct {
+	tokenURL string
+	issuer   string
+	userID   string
+	keyID    string
+	keyPath  string
+	scope    string
+	outPath  string
+}
+
+func main() {
+	var c config
+	flag.StringVar(&c.tokenURL, "token-url", env("TOKEN_URL", ""), "ZITADEL token endpoint")
+	flag.StringVar(&c.issuer, "issuer", env("ISSUER", ""), "ZITADEL issuer, the assertion audience")
+	flag.StringVar(&c.userID, "user-id", env("USER_ID", ""), "machine user id")
+	flag.StringVar(&c.keyID, "key-id", env("KEY_ID", ""), "machine key id")
+	flag.StringVar(&c.keyPath, "key-file", env("KEY_FILE", ""), "PEM private key, re-read on every mint")
+	flag.StringVar(&c.scope, "scope", env("SCOPE", "openid"), "requested scopes")
+	flag.StringVar(&c.outPath, "out", env("OUT", ""), "file the vendor image reads")
+	flag.Parse()
+
+	if c.tokenURL == "" || c.issuer == "" || c.userID == "" || c.keyPath == "" || c.outPath == "" {
+		log.Fatal("token-url, issuer, user-id, key-file and out are required")
+	}
+
+	for {
+		lifetime, err := mintAndWrite(c)
+		if err != nil {
+			// Exit rather than spin: a pod that cannot mint must be visible, and the
+			// vendor image must not keep serving the caller a stale token.
+			log.Fatalf("cannot mint a token: %v", err)
+		}
+		wait := time.Duration(float64(lifetime) * 0.8)
+		log.Printf("token written to %s, next mint in %s", c.outPath, wait.Round(time.Second))
+		time.Sleep(wait)
+	}
+}
+
+func mintAndWrite(c config) (time.Duration, error) {
+	pemBytes, err := os.ReadFile(c.keyPath)
+	if err != nil {
+		return 0, fmt.Errorf("cannot read the key %s: %w", c.keyPath, err)
+	}
+	assertion, err := assertion(c, time.Now(), pemBytes)
+	if err != nil {
+		return 0, err
+	}
+
+	form := url.Values{}
+	form.Set("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer")
+	form.Set("assertion", assertion)
+	form.Set("scope", c.scope)
+
+	resp, err := http.PostForm(c.tokenURL, form)
+	if err != nil {
+		return 0, fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("token endpoint returned %d: %s", resp.StatusCode, string(body))
+	}
+	var out struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int    `json:"expires_in"`
+	}
+	if err := json.Unmarshal(body, &out); err != nil {
+		return 0, fmt.Errorf("cannot decode the token response: %w", err)
+	}
+	if out.AccessToken == "" {
+		return 0, fmt.Errorf("the token response holds no access_token")
+	}
+
+	// Write beside the target, then rename. A rename is atomic, so the vendor either
+	// reads the previous token or the new one, never a half-written file.
+	tmp, err := os.CreateTemp(filepath.Dir(c.outPath), ".token-*")
+	if err != nil {
+		return 0, fmt.Errorf("cannot create a temporary file: %w", err)
+	}
+	defer os.Remove(tmp.Name())
+	if _, err := tmp.WriteString(out.AccessToken); err != nil {
+		tmp.Close()
+		return 0, fmt.Errorf("cannot write the token: %w", err)
+	}
+	tmp.Close()
+	if err := os.Chmod(tmp.Name(), 0o644); err != nil {
+		return 0, fmt.Errorf("cannot set the file mode: %w", err)
+	}
+	if err := os.Rename(tmp.Name(), c.outPath); err != nil {
+		return 0, fmt.Errorf("cannot replace %s: %w", c.outPath, err)
+	}
+
+	lifetime := time.Duration(out.ExpiresIn) * time.Second
+	if lifetime <= 0 {
+		lifetime = 5 * time.Minute
+	}
+	return lifetime, nil
+}
+
+func assertion(c config, now time.Time, pemBytes []byte) (string, error) {
+	block, _ := pem.Decode(pemBytes)
+	if block == nil {
+		return "", fmt.Errorf("the key file is not PEM")
+	}
+	var key *rsa.PrivateKey
+	if k, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil {
+		key = k
+	} else {
+		parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+		if err != nil {
+			return "", fmt.Errorf("the key is not an RSA private key: %w", err)
+		}
+		rk, ok := parsed.(*rsa.PrivateKey)
+		if !ok {
+			return "", fmt.Errorf("the key holds %T, not RSA", parsed)
+		}
+		key = rk
+	}
+
+	enc := func(v any) (string, error) {
+		raw, err := json.Marshal(v)
+		if err != nil {
+			return "", err
+		}
+		return base64.RawURLEncoding.EncodeToString(raw), nil
+	}
+	header := map[string]any{"alg": "RS256", "typ": "JWT"}
+	if c.keyID != "" {
+		header["kid"] = c.keyID
+	}
+	claims := map[string]any{
+		"iss": c.userID,
+		"sub": c.userID,
+		"aud": c.issuer,
+		// Backdated, so a clock that is a few seconds fast cannot make the assertion
+		// "not yet valid".
+		"iat": now.Add(-5 * time.Second).Unix(),
+		"exp": now.Add(5 * time.Minute).Unix(),
+	}
+	h, err := enc(header)
+	if err != nil {
+		return "", err
+	}
+	cl, err := enc(claims)
+	if err != nil {
+		return "", err
+	}
+	signing := h + "." + cl
+	digest := sha256.Sum256([]byte(signing))
+	sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:])
+	if err != nil {
+		return "", fmt.Errorf("cannot sign the assertion: %w", err)
+	}
+	return signing + "." + base64.RawURLEncoding.EncodeToString(sig), nil
+}
+
+func env(key, fallback string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return fallback
+}

k8s/apps/llm-token-vendor/token_vendor_test.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"strings"
+	"testing"
+	"time"
+)
+
+func key(t *testing.T) []byte {
+	t.Helper()
+	k, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("cannot generate a key: %v", err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)})
+}
+
+func seg(t *testing.T, s string, out any) {
+	t.Helper()
+	raw, err := base64.RawURLEncoding.DecodeString(s)
+	if err != nil {
+		t.Fatalf("not base64url: %v", err)
+	}
+	if err := json.Unmarshal(raw, out); err != nil {
+		t.Fatalf("not JSON: %v", err)
+	}
+}
+
+func TestAssertionCarriesTheKeyAndTheAudience(t *testing.T) {
+	c := config{issuer: "https://zitadel.example", userID: "3970", keyID: "3971"}
+	tok, err := assertion(c, time.Unix(1_700_000_000, 0), key(t))
+	if err != nil {
+		t.Fatalf("signing failed: %v", err)
+	}
+	parts := strings.Split(tok, ".")
+	if len(parts) != 3 {
+		t.Fatalf("a JWT has three segments, got %d", len(parts))
+	}
+	var h map[string]any
+	seg(t, parts[0], &h)
+	if h["kid"] != "3971" || h["alg"] != "RS256" {
+		t.Fatalf("header must name the key and the algorithm, got %v", h)
+	}
+	var cl map[string]any
+	seg(t, parts[1], &cl)
+	if cl["iss"] != "3970" || cl["sub"] != "3970" || cl["aud"] != "https://zitadel.example" {
+		t.Fatalf("claims are wrong: %v", cl)
+	}
+	if int64(cl["iat"].(float64)) != 1_699_999_995 {
+		t.Fatalf("iat must be backdated five seconds, got %v", cl["iat"])
+	}
+}
+
+func TestAssertionRejectsAKeyThatIsNotPEM(t *testing.T) {
+	if _, err := assertion(config{userID: "u", issuer: "i"}, time.Now(), []byte("nope")); err == nil {
+		t.Fatalf("a non-PEM key must be rejected")
+	}
+}

k8s/apps/paperless/paperless-ai.yaml

@@ -36,53 +36,54 @@ spec:
         - name: paperless-ai-env-config
           configMap:
             name: paperless-ai-env
-      initContainers:
-        - name: fetch-llm-token
-          image: curlimages/curl:8.12.0
-          securityContext:
-            runAsUser: 0
-            fsGroup: 0
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
-              
-              # Get JWT token via client_credentials grant
-              # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
-                -d "grant_type=client_credentials" \
-                -d "client_id=396757803559027071" \
-                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
-                -d "scope=openid llm:inference" 2>/dev/null)
-              
-              # Extract token from JSON response using sed (handles spaces after colons)
-              TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')
-              
-              if [ -z "$TOKEN" ]; then
-                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
-                exit 1
-              fi
-              
-              # Store token in file for main container to read
-              # Write to both locations for compatibility
-              mkdir -p /tmp/llm-token /app/data 2>/dev/null || true
-              echo "$TOKEN" | tee /tmp/llm-token/llm_token.txt > /dev/null 2>&1
-              echo "$TOKEN" > /app/data/llm_token.txt 2>/dev/null || true
-              echo "[init] Token fetched and stored"
-              [ -f /tmp/llm-token/llm_token.txt ] && echo "  -> /tmp/llm-token/llm_token.txt"
-              [ -f /app/data/llm_token.txt ] && echo "  -> /app/data/llm_token.txt"
-          env:
-            - name: LLM_AUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-ai-config
-                  key: LLM_AUTH_CLIENT_SECRET
-          volumeMounts:
-            - name: paperless-ai-data
-              mountPath: /app/data
+        - name: llm-key
+          secret:
+            secretName: paperless-llm-key
+            # 0440 with the pod's fsGroup. A Secret volume is owned by root, and the
+            # vendor container runs as 65532, so 0400 left the sidecar unable to read
+            # its own key: "permission denied" on every start.
+            defaultMode: 0440
+      # No init container. The token used to be fetched once at pod start with a
+      # client_credentials grant against a web app, which cannot work: a web app has
+      # no such grant, and even a token that was minted would be dead within the hour
+      # while this pod runs for weeks.
+      #
+      # llm-token-vendor runs beside the app instead. It signs a machine-key assertion,
+      # exchanges it for a token, and rewrites /app/data/llm_token.txt at 80% of the
+      # token lifetime. The vendor image keeps reading CUSTOM_API_KEY_FILE unchanged.
+      # The key is re-read on every mint, so a rotation lands without a restart.
+      securityContext:
+        fsGroup: 65532
+        fsGroupChangePolicy: OnRootMismatch
       containers:
+        - name: llm-token-vendor
+          image: forgejo.riotpiao.com/riotpiao-poimen/llm-token-vendor:0.1.0
+          imagePullPolicy: Always
+          args:
+            - --token-url=https://zitadel.riotpiao.com/oauth/v2/token
+            - --issuer=https://zitadel.riotpiao.com
+            - --user-id=397072668735570071
+            - --key-id=397072674272051351
+            - --key-file=/var/run/llm-key/key
+            # No project audience. ZITADEL puts the requested audience in `aud`, and
+            # the gateway compares `aud` with its allow-list of client ids. Asking for
+            # the project id made every token carry the project, so the gateway
+            # answered "audience not allowed: [396757796898471937]". With plain
+            # `openid` the token names its caller, which is the entry that exists.
+            - --scope=openid
+            - --out=/app/data/llm_token.txt
+          volumeMounts:
+            - {name: paperless-ai-data, mountPath: /app/data}
+            - {name: llm-key, mountPath: /var/run/llm-key, readOnly: true}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities: {drop: ["ALL"]}
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile: {type: RuntimeDefault}
+          resources:
+            requests: {cpu: 10m, memory: 32Mi}
+            limits: {cpu: 100m, memory: 64Mi}
         - name: paperless-ai
           image: clusterzx/paperless-ai:latest
           env:

k8s/apps/secret-rotation-controller/crd.yaml

@@ -1,144 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: secretrotations.homelab.riotpiao.com
-spec:
-  group: homelab.riotpiao.com
-  names:
-    kind: SecretRotation
-    plural: secretrotations
-  scope: Namespaced
-  versions:
-  - name: v1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        properties:
-          metadata:
-            type: object
-          spec:
-            type: object
-            required:
-            - provider
-            - rotationInterval
-            properties:
-              # External system: authentik | forgejo | minio | vault
-              provider:
-                type: string
-                enum: [authentik, forgejo, minio, vault]
-              
-              # How often to rotate (hours)
-              rotationInterval:
-                type: integer
-                minimum: 24
-              
-              # Application ID in external system
-              appId:
-                type: string
-              
-              # k8s Secret to update (name, namespace, key)
-              secretRef:
-                type: object
-                required: [name, namespace]
-                properties:
-                  name:
-                    type: string
-                  namespace:
-                    type: string
-                  key:
-                    type: string
-                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
-              
-              # Path to git file that holds the secret (for .enc.yaml files)
-              gitPath:
-                type: string
-                description: "Path in homelab repo to .enc.yaml file"
-              
-              # Ansible template values to substitute
-              templateValues:
-                type: object
-                additionalProperties:
-                  type: string
-          
-          status:
-            type: object
-            properties:
-              lastRotationTime:
-                type: string
-                format: date-time
-              nextRotationTime:
-                type: string
-                format: date-time
-              lastRotationStatus:
-                type: string
-                enum: [Success, Failed, Pending]
-              lastRotationError:
-                type: string
-              lastCommitHash:
-                type: string
-
----
-# Example usage:
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160  # 90 days in hours
-  appId: minio
-  secretRef:
-    name: minio-oidc
-    namespace: storage
-    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: portfolio-agent-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160
-  appId: portfolio-agent
-  secretRef:
-    name: portfolio-agent-oidc
-    namespace: portfolio
-    key: CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: forgejo-registry-token
-  namespace: secret-rotation
-spec:
-  provider: forgejo
-  rotationInterval: 2160
-  appId: rock/riotpiao.com
-  secretRef:
-    name: forgejo-registry-secret
-    namespace: kube-system
-    key: REGISTRY_TOKEN
-  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-root-credentials
-  namespace: secret-rotation
-spec:
-  provider: minio
-  rotationInterval: 4320  # 180 days in hours
-  appId: root
-  secretRef:
-    name: minio-creds
-    namespace: storage
-  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -1,92 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: secret-rotation-controller
-  template:
-    metadata:
-      labels:
-        app: secret-rotation-controller
-    spec:
-      serviceAccountName: secret-rotation-controller
-      containers:
-      - name: controller
-        image: secret-rotation-controller:latest
-        imagePullPolicy: IfNotPresent
-        env:
-        # SOPS reads age key from this file
-        - name: SOPS_AGE_KEY_FILE
-          value: /etc/sops/age/private-key.txt
-        
-        # Vault auth (token in projected volume)
-        - name: VAULT_ADDR
-          value: http://vault.vault.svc.cluster.local:8200
-        - name: VAULT_TOKEN_FILE
-          value: /var/run/secrets/vault/token
-        
-        # Authentik
-        - name: AUTHENTIK_URL
-          value: http://authentik-server.iam.svc.cluster.local
-        - name: AUTHENTIK_BOOTSTRAP_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: authentik-bootstrap
-              key: token
-        
-        # Git
-        - name: GIT_REPO
-          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-        - name: GIT_AUTHOR_EMAIL
-          value: [email protected]
-        - name: GIT_AUTHOR_NAME
-          value: Secret Rotation Controller
-        - name: FORGEJO_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: forgejo-registry-secret
-              key: REGISTRY_TOKEN
-        
-        volumeMounts:
-        # Age key from ExternalSecret (synced from Vault)
-        - name: age-key
-          mountPath: /etc/sops/age
-          readOnly: true
-        
-        # Vault auth token (projected)
-        - name: vault-token
-          mountPath: /var/run/secrets/vault
-          readOnly: true
-        
-        # Temp working dir
-        - name: tmp
-          mountPath: /tmp
-        
-        resources:
-          requests:
-            cpu: 100m
-            memory: 256Mi
-          limits:
-            cpu: 500m
-            memory: 512Mi
-      
-      volumes:
-      - name: age-key
-        secret:
-          secretName: sops-age-key
-          defaultMode: 0400
-      
-      - name: vault-token
-        projected:
-          sources:
-          - serviceAccountToken:
-              path: token
-              audience: vault
-              expirationSeconds: 3600
-      
-      - name: tmp
-        emptyDir: {}

k8s/apps/secret-rotation-controller/kustomization.yaml

@@ -1,15 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: secret-rotation
-
-resources:
-  - rbac.yaml
-  - crd.yaml
-  - external-secret.yaml
-  - deployment.yaml
-
-commonLabels:
-  app.kubernetes.io/name: secret-rotation-controller
-  app.kubernetes.io/component: automation
-  managed-by: argocd

k8s/apps/secret-rotation-controller/rbac.yaml

@@ -1,53 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: secret-rotation-controller
-rules:
-# Read SecretRotation CRDs
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations"]
-  verbs: ["get", "list", "watch"]
-
-# Update status
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations/status"]
-  verbs: ["get", "patch", "update"]
-
-# Read k8s secrets that will be rotated
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["get", "list"]
-
-# For recording events
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create", "patch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: secret-rotation-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: secret-rotation-controller
-subjects:
-- kind: ServiceAccount
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: secret-rotation
-  labels:
-    kubernetes.io/metadata.name: secret-rotation

k8s/argocd/apps/08-secret-rotation.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: secret-rotation
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: secret-rotation
-spec:
-  project: homelab
-  
-  sources:
-  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    path: k8s/apps/secret-rotation-controller
-    targetRevision: main
-  
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: secret-rotation
-  
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    - RespectIgnoreDifferences=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/secrets/forgejo-db-bootstrap.yaml

@@ -0,0 +1,39 @@
+# Bootstrap credential for the Forgejo database — the one secret in the CNPG path
+# that the cluster can lose without any controller to recreate it.
+#
+# forgejo-db-app is OWNED by the Cluster CR (ownerReferences: Cluster/forgejo-db,
+# managed-by: cloudnative-pg) and was GENERATED by CNPG: spec.bootstrap.initdb.secret
+# is empty, so nothing but cloudnative-pg ever knew this value.
+#
+# CNPG reads spec.bootstrap.initdb.secret ONLY at cluster creation. So if the Cluster
+# CR is ever deleted and recreated, CNPG mints a new password while the data directory
+# on the PVC keeps the old one. Forgejo then starts, stays Ready, and fails every
+# query — a credential fault that looks like a network fault. This file is the record
+# that makes a rebuild reproducible: point spec.bootstrap.initdb.secret at it.
+#
+# Keep this file. It cannot rotate on its own; rotation is a manual CNPG ALTER ROLE
+# followed by updating this file and the Secret.
+apiVersion: v1
+kind: Secret
+metadata:
+    name: forgejo-db-bootstrap
+    namespace: cicd
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:GFRjVvq3iQ==,iv:7ZxKdHTvKt6TYlsQYu57dt9C14h59xyfrL3+o/deon0=,tag:Ame/mfgDlHk1oPp80hE3LQ==,type:str]
+    password: ENC[AES256_GCM,data:11t2yglSWQGKeMQ0ZRaw3L5s/o3iHEYArFWdfAQBYSnz1SMOmZ2XTl9yDDHFMUEgHK4K4mP4fJaq4nsgfgr5pA==,iv:aCQ4nkuUX6jDNiOsRW5nzST7HgNZmhjRidmskfNArAg=,tag:ddN0lf+7i4LpS6+yK+za+A==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3ZVQzajgzcmE2MnFzb1pJ
+            VFR4NjNtU2lWYjZGdGNuOVVnTkNxRndadXhzCkhrU3pPZi9NUlRkRWNUdHNKbW9a
+            WTcvVnVRM2tWZE1OMDBGcTd0NjhzeVkKLS0tIG5PQlg3ZlBNMDhQV2hxdzB4K1g3
+            SHBtZ284bTJNV3FvSWFtbDRudGUzbTQKNPM04xB1DWyIE/6/qjUCvXw2M8zkhVkP
+            xV8bVIp81R0pnMuEGB60L38W81SfziSspzpU0YMUmgZcU2XmO73AdQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-28T10:09:09Z"
+    mac: ENC[AES256_GCM,data:oNYDrN9wzVbDrDTYcLEKwCqTruybeU+++58Duki5P8xddU1anjHzfBg9DXgXd6CaoYJAU2jEcrqX9YXN8zmJtGaTpgr2GUdgFWx6O1JFoaNM32RNcp7u8FZWpcFYecWyYRtRFMolRaLM/CxkXr2lLa/V3Azxe1xlJrf7EjyI3dY=,iv:7ztpZquVBNZFt59OcNM2IXiDdmMjE8AGs1lTDZzeNWs=,tag:xpak7lbN6hrXaSKoWSbHYQ==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml

@@ -1,28 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: grafana-oidc
-    namespace: logging
-type: Opaque
-stringData:
-    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:fGqA1ofVme1lVLxFjWIiU7sbvE9ip2Af7C1ua3+HdidlcVgkwW59vVlXZhgBRdoif86C8LKQSyoCGUqiO+740g==,iv:6Kv2ybbJg1GT8DLqz9P5y+E1zr7vcIBDMUErmvxNTaw=,tag:0wK28B3G/owCVPPqb2sLmQ==,type:str]
-sops:
-    kms: []
-    gcp_kms: []
-    azure_kv: []
-    hc_vault: []
-    age:
-        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTXMxMWdVRHIvbWh1bVNk
-            bEFFNlgzNU9zTjlzY2o3VGdnbFpnU2lkYmxrCjF4aWVGSlJaNkcxODJ6RDkrVFo2
-            UkV2RUwwVHFCd0xMNG9RWXdTUHZGaXcKLS0tIHVaWjhaNHRtS2c0UkVzRk83ckhj
-            OCtBVFR0d1F5amVhV2E0emRZWThvYjAKFW46i9brIL9ZoO1CEn+IewIdoUhvYK/A
-            RqzZ6xV+mhlDj0zC+SpVcEm/Ki83nMYOj4+vQingZlr/Kr3Nwb0iLQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-10-27T19:44:32Z"
-    mac: ENC[AES256_GCM,data:EJew+p/kJrurcNpLlFxJGTl69hQYxn3IkzP7EDtdSriW/WI7X1xNLC2dp/dTiLomGmG1uh6+2JI9Sc8gXmrRAWLELAbiOaYbcoU6gi89BjTCWGDAMX4lNSIuH40rsGs4YtYj5U7SGfgx5qto4ZH7LxjBmxzPWM/Xz2Uq/I5jV4k=,iv:LdhjhwCfPUIa7D0gved8Twf7ujN+dg3lN8lCnFBFs0g=,tag:bjkwUCMtR8IBL9BkbuXcnw==,type:str]
-    pgp: []
-    encrypted_regex: ^(data|stringData)$
-    version: 3.9.3

k8s/argocd/secrets/minio-secrets.enc.yaml

@@ -1,3 +1,6 @@
+# minio-oidc is not in this file any more: the KeyRotation object iam/minio owns it.
+# A prune of the sops-secrets application would delete that Secret, and the
+# controller recreates it on its next rotation.
 apiVersion: v1
 kind: Secret
 metadata:
@@ -5,46 +8,27 @@ metadata:
     namespace: storage
 type: Opaque
 stringData:
-    config.env: ENC[AES256_GCM,data:1yogGyBdwrlg27H8R5Kr9VcPu+xwS5/nPhi7eLcmVYtXmPlNokCsWHA70qnn/+I6i4ODys8GML++VeZ/lmcHiP7RIbY+G6xmw6nfBCXitgE8kQwTPsUUzn9bobOn4Sr1YMBf7FQ/Q0N8aupGJA==,iv:EfCdim4TLkP/FP1u1yoKZCq8YGFzVh7ca4yu1+WzXwE=,tag:rTYe0SIAEsUhh0KHQIEKNw==,type:str]
+    config.env: ENC[AES256_GCM,data:4NQYmVEnkv4nO6FnMAEXVCUhMM+DLCGG2tFC7SGJ0lGnfFQiISdzWwVXD0y9/gpK4qn5zv0Cs1MgOfDVBKH6UR1ONXQnIjUnvXnq4D5+P0BgVUlYD/7aComva9Ow1IEChvD3fBTmDW7FMEZrtA==,iv:/LAV+ukcE3pwgJMkEkR8z9h+k6pSvQ7a/FkxLvMK6X8=,tag:BnvsL05XttadRqqS3TU2Uw==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQ1BqbDZoYjVQWGlTVEFD
+            dVVWT0xFUlAwUXJUcHZOMGN1cG85S3NtcVZRCmpUZnM5dXpWUDh5UHhGUTcyYmNr
+            UWNlc2VDazUvd3k0VHVWR1NBUE13eTAKLS0tIG9VcGYvN1Z1UEo3Um1SMHZOSHF3
+            eUp5dE04M3lvTVlHc3lPZnMyUWU0bE0KPXwE7wOJNSrqM6kfO1BvXqHApvQI4hdz
+            5GWmLjmjQL/ZZlkF07uK1GDXIBC3N3K0vYrfQHjMUImw9Dwn9DnL5w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-28T03:00:02Z"
+    mac: ENC[AES256_GCM,data:DGglizsoj/Fmi+r1NvfYD1sSivtedJ+w0RKFioJ1K7ektG/I68hprcMkmrs9vY1H2zlP/1Io/Zf0wOMqIPVK+uyEHIDGr+kk5Y/vPkrPi1+TpVDHRJVwvfkFWWVBrnlMbpZ89076dl1cyphdi4XvSHq2z30Qa1oRnxNkPO2HrhY=,iv:je0iyd/AkWYeyL2vA6ipmvWL6c0X+4h1zeLMeE60Pks=,tag:Q3X/jIWQWFtiSGhkCbzBDA==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2
----
-apiVersion: v1
-kind: Secret
-metadata:
-    name: minio-oidc
-    namespace: storage
-type: Opaque
-stringData:
-    MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:eFfwW2ZnV+o3/6XHjUvyFsDbaIAdosY2PXOnRdN4Ie49YDIjfJPe+Mg1FsbgoEHEjuB33hfxesb4TWOJQfW8Og==,iv:+kRr9A3auts1XHgZWa+t5b2m4lF0x9dVhiRpB17fzv4=,tag:tH/jszFzLQdqORGr9vfTGw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2
+    version: 3.9.3
 ---
 apiVersion: v1
 kind: Secret
@@ -53,20 +37,25 @@ metadata:
     namespace: storage
 type: Opaque
 stringData:
-    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:Zxr/Nj0SUKSJbBD1cFjt3GE=,iv:r/B9ULn68hsXdH8uRQkKAoQTJIJIYKMoJuWOZfxS2I8=,tag:Xo4i+IVxuqrp+qDpYQysPw==,type:str]
-    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:PoIDNZ+Xs6BdhGm10y1HdL5jy47EVHfHHIJhGy/KX6Q=,iv:voAv32GoTKi9s9/4MQy2yT6i9VY4/igA5XXzehwUqRQ=,tag:DmPyk5biVvlxHjeyDyrLbQ==,type:str]
+    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:6mmBluC+4QmlMxPJbfNh0kA=,iv:CuWy1gVnd19vB1rOS9HRVKegX3CnmCEa6cpk+S7ZX00=,tag:OqZK09bVMZGbMXwnDmr77Q==,type:str]
+    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:Fyd5EdZZP7/q3QOVv3FU1bkd8960p6UiRNNl17LJltw=,iv:/QGSkVVfXkCAKSV1+RNTZEI+/riAFVLV6SiaL9DvbSo=,tag:lFTJpifGorsl2w8lG7beCw==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxQ1BqbDZoYjVQWGlTVEFD
+            dVVWT0xFUlAwUXJUcHZOMGN1cG85S3NtcVZRCmpUZnM5dXpWUDh5UHhGUTcyYmNr
+            UWNlc2VDazUvd3k0VHVWR1NBUE13eTAKLS0tIG9VcGYvN1Z1UEo3Um1SMHZOSHF3
+            eUp5dE04M3lvTVlHc3lPZnMyUWU0bE0KPXwE7wOJNSrqM6kfO1BvXqHApvQI4hdz
+            5GWmLjmjQL/ZZlkF07uK1GDXIBC3N3K0vYrfQHjMUImw9Dwn9DnL5w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-28T03:00:02Z"
+    mac: ENC[AES256_GCM,data:DGglizsoj/Fmi+r1NvfYD1sSivtedJ+w0RKFioJ1K7ektG/I68hprcMkmrs9vY1H2zlP/1Io/Zf0wOMqIPVK+uyEHIDGr+kk5Y/vPkrPi1+TpVDHRJVwvfkFWWVBrnlMbpZ89076dl1cyphdi4XvSHq2z30Qa1oRnxNkPO2HrhY=,iv:je0iyd/AkWYeyL2vA6ipmvWL6c0X+4h1zeLMeE60Pks=,tag:Q3X/jIWQWFtiSGhkCbzBDA==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
-    version: 3.13.2
+    version: 3.9.3

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -5,23 +5,25 @@ metadata:
     name: oauth2-credentials
     namespace: iam
 data:
-    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
-    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
-    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
-    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
-    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+    api-gw-client-secret: ENC[AES256_GCM,data:q41OBQ6WvxsmHmXFhrdzeNUlQyPKuOQQa0vWdhuioQwTsNJ5+l9rYZKBAcZxzvd4P066aRN6yVPmEZhq,iv:Dnfgy8N9NVOM2x656Qei8960dzu7icAU3BZLXmj7N5o=,tag:deilBSCC9Mg/Eo8K0w6rMw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:2k28ui+Mbu3qd4ar6/NidO4bWO8g7JQUjDQBD8liywpCm7rA8TSrmU51im/I5kEvRRhl+RByEo5wpozo,iv:JFrZYF7cP/tQL8gkQJ5GvtdOt1kKcAdHE7L4l+uT9M0=,tag:DSiv8kv7MINUxi7fO/093w==,type:str]
 sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
     age:
-        - enc: |
+        - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
-            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
-            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
-            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
-            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5WmthakxzRk5aeGJWZnRF
+            bzdqM1o5Q1h2MFlpNEJvRUFZVU1wWjRaVWowClVKYkp0YUJCbUM0MWduaCtBSExH
+            MDRWUGZkYTdndTZTeEM3ckxrMURFMUEKLS0tIHF3YUlLNndSWmxGejFPbTVjQnRS
+            THF4RFBNbDJCdjVKRDJ6cEJlZGVBM0UKEDz1sKDON+lelq1zbCO4CETaK77T4G8y
+            fly66sR5uIvAXdwXy14L9ssZsM1jOHqdkXnpbo6mzh3toB/ROzoL7w==
             -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-10-27T21:47:38Z"
+    mac: ENC[AES256_GCM,data:bgZD+/LAPKp6DXyRMnbFWyxmiRjWxYO0ofxeGpO7nyw1DMSTD9s8li43eh/IP7wW4NORwzfIfzxUvOFmVfAiQKfkzpYRDFIKqK+aa/+43zppwpVT0UJ3CcBt2qgdJRHKnv9qeUOx4M8ulY7Xcsoevw8EGdrXCc1KAcDIwoIAp0Q=,iv:djYQ6YnGPrYGRKgpfbdf0beW6QHYTq5CJ3nes7+C4p4=,tag:N9n4VKyMNzGCmuYpwBLPng==,type:str]
+    pgp: []
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
-    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
-    version: 3.13.2
+    version: 3.9.3

k8s/argocd/secrets/secret-generator.yaml

@@ -15,7 +15,6 @@ files:
     - forgejo-registry-pull.enc.yaml
     - forgejo-runner-token.enc.yaml
     - forgejo-secrets.enc.yaml
-    - grafana-oidc-secrets.enc.yaml
     - grafana-secrets.enc.yaml
     - homelab-ca-secrets.enc.yaml
     - loki-secrets.enc.yaml

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -34,6 +34,8 @@ service:
       lbipam.cilium.io/ips: "192.168.1.161"
 
 gitea:
+  podAnnotations:
+    reloader.stakater.com/auto: "true"
   admin:
     existingSecret: forgejo-admin
 

scripts/iam/rotate-oidc.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Rotate the OIDC client secret of every owned application, one at a time.
+
+This is the manual execution of what a KeyRotation object declares. It exists so
+the credentials can rotate today, before the controller image is built, and so the
+controller has a reference implementation to match.
+
+For each application: generate a new secret in ZITADEL, write it into the
+Kubernetes Secret, roll the consumer, then prove the consumer still talks to
+ZITADEL. A failure stops the run, because the next application should not start
+while one is in an unknown state.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+import importlib.util
+
+spec = importlib.util.spec_from_file_location("zp", Path(__file__).with_name("zitadel-provision.py"))
+zp = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(zp)
+
+PROJECT = "396757796898471937"
+ISSUER = "https://zitadel.riotpiao.com"
+
+# app key, ZITADEL app id, Secret name, namespace, Secret key, consumer kind, consumer name
+APPS = [
+    ("grafana",   "grafana",   "grafana-oidc",         "logging",   "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "deploy", "grafana"),
+    ("argocd",    "argocd",    "oidc-secret",           "argocd",    "client-secret",                       "deploy", "argocd-server"),
+    ("comfyui",   "comfyui",   "comfyui-oauth2-proxy", "comfyui",   "client-secret",                       "deploy", "comfyui-oauth2-proxy"),
+    ("minio",     "minio",     "minio-oidc",           "storage",   "MINIO_IDENTITY_OPENID_CLIENT_SECRET",  "sts",    "minio-cluster-az-a"),
+    ("paperless", "paperless", "paperless-oidc",       "paperless", "client-secret",                       "deploy", "paperless"),
+    ("immich",    "immich",    "immich-oidc",          "immich",    "client-secret",                       "deploy", "immich-server"),
+    ("forgejo",   "forgejo",   "forgejo-oidc",         "cicd",      "CLIENT_SECRET",                       "deploy", "forgejo-gitea"),
+]
+
+
+def sh(*args: str, timeout: int = 120) -> tuple[int, str]:
+    p = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+    return p.returncode, (p.stdout + p.stderr).strip()
+
+
+def state() -> dict:
+    return json.load(open(Path.home() / ".config/zitadel/state.json"))
+
+
+def app_ids(token: str) -> dict[str, str]:
+    """Map clientId to the app resource id. The two are different, and the secret
+    endpoint takes the resource id."""
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/_search", token,
+                         {"query": {"limit": 100, "offset": 0}})
+    out = {}
+    for app in body.get("result", []) or []:
+        cid = (app.get("oidcConfig") or {}).get("clientId")
+        if cid and app.get("id"):
+            out[cid] = app["id"]
+    return out
+
+
+def new_secret(token: str, app_id: str) -> str | None:
+    code, body = zp._api("POST", f"/management/v1/projects/{PROJECT}/apps/{app_id}/oidc_config/_generate_client_secret", token)
+    if code != 200:
+        print(f"    generate failed: {code} {str(body)[:120]}")
+        return None
+    return body.get("clientSecret")
+
+
+def patch_secret(ns: str, name: str, key: str, value: str) -> bool:
+    import base64
+    enc = base64.b64encode(value.encode()).decode()
+    patch = json.dumps({"data": {key: enc}})
+    code, out = sh("kubectl", "-n", ns, "patch", "secret", name, "--type=merge", "-p", patch)
+    if code != 0:
+        print(f"    secret patch failed: {out[:120]}")
+    return code == 0
+
+
+def roll(kind: str, ns: str, name: str) -> bool:
+    code, out = sh("kubectl", "-n", ns, "rollout", "restart", f"{kind}/{name}", timeout=120)
+    if code != 0:
+        print(f"    rollout restart failed: {out[:120]}")
+        return False
+    code, out = sh("kubectl", "-n", ns, "rollout", "status", f"{kind}/{name}", "--timeout=90s", timeout=120)
+    return code == 0
+
+
+def main() -> int:
+    only = sys.argv[1:] or None
+    token = zp.bootstrap_token()
+    if not token:
+        print("no bootstrap token")
+        return 2
+    cids = state()["client_ids"]["value"]
+    ids = app_ids(token)
+    print(f"resolved {len(ids)} app resource ids")
+    done, failed = [], []
+    for key, app_id, sname, ns, skey, kind, cname in APPS:
+        if only and key not in only:
+            continue
+        print(f"[{key}] issuer={ISSUER} client_id={cids[key]}")
+        rid = ids.get(cids[key])
+        if not rid:
+            print(f"[{key}] no app resource id for client {cids[key]}")
+            failed.append(key)
+            continue
+        secret = new_secret(token, rid)
+        if not secret:
+            failed.append(key)
+            continue
+        if not patch_secret(ns, sname, skey, secret):
+            failed.append(key)
+            continue
+        if not roll(kind, ns, cname):
+            failed.append(key)
+            print(f"[{key}] FAILED: consumer did not become ready")
+            continue
+        code, out = sh("kubectl", "-n", ns, "get", "pods", "-l",
+                       f"app={cname}" if kind == "deploy" else f"app.kubernetes.io/name={cname}",
+                       "--no-headers")
+        ready = "Running" in out
+        print(f"[{key}] rotated, consumer {'ready' if ready else 'NOT ready'}")
+        (done if ready else failed).append(key)
+    print(f"\nrotated: {len(done)} {done}\nfailed: {len(failed)} {failed}")
+    return 0 if not failed else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/iam/zitadel-provision.py

@@ -87,6 +87,12 @@ ROLES = {
     # LLM gate for the api-gw. The colon form matches the Authentik-era
     # permission string, so consumer manifests did not change.
     "llm:inference": "LLM inference access (api-gw)",
+    # Capability for machine-to-machine callers. Kept separate from the human roles
+    # so a service account can be identified by its grant alone. Note that ZITADEL
+    # does not assert roles into a machine-user access token on this instance (see
+    # MACHINE_CALLERS), so this grant documents intent; the gateway authorises the
+    # caller from its allow-list entry instead.
+    "svc-caller": "Machine-to-machine caller",
     "llm-inference": "LLM inference access (dash alias)",
     # Per-service capabilities. The gateway derives the required capability from
     # the verb: GET/HEAD/OPTIONS -> "<service>:read", anything else ->
@@ -122,6 +128,11 @@ OIDC_APPS = {
         "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
         "http://localhost:3000/user/oauth2/authentik/callback",
         "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
+    # ComfyUI has no native OIDC. oauth2-proxy is its front door and this app is
+    # the proxy's client, so the callback path belongs to the proxy.
+    "comfyui": {**_WEB, "redirect_uris": [
+        "http://localhost:4180/oauth2/callback",
+        "https://comfyui.riotpiao.com/oauth2/callback"]},
     "grafana": {**_WEB, "redirect_uris": [
         "http://localhost:3000/login/generic_oauth",
         "https://grafana.riotpiao.com/login/generic_oauth"]},
@@ -138,8 +149,13 @@ OIDC_APPS = {
         "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
         "http://localhost:8000/accounts/oidc/authentik/login/callback/",
         "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    # The frontend calls back at /auth/callback (see OAUTH_CALLBACK_PATH in
+    # poimen-frontend/lib/oauth.ts). The bare /callback spelling stays registered so
+    # an older image still completes its login.
     "poimen": {**_WEB, "redirect_uris": [
-        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback",
+        "http://localhost:3000/auth/callback",
+        "https://poimen.riotpiao.com/auth/callback"]},
     "queue-sqs": {**_WEB, "redirect_uris": [
         "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
     "vault": {**_WEB, "redirect_uris": [
@@ -335,6 +351,55 @@ def _ensure_builtin_login(token: str) -> None:
     print("login page    : built-in (loginV2.required=false)")
 
 
+# One machine user per cross-service caller. Each has its own key, so revoking one
+# service leaves the others working, and ZITADEL can overlap keys during a rotation.
+MACHINE_CALLERS = {
+    "poimen-llm": {
+        "name": "Poimen LLM caller",
+        "description": "poimen calls api-gateway inference",
+        "roles": ["llm:inference", "svc-caller"],
+    },
+}
+
+
+def _ensure_machine_users(token: str, project_id: str) -> dict[str, dict]:
+    """Create each caller once and grant its roles. Returns id and key id per caller.
+
+    Roles are granted even though ZITADEL does not assert them into a machine-user
+    token on this instance: the grant is the record of what the account may do, and
+    it becomes effective the moment that behaviour changes.
+    """
+    out: dict[str, dict] = {}
+    for user_name, spec in MACHINE_CALLERS.items():
+        code, found = _api("POST", "/management/v1/users/_search", token,
+                           {"queries": [{"userNameQuery": {"userName": user_name,
+                                                           "method": "TEXT_QUERY_METHOD_EQUALS"}}]})
+        user_id = ""
+        for user in (found.get("result") or []):
+            user_id = user.get("id", "")
+        if not user_id:
+            code, created = _api("POST", "/management/v1/users/machine", token, {
+                "userName": user_name,
+                "name": spec["name"],
+                "description": spec["description"],
+                "accessTokenType": "ACCESS_TOKEN_TYPE_JWT",
+            })
+            user_id = created.get("userId", "")
+        if not user_id:
+            continue
+        code, grant = _api("POST", "/management/v1/users/grants/_search", token,
+                           {"query": {"limit": 100, "offset": 0}})
+        existing = next((g for g in (grant.get("result") or []) if g.get("userId") == user_id), None)
+        if existing:
+            _api("PUT", f"/management/v1/users/{user_id}/grants/{existing['id']}", token,
+                 {"roleKeys": spec["roles"]})
+        else:
+            _api("POST", f"/management/v1/users/{user_id}/grants", token,
+                 {"projectId": project_id, "roleKeys": spec["roles"]})
+        out[user_name] = {"userId": user_id}
+    return out
+
+
 def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
     code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
     # ZITADEL answers with `key`, not `roleKey`. Reading the wrong field left
@@ -492,6 +557,9 @@ def phase_apply(_args: argparse.Namespace) -> int:
     project_id = _ensure_project(token, PROJECT_NAME)
     print(f"project   : {PROJECT_NAME} ({project_id})")
     _ensure_roles(token, project_id, ROLES)
+    machine_users = _ensure_machine_users(token, project_id)
+    for name, info in machine_users.items():
+        print(f"machine user {name}: {info['userId']}")
     print(f"roles     : {len(ROLES)} ensured")
     known: dict[str, str] = {}
     try: