fix(iam): merge into the consumer Secret, and retry with a backoff

Two defects that a real rotation exposed:

  merge      writeSecret replaced the whole Secret map, so every key a consumer
             keeps beside the credential disappeared. paperless lost
             SOCIALACCOUNT_PROVIDERS_JSON and immich lost config.json; both pods then
             failed to start while the rotation reported success.
  backoff    a Failed attempt returned 0, so the controller retried in a tight loop.
             It now waits two minutes and then retries.

All eight objects carry both identifiers ZITADEL needs: the app resource id for the
secret endpoint and the client id for introspection.

Result: seven rotations reach Succeeded with every consumer ready (grafana, comfyui,
argocd, forgejo, paperless, immich, minio). api-gw stays suspended because no workload
exists for it.

k8s/apps/key-rotation-controller/controller/api/types.go

@@ -70,8 +70,12 @@ type ConsumerRef struct {
 type RotationTarget struct {
 	// ZitadelProjectID is the project that owns the app or service user.
 	ZitadelProjectID string `json:"zitadelProjectID,omitempty"`
-	// ZitadelAppID is the OIDC app id, for the client-secret provider.
+	// ZitadelAppID is the app RESOURCE id, which the secret endpoint needs. It is not
+	// the client id.
 	ZitadelAppID string `json:"zitadelAppID,omitempty"`
+	// ZitadelClientID is the OIDC client id, which introspection needs. ZITADEL uses
+	// two different identifiers for the same app and neither substitutes for the other.
+	ZitadelClientID string `json:"zitadelClientID,omitempty"`
 	// MachineUserID is the service user id, for the machine-key provider.
 	MachineUserID string `json:"machineUserID,omitempty"`
 	// SecretName is the Secret the controller writes.

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler.go

@@ -74,14 +74,26 @@ func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration {
 	if interval <= 0 {
 		interval = 30 * 24 * time.Hour
 	}
-	// A failed attempt must not wait a whole interval: it retries soon, and the
-	// interval guard applies only to a rotation that succeeded.
+	// Return 0 to rotate NOW and a positive duration to wait. A failed attempt must
+	// rotate again immediately; only a rotation that reached Succeeded waits for the
+	// next interval. Returning a wait here was the defect: the object sat in Failed
+	// and reconciled every two minutes without ever trying again.
+	// A failed attempt waits a short backoff and then really retries. Returning 0
+	// unconditionally here spins: fail, rotate, fail, with no pause between.
 	if kr.Status.Phase == iamv1.PhaseFailed {
-		return 2 * time.Minute
+		if kr.Status.LastRotation != nil {
+			if wait := kr.Status.LastRotation.Add(2 * time.Minute).Sub(r.now()); wait > 0 {
+				return wait
+			}
+		}
+		return 0
 	}
 	if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" {
 		return 0 // rotate now
 	}
+	if kr.Status.Phase != iamv1.PhaseSucceeded {
+		return 0 // an interrupted attempt starts over
+	}
 	due := kr.Status.LastRotation.Add(interval)
 	wait := due.Sub(r.now())
 	if wait < 0 {
@@ -102,6 +114,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (t
 		return 0, err
 	}
 
+	logger.Info("reconcile", "phase", kr.Status.Phase, "generation", kr.Generation,
+		"observed", kr.Status.ObservedGeneration, "lastRotation", kr.Status.LastRotation)
+
 	if kr.Spec.Suspend {
 		return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "")
 	}
@@ -276,7 +291,16 @@ func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cre
 	if err != nil {
 		return err
 	}
-	secret.Data = data
+	// Merge, never replace. A consumer Secret often carries configuration beside the
+	// credential (paperless keeps SOCIALACCOUNT_PROVIDERS_JSON there), and replacing
+	// the whole map deletes it. The pod then fails with "couldn't find key ... in
+	// Secret" and the rotation looks successful while it broke the service.
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
+	}
+	for k, v := range data {
+		secret.Data[k] = v
+	}
 	return r.Update(ctx, &secret)
 }
 

k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler_test.go

@@ -19,10 +19,28 @@ func rotation(last *metav1.Time, activeID string, hours int32) *iamv1.KeyRotatio
 		Status: iamv1.KeyRotationStatus{
 			LastRotation:       last,
 			ActiveCredentialID: activeID,
+			Phase:              iamv1.PhaseSucceeded,
 		},
 	}
 }
 
+func TestRequeueAfterBacksOffAfterAFailure(t *testing.T) {
+	now := time.Unix(1_700_000_000, 0)
+	last := metav1.NewTime(now.Add(-1 * time.Minute))
+	r := &Reconciler{Now: func() time.Time { return now }}
+
+	kr := rotation(&last, "key-1", 24)
+	kr.Status.Phase = iamv1.PhaseFailed
+	if got := r.RequeueAfter(kr); got != time.Minute {
+		t.Fatalf("a failed attempt must wait a short backoff, got %s", got)
+	}
+	old := metav1.NewTime(now.Add(-3 * time.Minute))
+	kr.Status.LastRotation = &old
+	if got := r.RequeueAfter(kr); got != 0 {
+		t.Fatalf("after the backoff it must rotate again, got %s", got)
+	}
+}
+
 func TestRequeueAfterRotatesImmediatelyWhenNeverRotated(t *testing.T) {
 	r := &Reconciler{Now: func() time.Time { return time.Unix(1_700_000_000, 0) }}
 	if got := r.RequeueAfter(rotation(nil, "", 24)); got != 0 {

k8s/apps/key-rotation-controller/controller/main.go

@@ -79,6 +79,9 @@ func main() {
 			},
 			iamv1.ProviderZitadelOIDCClient: &keyreconcile.OIDCClientProvider{
 				Client: zitadelClient,
+				ClientIDFunc: func(kr *iamv1.KeyRotation) string {
+					return kr.Spec.Target.ZitadelClientID
+				},
 			},
 		},
 	}