// Package reconcile turns a KeyRotation object into a rotated credential. // // The state machine is identical for every provider. Only three calls differ: // create, verify, retire. That is what makes one controller cover machine keys, // client secrets and, later, the token-style providers. // // Order is the safety property: // // Prepare create the new credential // Publish write the Secret, roll the consumer when it cannot reload // Verify prove the new credential works against the provider // Retire delete the old credential, only for overlapping providers // // A failure before Retire leaves the old credential in place, so a broken // rotation never removes the thing that currently works. package reconcile import ( "context" "fmt" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" iamv1 "github.com/riotpiao/key-rotation-controller/api" ) // Credential is fresh credential material, before it is written to a Secret. type Credential struct { // ID identifies the credential at the provider, so it can be retired later. ID string // Fields maps a Secret key to its value. Fields map[string][]byte } // Provider creates, verifies and retires one class of credential. type Provider interface { // Name is the provider name from the spec. Name() string // Strategy is overlap or atomic. Atomic providers replace in place. Strategy() string // Prepare creates a new credential at the provider. Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error) // Verify proves the credential works, using the provider's own check. Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error // Retire deletes a superseded credential. It must be safe to call twice. Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error } // Reconciler rotates credentials for KeyRotation objects. type Reconciler struct { client.Client // Providers keyed by spec.provider. Providers map[string]Provider // Now is injectable so scheduling is testable. Now func() time.Time // RolloutTimeout bounds the wait for a consumer to become available. RolloutTimeout time.Duration } // previousKeySuffix keeps the superseded value beside the current one, so an // atomic provider can be rolled back without storing a secret in the object. const previousKeySuffix = ".previous" // RequeueAfter is how long to wait before checking a resource again. func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration { interval := time.Duration(kr.Spec.IntervalHours) * time.Hour if interval <= 0 { interval = 30 * 24 * time.Hour } if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" { return 0 // rotate now } due := kr.Status.LastRotation.Add(interval) wait := due.Sub(r.now()) if wait < 0 { return 0 } return wait } // Reconcile performs one rotation when the resource is due. func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (time.Duration, error) { logger := log.FromContext(ctx).WithValues("keyrotation", req.String()) var kr iamv1.KeyRotation if err := r.Get(ctx, req, &kr); err != nil { if apierrors.IsNotFound(err) { return 0, nil } return 0, err } if kr.Spec.Suspend { return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "") } provider := r.Providers[kr.Spec.Provider] if provider == nil { return 0, r.setPhase(ctx, &kr, iamv1.PhaseFailed, fmt.Sprintf("no provider named %q", kr.Spec.Provider)) } wait := r.RequeueAfter(&kr) if wait > 0 { return wait, nil } logger.Info("rotating credential", "provider", kr.Spec.Provider, "strategy", kr.Spec.Strategy) previousID := kr.Status.ActiveCredentialID previousFields := r.readCurrentSecret(ctx, &kr) // Prepare if err := r.setPhase(ctx, &kr, iamv1.PhasePreparing, ""); err != nil { return 0, err } cred, err := provider.Prepare(ctx, &kr) if err != nil { return r.fail(ctx, &kr, iamv1.PhasePreparing, err) } // Publish. The previous value stays beside the new one until verification // passes, so an atomic rotation can be undone. if err := r.setPhase(ctx, &kr, iamv1.PhasePropagating, ""); err != nil { return 0, err } if err := r.writeSecret(ctx, &kr, cred, previousFields); err != nil { return r.fail(ctx, &kr, iamv1.PhasePropagating, err) } if kr.Spec.Target.Reload != iamv1.ReloadHot { if err := r.rollConsumers(ctx, &kr, cred.ID); err != nil { return r.fail(ctx, &kr, iamv1.PhasePropagating, err) } } // Verify if err := r.setPhase(ctx, &kr, iamv1.PhaseVerifying, ""); err != nil { return 0, err } if err := provider.Verify(ctx, &kr, cred); err != nil { // Put the previous value back and roll again, then report the failure. r.restorePrevious(ctx, &kr, previousFields) return r.fail(ctx, &kr, iamv1.PhaseVerifying, err) } // Complete. Retire only when the provider can overlap, because retiring an // atomic credential would delete the one now in use. if err := r.setPhase(ctx, &kr, iamv1.PhaseCompleting, ""); err != nil { return 0, err } if provider.Strategy() == iamv1.StrategyOverlap && previousID != "" && previousID != cred.ID { if err := provider.Retire(ctx, &kr, previousID); err != nil { logger.Error(err, "retiring the previous credential failed; it stays valid", "credential", previousID) } } r.clearPrevious(ctx, &kr) now := metav1.NewTime(r.now()) kr.Status.LastRotation = &now kr.Status.ActiveCredentialID = cred.ID kr.Status.PreviousCredentialID = previousID kr.Status.LastError = "" kr.Status.ObservedGeneration = kr.Generation if err := r.Status().Update(ctx, &kr); err != nil { return 0, err } if err := r.setPhase(ctx, &kr, iamv1.PhaseSucceeded, ""); err != nil { return 0, err } logger.Info("rotation complete", "credential", cred.ID, "previous", previousID) return r.RequeueAfter(&kr), nil } func (r *Reconciler) now() time.Time { if r.Now != nil { return r.Now() } return time.Now() } func (r *Reconciler) setPhase(ctx context.Context, kr *iamv1.KeyRotation, phase, message string) error { kr.Status.Phase = phase kr.Status.LastError = message kr.Status.ObservedGeneration = kr.Generation if err := r.Status().Update(ctx, kr); err != nil && !apierrors.IsConflict(err) { return err } return nil } func (r *Reconciler) fail(ctx context.Context, kr *iamv1.KeyRotation, phase string, cause error) (time.Duration, error) { kr.Status.Phase = iamv1.PhaseFailed kr.Status.LastError = fmt.Sprintf("%s: %v", phase, cause) _ = r.Status().Update(ctx, kr) // Retry soon, but not in a tight loop. return 5 * time.Minute, nil } // secretData assembles the Secret payload: the new values, plus the superseded // values under a shadow key so an atomic rotation can be undone. func secretData(kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) map[string][]byte { data := make(map[string][]byte, len(cred.Fields)*2) for logical, value := range cred.Fields { key := secretKeyFor(kr, logical) data[key] = value if old, ok := previous[key]; ok && len(old) > 0 { data[key+previousKeySuffix] = old } } return data } // secretKeyFor maps a logical field name to the Secret key, defaulting to the // same name so a simple credential needs no mapping. func secretKeyFor(kr *iamv1.KeyRotation, logical string) string { if key, ok := kr.Spec.Target.SecretKeys[logical]; ok && key != "" { return key } return logical } func (r *Reconciler) readCurrentSecret(ctx context.Context, kr *iamv1.KeyRotation) map[string][]byte { var secret corev1.Secret err := r.Get(ctx, types.NamespacedName{ Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName, }, &secret) if err != nil { return nil } return secret.Data } func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) error { data := secretData(kr, cred, previous) var secret corev1.Secret key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName} err := r.Get(ctx, key, &secret) if apierrors.IsNotFound(err) { secret = corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, Labels: map[string]string{"app.kubernetes.io/managed-by": "key-rotation-controller"}, }, Type: corev1.SecretTypeOpaque, Data: data, } return r.Create(ctx, &secret) } if err != nil { return err } secret.Data = data return r.Update(ctx, &secret) } // restorePrevious puts the shadow values back in place after a failed check. func (r *Reconciler) restorePrevious(ctx context.Context, kr *iamv1.KeyRotation, previous map[string][]byte) { if len(previous) == 0 { return } var secret corev1.Secret key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName} if err := r.Get(ctx, key, &secret); err != nil { return } // Restore every shadowed value over the current one. for k, v := range secret.Data { base := k if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix { base = k[:len(k)-len(previousKeySuffix)] secret.Data[base] = v } } _ = r.Update(ctx, &secret) _ = r.rollConsumers(ctx, kr, "rollback") } // clearPrevious removes the shadow values once verification passed. func (r *Reconciler) clearPrevious(ctx context.Context, kr *iamv1.KeyRotation) { var secret corev1.Secret key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName} if err := r.Get(ctx, key, &secret); err != nil { return } changed := false for k := range secret.Data { if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix { delete(secret.Data, k) changed = true } } if changed { _ = r.Update(ctx, &secret) } } // rollConsumers restarts each consumer so it reads the new value. func (r *Reconciler) rollConsumers(ctx context.Context, kr *iamv1.KeyRotation, revision string) error { for _, consumer := range kr.Spec.Target.Consumers { ns := consumer.Namespace if ns == "" { ns = kr.Namespace } switch consumer.Kind { case "Deployment": var dep appsv1.Deployment if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil { return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err) } base := dep.DeepCopy() if dep.Spec.Template.Annotations == nil { dep.Spec.Template.Annotations = map[string]string{} } dep.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision if err := r.Patch(ctx, &dep, client.MergeFrom(base)); err != nil { return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err) } case "StatefulSet": var sts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &sts); err != nil { return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err) } base := sts.DeepCopy() if sts.Spec.Template.Annotations == nil { sts.Spec.Template.Annotations = map[string]string{} } sts.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision if err := r.Patch(ctx, &sts, client.MergeFrom(base)); err != nil { return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err) } default: return fmt.Errorf("consumer kind %q is not supported", consumer.Kind) } } return r.waitForConsumers(ctx, kr) } // waitForConsumers blocks until every consumer reports all replicas available. func (r *Reconciler) waitForConsumers(ctx context.Context, kr *iamv1.KeyRotation) error { deadline := r.now().Add(r.rolloutTimeout()) for { ready := true for _, consumer := range kr.Spec.Target.Consumers { ns := consumer.Namespace if ns == "" { ns = kr.Namespace } var dep appsv1.Deployment if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil { continue } desired := int32(1) if dep.Spec.Replicas != nil { desired = *dep.Spec.Replicas } if dep.Status.UpdatedReplicas < desired || dep.Status.AvailableReplicas < desired { ready = false break } } if ready { return nil } if r.now().After(deadline) { return fmt.Errorf("consumer did not become available within %s", r.rolloutTimeout()) } select { case <-ctx.Done(): return ctx.Err() case <-time.After(2 * time.Second): } } } func (r *Reconciler) rolloutTimeout() time.Duration { if r.RolloutTimeout > 0 { return r.RolloutTimeout } return 3 * time.Minute }