homelab/k8s/apps/key-rotation-controller/controller/internal/reconcile/reconciler.go

// Package reconcile turns a KeyRotation object into a rotated credential.
//
// The state machine is identical for every provider. Only three calls differ:
// create, verify, retire. That is what makes one controller cover machine keys,
// client secrets and, later, the token-style providers.
//
// Order is the safety property:
//
//	Prepare    create the new credential
//	Publish    write the Secret, roll the consumer when it cannot reload
//	Verify     prove the new credential works against the provider
//	Retire     delete the old credential, only for overlapping providers
//
// A failure before Retire leaves the old credential in place, so a broken
// rotation never removes the thing that currently works.
package reconcile

import (
	"context"
	"fmt"
	"time"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"

	iamv1 "github.com/riotpiao/key-rotation-controller/api"
)

// Credential is fresh credential material, before it is written to a Secret.
type Credential struct {
	// ID identifies the credential at the provider, so it can be retired later.
	ID string
	// Fields maps a Secret key to its value.
	Fields map[string][]byte
}

// Provider creates, verifies and retires one class of credential.
type Provider interface {
	// Name is the provider name from the spec.
	Name() string
	// Strategy is overlap or atomic. Atomic providers replace in place.
	Strategy() string
	// Prepare creates a new credential at the provider.
	Prepare(ctx context.Context, kr *iamv1.KeyRotation) (*Credential, error)
	// Verify proves the credential works, using the provider's own check.
	Verify(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential) error
	// Retire deletes a superseded credential. It must be safe to call twice.
	Retire(ctx context.Context, kr *iamv1.KeyRotation, credentialID string) error
}

// Reconciler rotates credentials for KeyRotation objects.
type Reconciler struct {
	client.Client
	// Providers keyed by spec.provider.
	Providers map[string]Provider
	// Now is injectable so scheduling is testable.
	Now func() time.Time
	// RolloutTimeout bounds the wait for a consumer to become available.
	RolloutTimeout time.Duration
}

// previousKeySuffix keeps the superseded value beside the current one, so an
// atomic provider can be rolled back without storing a secret in the object.
const previousKeySuffix = ".previous"

// RequeueAfter is how long to wait before checking a resource again.
func (r *Reconciler) RequeueAfter(kr *iamv1.KeyRotation) time.Duration {
	interval := time.Duration(kr.Spec.IntervalHours) * time.Hour
	if interval <= 0 {
		interval = 30 * 24 * time.Hour
	}
	// Return 0 to rotate NOW and a positive duration to wait. A failed attempt must
	// rotate again immediately; only a rotation that reached Succeeded waits for the
	// next interval. Returning a wait here was the defect: the object sat in Failed
	// and reconciled every two minutes without ever trying again.
	// A failed attempt waits a short backoff and then really retries. Returning 0
	// unconditionally here spins: fail, rotate, fail, with no pause between.
	if kr.Status.Phase == iamv1.PhaseFailed {
		if kr.Status.LastRotation != nil {
			if wait := kr.Status.LastRotation.Add(2 * time.Minute).Sub(r.now()); wait > 0 {
				return wait
			}
		}
		return 0
	}
	if kr.Status.LastRotation == nil || kr.Status.ActiveCredentialID == "" {
		return 0 // rotate now
	}
	if kr.Status.Phase != iamv1.PhaseSucceeded {
		return 0 // an interrupted attempt starts over
	}
	due := kr.Status.LastRotation.Add(interval)
	wait := due.Sub(r.now())
	if wait < 0 {
		return 0
	}
	return wait
}

// Reconcile performs one rotation when the resource is due.
func (r *Reconciler) Reconcile(ctx context.Context, req types.NamespacedName) (time.Duration, error) {
	logger := log.FromContext(ctx).WithValues("keyrotation", req.String())

	var kr iamv1.KeyRotation
	if err := r.Get(ctx, req, &kr); err != nil {
		if apierrors.IsNotFound(err) {
			return 0, nil
		}
		return 0, err
	}

	logger.Info("reconcile", "phase", kr.Status.Phase, "generation", kr.Generation,
		"observed", kr.Status.ObservedGeneration, "lastRotation", kr.Status.LastRotation)

	if kr.Spec.Suspend {
		return 0, r.setPhase(ctx, &kr, iamv1.PhaseSuspended, "")
	}

	provider := r.Providers[kr.Spec.Provider]
	if provider == nil {
		return 0, r.setPhase(ctx, &kr, iamv1.PhaseFailed,
			fmt.Sprintf("no provider named %q", kr.Spec.Provider))
	}

	wait := r.RequeueAfter(&kr)
	if wait > 0 {
		return wait, nil
	}

	logger.Info("rotating credential", "provider", kr.Spec.Provider, "strategy", kr.Spec.Strategy)

	previousID := kr.Status.ActiveCredentialID
	previousFields := r.readCurrentSecret(ctx, &kr)

	// Prepare
	if err := r.setPhase(ctx, &kr, iamv1.PhasePreparing, ""); err != nil {
		return 0, err
	}
	cred, err := provider.Prepare(ctx, &kr)
	if err != nil {
		return r.fail(ctx, &kr, iamv1.PhasePreparing, err)
	}

	// Record the credential and the attempt time before anything else can fail.
	// Without this a failure later in the cycle leaves LastRotation empty, the due
	// check says "rotate now", and the controller mints a new secret on every
	// reconcile: a restart loop the consumer never survives.
	started := metav1.NewTime(r.now())
	kr.Status.ActiveCredentialID = cred.ID
	kr.Status.LastRotation = &started
	if err := r.Status().Update(ctx, &kr); err != nil && !apierrors.IsConflict(err) {
		return 0, err
	}

	// Publish. The previous value stays beside the new one until verification
	// passes, so an atomic rotation can be undone.
	if err := r.setPhase(ctx, &kr, iamv1.PhasePropagating, ""); err != nil {
		return 0, err
	}
	if err := r.writeSecret(ctx, &kr, cred, previousFields); err != nil {
		return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
	}
	if kr.Spec.Target.Reload != iamv1.ReloadHot {
		if err := r.rollConsumers(ctx, &kr, cred.ID); err != nil {
			return r.fail(ctx, &kr, iamv1.PhasePropagating, err)
		}
	}

	// Verify
	if err := r.setPhase(ctx, &kr, iamv1.PhaseVerifying, ""); err != nil {
		return 0, err
	}
	if err := provider.Verify(ctx, &kr, cred); err != nil {
		// Put the previous value back and roll again, then report the failure.
		r.restorePrevious(ctx, &kr, previousFields)
		return r.fail(ctx, &kr, iamv1.PhaseVerifying, err)
	}

	// Complete. Retire only when the provider can overlap, because retiring an
	// atomic credential would delete the one now in use.
	if err := r.setPhase(ctx, &kr, iamv1.PhaseCompleting, ""); err != nil {
		return 0, err
	}
	if provider.Strategy() == iamv1.StrategyOverlap && previousID != "" && previousID != cred.ID {
		if err := provider.Retire(ctx, &kr, previousID); err != nil {
			logger.Error(err, "retiring the previous credential failed; it stays valid", "credential", previousID)
		}
	}
	r.clearPrevious(ctx, &kr)

	now := metav1.NewTime(r.now())
	kr.Status.LastRotation = &now
	kr.Status.ActiveCredentialID = cred.ID
	kr.Status.PreviousCredentialID = previousID
	kr.Status.LastError = ""
	kr.Status.ObservedGeneration = kr.Generation
	if err := r.Status().Update(ctx, &kr); err != nil {
		return 0, err
	}
	if err := r.setPhase(ctx, &kr, iamv1.PhaseSucceeded, ""); err != nil {
		return 0, err
	}
	logger.Info("rotation complete", "credential", cred.ID, "previous", previousID)

	return r.RequeueAfter(&kr), nil
}

func (r *Reconciler) now() time.Time {
	if r.Now != nil {
		return r.Now()
	}
	return time.Now()
}

func (r *Reconciler) setPhase(ctx context.Context, kr *iamv1.KeyRotation, phase, message string) error {
	kr.Status.Phase = phase
	kr.Status.LastError = message
	kr.Status.ObservedGeneration = kr.Generation
	if err := r.Status().Update(ctx, kr); err != nil && !apierrors.IsConflict(err) {
		return err
	}
	return nil
}

func (r *Reconciler) fail(ctx context.Context, kr *iamv1.KeyRotation, phase string, cause error) (time.Duration, error) {
	kr.Status.Phase = iamv1.PhaseFailed
	kr.Status.LastError = fmt.Sprintf("%s: %v", phase, cause)
	_ = r.Status().Update(ctx, kr)
	// Retry soon, but not in a tight loop.
	return 5 * time.Minute, nil
}

// secretData assembles the Secret payload: the new values, plus the superseded
// values under a shadow key so an atomic rotation can be undone.
func secretData(kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) map[string][]byte {
	data := make(map[string][]byte, len(cred.Fields)*2)
	for logical, value := range cred.Fields {
		key := secretKeyFor(kr, logical)
		data[key] = value
		if old, ok := previous[key]; ok && len(old) > 0 {
			data[key+previousKeySuffix] = old
		}
	}
	return data
}

// secretKeyFor maps a logical field name to the Secret key, defaulting to the
// same name so a simple credential needs no mapping.
func secretKeyFor(kr *iamv1.KeyRotation, logical string) string {
	if key, ok := kr.Spec.Target.SecretKeys[logical]; ok && key != "" {
		return key
	}
	return logical
}

func (r *Reconciler) readCurrentSecret(ctx context.Context, kr *iamv1.KeyRotation) map[string][]byte {
	var secret corev1.Secret
	err := r.Get(ctx, types.NamespacedName{
		Namespace: kr.Spec.Target.SecretNamespace,
		Name:      kr.Spec.Target.SecretName,
	}, &secret)
	if err != nil {
		return nil
	}
	return secret.Data
}

func (r *Reconciler) writeSecret(ctx context.Context, kr *iamv1.KeyRotation, cred *Credential, previous map[string][]byte) error {
	data := secretData(kr, cred, previous)

	var secret corev1.Secret
	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
	err := r.Get(ctx, key, &secret)
	if apierrors.IsNotFound(err) {
		secret = corev1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Name:      key.Name,
				Namespace: key.Namespace,
				Labels:    map[string]string{"app.kubernetes.io/managed-by": "key-rotation-controller"},
			},
			Type: corev1.SecretTypeOpaque,
			Data: data,
		}
		return r.Create(ctx, &secret)
	}
	if err != nil {
		return err
	}
	// Merge, never replace. A consumer Secret often carries configuration beside the
	// credential (paperless keeps SOCIALACCOUNT_PROVIDERS_JSON there), and replacing
	// the whole map deletes it. The pod then fails with "couldn't find key ... in
	// Secret" and the rotation looks successful while it broke the service.
	if secret.Data == nil {
		secret.Data = map[string][]byte{}
	}
	for k, v := range data {
		secret.Data[k] = v
	}
	return r.Update(ctx, &secret)
}

// restorePrevious puts the shadow values back in place after a failed check.
func (r *Reconciler) restorePrevious(ctx context.Context, kr *iamv1.KeyRotation, previous map[string][]byte) {
	if len(previous) == 0 {
		return
	}
	var secret corev1.Secret
	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
	if err := r.Get(ctx, key, &secret); err != nil {
		return
	}
	// Restore every shadowed value over the current one.
	for k, v := range secret.Data {
		base := k
		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
			base = k[:len(k)-len(previousKeySuffix)]
			secret.Data[base] = v
		}
	}
	_ = r.Update(ctx, &secret)
	_ = r.rollConsumers(ctx, kr, "rollback")
}

// clearPrevious removes the shadow values once verification passed.
func (r *Reconciler) clearPrevious(ctx context.Context, kr *iamv1.KeyRotation) {
	var secret corev1.Secret
	key := types.NamespacedName{Namespace: kr.Spec.Target.SecretNamespace, Name: kr.Spec.Target.SecretName}
	if err := r.Get(ctx, key, &secret); err != nil {
		return
	}
	changed := false
	for k := range secret.Data {
		if len(k) > len(previousKeySuffix) && k[len(k)-len(previousKeySuffix):] == previousKeySuffix {
			delete(secret.Data, k)
			changed = true
		}
	}
	if changed {
		_ = r.Update(ctx, &secret)
	}
}

// rollConsumers restarts each consumer so it reads the new value.
func (r *Reconciler) rollConsumers(ctx context.Context, kr *iamv1.KeyRotation, revision string) error {
	for _, consumer := range kr.Spec.Target.Consumers {
		ns := consumer.Namespace
		if ns == "" {
			ns = kr.Namespace
		}
		switch consumer.Kind {
		case "Deployment":
			var dep appsv1.Deployment
			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
			}
			base := dep.DeepCopy()
			if dep.Spec.Template.Annotations == nil {
				dep.Spec.Template.Annotations = map[string]string{}
			}
			dep.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
			if err := r.Patch(ctx, &dep, client.MergeFrom(base)); err != nil {
				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
			}
		case "StatefulSet":
			var sts appsv1.StatefulSet
			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &sts); err != nil {
				return fmt.Errorf("consumer %s/%s: %w", ns, consumer.Name, err)
			}
			base := sts.DeepCopy()
			if sts.Spec.Template.Annotations == nil {
				sts.Spec.Template.Annotations = map[string]string{}
			}
			sts.Spec.Template.Annotations["iam.homelab.riotpiao.com/credential-revision"] = revision
			if err := r.Patch(ctx, &sts, client.MergeFrom(base)); err != nil {
				return fmt.Errorf("rolling %s/%s: %w", ns, consumer.Name, err)
			}
		default:
			return fmt.Errorf("consumer kind %q is not supported", consumer.Kind)
		}
	}
	return r.waitForConsumers(ctx, kr)
}

// waitForConsumers blocks until every consumer reports all replicas available.
func (r *Reconciler) waitForConsumers(ctx context.Context, kr *iamv1.KeyRotation) error {
	deadline := r.now().Add(r.rolloutTimeout())
	for {
		ready := true
		for _, consumer := range kr.Spec.Target.Consumers {
			ns := consumer.Namespace
			if ns == "" {
				ns = kr.Namespace
			}
			var dep appsv1.Deployment
			if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: consumer.Name}, &dep); err != nil {
				continue
			}
			desired := int32(1)
			if dep.Spec.Replicas != nil {
				desired = *dep.Spec.Replicas
			}
			if dep.Status.UpdatedReplicas < desired || dep.Status.AvailableReplicas < desired {
				ready = false
				break
			}
		}
		if ready {
			return nil
		}
		if r.now().After(deadline) {
			return fmt.Errorf("consumer did not become available within %s", r.rolloutTimeout())
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(2 * time.Second):
		}
	}
}

func (r *Reconciler) rolloutTimeout() time.Duration {
	if r.RolloutTimeout > 0 {
		return r.RolloutTimeout
	}
	// Grafana needs about four minutes from container start to binding :3000, so a
	// three-minute budget fails a rotation that is actually working.
	return 8 * time.Minute
}