Files
homelab/k8s/apps/key-rotation-controller/controller/main.go
T
rock c5058dadc6
key-rotation-controller / image (push) Failing after 6s
fix(iam): merge into the consumer Secret, and retry with a backoff
Two defects that a real rotation exposed:

  merge      writeSecret replaced the whole Secret map, so every key a consumer
             keeps beside the credential disappeared. paperless lost
             SOCIALACCOUNT_PROVIDERS_JSON and immich lost config.json; both pods then
             failed to start while the rotation reported success.
  backoff    a Failed attempt returned 0, so the controller retried in a tight loop.
             It now waits two minutes and then retries.

All eight objects carry both identifiers ZITADEL needs: the app resource id for the
secret endpoint and the client id for introspection.

Result: seven rotations reach Succeeded with every consumer ready (grafana, comfyui,
argocd, forgejo, paperless, immich, minio). api-gw stays suspended because no workload
exists for it.
2026-10-27 19:59:23 -07:00

156 lines
4.9 KiB
Go

// Command key-rotation-controller rotates credentials declared by KeyRotation
// objects.
//
// The controller authenticates to ZITADEL with its own machine key, read from a
// mounted Secret. That key is the single bootstrap credential: everything else is
// created by this process and never committed to git.
package main
import (
"context"
"flag"
"fmt"
"net/http"
"os"
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
"sigs.k8s.io/controller-runtime/pkg/source"
iamv1 "github.com/riotpiao/key-rotation-controller/api"
keyreconcile "github.com/riotpiao/key-rotation-controller/internal/reconcile"
"github.com/riotpiao/key-rotation-controller/internal/zitadel"
)
func main() {
var (
metricsAddr = flag.String("metrics-bind-address", ":8080", "address for the metrics endpoint")
healthAddr = flag.String("health-probe-bind-address", ":8081", "address for probes")
interval = flag.Duration("scan-interval", 10*time.Minute, "how often to re-check every resource")
)
flag.Parse()
ctrl.SetLogger(zap.New(zap.UseDevMode(false)))
logger := ctrl.Log.WithName("setup")
scheme := runtime.NewScheme()
if err := clientgoscheme.AddToScheme(scheme); err != nil {
logger.Error(err, "cannot register the Kubernetes types")
os.Exit(1)
}
if err := iamv1.AddToScheme(scheme); err != nil {
logger.Error(err, "cannot register the KeyRotation types")
os.Exit(1)
}
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Metrics: metricsserver.Options{BindAddress: *metricsAddr},
HealthProbeBindAddress: *healthAddr,
LeaderElection: true,
LeaderElectionID: "key-rotation-controller.iam.homelab.riotpiao.com",
})
if err != nil {
logger.Error(err, "cannot create the manager")
os.Exit(1)
}
zitadelClient, err := buildZitadelClient()
if err != nil {
logger.Error(err, "cannot build the ZITADEL client")
os.Exit(1)
}
reconciler := &keyreconcile.Reconciler{
Client: mgr.GetClient(),
Providers: map[string]keyreconcile.Provider{
iamv1.ProviderZitadelMachineKey: &keyreconcile.MachineKeyProvider{
Client: zitadelClient,
AuthClientID: os.Getenv("ZITADEL_CLIENT_ID"),
},
iamv1.ProviderZitadelOIDCClient: &keyreconcile.OIDCClientProvider{
Client: zitadelClient,
ClientIDFunc: func(kr *iamv1.KeyRotation) string {
return kr.Spec.Target.ZitadelClientID
},
},
},
}
c, err := controller.New("keyrotation", mgr, controller.Options{
Reconciler: ctrlreconcile.Func(func(ctx context.Context, req ctrlreconcile.Request) (ctrlreconcile.Result, error) {
wait, err := reconciler.Reconcile(ctx, types.NamespacedName(req.NamespacedName))
if err != nil {
return ctrlreconcile.Result{}, err
}
if wait <= 0 {
wait = *interval
}
return ctrlreconcile.Result{RequeueAfter: wait}, nil
}),
})
if err != nil {
logger.Error(err, "cannot create the controller")
os.Exit(1)
}
if err := c.Watch(source.Kind(mgr.GetCache(), client.Object(&iamv1.KeyRotation{}), &handler.EnqueueRequestForObject{})); err != nil {
logger.Error(err, "cannot watch KeyRotation objects")
os.Exit(1)
}
if err := mgr.AddHealthzCheck("healthz", func(*http.Request) error { return nil }); err != nil {
logger.Error(err, "cannot add the health check")
os.Exit(1)
}
logger.Info("starting the key rotation controller")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
logger.Error(err, "manager stopped")
os.Exit(1)
}
}
// buildZitadelClient reads the controller's own credential from the environment.
//
// These are the only values the controller cannot create for itself. They arrive
// from one Secret, created once when the cluster is bootstrapped.
func buildZitadelClient() (*zitadel.Client, error) {
issuer := os.Getenv("ZITADEL_ISSUER")
clientID := os.Getenv("ZITADEL_CLIENT_ID")
userID := os.Getenv("ZITADEL_USER_ID")
keyID := os.Getenv("ZITADEL_KEY_ID")
keyFile := envOr("ZITADEL_KEY_FILE", "/var/run/keys/controller/key")
if issuer == "" || clientID == "" || keyID == "" {
return nil, fmt.Errorf("ZITADEL_ISSUER, ZITADEL_CLIENT_ID and ZITADEL_KEY_ID are required")
}
pem, err := os.ReadFile(keyFile)
if err != nil {
return nil, fmt.Errorf("cannot read the controller key: %w", err)
}
return zitadel.New(zitadel.Config{
Issuer: issuer,
ClientID: clientID,
UserID: userID,
KeyID: keyID,
PrivateKeyPEM: pem,
}), nil
}
func envOr(name, fallback string) string {
if v := os.Getenv(name); v != "" {
return v
}
return fallback
}