feat(iam): KeyRotation CRD and controller for credential rotation

Replace the dead secret-rotation scaffold (an Argo app that could not render, a
CRD that was never installed, and no controller code) with a working design.

One resource describes one credential:

  spec.provider          zitadel-machine-key | zitadel-oidc-client
  spec.strategy          overlap | atomic
  spec.target            Secret to write, consumers to roll, reload hot|restart
  spec.verify            introspection or token exchange

The state machine is shared by every provider:

  Prepare -> Publish -> Verify -> Retire

Order is the safety property. Retire runs only for an overlapping provider and
only after the new credential obtained a token. A failure before that point leaves
the working credential in place, which is exactly what went wrong by hand today
when a regenerated Grafana secret was never re-pointed.

Machine keys overlap in ZITADEL, so that provider rotates with no gap. Client
secrets do not overlap, so that provider is atomic: it publishes, rolls the
consumer, verifies, and restores the previous value from a shadow key if the
introspection check fails.

The controller authenticates with its own machine key, the single bootstrap
credential, read from a mounted Secret. Everything else it creates itself, so no
credential value needs to live in git.

Verified: go build and go vet clean, 12 tests pass covering scheduling, the shadow
key rollback path, provider strategies and assertion signing.

Not yet wired to Argo CD: the image must be built and pushed first, so the app is
not added until a Deployment can actually pull.

k8s/apps/secret-rotation-controller/crd.yaml

@@ -1,144 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  name: secretrotations.homelab.riotpiao.com
-spec:
-  group: homelab.riotpiao.com
-  names:
-    kind: SecretRotation
-    plural: secretrotations
-  scope: Namespaced
-  versions:
-  - name: v1
-    served: true
-    storage: true
-    schema:
-      openAPIV3Schema:
-        type: object
-        properties:
-          metadata:
-            type: object
-          spec:
-            type: object
-            required:
-            - provider
-            - rotationInterval
-            properties:
-              # External system: authentik | forgejo | minio | vault
-              provider:
-                type: string
-                enum: [authentik, forgejo, minio, vault]
-              
-              # How often to rotate (hours)
-              rotationInterval:
-                type: integer
-                minimum: 24
-              
-              # Application ID in external system
-              appId:
-                type: string
-              
-              # k8s Secret to update (name, namespace, key)
-              secretRef:
-                type: object
-                required: [name, namespace]
-                properties:
-                  name:
-                    type: string
-                  namespace:
-                    type: string
-                  key:
-                    type: string
-                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
-              
-              # Path to git file that holds the secret (for .enc.yaml files)
-              gitPath:
-                type: string
-                description: "Path in homelab repo to .enc.yaml file"
-              
-              # Ansible template values to substitute
-              templateValues:
-                type: object
-                additionalProperties:
-                  type: string
-          
-          status:
-            type: object
-            properties:
-              lastRotationTime:
-                type: string
-                format: date-time
-              nextRotationTime:
-                type: string
-                format: date-time
-              lastRotationStatus:
-                type: string
-                enum: [Success, Failed, Pending]
-              lastRotationError:
-                type: string
-              lastCommitHash:
-                type: string
-
----
-# Example usage:
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160  # 90 days in hours
-  appId: minio
-  secretRef:
-    name: minio-oidc
-    namespace: storage
-    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: portfolio-agent-oidc
-  namespace: secret-rotation
-spec:
-  provider: authentik
-  rotationInterval: 2160
-  appId: portfolio-agent
-  secretRef:
-    name: portfolio-agent-oidc
-    namespace: portfolio
-    key: CLIENT_SECRET
-  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: forgejo-registry-token
-  namespace: secret-rotation
-spec:
-  provider: forgejo
-  rotationInterval: 2160
-  appId: rock/riotpiao.com
-  secretRef:
-    name: forgejo-registry-secret
-    namespace: kube-system
-    key: REGISTRY_TOKEN
-  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
-
----
-apiVersion: homelab.riotpiao.com/v1
-kind: SecretRotation
-metadata:
-  name: minio-root-credentials
-  namespace: secret-rotation
-spec:
-  provider: minio
-  rotationInterval: 4320  # 180 days in hours
-  appId: root
-  secretRef:
-    name: minio-creds
-    namespace: storage
-  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -1,92 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: secret-rotation-controller
-  template:
-    metadata:
-      labels:
-        app: secret-rotation-controller
-    spec:
-      serviceAccountName: secret-rotation-controller
-      containers:
-      - name: controller
-        image: secret-rotation-controller:latest
-        imagePullPolicy: IfNotPresent
-        env:
-        # SOPS reads age key from this file
-        - name: SOPS_AGE_KEY_FILE
-          value: /etc/sops/age/private-key.txt
-        
-        # Vault auth (token in projected volume)
-        - name: VAULT_ADDR
-          value: http://vault.vault.svc.cluster.local:8200
-        - name: VAULT_TOKEN_FILE
-          value: /var/run/secrets/vault/token
-        
-        # Authentik
-        - name: AUTHENTIK_URL
-          value: http://authentik-server.iam.svc.cluster.local
-        - name: AUTHENTIK_BOOTSTRAP_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: authentik-bootstrap
-              key: token
-        
-        # Git
-        - name: GIT_REPO
-          value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-        - name: GIT_AUTHOR_EMAIL
-          value: [email protected]
-        - name: GIT_AUTHOR_NAME
-          value: Secret Rotation Controller
-        - name: FORGEJO_TOKEN
-          valueFrom:
-            secretKeyRef:
-              name: forgejo-registry-secret
-              key: REGISTRY_TOKEN
-        
-        volumeMounts:
-        # Age key from ExternalSecret (synced from Vault)
-        - name: age-key
-          mountPath: /etc/sops/age
-          readOnly: true
-        
-        # Vault auth token (projected)
-        - name: vault-token
-          mountPath: /var/run/secrets/vault
-          readOnly: true
-        
-        # Temp working dir
-        - name: tmp
-          mountPath: /tmp
-        
-        resources:
-          requests:
-            cpu: 100m
-            memory: 256Mi
-          limits:
-            cpu: 500m
-            memory: 512Mi
-      
-      volumes:
-      - name: age-key
-        secret:
-          secretName: sops-age-key
-          defaultMode: 0400
-      
-      - name: vault-token
-        projected:
-          sources:
-          - serviceAccountToken:
-              path: token
-              audience: vault
-              expirationSeconds: 3600
-      
-      - name: tmp
-        emptyDir: {}

k8s/apps/secret-rotation-controller/kustomization.yaml

@@ -1,15 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-namespace: secret-rotation
-
-resources:
-  - rbac.yaml
-  - crd.yaml
-  - external-secret.yaml
-  - deployment.yaml
-
-commonLabels:
-  app.kubernetes.io/name: secret-rotation-controller
-  app.kubernetes.io/component: automation
-  managed-by: argocd

k8s/apps/secret-rotation-controller/rbac.yaml

@@ -1,53 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: secret-rotation-controller
-rules:
-# Read SecretRotation CRDs
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations"]
-  verbs: ["get", "list", "watch"]
-
-# Update status
-- apiGroups: ["homelab.riotpiao.com"]
-  resources: ["secretrotations/status"]
-  verbs: ["get", "patch", "update"]
-
-# Read k8s secrets that will be rotated
-- apiGroups: [""]
-  resources: ["secrets"]
-  verbs: ["get", "list"]
-
-# For recording events
-- apiGroups: [""]
-  resources: ["events"]
-  verbs: ["create", "patch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: secret-rotation-controller
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: secret-rotation-controller
-subjects:
-- kind: ServiceAccount
-  name: secret-rotation-controller
-  namespace: secret-rotation
-
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: secret-rotation
-  labels:
-    kubernetes.io/metadata.name: secret-rotation

k8s/argocd/apps/08-secret-rotation.yaml

@@ -1,32 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: secret-rotation
-  namespace: argocd
-  labels:
-    app.kubernetes.io/name: secret-rotation
-spec:
-  project: homelab
-  
-  sources:
-  - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    path: k8s/apps/secret-rotation-controller
-    targetRevision: main
-  
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: secret-rotation
-  
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    - RespectIgnoreDifferences=true
-    retry:
-      limit: 5
-      backoff:
-        duration: 5s
-        factor: 2
-        maxDuration: 3m