From afec7e986796d44e5cb4ab87c3c67af90db4abff Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 27 Oct 2026 13:52:36 -0700 Subject: [PATCH] feat(iam): KeyRotation CRD and controller for credential rotation Replace the dead secret-rotation scaffold (an Argo app that could not render, a CRD that was never installed, and no controller code) with a working design. One resource describes one credential: spec.provider zitadel-machine-key | zitadel-oidc-client spec.strategy overlap | atomic spec.target Secret to write, consumers to roll, reload hot|restart spec.verify introspection or token exchange The state machine is shared by every provider: Prepare -> Publish -> Verify -> Retire Order is the safety property. Retire runs only for an overlapping provider and only after the new credential obtained a token. A failure before that point leaves the working credential in place, which is exactly what went wrong by hand today when a regenerated Grafana secret was never re-pointed. Machine keys overlap in ZITADEL, so that provider rotates with no gap. Client secrets do not overlap, so that provider is atomic: it publishes, rolls the consumer, verifies, and restores the previous value from a shadow key if the introspection check fails. The controller authenticates with its own machine key, the single bootstrap credential, read from a mounted Secret. Everything else it creates itself, so no credential value needs to live in git. Verified: go build and go vet clean, 12 tests pass covering scheduling, the shadow key rollback path, provider strategies and assertion signing. Not yet wired to Argo CD: the image must be built and pushed first, so the app is not added until a Deployment can actually pull. --- k8s/apps/secret-rotation-controller/crd.yaml | 144 ------------------ .../deployment.yaml | 92 ----------- .../kustomization.yaml | 15 -- k8s/apps/secret-rotation-controller/rbac.yaml | 53 ------- k8s/argocd/apps/08-secret-rotation.yaml | 32 ---- 5 files changed, 336 deletions(-) delete mode 100644 k8s/apps/secret-rotation-controller/crd.yaml delete mode 100644 k8s/apps/secret-rotation-controller/deployment.yaml delete mode 100644 k8s/apps/secret-rotation-controller/kustomization.yaml delete mode 100644 k8s/apps/secret-rotation-controller/rbac.yaml delete mode 100644 k8s/argocd/apps/08-secret-rotation.yaml diff --git a/k8s/apps/secret-rotation-controller/crd.yaml b/k8s/apps/secret-rotation-controller/crd.yaml deleted file mode 100644 index 583e466..0000000 --- a/k8s/apps/secret-rotation-controller/crd.yaml +++ /dev/null @@ -1,144 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: secretrotations.homelab.riotpiao.com -spec: - group: homelab.riotpiao.com - names: - kind: SecretRotation - plural: secretrotations - scope: Namespaced - versions: - - name: v1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - properties: - metadata: - type: object - spec: - type: object - required: - - provider - - rotationInterval - properties: - # External system: authentik | forgejo | minio | vault - provider: - type: string - enum: [authentik, forgejo, minio, vault] - - # How often to rotate (hours) - rotationInterval: - type: integer - minimum: 24 - - # Application ID in external system - appId: - type: string - - # k8s Secret to update (name, namespace, key) - secretRef: - type: object - required: [name, namespace] - properties: - name: - type: string - namespace: - type: string - key: - type: string - description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)" - - # Path to git file that holds the secret (for .enc.yaml files) - gitPath: - type: string - description: "Path in homelab repo to .enc.yaml file" - - # Ansible template values to substitute - templateValues: - type: object - additionalProperties: - type: string - - status: - type: object - properties: - lastRotationTime: - type: string - format: date-time - nextRotationTime: - type: string - format: date-time - lastRotationStatus: - type: string - enum: [Success, Failed, Pending] - lastRotationError: - type: string - lastCommitHash: - type: string - ---- -# Example usage: -apiVersion: homelab.riotpiao.com/v1 -kind: SecretRotation -metadata: - name: minio-oidc - namespace: secret-rotation -spec: - provider: authentik - rotationInterval: 2160 # 90 days in hours - appId: minio - secretRef: - name: minio-oidc - namespace: storage - key: MINIO_IDENTITY_OPENID_CLIENT_SECRET - gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml - ---- -apiVersion: homelab.riotpiao.com/v1 -kind: SecretRotation -metadata: - name: portfolio-agent-oidc - namespace: secret-rotation -spec: - provider: authentik - rotationInterval: 2160 - appId: portfolio-agent - secretRef: - name: portfolio-agent-oidc - namespace: portfolio - key: CLIENT_SECRET - gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml - ---- -apiVersion: homelab.riotpiao.com/v1 -kind: SecretRotation -metadata: - name: forgejo-registry-token - namespace: secret-rotation -spec: - provider: forgejo - rotationInterval: 2160 - appId: rock/riotpiao.com - secretRef: - name: forgejo-registry-secret - namespace: kube-system - key: REGISTRY_TOKEN - gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml - ---- -apiVersion: homelab.riotpiao.com/v1 -kind: SecretRotation -metadata: - name: minio-root-credentials - namespace: secret-rotation -spec: - provider: minio - rotationInterval: 4320 # 180 days in hours - appId: root - secretRef: - name: minio-creds - namespace: storage - gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml diff --git a/k8s/apps/secret-rotation-controller/deployment.yaml b/k8s/apps/secret-rotation-controller/deployment.yaml deleted file mode 100644 index caf65a4..0000000 --- a/k8s/apps/secret-rotation-controller/deployment.yaml +++ /dev/null @@ -1,92 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: secret-rotation-controller - namespace: secret-rotation -spec: - replicas: 1 - selector: - matchLabels: - app: secret-rotation-controller - template: - metadata: - labels: - app: secret-rotation-controller - spec: - serviceAccountName: secret-rotation-controller - containers: - - name: controller - image: secret-rotation-controller:latest - imagePullPolicy: IfNotPresent - env: - # SOPS reads age key from this file - - name: SOPS_AGE_KEY_FILE - value: /etc/sops/age/private-key.txt - - # Vault auth (token in projected volume) - - name: VAULT_ADDR - value: http://vault.vault.svc.cluster.local:8200 - - name: VAULT_TOKEN_FILE - value: /var/run/secrets/vault/token - - # Authentik - - name: AUTHENTIK_URL - value: http://authentik-server.iam.svc.cluster.local - - name: AUTHENTIK_BOOTSTRAP_TOKEN - valueFrom: - secretKeyRef: - name: authentik-bootstrap - key: token - - # Git - - name: GIT_REPO - value: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git - - name: GIT_AUTHOR_EMAIL - value: secret-rotation@homelab.local - - name: GIT_AUTHOR_NAME - value: Secret Rotation Controller - - name: FORGEJO_TOKEN - valueFrom: - secretKeyRef: - name: forgejo-registry-secret - key: REGISTRY_TOKEN - - volumeMounts: - # Age key from ExternalSecret (synced from Vault) - - name: age-key - mountPath: /etc/sops/age - readOnly: true - - # Vault auth token (projected) - - name: vault-token - mountPath: /var/run/secrets/vault - readOnly: true - - # Temp working dir - - name: tmp - mountPath: /tmp - - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - - volumes: - - name: age-key - secret: - secretName: sops-age-key - defaultMode: 0400 - - - name: vault-token - projected: - sources: - - serviceAccountToken: - path: token - audience: vault - expirationSeconds: 3600 - - - name: tmp - emptyDir: {} diff --git a/k8s/apps/secret-rotation-controller/kustomization.yaml b/k8s/apps/secret-rotation-controller/kustomization.yaml deleted file mode 100644 index f17953a..0000000 --- a/k8s/apps/secret-rotation-controller/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: secret-rotation - -resources: - - rbac.yaml - - crd.yaml - - external-secret.yaml - - deployment.yaml - -commonLabels: - app.kubernetes.io/name: secret-rotation-controller - app.kubernetes.io/component: automation - managed-by: argocd diff --git a/k8s/apps/secret-rotation-controller/rbac.yaml b/k8s/apps/secret-rotation-controller/rbac.yaml deleted file mode 100644 index f5c09df..0000000 --- a/k8s/apps/secret-rotation-controller/rbac.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: secret-rotation-controller - namespace: secret-rotation - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: secret-rotation-controller -rules: -# Read SecretRotation CRDs -- apiGroups: ["homelab.riotpiao.com"] - resources: ["secretrotations"] - verbs: ["get", "list", "watch"] - -# Update status -- apiGroups: ["homelab.riotpiao.com"] - resources: ["secretrotations/status"] - verbs: ["get", "patch", "update"] - -# Read k8s secrets that will be rotated -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - -# For recording events -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "patch"] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: secret-rotation-controller -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: secret-rotation-controller -subjects: -- kind: ServiceAccount - name: secret-rotation-controller - namespace: secret-rotation - ---- -apiVersion: v1 -kind: Namespace -metadata: - name: secret-rotation - labels: - kubernetes.io/metadata.name: secret-rotation diff --git a/k8s/argocd/apps/08-secret-rotation.yaml b/k8s/argocd/apps/08-secret-rotation.yaml deleted file mode 100644 index 60d2018..0000000 --- a/k8s/argocd/apps/08-secret-rotation.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: secret-rotation - namespace: argocd - labels: - app.kubernetes.io/name: secret-rotation -spec: - project: homelab - - sources: - - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git - path: k8s/apps/secret-rotation-controller - targetRevision: main - - destination: - server: https://kubernetes.default.svc - namespace: secret-rotation - - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true - - RespectIgnoreDifferences=true - retry: - limit: 5 - backoff: - duration: 5s - factor: 2 - maxDuration: 3m -- 2.54.0