From 7f29d640fde54528ecebfebddab4baf6c78a5206 Mon Sep 17 00:00:00 2001 From: rock Date: Mon, 26 Oct 2026 23:54:57 -0700 Subject: [PATCH] feat(iam): own the ZITADEL core objects in GitOps The chart declares its ServiceAccount, ConfigMap, Role and RoleBinding as pre-install/pre-upgrade Helm hooks with hook-delete-policy before-hook-creation. ArgoCD maps those to PreSync and prunes them when a sync ends, so every sync deleted the ServiceAccount and ConfigMap the ZITADEL Deployment mounts. New pods failed with `serviceaccount "zitadel" not found`, leaving the IdP one restart from an outage. Add a zatadel-core Application owning hook-free copies, keeping them as ordinary desired state that survives every sync. Prune is off for this path: deleting a ServiceAccount here would recreate the outage. The ConfigMap also carries the notification settings, pointing DefaultInstance.SMTPConfiguration at the smtp-gotify relay, with SMTPSenderAddressMatchesInstanceDomain false because the sender zitadel@riotpiao.com differs from the instance domain zatadel.riotpiao.com. Drop the envVarsSecret approach: ZITADEL created no SMTP provider from those env vars (no instance.smtp.config.added event). --- k8s/argocd/apps/30-security.yaml | 33 +++++ k8s/argocd/secrets/secret-generator.yaml | 1 - .../secrets/zitadel-smtp-secrets.enc.yaml | 25 ---- k8s/infra/iam/zitadel-values.yaml | 16 +-- k8s/infra/zitadel-core/kustomization.yaml | 4 + k8s/infra/zitadel-core/zitadel-core.yaml | 133 ++++++++++++++++++ 6 files changed, 173 insertions(+), 39 deletions(-) delete mode 100644 k8s/argocd/secrets/zitadel-smtp-secrets.enc.yaml create mode 100644 k8s/infra/zitadel-core/kustomization.yaml create mode 100644 k8s/infra/zitadel-core/zitadel-core.yaml diff --git a/k8s/argocd/apps/30-security.yaml b/k8s/argocd/apps/30-security.yaml index 511d5d5..a015dce 100644 --- a/k8s/argocd/apps/30-security.yaml +++ b/k8s/argocd/apps/30-security.yaml @@ -79,6 +79,39 @@ spec: syncOptions: - CreateNamespace=true --- +# ZITADEL core objects (ServiceAccount, Role, RoleBinding, ConfigMap). +# +# The chart declares these as pre-install/pre-upgrade Helm hooks, and ArgoCD +# prunes hook resources when a sync ends. That deleted the ServiceAccount and +# ConfigMap the ZITADEL Deployment references, so new pods failed to start with +# `serviceaccount "zitadel" not found`. This Application owns hook-free copies, +# which makes them ordinary desired state that survives every sync. +# +# Wave 2: before the zatadel chart itself, so the ServiceAccount exists by the +# time its Deployment is created. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: zitadel-core + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" +spec: + project: homelab + source: + repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git + targetRevision: main + path: k8s/infra/zitadel-core + destination: + server: https://kubernetes.default.svc + namespace: iam + syncPolicy: + automated: + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=true +--- # Raw IAM support manifests: the SMTP -> Gotify notification relay that ZITADEL # points at, so verification and password-reset codes reach the phone. Wave 3 # alongside ZITADEL itself; the Secret it reads is created by sops-secrets at diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml index 03bcc74..77211d2 100644 --- a/k8s/argocd/secrets/secret-generator.yaml +++ b/k8s/argocd/secrets/secret-generator.yaml @@ -36,7 +36,6 @@ files: - etcd-defrag-creds.enc.yaml - zitadel-masterkey.enc.yaml - smtp-gotify-secrets.enc.yaml - - zitadel-smtp-secrets.enc.yaml # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD # owns it. Without this, the chart re-generates the keypair on every render, # which changes checksum/secret-login-service-key and leaves both Zitadel diff --git a/k8s/argocd/secrets/zitadel-smtp-secrets.enc.yaml b/k8s/argocd/secrets/zitadel-smtp-secrets.enc.yaml deleted file mode 100644 index 3fecdb6..0000000 --- a/k8s/argocd/secrets/zitadel-smtp-secrets.enc.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: zitadel-smtp-env - namespace: iam -stringData: - ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_SMTP_HOST: ENC[AES256_GCM,data:c1O6xqe9O7b6OzDCYSQJjqkCvc7cisM/dIgGu3aC6yZGIFuuvfE=,iv:aAVqKCgi8NspPAiIKsgK4yfy8CDqAO2Y/GVdUKpVoGc=,tag:2t1GkBBVyimKorM291cR6A==,type:str] - ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_TLS: ENC[AES256_GCM,data:Pni4WpI=,iv:g11iVRNTYs9VHjX7TATYkzgFR1UQ0QbpXlXxfkEwzQQ=,tag:VzJeOG58w7RqCMk7NZJNxw==,type:str] - ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_FROM: ENC[AES256_GCM,data:ldRZWrqh+geS9ooxaH4kXbkJinE=,iv:ZPOrJj5R5i5FOvRvObNE/Fw2oPKyH/lw7+pUSP1qE8o=,tag:fsICt7sB4De44VLy3v2nsQ==,type:str] - ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_FROMNAME: ENC[AES256_GCM,data:DpRonnBJpQ==,iv:+oucgXGstpvduMhZIZjmrvbbm8gixr2D2zET410fXRw=,tag:eTVFhvg+Xnr4m6o+iFWVAw==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1eVFtenRNQXBKNndNZXpO - TXJoWWF1V2xnNUV2dVpxeXZWelBNVFZnTlFRCmUzT1RNL1lXbmMvR25LdWtrTXBB - WWNZK3hLRCtDVWl0enhLZmVpVjBlclEKLS0tIGYyWTlMaXQzbG12aGxTQi9CcnRM - UjRUOGJmQkZiU0I4L1MzaG1seFNJdncKqI3pFbUKj16946H2p9AHSK2DSArvo5Ep - y8y/aYc+NNPBaJzy/cdo9aI2mT4tNuw0IsM4f0VXlfEIc3teR5hdNA== - -----END AGE ENCRYPTED FILE----- - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla - encrypted_regex: ^(data|stringData)$ - lastmodified: "2026-10-27T06:45:46Z" - mac: ENC[AES256_GCM,data:ec9gwKHWtajOyPO5+4ZEAhCuLSlmbQWgcJXL+I/E32/AIoc8NX4Ec8AoeURbfCiknYgeTizAKA2BAipJkZQPev/6j3tuwLI5A34k1g1ax7gdc1gGvEZEpJ03u5PUgJRXfHi4mgSSdF1MrasogOUe2tYSwVwF+HnByBTvxj6HOzk=,iv:8EuCICwZghS4u8PQ0bJwTwTOo12zlVPGOp9+qqJ/Kt0=,tag:CkJGpzS7lGe3SOBtRId9Kg==,type:str] - version: 3.13.2 diff --git a/k8s/infra/iam/zitadel-values.yaml b/k8s/infra/iam/zitadel-values.yaml index 6c609be..321ce49 100644 --- a/k8s/infra/iam/zitadel-values.yaml +++ b/k8s/infra/iam/zitadel-values.yaml @@ -145,19 +145,9 @@ setupJob: replicaCount: 2 revisionHistoryLimit: 3 -# Notification delivery. ZITADEL sends verification codes and password resets -# by SMTP only, and this cluster has no mail path, so every user sits at -# verified_email = empty and cannot finish a first login. -# -# The settings go in through envVarsSecret rather than configmapConfig on -# purpose: changing configmapConfig makes ArgoCD stop managing the rendered -# `zitadel-config-yaml` ConfigMap, which the Deployment still mounts, so the -# next pod restart fails. An envFrom secret overrides the same keys with no -# effect on that ConfigMap. -# -# Target is the in-cluster smtp-gotify relay, which forwards to Gotify so the -# codes arrive on the phone. -envVarsSecret: zitadel-smtp-env +# Notification delivery lives in the ConfigMap owned by the zitadel-core app +# (k8s/infra/zitadel-core/). Env overrides here produced no SMTP provider, and +# editing configmapConfig made ArgoCD drop the rendered ConfigMap. # Best practice: a PDB guarantees at least one API pod stays available during # voluntary disruptions (node drain, rolling upgrade). Without it a drain can diff --git a/k8s/infra/zitadel-core/kustomization.yaml b/k8s/infra/zitadel-core/kustomization.yaml new file mode 100644 index 0000000..6a5d826 --- /dev/null +++ b/k8s/infra/zitadel-core/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - zitadel-core.yaml diff --git a/k8s/infra/zitadel-core/zitadel-core.yaml b/k8s/infra/zitadel-core/zitadel-core.yaml new file mode 100644 index 0000000..bf8853d --- /dev/null +++ b/k8s/infra/zitadel-core/zitadel-core.yaml @@ -0,0 +1,133 @@ +# ZITADEL core objects, owned by this repository instead of the Helm chart. +# +# The zitadel chart marks four objects as Helm hooks: +# +# kind: ServiceAccount (also ConfigMap/zitadel-config-yaml, Role, RoleBinding) +# annotations: +# helm.sh/hook: pre-install,pre-upgrade +# helm.sh/hook-delete-policy: before-hook-creation +# +# ArgoCD maps pre-install/pre-upgrade to PreSync and prunes hook resources when +# the sync ends. Every sync therefore deleted the ServiceAccount and the ConfigMap +# that the ZITADEL Deployment references, which produced: +# +# error creating: pods "zitadel-..." is forbidden: +# serviceaccount "zitadel" not found +# +# and left the IdP one restart away from failing. +# +# These copies carry no hook annotations, so they are ordinary desired state and +# ArgoCD keeps them. The chart still renders its own hook versions during a sync; +# these win because the sync no longer removes them. +# +# ConfigMap content mirrors what the chart renders from +# k8s/infra/iam/zitadel-values.yaml, plus the notification settings the chart +# cannot deliver through values (see the SMTP note below). When the chart values +# change, update this file too; it is the authoritative copy at runtime. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: zitadel + namespace: iam + labels: + app.kubernetes.io/name: zitadel + app.kubernetes.io/instance: zitadel + app.kubernetes.io/managed-by: gitops +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: zitadel + namespace: iam + labels: + app.kubernetes.io/name: zitadel + app.kubernetes.io/instance: zitadel + app.kubernetes.io/managed-by: gitops +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "patch", "list", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: zitadel + namespace: iam + labels: + app.kubernetes.io/name: zitadel + app.kubernetes.io/instance: zitadel + app.kubernetes.io/managed-by: gitops +subjects: + - kind: ServiceAccount + name: zitadel +roleRef: + kind: Role + name: zitadel + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: zitadel-config-yaml + namespace: iam + labels: + app.kubernetes.io/name: zitadel + app.kubernetes.io/instance: zitadel + app.kubernetes.io/managed-by: gitops +data: + zitadel-config-yaml: | + ExternalDomain: zitadel.riotpiao.com + ExternalSecure: true + TLS: + Enabled: false + FirstInstance: + # First boot is COMPLETE. Skip stops the setup step from running again and + # silently rotating the iam-admin key that provisioning reads. + Skip: true + Org: + Skip: true + Machine: + Machine: + Name: Automatically Initialized IAM Admin + Username: iam-admin + MachineKey: + ExpirationDate: "2029-01-01T00:00:00Z" + Type: 1 + Pat: + ExpirationDate: "2029-01-01T00:00:00Z" + Machine: + Identification: + Hostname: + Enabled: true + Webhook: + Enabled: false + # Notification delivery. + # + # ZITADEL sends verification codes and password resets by SMTP only. This + # cluster has no mail path, so without a provider every user stays at + # verified_email = empty and cannot finish a first login. + # + # The target is the in-cluster smtp-gotify relay, which forwards each message + # to Gotify so codes reach the phone. + # + # Why here and not in chart values: injecting the same keys as + # ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_* env vars produced no + # instance.smtp.config.added event, so ZITADEL never created the provider. + # Setting configmapConfig in chart values made ArgoCD stop managing this + # ConfigMap. Owning the file here avoids both problems. + DefaultInstance: + DomainPolicy: + # The sender is zitadel@riotpiao.com while the instance domain is + # zitadel.riotpiao.com. Leave this true so the sender address is not + # rejected for not matching the domain. + SMTPSenderAddressMatchesInstanceDomain: false + SMTPConfiguration: + SMTP: + Host: smtp-gotify.iam.svc.cluster.local:2525 + TLS: false + From: zitadel@riotpiao.com + FromName: ZITADEL