diff --git a/k8s/argocd/apps/30-security.yaml b/k8s/argocd/apps/30-security.yaml index a955315..2ec6f46 100644 --- a/k8s/argocd/apps/30-security.yaml +++ b/k8s/argocd/apps/30-security.yaml @@ -57,6 +57,55 @@ spec: prune: true selfHeal: true --- +# ZITADEL — the Authentik replacement. Coexistence, not cutover: Authentik keeps +# serving in the same `iam` namespace until the last consumer moves, then this +# becomes the only IdP. Migration plan (outside git): +# ~/workplace/homelab-doc/ZITADEL_MIGRATION.html +# +# Wave 3 ordering: sops-secrets (wave 0) creates the `zitadel-masterkey` Secret, +# databases (wave 2) creates the `zitadel-db` CNPG cluster and its +# `zitadel-db-app` DSN secret. Both must exist before the init job starts. +# +# automated sync is deliberately absent for the first sync — same reason as +# forgejo. The chart ships `pre-install,pre-upgrade` Helm hooks (zitadel-init, +# zitadel-setup), which ArgoCD maps to PreSync and re-runs on every sync, manual +# or automated. A human reads the first sync's hook output before any repeat can +# run. The init job runs `init zitadel` only (CNPG owns the database and the app +# user), and the setup job creates the first org and the `iam-admin` machine key +# that Terraform consumes. Read the SECOND-SYNC HAZARD block at the top of +# k8s/infra/iam/zitadel-values.yaml before enabling automated sync. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: zitadel + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io + annotations: + argocd.argoproj.io/sync-wave: "3" +spec: + project: homelab + revisionHistoryLimit: 3 + sources: + - repoURL: https://charts.zitadel.com + chart: zitadel + # Pinned: the chart trails the upstream app release, and the verified + # masterkey/passwap facts were read at v4.19.x. A floating tag would + # change the app version without a diff anyone reviews. + targetRevision: 10.4.0 + helm: + valueFiles: + - $values/k8s/infra/iam/zitadel-values.yaml + - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git + targetRevision: main + ref: values + destination: + server: https://kubernetes.default.svc + namespace: iam + syncPolicy: + syncOptions: + - CreateNamespace=true +--- # Raw IAM manifests: key-rotation cronjob + authentik migration job. apiVersion: argoproj.io/v1alpha1 kind: Application diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml index 92a4eaa..29d958c 100644 --- a/k8s/argocd/secrets/secret-generator.yaml +++ b/k8s/argocd/secrets/secret-generator.yaml @@ -38,3 +38,4 @@ files: - forgejo-registry-token.enc.yaml - pi-sandbox-secrets.enc.yaml - etcd-defrag-creds.enc.yaml + - zitadel-masterkey.enc.yaml diff --git a/k8s/argocd/secrets/zitadel-masterkey.enc.yaml b/k8s/argocd/secrets/zitadel-masterkey.enc.yaml new file mode 100644 index 0000000..3337ceb --- /dev/null +++ b/k8s/argocd/secrets/zitadel-masterkey.enc.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Secret +metadata: + name: zitadel-masterkey + namespace: iam +type: Opaque +stringData: + masterkey: ENC[AES256_GCM,data:dGp0wSCUJORP6W5PHLKYaQEqrZsgiAwU1GqcuvuZV+g=,iv:Wwt6E/XXwsoVEQxe4gG1m5oJLOcNUXngftADQvZQepw=,tag:1Vget5gs+t455U9zLavdwg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBld0tUWHNWem13cDlxRG14 + QWRlTjA5SzZnS2pBd2M0WWkzQWxKZW9HZWdJCk00QWJJcWZFM1ZTSUlReUlGZG4z + RFBSUWRlVUx2NGN2OFAwazBYRTVVNkEKLS0tIHpSSW9IWHhNa0d3SmEydXU4akVj + MGJkN2U4WmtLbVFMZm55U1hEZElZMlUKKV9twnC9GC7MSmNLxEBHbPl4RN5Hsd+z + NViCKxUoQAB8apn2XVHZcDnyfKNfr1P3aSa3IIHyVY3prgo1OFDpdw== + -----END AGE ENCRYPTED FILE----- + recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + encrypted_regex: ^(data|stringData)$ + lastmodified: "2026-10-24T08:57:27Z" + mac: ENC[AES256_GCM,data:hXDAddVmnlf/yinY2c2Q8yfsjqi6n9cRbEbwykqYScd6mDPfdF7BriTiu9K+CV/IvIT4zcMTZbhBeQ/vTCAh0TZDp1lOVglwtNH/imgagyzPmcCvbCIkZKwF5vwWQIGHcDI6pk4/zVQkqt7zKTf8+Qj0pNh5UrAjVeEFE7cuJkQ=,iv:oIQvROFGbiESmMTthHOJiNM+Gy5BCfM5XtDWbszOPmc=,tag:AaYrJrwXuKD7w0p+TTa7FA==,type:str] + version: 3.13.2 diff --git a/k8s/infra/databases/kustomization.yaml b/k8s/infra/databases/kustomization.yaml index ff75276..a566c32 100644 --- a/k8s/infra/databases/kustomization.yaml +++ b/k8s/infra/databases/kustomization.yaml @@ -7,5 +7,6 @@ kind: Kustomization resources: - namespaces.yaml - authentik-db.yaml + - zitadel-db.yaml - temporal-db.yaml - paperless-db.yaml diff --git a/k8s/infra/databases/zitadel-db.yaml b/k8s/infra/databases/zitadel-db.yaml new file mode 100644 index 0000000..bb3f48f --- /dev/null +++ b/k8s/infra/databases/zitadel-db.yaml @@ -0,0 +1,42 @@ +# Dedicated CNPG Postgres for ZITADEL (GitOps, wave 2 — before the zitadel +# chart at wave 3). Same shape as authentik-db.yaml: CNPG generates the secret +# `zitadel-db-app` and the service `zitadel-db-rw` in ns iam, and +# k8s/infra/iam/zitadel-values.yaml reads the DSN from that secret locally (no +# cross-namespace secret copy). +# +# ZITADEL requires PostgreSQL 14+; the repo convention image is 16.2. +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: zitadel-db + namespace: iam + annotations: + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + instances: 3 + imageName: ghcr.io/cloudnative-pg/postgresql:16.2 + bootstrap: + initdb: + database: zitadel + owner: app + encoding: UTF8 + localeCollate: C + localeCType: C + enableSuperuserAccess: false + resources: + requests: { memory: "512Mi", cpu: "250m" } + limits: { memory: "2Gi", cpu: "1" } + storage: + size: 10Gi + storageClass: longhorn-cnpg + monitoring: + enablePodMonitor: true + affinity: + # preferred + tolerations: HA across nodes without deadlocking a partly- + # schedulable 3-CP (same as authentik-db and forgejo-db). + podAntiAffinityType: preferred + topologyKey: kubernetes.io/hostname + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule diff --git a/k8s/infra/iam/zitadel-values.yaml b/k8s/infra/iam/zitadel-values.yaml new file mode 100644 index 0000000..b3e3dbc --- /dev/null +++ b/k8s/infra/iam/zitadel-values.yaml @@ -0,0 +1,134 @@ +# k8s/infra/iam/zitadel-values.yaml +# ZITADEL — Authentik replacement. Coexistence is deliberate: Authentik keeps +# running, read-only, in the same `iam` namespace until the last consumer moves. +# Migration plan: ~/workplace/homelab-doc/ZITADEL_MIGRATION.html (outside git). +# +# Chart: zitadel/zitadel 10.4.0, app v4.19.2. The plan cites v4.19.4; the chart +# has not published it yet. Pin the chart, never `*`, until the v4.19.4 chart +# exists — a silent app bump invalidates the verified masterkey/passwap facts. +# +# URL: https://zitadel.riotpiao.com. TLS terminates at ingress-nginx, which +# serves the wildcard riotpiao-com-tls as its default-ssl-certificate, so +# ExternalSecure=true and TLS.Enabled=false (no per-rule tls: block needed). +# +# Database: CNPG cluster zitadel-db (wave 2). The DSN comes from CNPG's +# generated `zitadel-db-app` secret. When ZITADEL_DATABASE_POSTGRES_DSN is set, +# ZITADEL ignores the discrete Database.Postgres.* fields, so neither a +# password nor a host is written into this file. +# +# Masterkey: Secret `zitadel-masterkey` (k8s/argocd/secrets/ +# zitadel-masterkey.enc.yaml), key `masterkey`, exactly 32 bytes. It wraps the +# rows in system.encryption_keys, which encrypt every client secret and private +# key. Losing it makes those values unreadable: keep the offline sealed copy. +# +# Ingress uses the chart's own objects (controller: nginx adds the gRPC backend +# annotation ZITADEL needs for its HTTP/2 API). Do not hand-write a second +# Ingress for this host. + +zitadel: + configmapConfig: + ExternalSecure: true + ExternalDomain: zitadel.riotpiao.com + TLS: + Enabled: false + + # Rendered into a Secret by the chart; read by pods and init jobs alike. + masterkeySecretName: zitadel-masterkey + + resources: + requests: { memory: "512Mi", cpu: "250m" } + limits: { memory: "2Gi", cpu: "1" } + + # FirstInstance defaults stay for the FIRST sync: the chart creates org + # `zitadel` plus the `iam-admin` machine user and its machine-key Secret. + # Terraform consumes that key to declare projects, roles, OIDC apps and the + # groups action. + # + # SECOND-SYNC HAZARD — read before enabling automated sync. + # The chart marks eight objects as `pre-install,pre-upgrade` hooks (two + # ServiceAccounts, two ConfigMaps, Role, RoleBinding, zitadel-init, + # zitadel-setup), and one cleanup Job as `post-delete`. ArgoCD maps the first + # eight to PreSync with `before-hook-creation`: it deletes and re-creates them + # on EVERY sync, not only on a chart change. + # - zitadel-init (`init zitadel`) — schema step + # - zitadel-setup (`setup`) — first instance + iam-admin machine key + # On an initialized instance the setup step can fail, which aborts the sync + # before the Deployment is applied. If it succeeds instead, its + # machinekeyWriter container re-applies the iam-admin key Secret that + # Terraform reads, which silently rotates a credential in use. + # After the first sync reports healthy: + # 1. set configmapConfig.FirstInstance.Skip: true here — permanently: the + # setup steps are schema-only once the instance exists, and Skip alone + # stops the machinekeyWriter rotation + # 2. set initJob.enabled: false below + # 3. commit, sync once, then enable automated+selfHeal on the Application + # For a later schema upgrade, re-enable initJob.enabled only — never set + # FirstInstance.Skip back to false. + # ArgoCD never prunes a hook Job, so a Job that failed before step 2 stays and + # keeps ZitadelSetupJobFailed firing until a human deletes it. + +# DSN mode (chart renders this through the zitadel.dbEnv helper). CNPG's `uri` +# key is a postgresql:// URL, which ZITADEL accepts as a DSN. +env: + - name: ZITADEL_DATABASE_POSTGRES_DSN + valueFrom: + secretKeyRef: + name: zitadel-db-app + key: uri + +# Bundled Bitnami Postgres is disabled: the CNPG cluster above is the system of +# record. Enabling it would create a second, unmanaged database. +postgresql: + enabled: false + +# CNPG already created the database, the `app` user and its grants, so the init +# job runs only the ZITADEL-internal step. Running the full default command +# would try to create a database and a superuser it does not own. +initJob: + command: "zitadel" + activeDeadlineSeconds: 600 + +# Two API replicas: the read path survives one node drain, and both share the +# same masterkey and database. The login UI stays at one replica — a session +# cookie is not shared, so a second pod buys nothing. +replicaCount: 2 +revisionHistoryLimit: 3 + +ingress: + enabled: true + className: nginx + controller: nginx + +login: + enabled: true + replicaCount: 1 + ingress: + enabled: true + className: nginx + +# Chart-owned ServiceMonitor. The cluster's Prometheus runs with +# serviceMonitorSelectorNilUsesHelmValues=false, so the release label is not +# required for discovery. Endpoint is /debug/metrics on port http2-server. +metrics: + enabled: true + serviceMonitor: + enabled: true + scrapeInterval: 30s + scrapeTimeout: 10s + +# The 3-CP / 2-worker cluster mixes control-plane taints with worker capacity. +# Without the toleration a rolling upgrade can leave both replicas Pending. +tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: zitadel diff --git a/k8s/infra/monitoring/alerts/svc-zitadel-rules.yaml b/k8s/infra/monitoring/alerts/svc-zitadel-rules.yaml new file mode 100644 index 0000000..ba1fa01 --- /dev/null +++ b/k8s/infra/monitoring/alerts/svc-zitadel-rules.yaml @@ -0,0 +1,68 @@ +# ZITADEL alert parity with the Authentik rules it will replace. +# +# Only two signals are ported at this point. The metrics endpoint +# (/debug/metrics, scraped by the chart's own ServiceMonitor in ns iam) exposes +# no request or error counters that this cluster has verified, so no error-rate +# rule is written here. Writing one against a guessed series name would produce +# a permanently-firing or permanently-silent alert — both worse than none. +# AuthentikOutpostDown is deliberately not ported: it monitors an outpost that +# carries zero providers and is deleted with the migration, not migrated. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: zitadel-rules + namespace: iam +spec: + groups: + - name: zitadel.rules + interval: 30s + rules: + - alert: ZitadelDown + # absent() is required: equality alone cannot fire when the series is + # gone (ServiceMonitor deleted, Service selector changed, pods scaled + # to zero). Those are exactly the outages a target-down rule exists for. + expr: absent(up{job="zitadel"}) or up{job="zitadel"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "ZITADEL API target is down" + description: "Prometheus cannot scrape the ZITADEL API for 10 minutes. Token issuance and every OIDC login depend on it. Check the zitadel pods and the zitadel-db cluster in namespace iam. Value: {{ $value }}" + + - alert: ZitadelLoginUIDown + expr: absent(kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"}) or kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"} == 0 + for: 10m + labels: + severity: warning + annotations: + summary: "ZITADEL login UI has no available replica" + description: "The login UI serves /ui/v2/login. With zero available replicas a browser login completes at the IdP but cannot render. Value: {{ $value }}" + + - alert: ZitadelRestarting + # pod!~"zitadel-db-.*" matters: the CNPG database pods share the + # `zitadel` name prefix, and a database restart must not page as an + # IdP crash loop. + expr: absent(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}) or increase(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}[1h]) > 3 + for: 5m + labels: + severity: warning + annotations: + summary: "ZITADEL pod restarts more than 3 times per hour" + description: "A crash loop on the IdP is a cluster-wide login outage. Check the pod logs and the masterkey/database secret references. Value: {{ $value }}" + + # The chart's zitadel-init and zitadel-setup Jobs are PreSync hooks with + # restartPolicy: Never, so their restart counter stays 0. Without this + # rule a failed schema init or first-instance setup reaches nobody but + # the person who runs `kubectl -n iam get job`. + # No absent() guard here on purpose: the chart ships only + # `before-hook-creation`, so a successful hook Job is replaced at the + # next PreSync. Once the jobs are retired, absent() would fire on a + # healthy cluster. + - alert: ZitadelSetupJobFailed + expr: kube_job_status_failed{namespace="iam",job_name=~"zitadel-(init|setup).*"} > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "ZITADEL init/setup job failed" + description: "A PreSync job failed, so ArgoCD aborted the sync before the Deployment was applied. Read the job pod logs, then delete the Job object to let the next sync retry. Value: {{ $value }}" diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 754994d..e77d36b 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -8,6 +8,7 @@ resources: - alerts/ingress-alerts.yaml - alerts/svc-argocd-rules.yaml - alerts/svc-authentik-rules.yaml + - alerts/svc-zitadel-rules.yaml - alerts/svc-forgejo-rules.yaml - alerts/svc-grafana-rules.yaml - alerts/svc-minio-rules.yaml diff --git a/terraform/files/coredns/Corefile b/terraform/files/coredns/Corefile index dbdcb7c..1043df3 100644 --- a/terraform/files/coredns/Corefile +++ b/terraform/files/coredns/Corefile @@ -23,6 +23,7 @@ # via nginx (not the backend service) is critical: a direct rewrite bypasses # nginx TLS and exposes each app's own self-signed cert, trusted by nothing. rewrite name authentik.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local + rewrite name zitadel.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name grafana.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name minio.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name minio-api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local