From 39081222a37a08060af78b6cb6bb4e69cc71030f Mon Sep 17 00:00:00 2001 From: pi Date: Sun, 25 Oct 2026 19:10:32 -0700 Subject: [PATCH] fix(iam): stop the setup hook running after first boot A PreSync setup run is not harmless once the instance exists: it replays every repeatable migration on every sync, and interrupting one leaves a system.migration.started event with no matching done. ZITADEL then waits forever on 'migration already started'. That deadlocked this instance twice (projections.projects4, then repeatable_delete_stale_org_fields). Move the setup job's hook from pre-install,pre-upgrade to post-delete so ArgoCD sync never runs it. Re-enable deliberately for a schema upgrade. --- k8s/infra/iam/zitadel-values.yaml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/k8s/infra/iam/zitadel-values.yaml b/k8s/infra/iam/zitadel-values.yaml index 003d54a..dd6f15f 100644 --- a/k8s/infra/iam/zitadel-values.yaml +++ b/k8s/infra/iam/zitadel-values.yaml @@ -118,12 +118,27 @@ initJob: activeDeadlineSeconds: 600 # The setup job runs the full first-instance creation: schema migrations, the -# system org, the iam-admin machine key, and 13 projection replays. The chart -# default killed it at ~5.5 minutes on 2026-10-25 while it was still replaying -# projections.projects4, which failed the whole sync. 20 minutes leaves room for -# a cold database without masking a genuine hang. +# system org, the iam-admin machine key, and 13 projection replays. 20 minutes +# leaves room for a cold database without masking a genuine hang. +# +# DONE, 2026-10-26 — first boot completed. The annotations are moved from +# pre-install/pre-upgrade to post-delete because a PreSync setup run is not +# harmless after boot: +# - it re-runs every repeatable migration on every sync; +# - interrupting one (a sync aborted, or a human deleting the Job) leaves a +# `system.migration.started` event with no matching `done`, and ZITADEL then +# waits forever on `migration already started`. That is exactly how this +# instance deadlocked on projections.projects4 and later on +# repeatable_delete_stale_org_fields. +# A post-delete hook is never executed by ArgoCD sync, so setup runs only on +# first install. Re-enable it deliberately (and only with initJob too) for a +# schema upgrade, then revert. setupJob: activeDeadlineSeconds: 1200 + annotations: + helm.sh/hook: post-delete + helm.sh/hook-delete-policy: hook-succeeded + helm.sh/hook-weight: "2" # Two API replicas: the read path survives one node drain, and both share the # same masterkey and database. The login UI stays at one replica — a session