From 566dcafbf630a0fc76e06fff0e6eb5c556d7f951 Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Tue, 21 Jul 2026 17:06:46 -0700
Subject: [PATCH] fix(temporal): db-secret-sync Job deadlocked as PreSync hook

PreSync hooks run BEFORE an Application's own normal (non-hook) resources
are synced. This Job's ServiceAccount/ClusterRole/RoleBindings are plain
resources in the same Application, so marking the Job PreSync created a
chicken-and-egg deadlock: confirmed live, the Job sat 'Running' for 14
minutes producing zero pods, with job-controller repeatedly logging
'serviceaccount temporal/temporal-db-secret-sync not found' - because that
ServiceAccount hadn't been created yet (it's created during the normal Sync
phase, which comes after PreSync).

Fixed to PostSync. This app (sync-wave 7) still fully completes - including
this hook - before the temporal Application (sync-wave 8) begins, so the
ordering guarantee we need (secret exists before Temporal's pods try to
mount it) is unaffected; only the intra-app hook-vs-normal-resource
ordering was wrong.
---
 .../temporal/db-secret-sync/copy-job.yaml     | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/k8s/applications/temporal/db-secret-sync/copy-job.yaml b/k8s/applications/temporal/db-secret-sync/copy-job.yaml
index 2c1482d..0d63cde 100644
--- a/k8s/applications/temporal/db-secret-sync/copy-job.yaml
+++ b/k8s/applications/temporal/db-secret-sync/copy-job.yaml
@@ -11,10 +11,21 @@
 # Job's ddb-scoped RoleBinding back to temporal (same class of bug fixed
 # earlier in k8s/security/iam/kustomization.yaml - see that file's comments).
 #
-# PreSync + BeforeHookCreation: reruns on every ArgoCD sync of the temporal
-# app group, so it re-copies the password if CNPG ever rotates it. Runs
-# before the main `temporal` Application (sync-wave 8) since this app is
-# registered at sync-wave 7.
+# PostSync (not PreSync!) + BeforeHookCreation: reruns on every ArgoCD sync
+# of this app, re-copying the password if CNPG ever rotates it.
+#
+# IMPORTANT: this MUST be PostSync, not PreSync. The ServiceAccount/
+# ClusterRole/RoleBindings below are plain (non-hook) resources - ArgoCD
+# creates those during the normal "Sync" phase, which happens AFTER PreSync
+# hooks run. A PreSync-hooked Job here would try to start before its own
+# ServiceAccount exists (chicken-and-egg deadlock: confirmed live - the Job
+# sat 'Running' for 14 minutes, unable to create any pod at all, event log
+# showed "serviceaccount temporal/temporal-db-secret-sync not found" on
+# every attempt). PostSync runs after this app's own normal resources are
+# already applied, and this whole app (sync-wave 7) still fully completes
+# before the `temporal` Application (sync-wave 8) begins, so the ordering
+# guarantee we actually need (secret exists before Temporal's pods start)
+# is preserved regardless of PreSync vs PostSync here.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -64,7 +75,7 @@ metadata:
   name: temporal-db-secret-sync
   namespace: temporal
   annotations:
-    argocd.argoproj.io/hook: PreSync
+    argocd.argoproj.io/hook: PostSync
     argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
 spec:
   ttlSecondsAfterFinished: 600