refactor(temporal): adopt unified CNPG pattern - use 'app' user

CHANGES:
  - temporal-values.yaml: user 'app', existingSecret 'ddb-cluster-app'
  - bootstrap.sh: Copy ddb-cluster-app to temporal namespace
  - Removed db-secret-sync directory (obsolete PostSync Job)
  - 60-applications.yaml: Removed db-secret-sync source from temporal Application

PATTERN (same as Forgejo/Authentik):
  1. Database CR: owner app
  2. bootstrap.sh: Copy ddb-cluster-app to temporal namespace
  3. App values: Reference ddb-cluster-app secret
  4. No PostSync Jobs needed

FIXES:
  - Temporal schema CrashLoopBackOff (wrong credentials)
  - Dropped/recreated databases with app owner (clean state)

Following CLAUDE.md CNPG pattern documentation.

bootstrap.sh

@@ -88,6 +88,12 @@ kubectl get secret ddb-cluster-app -n ddb -o yaml \
   | sed 's/namespace: ddb/namespace: iam/' \
   | kubectl apply -f -
 
+
+# Copy DB secret to temporal namespace (for temporal)
+log "Copying ddb-cluster-app secret to temporal namespace..."
+kubectl get secret ddb-cluster-app -n ddb -o yaml \
+  | sed 's/namespace: ddb/namespace: temporal/' \
+  | kubectl apply -f -
 # 7. Wait for Forgejo
 log "Waiting for Forgejo to be ready..."
 kubectl wait --for=condition=available --timeout=600s \

k8s/applications/temporal/db-secret-sync/copy-job.yaml

@@ -1,134 +0,0 @@
-# Copies the CNPG-generated temporal-db-role Secret from the ddb namespace
-# into the temporal namespace, as a plain (non-SOPS) k8s Secret with the same
-# keys. Kubernetes Secrets are strictly namespace-scoped - a Deployment in
-# `temporal` cannot reference a Secret living in `ddb` via secretKeyRef, and
-# temporal-values.yaml's server.config.persistence.*.sql.existingSecret:
-# temporal-db-role expects to find it in ITS OWN namespace (temporal).
-#
-# Deliberately a standalone directory (no kustomization.yaml) applied as its
-# own small Application - avoids the k8s/applications/temporal/kustomization.yaml
-# `namespace: temporal` transformer, which would silently force-rewrite this
-# Job's ddb-scoped RoleBinding back to temporal (same class of bug fixed
-# earlier in k8s/security/iam/kustomization.yaml - see that file's comments).
-#
-# PostSync (not PreSync!) + BeforeHookCreation: reruns on every ArgoCD sync
-# of this app, re-copying the password if CNPG ever rotates it.
-#
-# IMPORTANT: this MUST be PostSync, not PreSync. The ServiceAccount/
-# ClusterRole/RoleBindings below are plain (non-hook) resources - ArgoCD
-# creates those during the normal "Sync" phase, which happens AFTER PreSync
-# hooks run. A PreSync-hooked Job here would try to start before its own
-# ServiceAccount exists (chicken-and-egg deadlock: confirmed live - the Job
-# sat 'Running' for 14 minutes, unable to create any pod at all, event log
-# showed "serviceaccount temporal/temporal-db-secret-sync not found" on
-# every attempt). PostSync runs after this app's own normal resources are
-# already applied, and this whole app (sync-wave 7) still fully completes
-# before the `temporal` Application (sync-wave 8) begins, so the ordering
-# guarantee we actually need (secret exists before Temporal's pods start)
-# is preserved regardless of PreSync vs PostSync here.
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: temporal-db-secret-sync
-  namespace: temporal
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: temporal-db-secret-sync
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list", "create", "update", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: temporal-db-secret-sync
-  namespace: ddb
-subjects:
-  - kind: ServiceAccount
-    name: temporal-db-secret-sync
-    namespace: temporal
-roleRef:
-  kind: ClusterRole
-  name: temporal-db-secret-sync
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: temporal-db-secret-sync
-  namespace: temporal
-subjects:
-  - kind: ServiceAccount
-    name: temporal-db-secret-sync
-    namespace: temporal
-roleRef:
-  kind: ClusterRole
-  name: temporal-db-secret-sync
-  apiGroup: rbac.authorization.k8s.io
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: temporal-db-secret-sync
-  namespace: temporal
-  annotations:
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-spec:
-  ttlSecondsAfterFinished: 600
-  backoffLimit: 5
-  template:
-    spec:
-      serviceAccountName: temporal-db-secret-sync
-      restartPolicy: Never
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: copy
-          # bitnami/kubectl:1.30 does NOT exist - Bitnami stopped publishing
-          # versioned tags in 2025 (only `latest` + sha256-pinned digests
-          # remain), confirmed live via Docker Hub API before this fix - the
-          # original tag caused an indefinite ImagePullBackOff. Using
-          # python:3.12-alpine + a stdlib urllib kubectl download instead,
-          # same pattern already proven working in
-          # k8s/security/iam/authentik-provision-job.yaml - avoids depending
-          # on any third party's tagging policy at all.
-          image: python:3.12-alpine
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-          command:
-            - /bin/sh
-            - -c
-            - |
-              set -e
-              echo "installing kubectl (pure python urllib, no apk - see"
-              echo "authentik-provision-job.yaml for why apk fails as non-root)..."
-              python3 -c "
-              import urllib.request, os, stat
-              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
-              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
-              urllib.request.urlretrieve(url, '/tmp/kubectl')
-              st = os.stat('/tmp/kubectl')
-              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
-              "
-              export PATH="/tmp:$PATH"
-              echo "waiting for ddb/temporal-db-role..."
-              until kubectl -n ddb get secret temporal-db-role >/dev/null 2>&1; do
-                echo "  not ready yet, retrying..."
-                sleep 5
-              done
-              USERNAME=$(kubectl -n ddb get secret temporal-db-role -o jsonpath='{.data.username}' | base64 -d)
-              PASSWORD=$(kubectl -n ddb get secret temporal-db-role -o jsonpath='{.data.password}' | base64 -d)
-              kubectl -n temporal create secret generic temporal-db-role \
-                --from-literal=username="$USERNAME" \
-                --from-literal=password="$PASSWORD" \
-                --dry-run=client -o yaml | kubectl apply -f -
-              echo "synced temporal-db-role -> temporal namespace"

k8s/applications/temporal/temporal-values.yaml

@@ -95,14 +95,15 @@ server:
           host: "ddb-cluster-rw.ddb.svc.cluster.local"
           port: 5432
           database: "temporal"
-          user: "temporal"
+          user: "app"
           # existingSecret + secretKey: point directly at the CNPG-generated
           # Secret (kubernetes.io/basic-auth, keys: username/password/...)
           # rather than duplicating the password in git as plaintext. When
           # existingSecret is set the chart's own server-secret.yaml Secret
           # template is skipped entirely (see templates/server-secret.yaml:
           # `not $driverConfig.existingSecret` guards its creation).
-          existingSecret: "temporal-db-role"
+          # Use unified ddb-cluster-app secret (copied to temporal namespace)
+          existingSecret: "ddb-cluster-app"
           secretKey: "password"
           maxConns: 20
           maxIdleConns: 10
@@ -120,8 +121,9 @@ server:
           host: "ddb-cluster-rw.ddb.svc.cluster.local"
           port: 5432
           database: "temporal_visibility"
-          user: "temporal"
+          user: "app"
-          existingSecret: "temporal-db-role"
+          # Use unified ddb-cluster-app secret (copied to temporal namespace)
+          existingSecret: "ddb-cluster-app"
           secretKey: "password"
           maxConns: 20
           maxIdleConns: 10

k8s/applications/temporal/temporal-values.yaml.bak

@@ -0,0 +1,140 @@
+# k8s/temporal/temporal-values.yaml
+# Temporal — workflow engine
+# Uses external CNPG PostgreSQL for persistence (ddb-cluster)
+# Visibility via same PostgreSQL instance, separate database.
+#
+# IMPORTANT — chart schema note (root-caused after Postgres never actually
+# taking effect despite looking configured):
+# We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in
+# k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence
+# schema:
+#   server.config.persistence.<default|visibility>.driver: "sql"|"cassandra"
+#   server.config.persistence.<default|visibility>.sql: {...}
+# NOT the newer `datastores:`-wrapped schema
+# (server.config.persistence.datastores.<store>.sql) shown in the current
+# chart's values/values.postgresql.yaml example - that key was introduced in
+# a later major version and doesn't exist in 0.74.0. Helm doesn't validate
+# unknown keys, so a `datastores:` block here is silently a no-op: Temporal
+# would keep defaulting to Cassandra (with empty hosts: []) regardless of
+# anything nested inside it. Verified via `helm template` against the actual
+# 0.74.0 chart before writing this file - see chat history for the
+# side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and
+# temporal-cassandra-tool commands using the old datastores:-based values).
+#
+# Likewise `schema.setup.enabled` / `schema.update.enabled` /
+# `schema.createDatabase.enabled` are the real toggles for the schema-setup
+# Job (all default true) - there is no `jobs.autoSetup` key in this chart.
+
+# ── Disable every bundled/optional sub-chart ─────────────────────────────────
+# postgresql/mysql: never enable - we never want the chart to deploy its own
+# DB, only to know how to talk to our external CNPG instance (which happens
+# via server.config.persistence.*.sql below, independent of these flags).
+postgresql:
+  enabled: false
+mysql:
+  enabled: false
+cassandra:
+  enabled: false
+elasticsearch:
+  enabled: false
+prometheus:
+  enabled: false
+grafana:
+  enabled: false
+
+# ── Schema setup/update Jobs ──────────────────────────────────────────────────
+# The `temporal` and `temporal_visibility` databases are provisioned
+# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml,
+# temporal-visibility-database.yaml), so createDatabase stays disabled (the
+# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as
+# the `temporal` owner against those existing DBs to install and migrate the
+# Temporal server schema — without them both DBs have zero tables and the
+# server dies on "no usable database connection found" (no schema_version row).
+schema:
+  createDatabase:
+    enabled: false
+  setup:
+    enabled: true
+  update:
+    enabled: true
+
+# ── Temporal server config (PostgreSQL persistence) ──────────────────────────
+server:
+  replicaCount: 1
+  # temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to
+  # built-in sprig config templating. The chart still defaults to the legacy
+  # configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a
+  # config the 1.30 server never loads — it then falls back to its embedded
+  # env-only template (Cassandra default) and dies with
+  # "Persistence.DataStores[default](value).Cassandra.Hosts: zero value".
+  # Switch to the sprig ConfigMap and point the server at it (chart's own
+  # recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath).
+  configMapsToMount: "sprig"
+  setConfigFilePath: true
+  jobService:
+    enabled: false
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          podAffinityTerm:
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/instance: temporal
+            topologyKey: kubernetes.io/hostname
+  config:
+    logLevel: "info"
+    persistence:
+      defaultStore: default
+      visibilityStore: visibility
+      numHistoryShards: 512
+      default:
+        driver: "sql"
+        sql:
+          driver: "postgres12"
+          host: "ddb-cluster-rw.ddb.svc.cluster.local"
+          port: 5432
+          database: "temporal"
+          user: "temporal"
+          # existingSecret + secretKey: point directly at the CNPG-generated
+          # Secret (kubernetes.io/basic-auth, keys: username/password/...)
+          # rather than duplicating the password in git as plaintext. When
+          # existingSecret is set the chart's own server-secret.yaml Secret
+          # template is skipped entirely (see templates/server-secret.yaml:
+          # `not $driverConfig.existingSecret` guards its creation).
+          existingSecret: "temporal-db-role"
+          secretKey: "password"
+          maxConns: 20
+          maxIdleConns: 10
+          maxConnLifetime: "1h"
+          # NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation
+          # is a MySQL-only connection parameter. The Postgres `pq` driver rejects
+          # it ("unrecognized configuration parameter"), which killed every DB
+          # connection (schema-setup job AND server) with the misleading
+          # "no usable database connection found". Postgres defaults to READ
+          # COMMITTED isolation anyway, so nothing is lost by omitting it.
+      visibility:
+        driver: "sql"
+        sql:
+          driver: "postgres12"
+          host: "ddb-cluster-rw.ddb.svc.cluster.local"
+          port: 5432
+          database: "temporal_visibility"
+          user: "temporal"
+          existingSecret: "temporal-db-role"
+          secretKey: "password"
+          maxConns: 20
+          maxIdleConns: 10
+          maxConnLifetime: "1h"
+  service:
+    type: ClusterIP
+
+# ── Temporal Web UI ────────────────────────────────────────────────────────
+web:
+  replicaCount: 1
+  service:
+    type: ClusterIP
+
+# ── Ingress ────────────────────────────────────────────────────────
+ingress:
+  enabled: false

k8s/argocd/apps/60-applications.yaml

@@ -2,9 +2,8 @@
 # helpers (cloudflared tunnel, duckdns updater) that are already running.
 # Experimental dirs (llm, forge, dev-tools, shadowsocks) are intentionally
 # NOT included yet — add them here once they're production-ready.
-# Consolidated: temporal-db-secret-sync + temporal → temporal
+# temporal using unified CNPG pattern (app user, ddb-cluster-app secret)
-# Syncs the CNPG-generated temporal-db-role Secret from ddb -> temporal ns
+# Secret copied by bootstrap.sh (like cicd/iam namespaces)
-# via PostSync hook (db-secret-sync/copy-job.yaml).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -24,9 +23,6 @@ spec:
     - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
       targetRevision: main
       ref: values
-    - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git
-      targetRevision: main
-      path: k8s/applications/temporal/db-secret-sync  # PostSync hook: copy-job.yaml
   destination:
     server: https://kubernetes.default.svc
     namespace: temporal