diff --git a/bootstrap.sh b/bootstrap.sh index 5ee59fd..0e429fd 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -88,6 +88,12 @@ kubectl get secret ddb-cluster-app -n ddb -o yaml \ | sed 's/namespace: ddb/namespace: iam/' \ | kubectl apply -f - + +# Copy DB secret to temporal namespace (for temporal) +log "Copying ddb-cluster-app secret to temporal namespace..." +kubectl get secret ddb-cluster-app -n ddb -o yaml \ + | sed 's/namespace: ddb/namespace: temporal/' \ + | kubectl apply -f - # 7. Wait for Forgejo log "Waiting for Forgejo to be ready..." kubectl wait --for=condition=available --timeout=600s \ diff --git a/k8s/applications/temporal/db-secret-sync/copy-job.yaml b/k8s/applications/temporal/db-secret-sync/copy-job.yaml deleted file mode 100644 index 4c474be..0000000 --- a/k8s/applications/temporal/db-secret-sync/copy-job.yaml +++ /dev/null @@ -1,134 +0,0 @@ -# Copies the CNPG-generated temporal-db-role Secret from the ddb namespace -# into the temporal namespace, as a plain (non-SOPS) k8s Secret with the same -# keys. Kubernetes Secrets are strictly namespace-scoped - a Deployment in -# `temporal` cannot reference a Secret living in `ddb` via secretKeyRef, and -# temporal-values.yaml's server.config.persistence.*.sql.existingSecret: -# temporal-db-role expects to find it in ITS OWN namespace (temporal). -# -# Deliberately a standalone directory (no kustomization.yaml) applied as its -# own small Application - avoids the k8s/applications/temporal/kustomization.yaml -# `namespace: temporal` transformer, which would silently force-rewrite this -# Job's ddb-scoped RoleBinding back to temporal (same class of bug fixed -# earlier in k8s/security/iam/kustomization.yaml - see that file's comments). -# -# PostSync (not PreSync!) + BeforeHookCreation: reruns on every ArgoCD sync -# of this app, re-copying the password if CNPG ever rotates it. -# -# IMPORTANT: this MUST be PostSync, not PreSync. The ServiceAccount/ -# ClusterRole/RoleBindings below are plain (non-hook) resources - ArgoCD -# creates those during the normal "Sync" phase, which happens AFTER PreSync -# hooks run. A PreSync-hooked Job here would try to start before its own -# ServiceAccount exists (chicken-and-egg deadlock: confirmed live - the Job -# sat 'Running' for 14 minutes, unable to create any pod at all, event log -# showed "serviceaccount temporal/temporal-db-secret-sync not found" on -# every attempt). PostSync runs after this app's own normal resources are -# already applied, and this whole app (sync-wave 7) still fully completes -# before the `temporal` Application (sync-wave 8) begins, so the ordering -# guarantee we actually need (secret exists before Temporal's pods start) -# is preserved regardless of PreSync vs PostSync here. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: temporal-db-secret-sync - namespace: temporal ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: temporal-db-secret-sync -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: temporal-db-secret-sync - namespace: ddb -subjects: - - kind: ServiceAccount - name: temporal-db-secret-sync - namespace: temporal -roleRef: - kind: ClusterRole - name: temporal-db-secret-sync - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: temporal-db-secret-sync - namespace: temporal -subjects: - - kind: ServiceAccount - name: temporal-db-secret-sync - namespace: temporal -roleRef: - kind: ClusterRole - name: temporal-db-secret-sync - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: temporal-db-secret-sync - namespace: temporal - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation -spec: - ttlSecondsAfterFinished: 600 - backoffLimit: 5 - template: - spec: - serviceAccountName: temporal-db-secret-sync - restartPolicy: Never - securityContext: - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: copy - # bitnami/kubectl:1.30 does NOT exist - Bitnami stopped publishing - # versioned tags in 2025 (only `latest` + sha256-pinned digests - # remain), confirmed live via Docker Hub API before this fix - the - # original tag caused an indefinite ImagePullBackOff. Using - # python:3.12-alpine + a stdlib urllib kubectl download instead, - # same pattern already proven working in - # k8s/security/iam/authentik-provision-job.yaml - avoids depending - # on any third party's tagging policy at all. - image: python:3.12-alpine - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - command: - - /bin/sh - - -c - - | - set -e - echo "installing kubectl (pure python urllib, no apk - see" - echo "authentik-provision-job.yaml for why apk fails as non-root)..." - python3 -c " - import urllib.request, os, stat - kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip() - url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl' - urllib.request.urlretrieve(url, '/tmp/kubectl') - st = os.stat('/tmp/kubectl') - os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC) - " - export PATH="/tmp:$PATH" - echo "waiting for ddb/temporal-db-role..." - until kubectl -n ddb get secret temporal-db-role >/dev/null 2>&1; do - echo " not ready yet, retrying..." - sleep 5 - done - USERNAME=$(kubectl -n ddb get secret temporal-db-role -o jsonpath='{.data.username}' | base64 -d) - PASSWORD=$(kubectl -n ddb get secret temporal-db-role -o jsonpath='{.data.password}' | base64 -d) - kubectl -n temporal create secret generic temporal-db-role \ - --from-literal=username="$USERNAME" \ - --from-literal=password="$PASSWORD" \ - --dry-run=client -o yaml | kubectl apply -f - - echo "synced temporal-db-role -> temporal namespace" diff --git a/k8s/applications/temporal/temporal-values.yaml b/k8s/applications/temporal/temporal-values.yaml index c48acae..4bb5dcc 100644 --- a/k8s/applications/temporal/temporal-values.yaml +++ b/k8s/applications/temporal/temporal-values.yaml @@ -95,14 +95,15 @@ server: host: "ddb-cluster-rw.ddb.svc.cluster.local" port: 5432 database: "temporal" - user: "temporal" + user: "app" # existingSecret + secretKey: point directly at the CNPG-generated # Secret (kubernetes.io/basic-auth, keys: username/password/...) # rather than duplicating the password in git as plaintext. When # existingSecret is set the chart's own server-secret.yaml Secret # template is skipped entirely (see templates/server-secret.yaml: # `not $driverConfig.existingSecret` guards its creation). - existingSecret: "temporal-db-role" + # Use unified ddb-cluster-app secret (copied to temporal namespace) + existingSecret: "ddb-cluster-app" secretKey: "password" maxConns: 20 maxIdleConns: 10 @@ -120,8 +121,9 @@ server: host: "ddb-cluster-rw.ddb.svc.cluster.local" port: 5432 database: "temporal_visibility" - user: "temporal" - existingSecret: "temporal-db-role" + user: "app" + # Use unified ddb-cluster-app secret (copied to temporal namespace) + existingSecret: "ddb-cluster-app" secretKey: "password" maxConns: 20 maxIdleConns: 10 diff --git a/k8s/applications/temporal/temporal-values.yaml.bak b/k8s/applications/temporal/temporal-values.yaml.bak new file mode 100644 index 0000000..c48acae --- /dev/null +++ b/k8s/applications/temporal/temporal-values.yaml.bak @@ -0,0 +1,140 @@ +# k8s/temporal/temporal-values.yaml +# Temporal — workflow engine +# Uses external CNPG PostgreSQL for persistence (ddb-cluster) +# Visibility via same PostgreSQL instance, separate database. +# +# IMPORTANT — chart schema note (root-caused after Postgres never actually +# taking effect despite looking configured): +# We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in +# k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence +# schema: +# server.config.persistence..driver: "sql"|"cassandra" +# server.config.persistence..sql: {...} +# NOT the newer `datastores:`-wrapped schema +# (server.config.persistence.datastores..sql) shown in the current +# chart's values/values.postgresql.yaml example - that key was introduced in +# a later major version and doesn't exist in 0.74.0. Helm doesn't validate +# unknown keys, so a `datastores:` block here is silently a no-op: Temporal +# would keep defaulting to Cassandra (with empty hosts: []) regardless of +# anything nested inside it. Verified via `helm template` against the actual +# 0.74.0 chart before writing this file - see chat history for the +# side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and +# temporal-cassandra-tool commands using the old datastores:-based values). +# +# Likewise `schema.setup.enabled` / `schema.update.enabled` / +# `schema.createDatabase.enabled` are the real toggles for the schema-setup +# Job (all default true) - there is no `jobs.autoSetup` key in this chart. + +# ── Disable every bundled/optional sub-chart ───────────────────────────────── +# postgresql/mysql: never enable - we never want the chart to deploy its own +# DB, only to know how to talk to our external CNPG instance (which happens +# via server.config.persistence.*.sql below, independent of these flags). +postgresql: + enabled: false +mysql: + enabled: false +cassandra: + enabled: false +elasticsearch: + enabled: false +prometheus: + enabled: false +grafana: + enabled: false + +# ── Schema setup/update Jobs ────────────────────────────────────────────────── +# The `temporal` and `temporal_visibility` databases are provisioned +# declaratively by CNPG Database CRs (k8s/data/temporal-database.yaml, +# temporal-visibility-database.yaml), so createDatabase stays disabled (the +# `temporal` role also lacks CREATEDB). setup/update run temporal-sql-tool as +# the `temporal` owner against those existing DBs to install and migrate the +# Temporal server schema — without them both DBs have zero tables and the +# server dies on "no usable database connection found" (no schema_version row). +schema: + createDatabase: + enabled: false + setup: + enabled: true + update: + enabled: true + +# ── Temporal server config (PostgreSQL persistence) ────────────────────────── +server: + replicaCount: 1 + # temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to + # built-in sprig config templating. The chart still defaults to the legacy + # configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a + # config the 1.30 server never loads — it then falls back to its embedded + # env-only template (Cassandra default) and dies with + # "Persistence.DataStores[default](value).Cassandra.Hosts: zero value". + # Switch to the sprig ConfigMap and point the server at it (chart's own + # recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath). + configMapsToMount: "sprig" + setConfigFilePath: true + jobService: + enabled: false + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: temporal + topologyKey: kubernetes.io/hostname + config: + logLevel: "info" + persistence: + defaultStore: default + visibilityStore: visibility + numHistoryShards: 512 + default: + driver: "sql" + sql: + driver: "postgres12" + host: "ddb-cluster-rw.ddb.svc.cluster.local" + port: 5432 + database: "temporal" + user: "temporal" + # existingSecret + secretKey: point directly at the CNPG-generated + # Secret (kubernetes.io/basic-auth, keys: username/password/...) + # rather than duplicating the password in git as plaintext. When + # existingSecret is set the chart's own server-secret.yaml Secret + # template is skipped entirely (see templates/server-secret.yaml: + # `not $driverConfig.existingSecret` guards its creation). + existingSecret: "temporal-db-role" + secretKey: "password" + maxConns: 20 + maxIdleConns: 10 + maxConnLifetime: "1h" + # NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation + # is a MySQL-only connection parameter. The Postgres `pq` driver rejects + # it ("unrecognized configuration parameter"), which killed every DB + # connection (schema-setup job AND server) with the misleading + # "no usable database connection found". Postgres defaults to READ + # COMMITTED isolation anyway, so nothing is lost by omitting it. + visibility: + driver: "sql" + sql: + driver: "postgres12" + host: "ddb-cluster-rw.ddb.svc.cluster.local" + port: 5432 + database: "temporal_visibility" + user: "temporal" + existingSecret: "temporal-db-role" + secretKey: "password" + maxConns: 20 + maxIdleConns: 10 + maxConnLifetime: "1h" + service: + type: ClusterIP + +# ── Temporal Web UI ──────────────────────────────────────────────────────── +web: + replicaCount: 1 + service: + type: ClusterIP + +# ── Ingress ──────────────────────────────────────────────────────── +ingress: + enabled: false diff --git a/k8s/argocd/apps/60-applications.yaml b/k8s/argocd/apps/60-applications.yaml index a719840..8434033 100644 --- a/k8s/argocd/apps/60-applications.yaml +++ b/k8s/argocd/apps/60-applications.yaml @@ -2,9 +2,8 @@ # helpers (cloudflared tunnel, duckdns updater) that are already running. # Experimental dirs (llm, forge, dev-tools, shadowsocks) are intentionally # NOT included yet — add them here once they're production-ready. -# Consolidated: temporal-db-secret-sync + temporal → temporal -# Syncs the CNPG-generated temporal-db-role Secret from ddb -> temporal ns -# via PostSync hook (db-secret-sync/copy-job.yaml). +# temporal using unified CNPG pattern (app user, ddb-cluster-app secret) +# Secret copied by bootstrap.sh (like cicd/iam namespaces) apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -24,9 +23,6 @@ spec: - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git targetRevision: main ref: values - - repoURL: https://forgejo.riotpiao.com/riotpiao.com/homelab.git - targetRevision: main - path: k8s/applications/temporal/db-secret-sync # PostSync hook: copy-job.yaml destination: server: https://kubernetes.default.svc namespace: temporal