From ac73035c9e66ed46d8d9a4ea581d038a0407471d Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 27 Oct 2026 12:45:13 -0700 Subject: [PATCH] fix(iam): stop rotating client secrets on every apply, repair Grafana OIDC Grafana sign-in failed with "Failed to get token from provider". The deployed client secret no longer matched, because _ensure_oidc_apps called _generate_client_secret for every app on every run, and ZITADEL stores exactly one client secret per app: each call replaced the previous value and invalidated every deployed copy. Two changes: 1. Generate a secret only when the app is created. For an app that already exists, reuse the value from state.json, because the API cannot read a secret back. 2. Re-encrypt grafana-oidc-secrets.enc.yaml with the live secret, so Grafana's token exchange succeeds again. Verified with the introspection endpoint: old deployed secret -> {"error":"unauthorized_client"} new secret -> {"active":false} (credentials accepted) --- .../secrets/grafana-oidc-secrets.enc.yaml | 39 +++++++++++-------- scripts/iam/zitadel-provision.py | 35 ++++++++++++++++- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml b/k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml index b6c4639..681e5f1 100644 --- a/k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml +++ b/k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml @@ -1,23 +1,28 @@ -apiVersion: ENC[AES256_GCM,data:rGU=,iv:w1yJeQvN0MPZlPx8VQZmC6zkRaHW4rtjjROea0Kx9X4=,tag:qgYgVUGMQ2BtN0RZ0nZYjQ==,type:str] -kind: ENC[AES256_GCM,data:wp2QP1he,iv:C8c/jKh5pnBOtHqXsTEchGf9DACU/5hOJ0Mp/+Jk1kg=,tag:QUZNeguRXdi3JO6Ov6GMDg==,type:str] +apiVersion: v1 +kind: Secret metadata: - name: ENC[AES256_GCM,data:jVsnAcFTlRiKxDoU,iv:bWEqo28kH3bxzfI3ipiY6dsqhIcl9wIxBjFu0mxVGLg=,tag:PVWTveI8xhvLxvXPldhimQ==,type:str] - namespace: ENC[AES256_GCM,data:ZHnVZyd3wA==,iv:3evUnFTQfRz/ZilUQIl/9dsCba3nsX6o/iQsGXMBfgM=,tag:+9kp+VRu1ksroDTbTGmGPA==,type:str] -type: ENC[AES256_GCM,data:bHX+Bu9o,iv:8qqNddsuw4YoHAuJfUcPIqjcqNySIh5Ln8OhsfZxuk8=,tag:P7QGUXo9Io2Y19YqlWv5sQ==,type:str] + name: grafana-oidc + namespace: logging +type: Opaque stringData: - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:nJz9pqa6w9ZGOjanZ32D38Bolg2YrdHs1crhb2uZkKyaaKDpklVb4FOKZ46K+NS1NXjoS6XDUAlV69CSvm1PeA==,iv:g9F/iLuzNC0WogtDJ/TAb3+97N6iGkbOTlb4SzYzwx8=,tag:4F3kUheGbtW2U6jgOvRO1A==,type:str] + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:fGqA1ofVme1lVLxFjWIiU7sbvE9ip2Af7C1ua3+HdidlcVgkwW59vVlXZhgBRdoif86C8LKQSyoCGUqiO+740g==,iv:6Kv2ybbJg1GT8DLqz9P5y+E1zr7vcIBDMUErmvxNTaw=,tag:0wK28B3G/owCVPPqb2sLmQ==,type:str] sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] age: - - enc: | + - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrUkxLMEV3aEpILytic0Ux - V2VEV2EvcENJVnNNRUtBbGkwT0d6eUxYeEFvCjZPSkU2MG5qME5jZ01mRVJwTUFL - M3M0V3c3QXBoRml1Vm9NWW5JTlNRSG8KLS0tIEJnNlFmdFJXb1BPaVJzTmo1RDQv - VXZOVHRHUEZJRUZ4eGZGM1RxSURyWXcK3ZyYCPhRUvpvT/pjPQGJoLIwaktZY6Tg - LS0cIDUhjPFbeu7qUATOrat6vMUi0UREbXSFZ1KyAjNAwJZaUusPMA== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArTXMxMWdVRHIvbWh1bVNk + bEFFNlgzNU9zTjlzY2o3VGdnbFpnU2lkYmxrCjF4aWVGSlJaNkcxODJ6RDkrVFo2 + UkV2RUwwVHFCd0xMNG9RWXdTUHZGaXcKLS0tIHVaWjhaNHRtS2c0UkVzRk83ckhj + OCtBVFR0d1F5amVhV2E0emRZWThvYjAKFW46i9brIL9ZoO1CEn+IewIdoUhvYK/A + RqzZ6xV+mhlDj0zC+SpVcEm/Ki83nMYOj4+vQingZlr/Kr3Nwb0iLQ== -----END AGE ENCRYPTED FILE----- - recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla - lastmodified: "2026-10-26T04:03:16Z" - mac: ENC[AES256_GCM,data:0cgVNtFMwhbEHBxu7T1XfstcQlgUTxijq9Mz36zC60E7kYXRNwGt2/0KoCaqaE0leypwAdrPn2OjeoDPeCN6x+n0vLRVX2tnli6HazBLbRsPHljDAV0ASF07iNhb0PvgtWay432Mm+sGATkel3ITdAaP0lISNLUYfEe1/TbYHkk=,iv:FZ9Bgtk7LOC3aOSOo1+50zwLGhlcUZYJaUQ9r05cAn0=,tag:tsLQ1frZ7li5l717CCtgoQ==,type:str] - unencrypted_suffix: _unencrypted - version: 3.13.2 + lastmodified: "2026-10-27T19:44:32Z" + mac: ENC[AES256_GCM,data:EJew+p/kJrurcNpLlFxJGTl69hQYxn3IkzP7EDtdSriW/WI7X1xNLC2dp/dTiLomGmG1uh6+2JI9Sc8gXmrRAWLELAbiOaYbcoU6gi89BjTCWGDAMX4lNSIuH40rsGs4YtYj5U7SGfgx5qto4ZH7LxjBmxzPWM/Xz2Uq/I5jV4k=,iv:LdhjhwCfPUIa7D0gved8Twf7ujN+dg3lN8lCnFBFs0g=,tag:bjkwUCMtR8IBL9BkbuXcnw==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.3 diff --git a/scripts/iam/zitadel-provision.py b/scripts/iam/zitadel-provision.py index d39861e..b6dcf68 100755 --- a/scripts/iam/zitadel-provision.py +++ b/scripts/iam/zitadel-provision.py @@ -88,6 +88,20 @@ ROLES = { # permission string, so consumer manifests did not change. "llm:inference": "LLM inference access (api-gw)", "llm-inference": "LLM inference access (dash alias)", + # Per-service capabilities. The gateway derives the required capability from + # the verb: GET/HEAD/OPTIONS -> ":read", anything else -> + # ":write" (internal/serviceadapter/router.go:142). The role key + # must match that string exactly, colon included. + "memory:read": "Poimen memory read (api-gw)", + "memory:write": "Poimen memory write (api-gw)", + "workflow:read": "Temporal workflow read (api-gw)", + "workflow:execute": "Temporal workflow execute (api-gw)", + "s3:read": "Object store read (api-gw)", + "s3:write": "Object store write (api-gw)", + "iam:read": "Identity read (api-gw)", + "iam:write": "Identity write (api-gw)", + "sqs:read": "Queue read (api-gw)", + "sqs:write": "Queue write (api-gw)", } _WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"} @@ -336,13 +350,16 @@ def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None: die(f"create role {key} failed: {code} {d}") -def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]: +def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict], + known_secrets: dict[str, str] | None = None) -> tuple[dict, dict]: code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {}) existing = {a.get("name"): a for a in (d.get("result") or [])} ids: dict[str, str] = {} secrets: dict[str, str] = {} + known_secrets = known_secrets or {} for name, spec in apps.items(): row = existing.get(name) + created = False if row: ids[name] = (row.get("oidcConfig") or {}).get("clientId", "") app_id = row.get("id") @@ -365,8 +382,16 @@ def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tup die(f"create app {name} failed: {code} {d}") app_id = d["appId"] ids[name] = d.get("clientId", "") + created = True if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE": continue + if not created: + # ZITADEL keeps one client secret per app, and the API cannot read it + # back. Generating a new one here would replace the deployed copy and + # break that app's login, so reuse the value already on disk. + if known_secrets.get(name): + secrets[name] = known_secrets[name] + continue code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/{app_id}" "/oidc_config/_generate_client_secret", token, {}) @@ -468,7 +493,13 @@ def phase_apply(_args: argparse.Namespace) -> int: print(f"project : {PROJECT_NAME} ({project_id})") _ensure_roles(token, project_id, ROLES) print(f"roles : {len(ROLES)} ensured") - ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS) + known: dict[str, str] = {} + try: + known = (json.loads(STATE_FILE.read_bytes()).get("client_secrets", {}) + .get("value", {}) or {}) + except (OSError, ValueError): + known = {} + ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS, known) print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued") state = {