fix(iam): make ZITADEL OIDC secrets and identity links IaC-owned (#85)

Co-authored-by: rock <[email protected]>
2026-10-31 20:53:53 +00:00
parent 41dcda819e
commit 4e84571f0c
9 changed files with 282 additions and 2 deletions
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: oidc-secret
+    namespace: argocd
+    labels:
+        app.kubernetes.io/managed-by: argocd
+        app.kubernetes.io/part-of: iam
+type: Opaque
+stringData:
+    client-id: ENC[AES256_GCM,data:KJmGTcCz/c60+aLs7BqwL6Je,iv:hpwBdXp9einCN8EdvMXgTyf4Tl6SN9VTmDQotdEC/qE=,tag:MmF076Vsm0ejZtQMfG+e5w==,type:str]
+    client-secret: ENC[AES256_GCM,data:0W2lS6WTzUpJUQo4AQSZk0iXOqZjKVqXCRv4HVuNrGpcZYf81oJVR8dM/FnApd4zS1Y11465aXYcImqcKo4xkg==,iv:ECIkWzeRTroWpugjqASD3eJHipn4m2JWthC/1gaXhkA=,tag:GU5GyKhUvutJRuxkhQNE0Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZZjVTSlN6c2dZbDZCeEFa
+            SU9YVy9HUEEzckRVb0QrRmh5NTN2Qm8ycFFVCkpwVnAxK3BGT29NdnN2VDlveDBG
+            RHo5YjVicllRc1NlRzNabS9vWlo3cVUKLS0tIHF4OE1GKzJUZlBhU3dNOFFaeWZC
+            M3QrRTRxbkNRZVMreWsyQytlMDdFUTAKwtVdzOk6nMDrauDcRUnun3JU/Cfi3qgB
+            jhftbXHaxcKhB5MJCwZkN7VmtpxOjpvYgpetj9cBNXx2yVVcS7G5Hg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-31T03:41:24Z"
+    mac: ENC[AES256_GCM,data:GHwigzuZRGkLIH6vaDjUsefjdnmhToePQGB9+3CIZW8DunlkgmGqk/jIsQee7Yo0fa1OOdMHRQH7+biooUP1Z5GANmsBnQhDT0GAnd3JPbIN2lIJKeManGDALqsbhLzmfcSBNByIJaJh5c3xakL/LmU+CoUWluWCLQqjpeeiMEk=,iv:uBRbVAgzXxxn3Xx7foUZWyK/Cq6yMd3/lkt+S+uY2bk=,tag:Mr0F5rp5aEVNd7pZqnyWUw==,type:str]
+    version: 3.13.2
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-oidc
+    namespace: logging
+    labels:
+        app.kubernetes.io/managed-by: argocd
+        app.kubernetes.io/part-of: iam
+type: Opaque
+stringData:
+    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:ZCO6gQGNRKtahbUctlLYCTx3TpRdz5lj8r6X4fUXkyvS5rbWXgip2mGqho4LUSHh8du3wo+bK8VyP1MhzIFRGA==,iv:O2aCivj8Z3cbKCqj/Q8uG6mEH9QJbCrDLPIjX6bLkjo=,tag:qpKgg9AAtIUvYzr5zwTO1g==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEc2xWSk1ldGxpYzlHUlF5
+            aFZQWnVxWXJVemZVd1llQlZUTkNrTFpVWGhrCjZtbW1XU1Z0U1ZlTHlGMjVSTUFp
+            NnFNTjdlYUUyTlhnN3QybFZJSGRuRG8KLS0tIG9DL3ZQYmErc2ZndWd6TUNlOStp
+            UTI0UDZZM2IwQU9oRmxsU25rQi9zUUkKYznGn96vdcEZSCpfyReNj4qLoS0t19ZQ
+            frodab7z/n6TadRwsb1ufb+YIHa4pGPlImmrlNGOIpnKvzXlmOCP4Q==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-31T03:41:24Z"
+    mac: ENC[AES256_GCM,data:uOPMvN1D/6AdxpWrR0aQYAxSEcC8/aJNg574cF+ypi/rt+srk68q2kAiKwA5L5GDvXs65rSZh2PMkJ0KV75g3ShFYOQSLaq0HIwYYTvIXRLtF43G8vMHaaZAhDC1pZeYPdD1KG61pDVjyCDc+qFZ/90kHqYiIeAdEI4Tia5YOJ4=,iv:SpU/YrCxORAJWJCLyTNSaNuIB0dAhq8+Bma58RUYL3U=,tag:s7fsEDQfqElTsE8EQsZfxw==,type:str]
+    version: 3.13.2
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: immich-oidc
+    namespace: immich
+    labels:
+        app.kubernetes.io/managed-by: argocd
+        app.kubernetes.io/part-of: iam
+type: Opaque
+stringData:
+    client-secret: ENC[AES256_GCM,data:SAxFOfR1lIeKFHII2BJskF2T46lXlFFu+qzLxyf/uBDiQACRDTkQGzeJccc2jNT5hagS5GDOidnZN4aXrHGjUA==,iv:V4sDhHbiOyB+cllFuBpfO0JsiEG5frWMj0lUZSCLRwk=,tag:tb/m9gFUSb7SGuiilYoZSg==,type:str]
+    config.json: ENC[AES256_GCM,data:6HCZ2bl5RavyHP15RMMRkKYaCGXIVwIEkIOkT5VZs2BFZ7b+Po2fLimM7efVsR1QAF4NZ3StAOnsdJoEaTrNeEfCf22pLdELYw4swodlJi0Ybty09q9N7JZzENoa+3Sor0wxPI0FE70m058Sg+NkliRXxwpdwMAFkh6nj4zwZy0dUiyaHVsQFs5CDoAMrKP5KCdQhAz6Qx622C2D03pWjZICAO/VrKhLU/2N8c8Jid9r2zJWFJxegazhEtJ+AE/1aM+h9rjb7IB9HObRUROd8tYypOgV5u3LBQMg9DP4PTHcPoDdyHbKtRYyCTP5adIt6/Xr37C+5o3dpEi8cR8yUfWIn2jYkcbhIUSVGCVE8b4e3b6jNhvUAlv5JZubjVmm+MJdAhGTPBi7NEPoU+1p0Bs8yx4mjvtXZRn4is2oei322vhUUpIq/OzAn5Km8v/8CnzyS6Zclsl5Oa+jRqWIqNJ3vVYSReytN11KbvCNtDmVsrNpPg1Set3mhk7CIrND3tZQqX6Q4P7/vllAvfsoorf7BZrI7z7/SrRxYWZZWziC7tsPnqul+yntymWxHt2NW203Fdk=,iv:6IymBA+UeVGjDffyKAxpiAA/1G9/eB6mJJmZ8Ntwq90=,tag:X38OSEIgoNbrSyHFADwADQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKV0Fta2VXVThvTnl1S09O
+            TkgwTjJBc040K3NRdXFRTlFEMGJqaVVVZmd3Ck9MMDc0VzNkRDhrbTFteHc1VFVo
+            OTNrMytLVFZYMkZYSGJjU2pnMjNlSk0KLS0tIDJrajNqZjVNQ1NjZ2FienJFcEVK
+            d3JIMk1lQjNuVE9LTmhoYzJlRHpua3MK17o8wpCg98eiFYJdZhWqHOl+riMcqLpg
+            pL+jzEAb6F8aOOvlog99TDyq5Pw0g1MvJBmw+2jyMTAZE2cIoX4CLg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-31T03:41:24Z"
+    mac: ENC[AES256_GCM,data:cv9uXDrbECSIPQLbEugfvYyGG6BoNxu/Cva7BmUYF6DkT34dCtCMNI2ey1nV5m8dJM1ote2lxpnWjt/6s1P41AMRsdyGkEaGrAW/62uP8NCX6PP+p519qTUvf4ZdGFQ24Uo0UVQ0AV/vCw1RAurmZXKcoLVvQaCzgDCn2EgfAiE=,iv:mpJ+YqFamDGaEiqy6Y3gT5IqMcKs2qR/FyfmxKM7BcM=,tag:U+UtCTLTX+b/FjZGfUrrgQ==,type:str]
+    version: 3.13.2
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: minio-oidc
+    namespace: storage
+    labels:
+        app.kubernetes.io/managed-by: argocd
+        app.kubernetes.io/part-of: iam
+type: Opaque
+stringData:
+    MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:M8iRH6BxzEz7r/eOwED+uiOdHwkXg5DD5s4xEaKJCNAn2HwlpuKM+pUVLyLNwSxeN7Kf6dYVMUWuuzL2N3BkEQ==,iv:tSa3MSSNkM38/avXIt4CHUDZ64zXGGqiInCJx3cP2gI=,tag:z5ODn9b3/464+O1bF5CoZg==,type:str]
+    client-secret: ENC[AES256_GCM,data:9/YiXaYw/AavqklSPZ0akuF6gNjPL9OrpEc1jMz8N5QgYD8aZJxol0mUXqhoLy6zldaifo7ZxtzRONdJkuGBwg==,iv:kEUzmeQMVbyI/2ag/9baRWp0XJ1VhIqBlpVxZB6OEog=,tag:kf34PPlQzHTWfd2ndGvP3A==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3S2JTSFF3Y2VMWkNncTIw
+            K2VKM1dDNnE1ZE5YSTNDN0ZFS2paNXNHSUU0ClFGWnBSVThrbWk3MGUyTVlPRGNB
+            cGJOMmxSMlZnNHNMT3cxNzFneEc2b0EKLS0tIG9lbURIdEdLNVZVR3JiWkFWVmMx
+            aUVDVUpOTENjK0ViYWFVRm5BNmcwbncKg5ThSLcmeJKY2mm7EkRlDg6FKnpBvFrE
+            eEwfEEX68KirboOsv6U1TPjyf2ZM0Wfj+1kVfiNQTLi7QEE3R377KQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-31T03:41:24Z"
+    mac: ENC[AES256_GCM,data:QA7Skxv7f9AOZCPWAZlEW+TR4eNSSX8LlxFc6zUqfiWu3oPPThMQxXiHJi4FEeUsjwe3Qy/FsPWF2182P0yZ/lfiXFi6bfPMzL9OeyYISrjfd/MUBm02+JG/MMrMXat+sxQgrTljl7z5H74X7BywHOLLGHjHDoWEEzqU3pJrFIQ=,iv:wecxDkPDPQmFwaRZjnxKD6zuTXQREjfkyZ34eNF/63M=,tag:IUDoRtpMCYa3amxWoyHL+g==,type:str]
+    version: 3.13.2
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: paperless-oidc
+    namespace: paperless
+    labels:
+        app.kubernetes.io/managed-by: argocd
+        app.kubernetes.io/part-of: iam
+type: Opaque
+stringData:
+    SOCIALACCOUNT_PROVIDERS_JSON: ENC[AES256_GCM,data:mz6fvwB8/E881pnl0fX2sXj5vdnGUoN0jQsYX7OqzRhZxAmXwvsIt3ZdqXPdoz8xXJOM+2P082aFV24q5jKjU6m4pGVxZT79kooNQdPImcthBl/1oSvgbuQrKnX9kDdlcfTaONckya1SycXyixyqWIx+dQmosLcPXQZgVrbJ0XgPYAnHAqdNB8S0yLNCPQMmf+isC+JQY/iw5gVYbVehTTZnJkZoh07d2amMFLbx1janzKuGWsSeXEEwFPfntBEDcq8CMD+uGfZmiaunTvztLZCIYfm/ZriBeWroY42nrncAhBcjMWfT7hFjK6I+SMY1+/uDjVuhlmg96C6uwnFBgzTDiqz46W8kg+qYVg3DAO1KdORL84N54vhMOj+NMoG0GxL1SW4M9JPoMCjnkpVvhE0C6m9MeNcV5CEDN+wPQuoDA60F5NrGmp9cnPD4GVXBOGqdUjr9QQbfkZeGrzIyREYS5krBD/89mx7a,iv:dCXz2VwJGXqKnETGEKrtg8afTjQZ80fdQetQkQbWFmA=,tag:3fSADqRQrqE2Kmap0cMYCQ==,type:str]
+    client-secret: ENC[AES256_GCM,data:960UNsowQT3xrBdQ0pD4VrFQSFFi/2yiN3RPVnK4a0yySDjcxlsZVLn0fWnLEVHbmAWEe10j36xhGS2pzTx06A==,iv:ztpJi0zHCKx7Jjk40wtDNHhatr2vCOMdD8XIUnfIzTU=,tag:efTUtQmPvteeEtX9ZOT1nw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXdW01dUY4ZXZYeWdWeU5B
+            Uk9kUkRDS2tLUXc5dVVMOFJJVUcxbmtEbjNVCk4vK3ZVYlppT0lwSGtTeXJiM1ZU
+            SEZyTWlrYjNVNFBxU01YcGs1Q2YrOEEKLS0tIHlyR2JnWDBpUkcyUTNDM2IxMjkr
+            NTROVkVyd3RpS014WktISi9Ba3dZOE0KoP8Q30KCrLF+mqVIIfp6k6B727yiUBIj
+            1H3krVQt6eOadTZZd+EPS0nvJA4wzUyaCco6oiB00gckhPnZu8m4jw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-31T03:41:24Z"
+    mac: ENC[AES256_GCM,data:xB4NS9KAjJ9e14kCrP87PcXyW5F4eBNwWnmkBOXYAVb9RMsFuEG6zAslENLur+LBO2TYyYE+YSV0HVvGICU2jJ8LxsJXdfCm0ADxrhkBcneBgLwqDPMURZDwQSCLJxdh3AyV2+vRDGdrjyjzUE507fWpYd84HPEgyrVJ4w+V6TA=,iv:foISE6qjTurY6Mll8FEdirvx9QXOalsUPdWx+sK59CM=,tag:3hmGRXsMFkKw4+VYcNypdA==,type:str]
+    version: 3.13.2
@@ -30,6 +30,16 @@ files:
    - paperless-ai-secrets.enc.yaml
    - memory-agent-oidc.enc.yaml
    - oauth2-credentials.enc.yaml
+    # ZITADEL OIDC client secrets for the consumer apps. These used to live only
+    # in the cluster, written by scripts/iam/zitadel-provision.py, so a rotation
+    # there left every consumer holding a stale secret and no git record of the
+    # real value. Declaring them here makes ArgoCD the owner and the rotation
+    # outcome reviewable in a diff.
+    - argocd-oidc.enc.yaml
+    - grafana-oidc.enc.yaml
+    - minio-oidc.enc.yaml
+    - immich-oidc.enc.yaml
+    - paperless-oidc.enc.yaml
    - forgejo-registry-token.enc.yaml
    - pi-sandbox-secrets.enc.yaml
    - etcd-defrag-creds.enc.yaml
@@ -0,0 +1,123 @@
+# Idempotent repair of the ZITADEL identity links that live inside three
+# application databases.
+#
+# Why this exists: Forgejo, Paperless and Immich each store the link between
+# their local account and the identity provider INSIDE their own database, not
+# in Kubernetes. When the IdP changed from Authentik to ZITADEL, every one of
+# those rows kept the Authentik subject and the provider name. The result was:
+#
+#   Forgejo   -> /user/link_account, never a login
+#   Paperless -> /accounts/3rdparty/signup/, never a login
+#   Immich    -> 400 "email already linked to different account"
+#
+# Each row is keyed on a stable, human-meaningful column (the account email or
+# username) rather than a numeric primary key, so a re-run converges and a
+# fresh database does not need the hard-coded ids this was first discovered on.
+#
+# The ZITADEL subject below is the `sub` claim of the operator account. It is
+# NOT a client id: every one of the three databases stores the subject.
+#
+# Re-run policy: safe. Every statement is an UPDATE that matches its row by
+# email/username, and the Job prunes itself so ArgoCD can recreate it on the
+# next sync.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: iam-identity-link-repair
+  namespace: iam
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 2
+  ttlSecondsAfterFinished: 600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: iam-identity-link-repair
+        app.kubernetes.io/part-of: iam
+    spec:
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 70 # postgres user in the image, so psql does not run as root
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: repair
+          image: postgres:16-alpine
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          env:
+            # The ZITADEL user that must own all three links.
+            - name: ZITADEL_SUB
+              value: "396924923823522416"
+            - name: LINK_EMAIL
+              value: "[email protected]"
+            - name: LINK_USERNAME
+              value: "rock"
+            # Forgejo
+            - name: FORGEJO_PGHOST
+              value: "forgejo-db-rw.cicd.svc.cluster.local"
+            - name: FORGEJO_PGUSER
+              valueFrom: { secretKeyRef: { name: forgejo-db-app, key: username } }
+            - name: FORGEJO_PGPASSWORD
+              valueFrom: { secretKeyRef: { name: forgejo-db-app, key: password } }
+            - name: FORGEJO_PGDATABASE
+              value: "forgejo"
+            # Paperless
+            - name: PAPERLESS_PGHOST
+              value: "paperless-db-rw.paperless.svc.cluster.local"
+            - name: PAPERLESS_PGUSER
+              valueFrom: { secretKeyRef: { name: paperless-db-app, key: username } }
+            - name: PAPERLESS_PGPASSWORD
+              valueFrom: { secretKeyRef: { name: paperless-db-app, key: password } }
+            - name: PAPERLESS_PGDATABASE
+              value: "paperless"
+            # Immich
+            - name: IMMICH_PGHOST
+              value: "immich-db-rw.immich.svc.cluster.local"
+            - name: IMMICH_PGUSER
+              valueFrom: { secretKeyRef: { name: immich-db-app, key: username } }
+            - name: IMMICH_PGPASSWORD
+              valueFrom: { secretKeyRef: { name: immich-db-app, key: password } }
+            - name: IMMICH_PGDATABASE
+              value: "immich"
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              psql_dsn() { printf 'postgresql://%s:%s@%s:5432/%s' "$2" "$3" "$1" "$4"; }
+
+              echo "=== Forgejo: link the OIDC subject to the local account ==="
+              # login_source.type = 6 is OAuth2/OIDC. The source may be named
+              # anything ("zitadel" today, "authentik" before), so select on type
+              # and take the single active row rather than assuming a name.
+              psql "$(psql_dsn "$FORGEJO_PGHOST" "$FORGEJO_PGUSER" "$FORGEJO_PGPASSWORD" "$FORGEJO_PGDATABASE")" -v ON_ERROR_STOP=1 -c \
+                "INSERT INTO external_login_user (external_id, user_id, login_source_id, provider) SELECT '$ZITADEL_SUB', u.id, ls.id, 'openidConnect' FROM \"user\" u CROSS JOIN (SELECT id FROM login_source WHERE type = 6 AND is_active LIMIT 1) ls WHERE lower(u.lower_name) = lower('$LINK_USERNAME') ON CONFLICT DO NOTHING;"
+
+              echo "=== Paperless: point the social link at the ZITADEL subject ==="
+              # provider must match the allauth provider_id in
+              # SOCIALACCOUNT_PROVIDERS_JSON, which is "zitadel".
+              psql "$(psql_dsn "$PAPERLESS_PGHOST" "$PAPERLESS_PGUSER" "$PAPERLESS_PGPASSWORD" "$PAPERLESS_PGDATABASE")" -v ON_ERROR_STOP=1 -c \
+                "UPDATE socialaccount_socialaccount sa SET uid = '$ZITADEL_SUB', provider = 'zitadel' FROM auth_user au WHERE au.id = sa.user_id AND lower(au.email) = lower('$LINK_EMAIL');"
+              psql "$(psql_dsn "$PAPERLESS_PGHOST" "$PAPERLESS_PGUSER" "$PAPERLESS_PGPASSWORD" "$PAPERLESS_PGDATABASE")" -v ON_ERROR_STOP=1 -c \
+                "UPDATE socialaccount_socialaccount sa SET provider = 'zitadel' FROM auth_user au WHERE au.id = sa.user_id AND lower(au.email) = lower('$LINK_EMAIL');"
+
+              echo "=== Immich: replace the stale provider subject on the existing user ==="
+              # Immich rejects a callback whose email is already attached to a
+              # different oauthId, so the row must be re-pointed, not inserted.
+              psql "$(psql_dsn "$IMMICH_PGHOST" "$IMMICH_PGUSER" "$IMMICH_PGPASSWORD" "$IMMICH_PGDATABASE")" -v ON_ERROR_STOP=1 -c \
+                "UPDATE \"user\" SET \"oauthId\" = '$ZITADEL_SUB' WHERE lower(email) = lower('$LINK_EMAIL');"
+
+              echo "=== verify ==="
+              psql "$(psql_dsn "$FORGEJO_PGHOST" "$FORGEJO_PGUSER" "$FORGEJO_PGPASSWORD" "$FORGEJO_PGDATABASE")" -tAc \
+                "SELECT 'forgejo:'||external_id FROM external_login_user WHERE external_id='$ZITADEL_SUB';"
+              psql "$(psql_dsn "$PAPERLESS_PGHOST" "$PAPERLESS_PGUSER" "$PAPERLESS_PGPASSWORD" "$PAPERLESS_PGDATABASE")" -tAc \
+                "SELECT 'paperless:'||provider||':'||uid FROM socialaccount_socialaccount WHERE uid='$ZITADEL_SUB';"
+              psql "$(psql_dsn "$IMMICH_PGHOST" "$IMMICH_PGUSER" "$IMMICH_PGPASSWORD" "$IMMICH_PGDATABASE")" -tAc \
+                "SELECT 'immich:'||\"oauthId\" FROM \"user\" WHERE \"oauthId\"='$ZITADEL_SUB';"
+              echo "=== done ==="
@@ -6,6 +6,11 @@ resources:
  # RoleBinding). Authentik is retired, so the file is gone; keep this list in
  # step with the directory or kustomize fails the whole app.
  - smtp-gotify.yaml
+  # Repairs the ZITADEL subject stored inside the Forgejo, Paperless and Immich
+  # databases. Those links are not Kubernetes objects, so without this Job a
+  # fresh cluster (or a provider change) leaves every one of them pointing at
+  # the old subject and SSO cannot complete.
+  - identity-link-repair-job.yaml

 # IAM provisioning is manual-only (security-sensitive).
 # Script: scripts/iam/zitadel-provision.py
@@ -107,6 +107,13 @@ spec:
  # Use external URL for config fetch (hairpins through ingress-nginx); browser
  # redirects use public URLs embedded in the OIDC metadata response.
  env:
+    # MinIO ships identity_openid DISABLED by default, whatever else is set.
+    # Without this key every other MINIO_IDENTITY_OPENID_* value below is
+    # inert and the console advertises password login only — it answers
+    # /api/v1/login with {"loginStrategy":"form"} and never renders an SSO
+    # button, which is exactly the symptom this tenant showed.
+    - name: MINIO_IDENTITY_OPENID_ENABLE
+      value: "on"
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
      # Must use external URL — well-known response contains external issuer/jwks_uri.
      # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
@@ -121,8 +128,9 @@ spec:
          key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
      value: "groups"
-    - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX
-      value: ""
+    # An empty claim prefix is worse than no key: it makes the claim resolver
+    # prepend nothing and then look for a nested property that cannot exist.
+    # Omit the key entirely unless a real prefix is needed.
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: "https://minio.riotpiao.com/oauth_callback"
    - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME