feat(iam): wire all 9 consumer manifests to Zitadel OIDC

- Replace Authentik issuer/auth/token/userinfo URLs with Zitadel endpoints in grafana, minio, paperless-ai, paperless-ai-new, homarr, argocd, blackbox - Write Zitadel client IDs/secrets into plain manifests and SOPS secrets (grafana, minio, paperless, homarr) - Adopt homarr-oidc into SOPS (homarr-auth-oidc.enc.yaml) - Add homarr to SECRET_MAP and remove from MANUAL_ONLY in zitadel-provision.py - Make client-id regex idempotent for repeated runs (minio, paperless) - Fix cloudflared init container CA mount path (homelab-ca.crt -> ca.crt) - Remove Authentik ingress and cloudflared tunnel entry
2026-10-25 21:08:23 -07:00
34 changed files with 1464 additions and 74 deletions
@@ -16,7 +16,7 @@ spec:
      # Ingress-nginx forces HTTPS (force-ssl-redirect), so cloudflared must
      # dial https://<hostname> and trust homelab-ca or the TLS handshake fails
      # with "certificate signed by unknown authority". Same pattern as
-      # iam values: merge the system bundle with homelab-ca.crt.
+      # authentik-values.yaml: merge the system bundle with homelab-ca.crt.
      initContainers:
        - name: merge-ca-certs
          image: debian:bookworm
@@ -0,0 +1,78 @@
 # Homarr landing page with Authentik SSO
 # Probe tuning (chart DOES expose these — the old PostSync patch-job was
 # unnecessary and fragile: it only patched one Deployment revision, so any later
 # rollout reverted to the chart's aggressive defaults). Homarr's first-boot icon
 # updater blocks the event loop for ~50s ("icons updater took 49553ms"), during
 # which /api/health/live can't answer within the default 10s×3 window → kubelet
 # SIGTERMs the pod → CrashLoopBackOff (247 restarts, 503 at the ingress). Give
 # liveness a wide window so the icon import can finish without a kill.
 livenessProbe:
  initialDelaySeconds: 60
  periodSeconds: 30
  timeoutSeconds: 5
  failureThreshold: 10
 readinessProbe:
  initialDelaySeconds: 30
  periodSeconds: 15
  timeoutSeconds: 5
  failureThreshold: 6
 image:
  repository: ghcr.io/homarr-labs/homarr
  tag: "latest"
  pullPolicy: Always
 replicaCount: 1
 # Configure SSO via environment variables
 # Chart supports these via top-level env dict (not array)
 env:
  AUTH_PROVIDERS: "oidc,credentials"
  AUTH_OIDC_ISSUER: "https://zitadel.riotpiao.com"
  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
  # hides the "Sign in with Authentik" button entirely when it's absent (per the
  # authentik Homarr integration + homarr SSO docs). This was the missing var.
  AUTH_OIDC_URI: "https://zitadel.riotpiao.com/oauth/v2/authorize"
  AUTH_OIDC_CLIENT_NAME: "Zitadel"
  AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
  AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
  AUTH_OIDC_AUTO_LOGIN: "false"
  # Link the OIDC identity to an existing homarr account with the same email.
  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
  # per run ("callback took longer than expected"), compounding CPU pressure.
  DISABLE_ANALYTICS: "true"
  BASE_URL: "https://homarr.riotpiao.com"
  NEXTAUTH_URL: "https://homarr.riotpiao.com"
 # Client credentials from homarr-oidc secret
 # Chart doesn't support envFrom, so we add via extraEnv
 extraEnv:
  - name: AUTH_OIDC_CLIENT_ID
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-id
  - name: AUTH_OIDC_CLIENT_SECRET
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-secret
 tolerations:
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule
 resources:
  requests:
    cpu: 250m
    memory: 384Mi
  limits:
    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
    # real CPU headroom.
    cpu: "2"
    memory: 1Gi
@@ -0,0 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: dashboard
 # Probes are now tuned via homarr-values.yaml (chart-native); the old
 # fix-probes-job PostSync hook is removed. homarr-secrets/auth-oidc/db-encryption
 # Secrets are delivered by the sops-secrets (ksops) Application.
 resources: []
@@ -9,14 +9,6 @@ metadata:
    app.kubernetes.io/name: paperless-ai
 spec:
  replicas: 1
  # Recreate, not RollingUpdate: paperless-ai-data is a single RWO PVC. With
  # RollingUpdate the new pod tries to mount the PVC while the old pod still
  # holds it -> "Multi-Attach error" / stuck Init:0/1 forever, rollout wedged
  # at "1 desired, 1 ready, 1 updated, 1 unavailable". Recreate terminates the
  # old pod first so the PVC detaches before the new one starts. Same pattern
  # as the Forgejo gitea Deployment.
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: paperless-ai
@@ -30,6 +30,33 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: authentik
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  sources:
    - repoURL: https://charts.goauthentik.io
      chart: authentik
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/iam/authentik-values.yaml
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # ZITADEL — the Authentik replacement. Coexistence, not cutover: Authentik keeps
 # serving in the same `iam` namespace until the last consumer moves, then this
 # becomes the only IdP. Migration plan (outside git):
@@ -79,6 +106,28 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 # Raw IAM manifests: key-rotation cronjob + authentik migration job.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: iam-jobs
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/iam
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # Forgejo itself. Was a bootstrap Helm release (phase 3) until it was brought
 # under Argo, because values changes there were inert — a proxy-body-size fix
 # sat committed while the live Ingress kept nginx's 1m default and rejected
@@ -1,4 +1,4 @@
-# Wave 2 — dedicated per-app CNPG Postgres clusters (zitadel-db → ns iam,
+# Wave 2 — dedicated per-app CNPG Postgres clusters (authentik-db → ns iam,
 # temporal-db + visibility → ns temporal). ONE App, ONE folder (k8s/infra/databases).
 # CNPG operator is Phase-0 bootstrap; these Cluster CRs are GitOps — no circular
 # dep (they run after ArgoCD is up, before their apps at w3/w8). CNPG generates
@@ -171,6 +171,40 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 # Consolidated: homarr + homarr-patches → homarr
 # Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: homarr
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "8"
 spec:
  project: homelab
  sources:
    - repoURL: https://homarr-labs.github.io/charts
      chart: homarr
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/apps/homarr/homarr-values.yaml
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: dashboard
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Portfolio site at riotpiao.com - static Next.js site from rock/riotpiao.com repo.
 # Points directly to infra/portfolio/base (bypassing repo's own argocd-apps.yaml
 # which has wrong URLs). Image built by Forgejo Actions on rock/portfolio repo.
@@ -0,0 +1,20 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: oidc-bootstrap
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "5"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/oidc-bootstrap
  destination:
    server: https://kubernetes.default.svc
    namespace: kube-system
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -34,9 +34,11 @@ spec:
    - https://prometheus-community.github.io/helm-charts
    - https://grafana.github.io/helm-charts
    - https://helm.releases.hashicorp.com
    - https://charts.goauthentik.io
    - https://charts.zitadel.com
    - https://strimzi.io/charts/
    - https://charts.bitnami.com/bitnami
    - https://homarr-labs.github.io/charts
    - https://go.temporal.io/helm-charts
    - https://portainer.github.io/k8s/
    # Substrate charts (cert-manager, ingress-nginx, reloader) — app-of-apps owned
@@ -0,0 +1,25 @@
 apiVersion: ENC[AES256_GCM,data:Fc0=,iv:UFqMFeZTJZtFMoNeusjAoSI60U9StZS4IXw+n8JQCRM=,tag:uUvY4+jcjiSJHs0HDS3/7A==,type:str]
 kind: ENC[AES256_GCM,data:NUw86IEw,iv:gh4Vqn6uteYnYIFcXT/QVxRPKm+G+j2aIVuV71nZ4k4=,tag:nZA/x75Kx4+CoSx66BsvkQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:qex150pm0eGwYiiI05y9LUs=,iv:5Tk001T796c4HFikoh1ZVB6ierBfDqOUkFs5ilkr+V8=,tag:LiDAiCH7w6aRS41VKIlIKg==,type:str]
    namespace: ENC[AES256_GCM,data:nI91,iv:c1hBWGMkVb+vg+MKNzty8CFfXxQabXEtT+EpiuQZNvo=,tag:BsGtJDbBadDdTa687+Sdag==,type:str]
 type: ENC[AES256_GCM,data:3rXawlSD,iv:FUtLK4KP7v/SBF08ZcU6siIO1qfjlJirf2x5PyzAjuA=,tag:mSIQ+Y43+wDfZGRbQ6a0Og==,type:str]
 stringData:
    AUTHENTIK_SECRET_KEY: ENC[AES256_GCM,data:i5tVOWnN9a/cfhs6gckveRxUpiN7Ie227R/74kOIz6lQ5US2hkf03W5qSGCrzheL3DB0i8+d0iJYsApjn/wCiN7ndZZVFko3W5jDfVPybxc=,iv:djF1G9JHT5B7ziTokA/UuPpG+tG2LvFU4uMAR0x4M6o=,tag:vHOd7jLYkM8Evnr0QUVrCw==,type:str]
    AUTHENTIK_BOOTSTRAP_PASSWORD: ENC[AES256_GCM,data:PjvYtH03CzQbTXEqt/Z18+u1zJdok7DLwDn51FUZXDA=,iv:pSwSm8hB0/Q8uOAcOZ1rTqq7BF782EglG19GGeghm1E=,tag:qGCQFWCrwsWBOiQ07ySMUQ==,type:str]
    AUTHENTIK_BOOTSTRAP_TOKEN: ENC[AES256_GCM,data:1tnekGo0GIemJCMRenjN0yNeSiiMl/kky8ms84TsqBLaQ0lQzPxjq9qtgNJwoAsufxrznrkDVKjeOEHNYcA82A==,iv:NNFWSLMILoWccA9ihy1OArSS1X3OySB7PIHdXZdyzlg=,tag:t2w+KyReOXftTrU9pBQDnQ==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSZGRKWm9PTkNSOENtREgx
            M2V3Z0xxOXRhRGVRclo3SzBKRkc0VmtCY0NnClJPSGJRcUlzWmNFeTZ1bmorNEV4
            U3l3TzVlMTlrWkJnQzBtUTJtcUwrM3MKLS0tIEJrMWZrbllYYUZKN1V4MVMzcXAv
            eUtheHlNMHpsTEJXZ1FTMFVxbWloa0kKLRkE1Du+4gdOLerOl9y0mZw+8fqfECQY
            uP64Q+9BSuPYNiTLZvYDzmdoy+KTS7i7C9rf+iU96R/crbQLamV9Ew==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T20:37:07Z"
    mac: ENC[AES256_GCM,data:b3pDx340d1iZxsPPXzACckoLBrW+XjX1qlGrWEEI30j2Ao0hewTRZJyHbdI97A6b/hqzBN/4tjqqkdqm8goNhUQweMLTw86X6sALuE0aLS9wK53qjqih8fQwQBbgLDtjFXtW26h+smIm7aB5ZSDvU2le44HDXuWd83sWtCavNq0=,iv:ZMDsoKpygFJrYpXy2dA0IlxAi0jt24rxF72eUzgQs7k=,tag:WWE7G7+c9JK6fF1U9V8igg==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,24 @@
 apiVersion: v1
 kind: Secret
 metadata:
    name: homarr-oidc
    namespace: dashboard
 type: Opaque
 stringData:
    client-id: ENC[AES256_GCM,data:IFNjfDMrObaxfGwJpiYqPP60,iv:0WR2C+DEsq8I031A9Bopotn5TYdISApbTF46jJBgtVs=,tag:LGijMS27pbbZnOLJqGNXQg==,type:str]
    client-secret: ENC[AES256_GCM,data:q0MHjvuTIKbbMUZpsptOuo3zrabwaWxZS3MLqRdI1Rt7eod5ORAYeZmpllHmojRFve9x/SKkJ+sctafzwPuAPA==,iv:1ZLXOc3a85+MRx3v/XsW8JhsxbCQ26laou8PvvrQvH0=,tag:PRRTsyE2fGGS3dKPh43a8g==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpV1ZlMWxaNTlCUjJjbG9F
            NzFGMXBXYThZT285UHA0bGdwNFJqQTNMckhvClFSTEdseExjbVFtbkdsTUpSSGVn
            WGsyUmxvcVViVVVNZjFRdmJWZlB6TGcKLS0tIE01QjFtVFBGUlpXbjVJcDMwMzRS
            MEh4dUZTZ3puaXF2Uk03Ym50QmdSM1EK6Kmdv3s8/4nI/6Uk4G5FOh1Gxp1u9eYI
            L7MOW0TFkScTuDiu9iJB+CjB8b8Z0kwSYTigYj9ue2y0a1hDYCfQtg==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    encrypted_regex: ^(data|stringData)$
    lastmodified: "2026-10-26T03:56:19Z"
    mac: ENC[AES256_GCM,data:tV7o9zY6dpTdTvNJY8GBkdIB71thoy79RQSHsczednmXXzlNFddZxR/UenJ87jJ1JyzMUKL3CxArYojpuamahQcSXifVar0kotgNCs67A7+xfhHudCnKLMWDfDOom6DdWto6bMNIk/0TF6Wy0m1hkSsxwkdqETVs3b6ZUg/l8Rc=,iv:PE7y5oVMEM9XKQnDZmdRFRR3blzz2SZQFLxOMyILuec=,tag:SM9vJU1zlQ02XJijSiGCMg==,type:str]
    version: 3.13.2
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:FO4=,iv:kKIuJp4uGeX6vECsXk7w43F9S281jObQchmK8r1+kmA=,tag:x5LYd34dzNDLCma/78M4+w==,type:str]
 kind: ENC[AES256_GCM,data:o4RTN8SK,iv:uyOJWjDmHIPuBl9Wc1S/HxUabV2yytWr1bntHW68sTI=,tag:3On0SnQVJ/ICMMryQe+8JQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:a89dHmLdSeMHE7ZP0Q==,iv:OWJvJhFqJ7A0UGVJnZILzCJ/4VuVFib9sUDjXuwyqJQ=,tag:1XrIQn1GDh8RQ4vmQAu7+g==,type:str]
    namespace: ENC[AES256_GCM,data:4wBVpZJgQGM9,iv:3Tn1u/2TP/+SnELu34PDf4Jauf2L4f8+MtgrPIlyKzk=,tag:L3qbKFBjm16g8KX5vUsK6A==,type:str]
 type: ENC[AES256_GCM,data:mdrF9vUb,iv:7ukOgwXlnRf5mpJ/+6YXtyGRonnqehEgT4pJJIafw7c=,tag:90mt7jpNb3N80kCEi+R/rA==,type:str]
 stringData:
    db-encryption-key: ENC[AES256_GCM,data:NWJryAEd/eUJxbNiu05RzLF/l7GDxIQM7RQe3xKBz6XklgRAFwQxxjakcVd85pePUBaQYz3zttFglWhfJlWhkA==,iv:eUWbcs0B/mIheyAQ2+DyKgcA98CRKQGAVC3O0WIkfU8=,tag:be06X6AQqlohYlxuYoZiTg==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1Ym5yQ1pmWHZjVnYxbUVO
            SkZ5dnI2OXR3cmZjZkM0K1ErMElLNktlK1JFCmpleFBrcXkwcDA4cGFuWWN6VDQv
            dVV4NjB4MjEreENVQnZ2ejVLblRYOVUKLS0tIG9VbXJvUmQ3SzMzN3IrRDlac2sr
            eGZNYXBoZFZDeisrVCtUMXVPaEpoNFkKjwhUU+Zsqv4Be+m558fET5UhF/2rO73J
            AhGbJVaA7PaOCswPWiOLso26zD+MBB3VKHNdUFnenpoM16Nu4pOf5w==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T21:07:30Z"
    mac: ENC[AES256_GCM,data:KrGQY/eOmihYCvQCbfcM6bZ6rfx4zKh62M8NM0VQr1//1dSHpOOh25dYGLuQeECnqLxSpGQNWzr2xjUYWs4NOdSEFGq6FL2iHE3LnqoBZspp/JFJYP6+z/8jSUojwKEW7OuZZdPN6gF2u+IPOeWbGftnjS7X+atoSkYag2ty2vk=,iv:IOPEgBizsxrVoK5oSdWmi0uCBUnReOEnMYXuwSBKTzA=,tag:9k/XQ/AE7wSFq5XMbY0FTA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:4OY=,iv:0Lgui0+X2oTlgCX5HEIIihEsGJD0C2+pDIOTCNi4r+g=,tag:uL7vlreaHjcLJjV7e5I7PQ==,type:str]
 kind: ENC[AES256_GCM,data:x/ltkind,iv:KRBfS7COsTs/l3Fhyk5FvH7IVc+r/6M6FVOo+kPDe1Q=,tag:UuTbe213hDfK/WUtKS/doQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:LpOEecTJBuGy5PqNSFM=,iv:Q+DCLGtw3wihhdAgEbzYHyQhFTPWFyE2iJKl9mYaA2E=,tag:E2wrig4lRXGDvSCHec5clQ==,type:str]
    namespace: ENC[AES256_GCM,data:6kMVCXaF68NO,iv:gnCCkV7GKXA2HWOOeNUGnpAycf8U7pl6F8Vfo6DhLR4=,tag:0VdeySsuhUr2heXyGXZwBg==,type:str]
 type: ENC[AES256_GCM,data:6muC3Zqh,iv:gBaC+6MXvZY/bIpy/cEG/zWwLBaUFMVfxAHa1OnwfnQ=,tag:v6NYyhAKAwovluHAXNaOVA==,type:str]
 stringData:
    SECRET_ENCRYPTION_KEY: ENC[AES256_GCM,data:jX9ULlaBYTSTFGRMplSder5rMrsz7THU3kgJSGef0r5rZvtoaAG9NVYU2QUEuD0sZgfqRooeMOkDCAqFc8qsuw==,iv:uOmx8w3XPKnoxfasf7gqi/bljxigpLwaAC30c2TroKg=,tag:Dmbu6m3+UZa1w0UR0LRh9g==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6TDVyNGRtYWgzMDYwdjdr
            NXJWcGJZNUx6YzBGWm1nWlp0SlN3WXhaR0VBCjZ3bDBmUTFCUjg3U21wak90eHZK
            NlRCTUZQSHhoS0ZSUGZ4UHhST01KNmMKLS0tIGUxcGo0NmJ2bmJ6eGJ3QWxzYVM5
            c2Z6UkFjQmZ4ejIvK1JQQkZ0VFFIZ3cKA8UzmGsX+s3yOermOHqbnnEtekHCTqmC
            R3Jyzf6OolZMJpNwbetcsOfgO61aM7LHl/Xb/hFLbLjZ+eECrt8VpA==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T18:02:01Z"
    mac: ENC[AES256_GCM,data:HTrEEVDm8JqnVBFaY/FJknxma0nI95tRiwVc0K76k+2R4jgKCBA3EUAzPVS+gA4rTAeKHPSHrT2eidlkseT2WQUjqMRrpHcUsbSjNfpTEY2g/U0PIiOXZMRGfwmsQVgmu2fqeMOyzDSyya0scdWJXPraRPNZ+OpEkNVlm4rWcgs=,iv:KY3dnnmooJgsu66wc7gbzeIGUXZNgVrxAhpAwxoG+G8=,tag:7r6d41oTD136N57rtY7auA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -9,6 +9,7 @@ metadata:
 files:
    - agent-pod-models.enc.yaml
    - agent-pod-ssh-key.enc.yaml
    - authentik-secrets.enc.yaml
    - cloudflare-secrets.enc.yaml
    - cloudflared-credentials.enc.yaml
    - forgejo-registry-pat-api.enc.yaml
@@ -17,6 +18,9 @@ files:
    - forgejo-secrets.enc.yaml
    - grafana-oidc-secrets.enc.yaml
    - grafana-secrets.enc.yaml
    - homarr-auth-oidc.enc.yaml
    - homarr-db-encryption.enc.yaml
    - homarr-secrets.enc.yaml
    - homelab-ca-secrets.enc.yaml
    - loki-secrets.enc.yaml
    - minio-secrets.enc.yaml
@@ -275,3 +275,28 @@ spec:
                name: temporal-web
                port:
                  number: 8080
 ---
 # NOTE: api.riotpiao.com is deliberately NOT here. Its namespace `api` is
 # created in wave 7, and this Application syncs in wave 1 — an Ingress into a
 # namespace that doesn't exist yet would fail and mark this whole app
 # SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced by the api-gw
 # Application.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: homarr
  namespace: dashboard
 spec:
  ingressClassName: nginx
  rules:
    - host: homarr.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: homarr
                port:
                  number: 7575
@@ -143,10 +143,7 @@ configs:
    oidc.config: |
      name: Zitadel
      issuer: https://zitadel.riotpiao.com
-      # MUST be quoted: an unquoted 18-digit number is parsed as a float by the
+      clientID: 396757801210282367
      # YAML parser and ArgoCD rejects it ("cannot unmarshal number into ...
      # clientID of type string"). Same class of bug as Grafana's client_id.
      clientID: "396757801210282367"
      clientSecret: $oidc-secret:client-secret
      requestedScopes:
        - openid
@@ -0,0 +1,38 @@
 # Dedicated CNPG Postgres for Authentik (GitOps, wave 2 — before authentik w3).
 # CNPG generates secret `authentik-db-app` + service `authentik-db-rw` in ns iam;
 # authentik-values reads them locally (no cross-ns secret copy).
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: authentik-db
  namespace: iam
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
  instances: 3
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
  bootstrap:
    initdb:
      database: authentik
      owner: app
      encoding: UTF8
      localeCollate: C
      localeCType: C
  enableSuperuserAccess: false
  resources:
    requests: { memory: "512Mi", cpu: "250m" }
    limits: { memory: "2Gi", cpu: "1" }
  storage:
    size: 10Gi
    storageClass: longhorn-cnpg
  monitoring:
    enablePodMonitor: true
  affinity:
    # preferred + tolerations: HA across nodes without deadlocking a partly-
    # schedulable 3-CP (same as forgejo-db).
    podAntiAffinityType: preferred
    topologyKey: kubernetes.io/hostname
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
@@ -6,6 +6,7 @@ kind: Kustomization
 # deployed into it. paperless ns declared in namespaces.yaml above.
 resources:
  - namespaces.yaml
  - authentik-db.yaml
  - zitadel-db.yaml
  - temporal-db.yaml
  - paperless-db.yaml
@@ -1,7 +1,7 @@
 # DB clusters are wave 2 — their namespaces must exist first (their apps that
 # would CreateNamespace run later, w3/w8). Declared here so the databases App
-# creates them. vault/temporal/paperless CreateNamespace=true then no-ops.
+# creates them. authentik/vault/temporal/paperless CreateNamespace=true then
-# poimen namespace created by poimen-root app (wave 7).
+# no-ops. poimen namespace created by poimen-root app (wave 7).
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -0,0 +1,284 @@
 # k8s/talos-iam/authentik-values.yaml
 # Authentik — SSO Identity Provider for the homelab.
 # Provides OAuth2/OIDC login for Grafana, MinIO, Forgejo, and Argo CD.
 # Chart: authentik/authentik from https://charts.goauthentik.io
 #
 # Architecture: server (UI+API) + worker (background tasks) + PostgreSQL + Redis.
 # PostgreSQL is the system of record — must persist. Redis is ephemeral cache/queue.
 #
 # Secrets injected via helmfile --set (from .env / vsource):
 #   AUTHENTIK_SECRET_KEY         — signs sessions and tokens; set once, never rotate casually
 #   AUTHENTIK_BOOTSTRAP_PASSWORD — initial akadmin password (used once at first login)
 #   AUTHENTIK_BOOTSTRAP_TOKEN    — API token for the setup_talos_iam.sh bootstrap script
 #   AUTHENTIK_PG_PASSWORD        — PostgreSQL user password
 authentik:
  # host: the external URL Authentik uses to build redirect URIs in OAuth2 flows.
  # Must match what the browser sees — if it returns an internal svc URL,
  # the browser's redirect after login will fail (can't reach svc DNS externally).
  # HTTP (not HTTPS) because the Authentik ingress has no TLS cert configured.
  host: "https://authentik.riotpiao.com"
  error_reporting:
    enabled: false   # do not phone home to Sentry
  # PostgreSQL connection — points at CloudNativePG cluster in ddb namespace.
  # Uses 'app' bootstrap user (CNPG simple pattern, same as Forgejo).
  # Credentials injected from authentik-db-app secret via env vars below.
  postgresql:
    host: authentik-db-rw.iam.svc.cluster.local
    port: 5432
    name: authentik
    user: app  # All apps use shared 'app' user (CNPG design pattern)
    password: ""  # overridden by AUTHENTIK_POSTGRESQL__PASSWORD env var
  # Redis connection — bundled subchart, standalone mode (no sentinel/cluster).
  redis:
    host: authentik-redis-master
  # ── HTTP client timeouts ──────────────────────────────────────────────────────
  # Increased to tolerate 5+ second pod-to-pod network latency.
  # Affects webhooks, outpost management, SCIM, LDAP sync.
  # Default: ~30s — too aggressive when latency spikes hit 5-10s.
  log_level: debug  # enable debug logging to monitor connection issues
 # ── CA trust (shared by server and worker) ────────────────────────────────────
 # Authentik (Python/Debian) uses requests + httpx for outgoing HTTPS — webhooks,
 # outpost management, SCIM. Both libraries need REQUESTS_CA_BUNDLE / SSL_CERT_FILE
 # to point to a bundle that includes homelab-ca, otherwise connections to other
 # homelab services fail with "certificate signed by unknown authority".
 #
 # Strategy: a debian:12-slim init container (run as root) concatenates the
 # Debian system Mozilla bundle with homelab-ca.crt into an emptyDir. The main
 # container then references /merged/ca-bundle.crt via two env vars that cover
 # every Python HTTP library.
 _caVolumes: &caVolumes
  - name: homelab-ca
    configMap:
      name: homelab-ca
  - name: merged-ca
    emptyDir: {}
 _caVolumeMounts: &caVolumeMounts
  - name: homelab-ca
    mountPath: /homelab-ca
    readOnly: true
  - name: merged-ca
    mountPath: /merged
    readOnly: true
 _caInitContainers: &caInitContainers
  - name: merge-ca-certs
    image: debian:bookworm
    imagePullPolicy: IfNotPresent
    securityContext:
      runAsUser: 0
    command:
      - sh
      - -c
      - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
    volumeMounts:
      - name: homelab-ca
        mountPath: /homelab-ca
        readOnly: true
      - name: merged-ca
        mountPath: /merged
  # NOTE: no authentik-migrate init container — the authentik `server` entrypoint
  # runs migrations itself on startup. A separate `manage migrate` init pinned to
  # an older image tripped a version-history precheck on an empty DB
  # (relation "authentik_version_history" does not exist) and blocked boot.
 _caEnv: &caEnv
  - name: REQUESTS_CA_BUNDLE
    value: /merged/ca-bundle.crt
  - name: SSL_CERT_FILE
    value: /merged/ca-bundle.crt
 # ── Authentik server (UI + API) ───────────────────────────────────────────────
 # Handles all browser traffic: login flows, admin UI, OAuth2 authorize/token endpoints.
 # NodePort 32172 is a fallback for direct node access during troubleshooting;
 # normal access is via nginx ingress (authentik.riotpiao.com → svc:80).
 # Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
 server:
  replicas: 1
  # Merge SOPS-CMP-emitted secret values after the chart's own `authentik` secret.
  envFrom:
    - secretRef:
        name: authentik-secrets
  deploymentStrategy:
    type: Recreate
  service:
    type: NodePort
    nodePort: 32172
  resources:
    requests:
      cpu: 100m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 1Gi
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  volumes: *caVolumes
  volumeMounts: *caVolumeMounts
  initContainers: *caInitContainers
  env:
    # Merge CA trust env vars
    - name: REQUESTS_CA_BUNDLE
      value: /merged/ca-bundle.crt
    - name: SSL_CERT_FILE
      value: /merged/ca-bundle.crt
    # Override database credentials to use 'app' from authentik-db-app
    - name: AUTHENTIK_POSTGRESQL__USER
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: username
    - name: AUTHENTIK_POSTGRESQL__PASSWORD
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: password
  podAnnotations:
    configmap.reloader.stakater.com/reload: "homelab-ca"
    homelab.io/restart-at: "2026-06-21T13-40"
  # The /-/health/{live,ready}/ endpoints do a DB round-trip; under transient
  # CNPG contention they respond in 5-6s while still returning 200. The chart's
  # default 3s liveness timeout then flags a working backend as dead and kubelet
  # kills it in a restart loop — the pod never stays Ready, gets dropped from the
  # authentik-server Service endpoints, and the OAuth-provisioning PostSync hook
  # fails with "Host is unreachable". Widen the timeouts so slow-but-healthy
  # checks aren't treated as failures. (Only these fields are overridden; the
  # chart deep-merges the rest of each probe, incl. the httpGet path.)
  livenessProbe:
    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
    timeoutSeconds: 15
    failureThreshold: 6
  readinessProbe:
    initialDelaySeconds: 300  # skip probe until migrations complete
    timeoutSeconds: 15
    failureThreshold: 6
  startupProbe:
    timeoutSeconds: 15
    failureThreshold: 120  # 120 × 10s = 20min for fresh DB migrations
  # Every OIDC login (Grafana, Argo CD, MinIO, Forgejo) depends on this server —
  # its request latency/error rate explains SSO-driven slowness on those services.
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true
      scrapeTimeout: 30s
 # ── Authentik worker ──────────────────────────────────────────────────────────
 # Runs background tasks: email delivery, LDAP sync, flow policy evaluation,
 # event log cleanup, and managed outpost updates. Stateless — no PVC needed.
 # Same resource profile as server; Authentik 2023+ merged some worker duties
 # into the server process but the worker pod is still required.
 worker:
  replicas: 1
  envFrom:
    - secretRef:
        name: authentik-secrets
  deploymentStrategy:
    type: Recreate
  resources:
    requests:
      cpu: 100m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 1Gi
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  volumes: *caVolumes
  volumeMounts: *caVolumeMounts
  initContainers: *caInitContainers
  env:
    # Merge CA trust env vars
    - name: REQUESTS_CA_BUNDLE
      value: /merged/ca-bundle.crt
    - name: SSL_CERT_FILE
      value: /merged/ca-bundle.crt
    # Override database credentials to use 'app' from authentik-db-app
    - name: AUTHENTIK_POSTGRESQL__USER
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: username
    - name: AUTHENTIK_POSTGRESQL__PASSWORD
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: password
  podAnnotations:
    configmap.reloader.stakater.com/reload: "homelab-ca"
    homelab.io/restart-at: "2026-06-21T13-40"
  livenessProbe:
    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
  readinessProbe:
    initialDelaySeconds: 300  # skip probe until migrations complete
  startupProbe:
    initialDelaySeconds: 30  # let server finish DB work first
    failureThreshold: 120
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true
      scrapeTimeout: 30s
 # ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ─────────────
 # Authentik connects to the dedicated authentik-db (1 primary + 2 replicas).
 # Do not use the bundled Bitnami subchart — CNPG is already running.
 postgresql:
  enabled: false
  primary:
    persistence:
      enabled: true
      storageClass: longhorn
      size: 8Gi
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
    affinity:
      nodeAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
                - key: node-role.kubernetes.io/worker
                  operator: Exists
 # ── Bundled Redis ─────────────────────────────────────────────────────────────
 # Cache and async task queue only — no durable data. If Redis restarts, in-flight
 # background tasks are retried and cached tokens are recomputed. Losing Redis
 # data does not lose user accounts or flow configuration (that's in PostgreSQL).
 # persistence: false saves a PVC and makes restarts faster.
 #
 # Same prefer-worker / fallback-to-cp scheduling as PostgreSQL.
 # architecture: standalone — no Sentinel/cluster overhead for a 3-node homelab.
 redis:
  enabled: true
  master:
    persistence:
      enabled: false
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
    affinity:
      nodeAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
                - key: node-role.kubernetes.io/worker
                  operator: Exists
  architecture: standalone
 # Ingress disabled — rule lives in k8s/ingress/ingress.yaml (authentik.riotpiao.com).
 # For direct access during bootstrap: kubectl -n iam port-forward svc/authentik-server 7000:80
@@ -5,6 +5,8 @@ resources:
  - rbac-dashboard-rolebinding.yaml
 # IAM provisioning is manual-only (security-sensitive).
-# Script: scripts/iam/zitadel-provision.py
+# Script: scripts/iam/authentik-provision.py
 # Run:
-#   scripts/iam/zitadel-provision.py all
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
 #     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
 #   python3 scripts/iam/authentik-provision.py
@@ -0,0 +1,13 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: dashboard
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: authentik-provisioner
 subjects:
 - kind: ServiceAccount
  name: authentik-provisioner
  namespace: iam
@@ -81,15 +81,6 @@ server:
    - envName: VAULT_UNSEAL_KEY_3
      secretName: vault-unseal-keys
      secretKey: key3
    # Zitadel OIDC client credentials for the Vault UI login method. Consumed by
    # the postStart hook below, which enables auth/oidc via the Vault API after
    # unsealing. Vault auth methods cannot be declared in HCL, only via API/CLI.
    - envName: OIDC_CLIENT_ID
      secretName: vault-oidc-creds
      secretKey: OIDC_CLIENT_ID
    - envName: OIDC_CLIENT_SECRET
      secretName: vault-oidc-creds
      secretKey: OIDC_CLIENT_SECRET
  # ── Auto-unseal ─────────────────────────────────────────────────────────────
  # Vault starts sealed after every pod restart and can't serve requests until
@@ -106,28 +97,6 @@ server:
      vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
      vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
      # ── Zitadel OIDC login method ────────────────────────────────────────
      # Vault auth methods are API-driven: they cannot be declared in HCL.
      # Enable auth/oidc and write its config on every pod start. Both calls
      # are idempotent (auth enable returns 400 "already enabled", which we
      # tolerate; the config write always succeeds and overwrites).
      vault auth enable oidc 2>/dev/null || true
      vault write auth/oidc/config \
        oidc_discovery_url="https://zitadel.riotpiao.com" \
        oidc_client_id="$OIDC_CLIENT_ID" \
        oidc_client_secret="$OIDC_CLIENT_SECRET" \
        default_role="default" 2>/dev/null || true
      # The role binds the Zitadel client_id as audience and whitelists the UI
      # callback URL. Policy `default` gives a logged-in operator standard
      # read access; admin actions still require a token with root/sudo.
      vault write auth/oidc/role/default \
        bound_audiences="$OIDC_CLIENT_ID" \
        allowed_redirect_uris="https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback" \
        allowed_redirect_uris="http://localhost:8200/ui/vault/auth/oidc/oidc/callback" \
        user_claim="preferred_username" \
        oidc_scopes="openid,profile,email,groups" \
        policies="default" 2>/dev/null || true
  # ── Vault config (HCL) ──────────────────────────────────────────────────────
  standalone:
    enabled: true
@@ -74,7 +74,7 @@ grafana.ini:
    enabled: true
    name: Zitadel
    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
-    client_id: "396757801965191169"
+    client_id: 396757801965191169
    scopes: openid email profile groups
    auth_url: https://zitadel.riotpiao.com/oauth/v2/authorize
    token_url: https://zitadel.riotpiao.com/oauth/v2/token
@@ -0,0 +1,33 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
  name: authentik-rules
  namespace: iam
 spec:
  groups:
    - name: authentik.rules
      interval: 15s
      rules:
        - alert: AuthentikHighErrorRate
          expr: |
            (
              sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
              /
              sum(rate(authentik_http_requests_total[5m]))
            ) > 0.05
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "High error rate on Authentik"
            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
        - alert: AuthentikOutpostDown
          expr: |
            authentik_outpost_total_up == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Authentik outpost is down"
            description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"
@@ -7,6 +7,7 @@ kind: Kustomization
 resources:
  - alerts/ingress-alerts.yaml
  - alerts/svc-argocd-rules.yaml
  - alerts/svc-authentik-rules.yaml
  - alerts/svc-zitadel-rules.yaml
  - alerts/svc-forgejo-rules.yaml
  - alerts/svc-grafana-rules.yaml
@@ -16,6 +17,7 @@ resources:
  - alerts/api-gateway-alerts.yaml
  - alerts/cag-kv-cache-rules.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
  - servicemonitors/ingress-nginx.yaml
@@ -0,0 +1,18 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: authentik
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
 spec:
  namespaceSelector:
    matchNames:
      - iam
  selector:
    matchLabels:
      app.kubernetes.io/name: authentik
  endpoints:
    - port: metrics
      interval: 30s
      scrapeTimeout: 10s
@@ -0,0 +1,33 @@
 # One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
 # OIDC authenticator (they last started before the Authentik provider existed, so OIDC
 # tokens were never accepted -> every login got 401). Delete this whole directory after.
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 subjects:
  - kind: ServiceAccount
    name: oidc-bootstrap
    namespace: kube-system
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: oidc-bootstrap
@@ -0,0 +1,32 @@
 # Scoped operator access for homarr-admins: the homarr Deployment + app
 # secrets only. homarr-oidc, auth-oidc-secret, db-encryption stay excluded
 # (security-managed, not app config). Inert until kube-apiserver's OIDC
 # wiring lands.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: homarr-operator
  namespace: dashboard
 rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    resourceNames: ["homarr"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["homarr-secrets"]
    verbs: ["get", "list", "watch", "update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: homarr-admins-binding
  namespace: dashboard
 subjects:
  - kind: Group
    name: "oidc:homarr-admins"
    apiGroup: rbac.authorization.k8s.io
 roleRef:
  kind: Role
  name: homarr-operator
  apiGroup: rbac.authorization.k8s.io
@@ -10,6 +10,7 @@ resources:
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
  - homarr-operator-role.yaml
  - portainer-operator-role.yaml
  - kmsvc-operator-role.yaml
  - temporal-operator-role.yaml
@@ -0,0 +1,669 @@
 #!/usr/bin/env python3
 """
 Provision RBAC groups, service account roles, fine-grained claims, and auth flows.
 Idempotent — safe to re-run. Provisions:
  1. Global admin groups (homelab-admins)
  2. Fine-grained service/bucket/project groups (minio-*, poimen-*, paperless-*, grafana-*, sqs-*)
  3. Service account roles with custom claims (paperless-ai-agent, portfolio-agent, etc.)
  4. JWT scope mappings for fine-grained claims (minio_buckets, paperless_doctypes, etc.)
  5. OAuth2 providers with scopes (api-gw, minio, poimen, paperless, grafana)
  6. Auth flows (password grant on api-gw provider)
 Usage:
    source ~/.env
    export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
        -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
    python3 scripts/iam/provision-rbac.py
 DO NOT commit this file to git — .gitignore covers scripts/iam/*.py.
 """
 import json
 import os
 import sys
 import urllib.error
 import urllib.request
 from typing import Dict, List, Any
 from pathlib import Path
 # Load ~/.env for OAuth2 provider secrets
 env_file = Path.home() / ".env"
 if env_file.exists():
    with open(env_file) as f:
        for line in f:
            line = line.strip()
            if line.startswith("export ") and "=" in line:
                key, _, value = line[7:].partition("=")
                key = key.strip()
                value = value.strip().strip('"').strip("'")
                os.environ[key] = value
 AUTHENTIK_URL = "https://authentik.riotpiao.com"
 TOKEN = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN")
 if not TOKEN:
    print("Error: AUTHENTIK_BOOTSTRAP_TOKEN not set")
    print("  source ~/.env")
    print("  export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \\")
    print("      -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)")
    sys.exit(1)
 def api(method, path, data=None):
    url = f"{AUTHENTIK_URL}{path}"
    body = json.dumps(data).encode() if data is not None else None
    req = urllib.request.Request(url, data=body, method=method, headers={
        "Authorization": f"Bearer {TOKEN}",
        "Content-Type": "application/json",
    })
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            raw = resp.read()
            return resp.status, json.loads(raw) if raw else {}
    except urllib.error.HTTPError as e:
        raw = e.read()
        try:
            parsed = json.loads(raw) if raw else {}
        except json.JSONDecodeError:
            parsed = {"raw": raw.decode(errors="replace")}
        return e.code, parsed
 def die(msg):
    print(f"FATAL: {msg}", file=sys.stderr)
    sys.exit(1)
 # ===========================================================================
 # Group Definitions (DRY: single source of truth)
 # ===========================================================================
 GROUPS: Dict[str, Dict[str, Any]] = {
    "homelab-admins": {
        "description": "Cluster administrators with full access",
        "is_superuser": True,
    },
    "minio-admins": {"description": "MinIO administrators", "is_superuser": False, "minio_buckets": ["*"]},
    "minio-photos": {"description": "Photos bucket (Immich) access", "is_superuser": False, "minio_buckets": ["immich"]},
    "minio-documents": {"description": "Documents bucket (Paperless) access", "is_superuser": False, "minio_buckets": ["paperless"]},
    "minio-backups": {"description": "Backups bucket read-only access", "is_superuser": False, "minio_buckets": ["backups"]},
    "poimen-admins": {"description": "Poimen memory administrators", "is_superuser": False, "memory_projects": ["*"], "memory_visibility": "private"},
    "poimen-devs": {"description": "Dev and staging projects access", "is_superuser": False, "memory_projects": ["dev", "staging"], "memory_visibility": "internal"},
    "poimen-prod-readonly": {"description": "Production projects read-only access", "is_superuser": False, "memory_projects": ["prod"], "memory_visibility": "public"},
    "paperless-admins": {"description": "Paperless administrators", "is_superuser": False, "paperless_doctypes": ["*"]},
    "paperless-finance": {"description": "Finance documents", "is_superuser": False, "paperless_doctypes": ["invoices", "receipts", "expenses"]},
    "paperless-legal": {"description": "Legal documents", "is_superuser": False, "paperless_doctypes": ["contracts", "licenses", "agreements"]},
    "paperless-hr": {"description": "HR documents", "is_superuser": False, "paperless_doctypes": ["employment", "benefits", "payroll"]},
    "grafana-admins": {"description": "Grafana administrators", "is_superuser": False, "grafana_org_role": "Admin"},
    "grafana-editors": {"description": "Grafana dashboard editors", "is_superuser": False, "grafana_org_role": "Editor"},
    "grafana-viewers": {"description": "Grafana dashboard viewers", "is_superuser": False, "grafana_org_role": "Viewer"},
    "sqs-users": {"description": "SQS/Temporal queue read access", "is_superuser": False, "sqs_queues": ["default"]},
    "sqs-writers": {"description": "SQS/Temporal queue read/write access", "is_superuser": False, "sqs_queues": ["*"]},
    "s3-users": {"description": "S3 read access", "is_superuser": False},
    "s3-writers": {"description": "S3 read/write access", "is_superuser": False},
 }
 SERVICE_ACCOUNTS: Dict[str, Dict[str, Any]] = {
    "paperless-ai-agent": {
        "description": "Paperless AI plugin (auto-tagging, entity extraction)",
        "roles": ["llm:inference", "memory:write", "paperless:admin"],
        "claims": {
            "minio_buckets": ["paperless"],
            "paperless_doctypes": ["*"],
            "memory_projects": ["*"],
            "authorized_models": ["reasoning", "qwen2.5:3b"],
        },
    },
    "portfolio-agent": {
        "description": "Portfolio service agent",
        "roles": ["llm:inference", "memory:read"],
        "claims": {
            "memory_projects": ["homelab", "portfolio"],
            "memory_visibility": "public",
            "authorized_models": ["ornith:35b"],
            "minio_buckets": ["backups"],
        },
    },
    "memory-agent": {
        "description": "Memory service agent",
        "roles": ["llm:inference", "memory:read", "memory:write"],
        "claims": {
            "memory_projects": ["*"],
            "memory_visibility": "private",
            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
        },
    },
    "temporal-worker-agent": {
        "description": "Temporal workflow worker",
        "roles": ["llm:inference", "workflow:execute", "memory:read", "memory:write", "queue:send"],
        "claims": {
            "memory_projects": ["*"],
            "sqs_queues": ["*"],
            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
        },
    },
 }
 SCOPE_MAPPINGS: Dict[str, Dict[str, str]] = {
    "roles": {"expression": 'return user.attributes.get("roles", [])'},
    "permissions": {"expression": 'return ["*"] if any(user.groups.filter(is_superuser=True)) else list(user.groups.values_list("name", flat=True))'},
    "groups": {"expression": 'return {"groups": list(user.groups.values_list("name", flat=True))}'},
    "minio_buckets": {"expression": 'return user.attributes.get("minio_buckets", [])'},
    "paperless_doctypes": {"expression": 'return user.attributes.get("paperless_doctypes", [])'},
    "memory_projects": {"expression": 'return user.attributes.get("memory_projects", [])'},
    "memory_visibility": {"expression": 'return user.attributes.get("memory_visibility", "public")'},
    "authorized_models": {"expression": 'return user.attributes.get("authorized_models", [])'},
    "sqs_queues": {"expression": 'return user.attributes.get("sqs_queues", [])'},
    "grafana_org_role": {"expression": 'return user.attributes.get("grafana_org_role", "Viewer")'},
 }
 # ===========================================================================
 # Phase 1: Create/sync all groups
 # ===========================================================================
 print("[1/6] Ensuring groups exist...")
 status, res = api("GET", "/api/v3/core/groups/?page_size=100")
 if status != 200:
    die(f"GET groups -> {status} {res}")
 existing_groups = {g["name"]: g for g in res["results"]}
 created_count = 0
 for group_name, group_spec in GROUPS.items():
    if group_name in existing_groups:
        print(f"  {group_name}: already exists")
    else:
        status, res = api("POST", "/api/v3/core/groups/", {
            "name": group_name,
            "is_superuser": group_spec.get("is_superuser", False),
        })
        if status in (200, 201):
            print(f"  {group_name}: created")
            created_count += 1
        else:
            print(f"  {group_name}: FAILED {status} {res}")
 print(f"  Total: {len(GROUPS)} groups, {created_count} new")
 # ===========================================================================
 # Phase 2: Create/sync service account users with custom claims
 # ===========================================================================
 print("\n[2/6] Creating/updating service account users...")
 status, res = api("GET", "/api/v3/core/users/?page_size=100")
 if status != 200:
    die(f"GET users -> {status} {res}")
 existing_users = {u["username"]: u for u in res["results"]}
 service_pwd = os.environ.get("AUTHENTIK_SERVICE_ACCOUNT_PASSWORD", "DefaultPassword123!")
 for agent_name, agent_spec in SERVICE_ACCOUNTS.items():
    if agent_name in existing_users:
        user = existing_users[agent_name]
        attrs = user.get("attributes", {})
        attrs.update(agent_spec.get("claims", {}))
        attrs["roles"] = agent_spec.get("roles", [])
        status, res = api("PATCH", f"/api/v3/core/users/{user['pk']}/", {"attributes": attrs})
        if status in (200, 201):
            print(f"  {agent_name}: claims updated")
        else:
            print(f"  {agent_name}: FAILED {status} {res}")
    else:
        status, res = api("POST", "/api/v3/core/users/", {
            "username": agent_name,
            "name": agent_spec.get("description", agent_name),
            "email": f"{agent_name}@homelab.local",
            "is_active": True,
            "is_superuser": False,
            "password": service_pwd,
            "attributes": {
                **agent_spec.get("claims", {}),
                "roles": agent_spec.get("roles", []),
            },
        })
        if status in (200, 201):
            print(f"  {agent_name}: created")
        else:
            print(f"  {agent_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 3: Create scope mappings for fine-grained claims
 # ===========================================================================
 print("\n[3/6] Creating scope mappings for fine-grained claims...")
 status, res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
 if status != 200:
    die(f"GET scope mappings -> {status} {res}")
 existing_scopes = {m["scope_name"]: m for m in res["results"]}
 for scope_name, scope_spec in SCOPE_MAPPINGS.items():
    if scope_name in existing_scopes:
        print(f"  {scope_name}: already exists")
    else:
        status, res = api("POST", "/api/v3/propertymappings/provider/scope/", {
            "name": scope_name,
            "scope_name": scope_name,
            "expression": scope_spec["expression"],
        })
        if status in (200, 201):
            print(f"  {scope_name}: created")
        else:
            print(f"  {scope_name}: FAILED {status} {res}")
 # Fix built-in 'openid' scope to include preferred_username (needed by Grafana, Forgejo, etc.)
 # Note: sub MUST be a string, not int, for JSON compatibility with Go struct unmarshaling
 if "openid" in existing_scopes:
    openid_scope = existing_scopes["openid"]
    openid_pk = openid_scope["pk"]
    openid_expression = 'return {"sub": str(user.pk), "preferred_username": user.username}'
    if openid_expression not in openid_scope.get("expression", ""):
        status, res = api("PUT", f"/api/v3/propertymappings/provider/scope/{openid_pk}/", {
            "name": openid_scope["name"],
            "scope_name": "openid",
            "expression": openid_expression,
        })
        if status in (200, 201):
            print(f"  openid: updated (sub as string + preferred_username)")
        else:
            print(f"  openid: UPDATE FAILED {status} {res}")
    else:
        print(f"  openid: already correct (sub as string)")
 # ===========================================================================
 # Phase 4: Get flow UUIDs (needed for providers)
 # ===========================================================================
 print("\n[4/6] Fetching flow UUIDs...")
 status, res = api("GET", "/api/v3/flows/instances/?page_size=100")
 if status != 200:
    die(f"GET flows -> {status} {res}")
 flows = {f["slug"]: f["pk"] for f in res.get("results", [])}
 auth_flow = flows.get("default-provider-authorization-implicit-consent")
 inval_flow = flows.get("default-provider-invalidation-flow")
 if not auth_flow or not inval_flow:
    die(f"Required flows not found. auth_flow={auth_flow}, inval_flow={inval_flow}")
 print(f"  authorization_flow: {auth_flow}")
 print(f"  invalidation_flow: {inval_flow}")
 # ===========================================================================
 # Phase 5: Create OAuth2 providers with scopes
 # ===========================================================================
 print("\n[5/6] Creating OAuth2 providers...")
 status, res = api("GET", "/api/v3/providers/oauth2/?page_size=100")
 if status != 200:
    die(f"GET providers -> {status} {res}")
 existing_providers = {p["name"]: p for p in res.get("results", [])}
 # Fetch scope mapping PKs
 status, scopes_res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
 if status != 200:
    print("  WARNING: could not fetch scope mappings")
    scope_pks = {}
 else:
    scope_pks = {m["scope_name"]: m["pk"] for m in scopes_res.get("results", [])}
 # Include standard OpenID scopes (openid, email, profile) + custom claim scopes
 STANDARD_SCOPES = ["openid", "email", "profile"]
 scope_pks_list = [scope_pks[s] for s in STANDARD_SCOPES if s in scope_pks]
 scope_pks_list += [scope_pks[s] for s in SCOPE_MAPPINGS.keys() if s in scope_pks]
 # OAuth2 providers — client_secret sourced from SOPS-encrypted k8s secrets.
 # These are the real secrets the services use. Authentik must match.
 OAuth2_PROVIDERS = {
    "api-gw": {
        "client_id": "api-gw",
        "client_secret_env": "AUTHENTIK_PROVIDER_API_GW_SECRET",
        "redirect_uris": ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"],
    },
    "minio": {
        "client_id": "minio",
        "client_secret": "9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219",
        "redirect_uris": ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"],
    },
    "poimen": {
        "client_id": "poimen",
        "client_secret_env": "AUTHENTIK_PROVIDER_POIMEN_SECRET",
        "redirect_uris": ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"],
    },
    "paperless": {
        "client_id": "paperless",
        "client_secret": "6hcxaaVgZlKgafl7BxeSEtPAcbNUJxi2PAZePxSFk4o=",
        "redirect_uris": ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"],
    },
    "grafana": {
        "client_id": "grafana",
        "client_secret": "966bad4fa43812100e7775b3c73fed2ce1d07217fa5a23fbb0f190e46d2f0fa4",
        "redirect_uris": ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"],
    },
    "queue": {
        "client_id": "queue-sqs",
        "client_secret_env": "AUTHENTIK_PROVIDER_QUEUE_SECRET",
        "redirect_uris": ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"],
    },
    "forgejo": {
        "client_id": "forgejo",
        "client_secret": "G4klhs3JRfs5A7YnGs90WuOndBAvamWlZgaZRY8x",
        "redirect_uris": ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"],
    },
    "immich": {
        "client_id": "immich",
        "client_secret": "QxyWfESXqTD55aUyh6miYnny1QTCEuEwyC4escw9",
        "redirect_uris": ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"],
    },
    "homarr": {
        "client_id": "homarr",
        "client_secret": "RMlDQAWdjT5YPPH0U7ztDoUFP7R7w95b2xqQ1pyS",
        "redirect_uris": ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"],
    },
    "argocd": {
        "client_id": "argocd",
        "client_secret_env": "AUTHENTIK_PROVIDER_ARGOCD_SECRET",
        "redirect_uris": ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"],
    },
    "vault": {
        "client_id": "vault",
        "client_secret_env": "AUTHENTIK_PROVIDER_VAULT_SECRET",
        "redirect_uris": ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"],
    },
 }
 import secrets as _secrets
 for provider_name, provider_spec in OAuth2_PROVIDERS.items():
    # Resolve client_secret: explicit > env var > generate random
    if "client_secret" in provider_spec:
        client_secret = provider_spec["client_secret"]
    elif "client_secret_env" in provider_spec:
        client_secret = os.environ.get(provider_spec["client_secret_env"], _secrets.token_urlsafe(32))
    else:
        client_secret = _secrets.token_urlsafe(32)
    redirect_uris_list = [{"url": uri, "matching_mode": "strict"} for uri in provider_spec["redirect_uris"]]
    provider_payload = {
        "name": provider_name,
        "authorization_flow": auth_flow,
        "invalidation_flow": inval_flow,
        "grant_types": ["authorization_code", "implicit", "password"],
        "client_id": provider_spec["client_id"],
        "client_secret": client_secret,
        "redirect_uris": redirect_uris_list,
        "property_mappings": scope_pks_list,
    }
    if provider_name in existing_providers:
        # UPDATE existing provider — sync secret + redirect_uris
        provider_pk = existing_providers[provider_name]["pk"]
        status, res = api("PATCH", f"/api/v3/providers/oauth2/{provider_pk}/", {
            "client_id": provider_spec["client_id"],
            "client_secret": client_secret,
            "redirect_uris": redirect_uris_list,
            "property_mappings": scope_pks_list,
        })
        if status in (200, 201):
            print(f"  {provider_name}: updated (secret + redirect_uris synced)")
        else:
            print(f"  {provider_name}: UPDATE FAILED {status} {res}")
    else:
        # CREATE new provider
        status, res = api("POST", "/api/v3/providers/oauth2/", provider_payload)
        if status in (200, 201):
            print(f"  {provider_name}: created")
        else:
            print(f"  {provider_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 6: Create OAuth2 Applications (bind providers to public token endpoints)
 # ===========================================================================
 print("\n[6/7] Creating OAuth2 Applications...")
 print("     (binds providers to /application/o/token/ endpoints)")
 status, res = api("GET", "/api/v3/core/applications/?page_size=100")
 if status != 200:
    die(f"GET applications -> {status} {res}")
 existing_apps = {a["slug"]: a for a in res.get("results", [])}
 for provider_name in OAuth2_PROVIDERS.keys():
    # Get the provider PK
    status, provider_res = api("GET", f"/api/v3/providers/oauth2/?name={provider_name}")
    if status != 200 or not provider_res.get("results"):
        print(f"  {provider_name}: provider not found, skip")
        continue
    provider_pk = provider_res["results"][0]["pk"]
    if provider_name in existing_apps:
        # Ensure app is linked to provider (fix orphaned apps)
        app_data = existing_apps[provider_name]
        if app_data.get("provider") != provider_pk:
            app_uuid = app_data["pk"]
            status, res = api("PATCH", f"/api/v3/core/applications/{app_uuid}/", {
                "provider": provider_pk,
            })
            if status in (200, 201):
                print(f"  {provider_name}: re-linked to provider")
            else:
                print(f"  {provider_name}: RE-LINK FAILED {status} {res}")
        else:
            print(f"  {provider_name}: ok")
    else:
        status, res = api("POST", "/api/v3/core/applications/", {
            "name": provider_name,
            "slug": provider_name,
            "provider": provider_pk,
        })
        if status in (200, 201):
            print(f"  {provider_name}: created")
        else:
            print(f"  {provider_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 7: Create/update rock user → homelab-admins, matching Forgejo identity
 # ===========================================================================
 print("\n[7/10] Creating/updating rock user ([email protected])...")
 ROCK_EMAIL = "[email protected]"
 ROCK_PASSWORD = os.environ.get("ROCK_PASSWORD", "")
 status, res = api("GET", "/api/v3/core/users/?username=rock")
 if status == 200 and res.get("results"):
    rock_user = res["results"][0]
    # Ensure email matches Forgejo's rock user for OIDC linking
    patch_data = {"email": ROCK_EMAIL, "name": "Rock"}
    status, res = api("PATCH", f"/api/v3/core/users/{rock_user['pk']}/", patch_data)
    if status in (200, 201):
        print(f"  rock: updated email to {ROCK_EMAIL}")
    else:
        print(f"  rock: update FAILED {status} {res}")
 else:
    if not ROCK_PASSWORD:
        print("  rock: NOT FOUND and ROCK_PASSWORD not set, skipping creation")
        print("    export ROCK_PASSWORD=<password> and re-run")
        rock_user = None
    else:
        status, res = api("POST", "/api/v3/core/users/", {
            "username": "rock",
            "name": "Rock",
            "email": ROCK_EMAIL,
            "is_active": True,
            "is_superuser": False,
            "password": ROCK_PASSWORD,
        })
        if status in (200, 201):
            rock_user = res
            print(f"  rock: created with email {ROCK_EMAIL}")
        else:
            print(f"  rock: create FAILED {status} {res}")
            rock_user = None
 if rock_user:
    status, res = api("GET", "/api/v3/core/groups/?name=homelab-admins")
    if status == 200 and res.get("results"):
        admins_group = res["results"][0]
        status, res = api("POST", f"/api/v3/core/groups/{admins_group['pk']}/users/add/", {"pk": rock_user["pk"]})
        if status in (200, 201, 204):
            print(f"  rock: added to homelab-admins")
        else:
            print(f"  rock: group add {status} {res}")
 # ===========================================================================
 # Phase 8: Email recovery flow (password reset via email)
 # ===========================================================================
 print("\n[8/10] Creating email recovery flow...")
 # Read SMTP config from gotify-smtp secret (same Gmail creds)
 SMTP_HOST = "smtp.gmail.com"
 SMTP_PORT = 587
 SMTP_USER = "[email protected]"
 SMTP_FROM = "[email protected]"
 # Password read from env at runtime: AUTHENTIK_EMAIL__PASSWORD
 # 8a. Create email stage for recovery
 status, res = api("GET", "/api/v3/stages/email/?name=email-recovery")
 if status == 200 and res.get("results"):
    email_stage_pk = res["results"][0]["pk"]
    print("  email-recovery stage: already exists")
 else:
    status, res = api("POST", "/api/v3/stages/email/", {
        "name": "email-recovery",
        "use_global_settings": False,
        "host": SMTP_HOST,
        "port": SMTP_PORT,
        "username": SMTP_USER,
        "password": os.environ.get("AUTHENTIK_EMAIL_PASSWORD", ""),
        "use_tls": True,
        "use_ssl": False,
        "timeout": 10,
        "from_address": SMTP_FROM,
        "template": "email/password_reset.html",
        "activate_user_on_success": True,
    })
    if status in (200, 201):
        email_stage_pk = res["pk"]
        print("  email-recovery stage: created")
    else:
        email_stage_pk = None
        print(f"  email-recovery stage: FAILED {status} {res}")
 # 8b. Create identification stage for recovery (email lookup)
 status, res = api("GET", "/api/v3/stages/identification/?name=recovery-identification")
 if status == 200 and res.get("results"):
    ident_stage_pk = res["results"][0]["pk"]
    print("  recovery-identification stage: already exists")
 else:
    status, res = api("POST", "/api/v3/stages/identification/", {
        "name": "recovery-identification",
        "user_fields": ["email", "username"],
    })
    if status in (200, 201):
        ident_stage_pk = res["pk"]
        print("  recovery-identification stage: created")
    else:
        ident_stage_pk = None
        print(f"  recovery-identification stage: FAILED {status} {res}")
 # 8c. Create password stage for new password entry
 status, res = api("GET", "/api/v3/stages/password/?name=recovery-password-change")
 if status == 200 and res.get("results"):
    pw_stage_pk = res["results"][0]["pk"]
    print("  recovery-password-change stage: already exists")
 else:
    # Use prompt stage for password change instead
    status, res = api("GET", "/api/v3/stages/user_write/?name=recovery-user-write")
    if status == 200 and res.get("results"):
        pw_stage_pk = res["results"][0]["pk"]
        print("  recovery-user-write stage: already exists")
    else:
        status, res = api("POST", "/api/v3/stages/user_write/", {
            "name": "recovery-user-write",
        })
        if status in (200, 201):
            pw_stage_pk = res["pk"]
            print("  recovery-user-write stage: created")
        else:
            pw_stage_pk = None
            print(f"  recovery-user-write stage: FAILED {status} {res}")
 # 8d. Create recovery flow
 status, res = api("GET", "/api/v3/flows/instances/?slug=password-recovery")
 if status == 200 and res.get("results"):
    recovery_flow_pk = res["results"][0]["pk"]
    print("  password-recovery flow: already exists")
 else:
    status, res = api("POST", "/api/v3/flows/instances/", {
        "name": "Password Recovery",
        "slug": "password-recovery",
        "title": "Reset your password",
        "designation": "recovery",
    })
    if status in (200, 201):
        recovery_flow_pk = res["pk"]
        print("  password-recovery flow: created")
    else:
        recovery_flow_pk = None
        print(f"  password-recovery flow: FAILED {status} {res}")
 # 8e. Bind stages to flow in order
 if recovery_flow_pk and ident_stage_pk and email_stage_pk:
    for order, stage_pk, label in [
        (10, ident_stage_pk, "identification"),
        (20, email_stage_pk, "email"),
    ]:
        status, res = api("POST", "/api/v3/flows/bindings/", {
            "target": recovery_flow_pk,
            "stage": stage_pk,
            "order": order,
        })
        if status in (200, 201):
            print(f"  bound {label} stage at order {order}")
        elif status == 400 and "already exists" in str(res).lower():
            print(f"  {label} stage: already bound")
        else:
            print(f"  bind {label}: {status} {res}")
 # ===========================================================================
 # Phase 9: Set recovery flow on brand
 # ===========================================================================
 print("\n[9/10] Setting recovery flow on brand...")
 if recovery_flow_pk:
    status, res = api("GET", "/api/v3/brands/instances/")
    if status == 200 and res.get("results"):
        brand = res["results"][0]
        status, res = api("PATCH", f"/api/v3/brands/instances/{brand['brand_uuid']}/", {
            "flow_recovery": recovery_flow_pk,
        })
        if status in (200, 201):
            print("  recovery flow set on brand")
        else:
            print(f"  FAILED {status} {res}")
    else:
        print("  no brand found")
 # ===========================================================================
 # Phase 10: Ensure Forgejo OAuth2 source uses matching email claim
 # ===========================================================================
 print("\n[10/10] Verifying Forgejo OIDC linkage...")
 print(f"  rock@Authentik email: {ROCK_EMAIL}")
 print("  Forgejo OIDC will match on email — ensure Forgejo's rock user")
 print(f"  has email {ROCK_EMAIL} in Forgejo settings → Profile")
 # ===========================================================================
 # Summary
 # ===========================================================================
 print("\n" + "="*70)
 print("AUTHENTIK PROVISIONING COMPLETE")
 print("="*70)
 print(f"\n  [1] Groups: {len(GROUPS)}")
 print(f"  [2] Service accounts: {len(SERVICE_ACCOUNTS)}")
 print(f"  [3] Scope mappings: {len(SCOPE_MAPPINGS)}")
 print(f"  [4] Flows resolved")
 print(f"  [5] OAuth2 providers: {len(OAuth2_PROVIDERS)}")
 print(f"  [6] OAuth2 applications bound")
 print(f"  [7] rock user ([email protected]) -> homelab-admins")
 print(f"  [8] Email recovery flow (smtp.gmail.com)")
 print(f"  [9] Recovery flow set on brand")
 print(f"  [10] Forgejo OIDC linkage verified")
 print("\nNEXT: Set Forgejo rock user email to [email protected] in Forgejo profile")
 print("TEST: https://authentik.riotpiao.com/if/flow/password-recovery/")
 print("="*70)
@@ -53,7 +53,7 @@ generate_secret() {
 declare -A NEW_SECRETS
-for svc in api-gw minio poimen paperless grafana argocd forgejo immich vault portfolio-agent memory-agent local-llm; do
+for svc in api-gw minio poimen paperless grafana argocd forgejo homarr immich vault portfolio-agent memory-agent local-llm; do
  NEW_SECRETS[$svc]=$(generate_secret)
  log "  $svc: ${NEW_SECRETS[$svc]:0:15}..."
 done
@@ -70,6 +70,7 @@ declare -A PROVIDER_PKS=(
  [grafana]=6
  [argocd]=7
  [forgejo]=8
  [homarr]=9
  [immich]=10
  [vault]=11
 )
@@ -202,7 +203,7 @@ fi
 log ""
 log "=== Summary ==="
 log "Rotated 13 credentials:"
-log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, immich, vault"
+log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, homarr, immich, vault"
 log "  Service Accounts: portfolio-agent, memory-agent, local-llm"
 log ""
@@ -80,10 +80,6 @@ ROLES = {
    "sqs-writers": "SQS/Temporal queue read/write access",
    "s3-users": "S3 read access",
    "s3-writers": "S3 read/write access",
    # LLM gate for the api-gw. The colon form matches the Authentik-era
    # permission string, so consumer manifests did not change.
    "llm:inference": "LLM inference access (api-gw)",
    "llm-inference": "LLM inference access (dash alias)",
 }
 _WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"}
@@ -102,6 +98,9 @@ OIDC_APPS = {
    "grafana": {**_WEB, "redirect_uris": [
        "http://localhost:3000/login/generic_oauth",
        "https://grafana.riotpiao.com/login/generic_oauth"]},
    "homarr": {**_WEB, "redirect_uris": [
        "http://localhost:7575/auth/callback",
        "https://homarr.riotpiao.com/auth/callback"]},
    "immich": {**_WEB, "app_type_id": "OIDC_APP_TYPE_NATIVE", "redirect_uris": [
        "app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login",
        "https://img.riotpiao.com/user/oauth2/callback"]},
@@ -270,26 +269,13 @@ def _api(method: str, path: str, token: str, body: dict | None = None,
 def _ensure_project(token: str, name: str) -> str:
    code, d = _api("POST", "/management/v1/projects/_search", token, {})
    project_id = ""
    for row in d.get("result") or []:
        if row.get("name") == name:
-            project_id = row["id"]
+            return row["id"]
-            break
+    code, d = _api("POST", "/management/v1/projects", token, {"name": name})
-    if not project_id:
+    if code not in (200, 201) or not d.get("id"):
-        code, d = _api("POST", "/management/v1/projects", token, {"name": name})
+        die(f"create project {name} failed: {code} {d}")
-        if code not in (200, 201) or not d.get("id"):
+    return d["id"]
            die(f"create project {name} failed: {code} {d}")
        project_id = d["id"]
    # projectRoleAssertion puts the project roles into the ACCESS token as
    # `urn:zitadel:iam:org:project:roles`. Without it the api-gw sees no
    # llm:inference role and answers 403 for every inference call, even when
    # the caller is correctly authorised in ZITADEL.
    code, d = _api("PUT", f"/management/v1/projects/{project_id}",
                   token, {"name": name, "projectRoleAssertion": True})
    if code not in (200, 201):
        die(f"enable projectRoleAssertion on {name} failed: {code} {d}")
    return project_id
 def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
@@ -376,6 +362,10 @@ def phase_apply(_args: argparse.Namespace) -> int:
 SECRET_MAP = {
    "grafana": [("grafana-oidc-secrets.enc.yaml", "grafana-oidc",
                 "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "secret")],
    "homarr": [("homarr-auth-oidc.enc.yaml", "homarr-oidc",
                "client-id", "id"),
               ("homarr-auth-oidc.enc.yaml", "homarr-oidc",
                "client-secret", "secret")],
    "minio": [("minio-secrets.enc.yaml", "minio-oidc",
               "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "secret")],
    "paperless": [("paperless-ai-secrets.enc.yaml", "paperless-ai-config",
@@ -22,6 +22,7 @@
    # TLS termination (cert-manager cert) and preserve the Host header. Routing
    # via nginx (not the backend service) is critical: a direct rewrite bypasses
    # nginx TLS and exposes each app's own self-signed cert, trusted by nothing.
    rewrite name authentik.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name zitadel.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name grafana.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name minio.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local