feat(iam): wire all 9 consumer manifests to Zitadel OIDC

- Replace Authentik issuer/auth/token/userinfo URLs with Zitadel endpoints in grafana, minio, paperless-ai, paperless-ai-new, homarr, argocd, blackbox - Write Zitadel client IDs/secrets into plain manifests and SOPS secrets (grafana, minio, paperless, homarr) - Adopt homarr-oidc into SOPS (homarr-auth-oidc.enc.yaml) - Add homarr to SECRET_MAP and remove from MANUAL_ONLY in zitadel-provision.py - Make client-id regex idempotent for repeated runs (minio, paperless) - Fix cloudflared init container CA mount path (homelab-ca.crt -> ca.crt) - Remove Authentik ingress and cloudflared tunnel entry
2026-10-25 21:08:23 -07:00
42 changed files with 1717 additions and 557 deletions
@@ -16,7 +16,7 @@ spec:
      # Ingress-nginx forces HTTPS (force-ssl-redirect), so cloudflared must
      # dial https://<hostname> and trust homelab-ca or the TLS handshake fails
      # with "certificate signed by unknown authority". Same pattern as
-      # iam values: merge the system bundle with homelab-ca.crt.
+      # authentik-values.yaml: merge the system bundle with homelab-ca.crt.
      initContainers:
        - name: merge-ca-certs
          image: debian:bookworm
@@ -0,0 +1,78 @@
 # Homarr landing page with Authentik SSO
 # Probe tuning (chart DOES expose these — the old PostSync patch-job was
 # unnecessary and fragile: it only patched one Deployment revision, so any later
 # rollout reverted to the chart's aggressive defaults). Homarr's first-boot icon
 # updater blocks the event loop for ~50s ("icons updater took 49553ms"), during
 # which /api/health/live can't answer within the default 10s×3 window → kubelet
 # SIGTERMs the pod → CrashLoopBackOff (247 restarts, 503 at the ingress). Give
 # liveness a wide window so the icon import can finish without a kill.
 livenessProbe:
  initialDelaySeconds: 60
  periodSeconds: 30
  timeoutSeconds: 5
  failureThreshold: 10
 readinessProbe:
  initialDelaySeconds: 30
  periodSeconds: 15
  timeoutSeconds: 5
  failureThreshold: 6
 image:
  repository: ghcr.io/homarr-labs/homarr
  tag: "latest"
  pullPolicy: Always
 replicaCount: 1
 # Configure SSO via environment variables
 # Chart supports these via top-level env dict (not array)
 env:
  AUTH_PROVIDERS: "oidc,credentials"
  AUTH_OIDC_ISSUER: "https://zitadel.riotpiao.com"
  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
  # hides the "Sign in with Authentik" button entirely when it's absent (per the
  # authentik Homarr integration + homarr SSO docs). This was the missing var.
  AUTH_OIDC_URI: "https://zitadel.riotpiao.com/oauth/v2/authorize"
  AUTH_OIDC_CLIENT_NAME: "Zitadel"
  AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
  AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
  AUTH_OIDC_AUTO_LOGIN: "false"
  # Link the OIDC identity to an existing homarr account with the same email.
  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
  # per run ("callback took longer than expected"), compounding CPU pressure.
  DISABLE_ANALYTICS: "true"
  BASE_URL: "https://homarr.riotpiao.com"
  NEXTAUTH_URL: "https://homarr.riotpiao.com"
 # Client credentials from homarr-oidc secret
 # Chart doesn't support envFrom, so we add via extraEnv
 extraEnv:
  - name: AUTH_OIDC_CLIENT_ID
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-id
  - name: AUTH_OIDC_CLIENT_SECRET
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-secret
 tolerations:
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule
 resources:
  requests:
    cpu: 250m
    memory: 384Mi
  limits:
    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
    # real CPU headroom.
    cpu: "2"
    memory: 1Gi
@@ -0,0 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: dashboard
 # Probes are now tuned via homarr-values.yaml (chart-native); the old
 # fix-probes-job PostSync hook is removed. homarr-secrets/auth-oidc/db-encryption
 # Secrets are delivered by the sops-secrets (ksops) Application.
 resources: []
@@ -0,0 +1,107 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # The konghq.com/{connect,read,write}-timeout annotations that used to live
    # here went with Kong (retired 2026-08-19). They existed because Kong read
    # its upstream timeouts off the Kubernetes Service, and its 60s default cut
    # off the first request after any pod restart — a restart flushes VRAM and
    # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
    # problem in steady state.
    #
    # The equivalent budget now belongs to the Go gateway's per-route timeout
    # config in rock/homelab-frontend, not to an annotation on this object.
  labels:
    app.kubernetes.io/name: llm-ornith
    app.kubernetes.io/part-of: llm-serving
  name: ornith
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - command:
      - /bin/sh
      - -c
      - 'set -e
        ollama serve &
        SERVE_PID=$!
        until ollama list >/dev/null 2>&1; do sleep 2; done
        ollama pull ornith:35b
        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
        wait $SERVE_PID
        '
      env:
      - name: OLLAMA_HOST
        value: 0.0.0.0:8080
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
        value: '65536'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      - name: OLLAMA_MAX_LOADED_MODELS
        value: '1'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    podMetadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
      labels:
        app.kubernetes.io/name: llm-ornith
        app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.
    # Frees 1 GPU for ComfyUI.
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,162 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then-
      # matmul is two slow kernel launches instead of one fused int4 GEMM,
      # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to
      # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same
      # hermes/qwen3 parser stack (no narration-bug risk, same as before),
      # only the quant format changes. Plain (non-Marlin) GPTQ kernel is
      # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try
      # to auto-upgrade to it, so --quantization is pinned explicitly to
      # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB
      # (summed from the real safetensors index, not bits-per-param math).
      # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN:
      # factor=4.0, original_max_position_embeddings=32768) -- 200k was
      # asked for but exceeds this architecturally regardless of VRAM.
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
      # metric after the rollout -- max_num_batched_tokens=8192 also raises
      # activation memory out of the same budget, so the post-boot
      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
      # only valid if it still covers 2x65k + 3x8k. The old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
      # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no
      # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth
      # trying later, but not risking a parallelism-strategy change in the
      # same rollout as the quant+context-length change.
      # This GPTQ requant's own config.json ships max_position_embeddings=
      # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw
      # config.json), the base Qwen3-32B repo's YaRN block did NOT carry
      # over during quantization. Re-applying it explicitly here restores
      # the same math the base model documents (32768 * 4.0 = 131072);
      # without this, --max-model-len=131072 fails ModelConfig validation
      # against the checkpoint's own (unscaled) 40960 ceiling.
      - --model=JunHowie/Qwen3-32B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
      - --gpu-memory-utilization=0.95
      - --max-num-seqs=5
      # Prefill scheduling -- see the measured starvation this fixes.
      # V1's scheduler has no prefill/decode phase: it walks self.running and
      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
      # per request is applied BEFORE the budget clamp:
      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
      #     take = min(take, token_budget)
      # The loop is `while ... and token_budget > 0`. With the threshold at its
      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
      # budget, token_budget hits 0, the loop exits and every other request is
      # skipped for that step -- measured as 262 tok/s contended prefill and
      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
      # Hence threshold must be strictly BELOW the budget or it never binds:
      # at 8192/8192 a single request is served per step (still starved), while
      # 8192/2048 serves four. Verified by replaying the loop.
      - --max-num-batched-tokens=8192
      - --long-prefill-token-threshold=2048
      - --enable-chunked-prefill
      - --enable-prefix-caching
      #
      # NOT set, deliberately: --max-num-partial-prefills and
      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
      # rejects any value other than the default for either one, calling
      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
      # -> AssertionError, because there is no V0 engine to fall back to.
      # The Deployment uses strategy Recreate, so setting them is an outage.
      # V1 does concurrent chunked prefill natively; it needs no flag, only the
      # budget/threshold ratio above.
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
      # hermes is the documented tool-call parser for general (non-Coder)
      # Qwen3 models -- native chat template support, not narrated text.
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 36Gi
          nvidia.com/gpu: '2'
        requests:
          cpu: '8'
          memory: 12Gi
          nvidia.com/gpu: '2'
      startupProbe:
        failureThreshold: 80
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    podMetadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
      labels:
        app.kubernetes.io/name: llm-reasoning
        app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm
@@ -9,14 +9,6 @@ metadata:
    app.kubernetes.io/name: paperless-ai
 spec:
  replicas: 1
  # Recreate, not RollingUpdate: paperless-ai-data is a single RWO PVC. With
  # RollingUpdate the new pod tries to mount the PVC while the old pod still
  # holds it -> "Multi-Attach error" / stuck Init:0/1 forever, rollout wedged
  # at "1 desired, 1 ready, 1 updated, 1 unavailable". Recreate terminates the
  # old pod first so the PVC detaches before the new one starts. Same pattern
  # as the Forgejo gitea Deployment.
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: paperless-ai
@@ -30,6 +30,33 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: authentik
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  sources:
    - repoURL: https://charts.goauthentik.io
      chart: authentik
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/iam/authentik-values.yaml
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # ZITADEL — the Authentik replacement. Coexistence, not cutover: Authentik keeps
 # serving in the same `iam` namespace until the last consumer moves, then this
 # becomes the only IdP. Migration plan (outside git):
@@ -79,47 +106,11 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
-# ZITADEL core objects (ServiceAccount, Role, RoleBinding, ConfigMap).
+# Raw IAM manifests: key-rotation cronjob + authentik migration job.
 #
 # The chart declares these as pre-install/pre-upgrade Helm hooks, and ArgoCD
 # prunes hook resources when a sync ends. That deleted the ServiceAccount and
 # ConfigMap the ZITADEL Deployment references, so new pods failed to start with
 # `serviceaccount "zitadel" not found`. This Application owns hook-free copies,
 # which makes them ordinary desired state that survives every sync.
 #
 # Wave 2: before the zatadel chart itself, so the ServiceAccount exists by the
 # time its Deployment is created.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: zitadel-core
+  name: iam-jobs
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "2"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/zitadel-core
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: false
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Raw IAM support manifests: the SMTP -> Gotify notification relay that ZITADEL
 # points at, so verification and password-reset codes reach the phone. Wave 3
 # alongside ZITADEL itself; the Secret it reads is created by sops-secrets at
 # wave 0.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: iam-support
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
@@ -1,4 +1,4 @@
-# Wave 2 — dedicated per-app CNPG Postgres clusters (zitadel-db → ns iam,
+# Wave 2 — dedicated per-app CNPG Postgres clusters (authentik-db → ns iam,
 # temporal-db + visibility → ns temporal). ONE App, ONE folder (k8s/infra/databases).
 # CNPG operator is Phase-0 bootstrap; these Cluster CRs are GitOps — no circular
 # dep (they run after ArgoCD is up, before their apps at w3/w8). CNPG generates
@@ -171,6 +171,40 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 # Consolidated: homarr + homarr-patches → homarr
 # Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: homarr
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "8"
 spec:
  project: homelab
  sources:
    - repoURL: https://homarr-labs.github.io/charts
      chart: homarr
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/apps/homarr/homarr-values.yaml
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      ref: values
    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
      targetRevision: main
      path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: dashboard
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Portfolio site at riotpiao.com - static Next.js site from rock/riotpiao.com repo.
 # Points directly to infra/portfolio/base (bypassing repo's own argocd-apps.yaml
 # which has wrong URLs). Image built by Forgejo Actions on rock/portfolio repo.
@@ -0,0 +1,20 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: oidc-bootstrap
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "5"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    targetRevision: main
    path: k8s/infra/oidc-bootstrap
  destination:
    server: https://kubernetes.default.svc
    namespace: kube-system
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -34,9 +34,11 @@ spec:
    - https://prometheus-community.github.io/helm-charts
    - https://grafana.github.io/helm-charts
    - https://helm.releases.hashicorp.com
    - https://charts.goauthentik.io
    - https://charts.zitadel.com
    - https://strimzi.io/charts/
    - https://charts.bitnami.com/bitnami
    - https://homarr-labs.github.io/charts
    - https://go.temporal.io/helm-charts
    - https://portainer.github.io/k8s/
    # Substrate charts (cert-manager, ingress-nginx, reloader) — app-of-apps owned
@@ -0,0 +1,25 @@
 apiVersion: ENC[AES256_GCM,data:Fc0=,iv:UFqMFeZTJZtFMoNeusjAoSI60U9StZS4IXw+n8JQCRM=,tag:uUvY4+jcjiSJHs0HDS3/7A==,type:str]
 kind: ENC[AES256_GCM,data:NUw86IEw,iv:gh4Vqn6uteYnYIFcXT/QVxRPKm+G+j2aIVuV71nZ4k4=,tag:nZA/x75Kx4+CoSx66BsvkQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:qex150pm0eGwYiiI05y9LUs=,iv:5Tk001T796c4HFikoh1ZVB6ierBfDqOUkFs5ilkr+V8=,tag:LiDAiCH7w6aRS41VKIlIKg==,type:str]
    namespace: ENC[AES256_GCM,data:nI91,iv:c1hBWGMkVb+vg+MKNzty8CFfXxQabXEtT+EpiuQZNvo=,tag:BsGtJDbBadDdTa687+Sdag==,type:str]
 type: ENC[AES256_GCM,data:3rXawlSD,iv:FUtLK4KP7v/SBF08ZcU6siIO1qfjlJirf2x5PyzAjuA=,tag:mSIQ+Y43+wDfZGRbQ6a0Og==,type:str]
 stringData:
    AUTHENTIK_SECRET_KEY: ENC[AES256_GCM,data:i5tVOWnN9a/cfhs6gckveRxUpiN7Ie227R/74kOIz6lQ5US2hkf03W5qSGCrzheL3DB0i8+d0iJYsApjn/wCiN7ndZZVFko3W5jDfVPybxc=,iv:djF1G9JHT5B7ziTokA/UuPpG+tG2LvFU4uMAR0x4M6o=,tag:vHOd7jLYkM8Evnr0QUVrCw==,type:str]
    AUTHENTIK_BOOTSTRAP_PASSWORD: ENC[AES256_GCM,data:PjvYtH03CzQbTXEqt/Z18+u1zJdok7DLwDn51FUZXDA=,iv:pSwSm8hB0/Q8uOAcOZ1rTqq7BF782EglG19GGeghm1E=,tag:qGCQFWCrwsWBOiQ07ySMUQ==,type:str]
    AUTHENTIK_BOOTSTRAP_TOKEN: ENC[AES256_GCM,data:1tnekGo0GIemJCMRenjN0yNeSiiMl/kky8ms84TsqBLaQ0lQzPxjq9qtgNJwoAsufxrznrkDVKjeOEHNYcA82A==,iv:NNFWSLMILoWccA9ihy1OArSS1X3OySB7PIHdXZdyzlg=,tag:t2w+KyReOXftTrU9pBQDnQ==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSZGRKWm9PTkNSOENtREgx
            M2V3Z0xxOXRhRGVRclo3SzBKRkc0VmtCY0NnClJPSGJRcUlzWmNFeTZ1bmorNEV4
            U3l3TzVlMTlrWkJnQzBtUTJtcUwrM3MKLS0tIEJrMWZrbllYYUZKN1V4MVMzcXAv
            eUtheHlNMHpsTEJXZ1FTMFVxbWloa0kKLRkE1Du+4gdOLerOl9y0mZw+8fqfECQY
            uP64Q+9BSuPYNiTLZvYDzmdoy+KTS7i7C9rf+iU96R/crbQLamV9Ew==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T20:37:07Z"
    mac: ENC[AES256_GCM,data:b3pDx340d1iZxsPPXzACckoLBrW+XjX1qlGrWEEI30j2Ao0hewTRZJyHbdI97A6b/hqzBN/4tjqqkdqm8goNhUQweMLTw86X6sALuE0aLS9wK53qjqih8fQwQBbgLDtjFXtW26h+smIm7aB5ZSDvU2le44HDXuWd83sWtCavNq0=,iv:ZMDsoKpygFJrYpXy2dA0IlxAi0jt24rxF72eUzgQs7k=,tag:WWE7G7+c9JK6fF1U9V8igg==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,24 @@
 apiVersion: v1
 kind: Secret
 metadata:
    name: homarr-oidc
    namespace: dashboard
 type: Opaque
 stringData:
    client-id: ENC[AES256_GCM,data:IFNjfDMrObaxfGwJpiYqPP60,iv:0WR2C+DEsq8I031A9Bopotn5TYdISApbTF46jJBgtVs=,tag:LGijMS27pbbZnOLJqGNXQg==,type:str]
    client-secret: ENC[AES256_GCM,data:q0MHjvuTIKbbMUZpsptOuo3zrabwaWxZS3MLqRdI1Rt7eod5ORAYeZmpllHmojRFve9x/SKkJ+sctafzwPuAPA==,iv:1ZLXOc3a85+MRx3v/XsW8JhsxbCQ26laou8PvvrQvH0=,tag:PRRTsyE2fGGS3dKPh43a8g==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpV1ZlMWxaNTlCUjJjbG9F
            NzFGMXBXYThZT285UHA0bGdwNFJqQTNMckhvClFSTEdseExjbVFtbkdsTUpSSGVn
            WGsyUmxvcVViVVVNZjFRdmJWZlB6TGcKLS0tIE01QjFtVFBGUlpXbjVJcDMwMzRS
            MEh4dUZTZ3puaXF2Uk03Ym50QmdSM1EK6Kmdv3s8/4nI/6Uk4G5FOh1Gxp1u9eYI
            L7MOW0TFkScTuDiu9iJB+CjB8b8Z0kwSYTigYj9ue2y0a1hDYCfQtg==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    encrypted_regex: ^(data|stringData)$
    lastmodified: "2026-10-26T03:56:19Z"
    mac: ENC[AES256_GCM,data:tV7o9zY6dpTdTvNJY8GBkdIB71thoy79RQSHsczednmXXzlNFddZxR/UenJ87jJ1JyzMUKL3CxArYojpuamahQcSXifVar0kotgNCs67A7+xfhHudCnKLMWDfDOom6DdWto6bMNIk/0TF6Wy0m1hkSsxwkdqETVs3b6ZUg/l8Rc=,iv:PE7y5oVMEM9XKQnDZmdRFRR3blzz2SZQFLxOMyILuec=,tag:SM9vJU1zlQ02XJijSiGCMg==,type:str]
    version: 3.13.2
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:FO4=,iv:kKIuJp4uGeX6vECsXk7w43F9S281jObQchmK8r1+kmA=,tag:x5LYd34dzNDLCma/78M4+w==,type:str]
 kind: ENC[AES256_GCM,data:o4RTN8SK,iv:uyOJWjDmHIPuBl9Wc1S/HxUabV2yytWr1bntHW68sTI=,tag:3On0SnQVJ/ICMMryQe+8JQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:a89dHmLdSeMHE7ZP0Q==,iv:OWJvJhFqJ7A0UGVJnZILzCJ/4VuVFib9sUDjXuwyqJQ=,tag:1XrIQn1GDh8RQ4vmQAu7+g==,type:str]
    namespace: ENC[AES256_GCM,data:4wBVpZJgQGM9,iv:3Tn1u/2TP/+SnELu34PDf4Jauf2L4f8+MtgrPIlyKzk=,tag:L3qbKFBjm16g8KX5vUsK6A==,type:str]
 type: ENC[AES256_GCM,data:mdrF9vUb,iv:7ukOgwXlnRf5mpJ/+6YXtyGRonnqehEgT4pJJIafw7c=,tag:90mt7jpNb3N80kCEi+R/rA==,type:str]
 stringData:
    db-encryption-key: ENC[AES256_GCM,data:NWJryAEd/eUJxbNiu05RzLF/l7GDxIQM7RQe3xKBz6XklgRAFwQxxjakcVd85pePUBaQYz3zttFglWhfJlWhkA==,iv:eUWbcs0B/mIheyAQ2+DyKgcA98CRKQGAVC3O0WIkfU8=,tag:be06X6AQqlohYlxuYoZiTg==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1Ym5yQ1pmWHZjVnYxbUVO
            SkZ5dnI2OXR3cmZjZkM0K1ErMElLNktlK1JFCmpleFBrcXkwcDA4cGFuWWN6VDQv
            dVV4NjB4MjEreENVQnZ2ejVLblRYOVUKLS0tIG9VbXJvUmQ3SzMzN3IrRDlac2sr
            eGZNYXBoZFZDeisrVCtUMXVPaEpoNFkKjwhUU+Zsqv4Be+m558fET5UhF/2rO73J
            AhGbJVaA7PaOCswPWiOLso26zD+MBB3VKHNdUFnenpoM16Nu4pOf5w==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T21:07:30Z"
    mac: ENC[AES256_GCM,data:KrGQY/eOmihYCvQCbfcM6bZ6rfx4zKh62M8NM0VQr1//1dSHpOOh25dYGLuQeECnqLxSpGQNWzr2xjUYWs4NOdSEFGq6FL2iHE3LnqoBZspp/JFJYP6+z/8jSUojwKEW7OuZZdPN6gF2u+IPOeWbGftnjS7X+atoSkYag2ty2vk=,iv:IOPEgBizsxrVoK5oSdWmi0uCBUnReOEnMYXuwSBKTzA=,tag:9k/XQ/AE7wSFq5XMbY0FTA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:4OY=,iv:0Lgui0+X2oTlgCX5HEIIihEsGJD0C2+pDIOTCNi4r+g=,tag:uL7vlreaHjcLJjV7e5I7PQ==,type:str]
 kind: ENC[AES256_GCM,data:x/ltkind,iv:KRBfS7COsTs/l3Fhyk5FvH7IVc+r/6M6FVOo+kPDe1Q=,tag:UuTbe213hDfK/WUtKS/doQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:LpOEecTJBuGy5PqNSFM=,iv:Q+DCLGtw3wihhdAgEbzYHyQhFTPWFyE2iJKl9mYaA2E=,tag:E2wrig4lRXGDvSCHec5clQ==,type:str]
    namespace: ENC[AES256_GCM,data:6kMVCXaF68NO,iv:gnCCkV7GKXA2HWOOeNUGnpAycf8U7pl6F8Vfo6DhLR4=,tag:0VdeySsuhUr2heXyGXZwBg==,type:str]
 type: ENC[AES256_GCM,data:6muC3Zqh,iv:gBaC+6MXvZY/bIpy/cEG/zWwLBaUFMVfxAHa1OnwfnQ=,tag:v6NYyhAKAwovluHAXNaOVA==,type:str]
 stringData:
    SECRET_ENCRYPTION_KEY: ENC[AES256_GCM,data:jX9ULlaBYTSTFGRMplSder5rMrsz7THU3kgJSGef0r5rZvtoaAG9NVYU2QUEuD0sZgfqRooeMOkDCAqFc8qsuw==,iv:uOmx8w3XPKnoxfasf7gqi/bljxigpLwaAC30c2TroKg=,tag:Dmbu6m3+UZa1w0UR0LRh9g==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6TDVyNGRtYWgzMDYwdjdr
            NXJWcGJZNUx6YzBGWm1nWlp0SlN3WXhaR0VBCjZ3bDBmUTFCUjg3U21wak90eHZK
            NlRCTUZQSHhoS0ZSUGZ4UHhST01KNmMKLS0tIGUxcGo0NmJ2bmJ6eGJ3QWxzYVM5
            c2Z6UkFjQmZ4ejIvK1JQQkZ0VFFIZ3cKA8UzmGsX+s3yOermOHqbnnEtekHCTqmC
            R3Jyzf6OolZMJpNwbetcsOfgO61aM7LHl/Xb/hFLbLjZ+eECrt8VpA==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-12T18:02:01Z"
    mac: ENC[AES256_GCM,data:HTrEEVDm8JqnVBFaY/FJknxma0nI95tRiwVc0K76k+2R4jgKCBA3EUAzPVS+gA4rTAeKHPSHrT2eidlkseT2WQUjqMRrpHcUsbSjNfpTEY2g/U0PIiOXZMRGfwmsQVgmu2fqeMOyzDSyya0scdWJXPraRPNZ+OpEkNVlm4rWcgs=,iv:KY3dnnmooJgsu66wc7gbzeIGUXZNgVrxAhpAwxoG+G8=,tag:7r6d41oTD136N57rtY7auA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -9,6 +9,7 @@ metadata:
 files:
    - agent-pod-models.enc.yaml
    - agent-pod-ssh-key.enc.yaml
    - authentik-secrets.enc.yaml
    - cloudflare-secrets.enc.yaml
    - cloudflared-credentials.enc.yaml
    - forgejo-registry-pat-api.enc.yaml
@@ -17,6 +18,9 @@ files:
    - forgejo-secrets.enc.yaml
    - grafana-oidc-secrets.enc.yaml
    - grafana-secrets.enc.yaml
    - homarr-auth-oidc.enc.yaml
    - homarr-db-encryption.enc.yaml
    - homarr-secrets.enc.yaml
    - homelab-ca-secrets.enc.yaml
    - loki-secrets.enc.yaml
    - minio-secrets.enc.yaml
@@ -35,7 +39,6 @@ files:
    - pi-sandbox-secrets.enc.yaml
    - etcd-defrag-creds.enc.yaml
    - zitadel-masterkey.enc.yaml
    - smtp-gotify-secrets.enc.yaml
    # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
    # owns it. Without this, the chart re-generates the keypair on every render,
    # which changes checksum/secret-login-service-key and leaves both Zitadel
@@ -1,23 +0,0 @@
 apiVersion: v1
 data:
    token: ENC[AES256_GCM,data:CUJ0rUtgye71fpFz50RSDXJeG+t8Wjrm6T6jboVPbt2FVuIRSscqKg/Pc3k/L8nvLvrkYlzz1rbYu9v5NAdNF2c35AQ=,iv:tsfQ+v4qwj6H7G/O79b6m4XMWRw3mwE67f+4LjDCOoY=,tag:SLGGBYtYoF45AQstETPYZA==,type:str]
 kind: Secret
 metadata:
    creationTimestamp: null
    name: smtp-gotify
    namespace: iam
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFWmU5TUxLeGU4YzUxRU9G
            ZFR5a2hqS3huYlJtempaQ25zUkljUDJLSENrCkx0bjQ4UVF3UWpWa0t3WHQ2bkNl
            eEp2d0dQRHJSOExIYjN4TmRTZWdQY00KLS0tIGF5enUxTmxRTHlQRUt1aTFIYTFB
            MEJaOWd6SnhWQU1PQXdUZDJqWmp2MGsKIOOrp52+d3M8AQlMw0VSnLQ2efDtC6Ye
            oC/TPqYi9SqYNxaHkODS2QZN9msyfRFcgUh/6ZXr8sivnpwRY8ZZtw==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    encrypted_regex: ^(data|stringData)$
    lastmodified: "2026-10-27T06:24:20Z"
    mac: ENC[AES256_GCM,data:9Jsnij+I1MhLhIPLt2hr1a6Jryf8XcAb4QAknGkX1hNEN+Z9HT5PSnPARqyhl0WTxe8813a/G0h93SJ0A419jIzZugv6zEj4STsOG4hMI8O2NniqOwiDWl+Az9SRXIQ0ahH9En7ssvEdiosIDGJSz31ufNc8FbwKnHvgWCwfK9Q=,iv:VNTOpQkDakoaCu2dG4lRbd+6AzimNZVNzggq/7C2eM8=,tag:cJI9WdnKiG3K3J4LRDqQxQ==,type:str]
    version: 3.13.2
@@ -275,3 +275,28 @@ spec:
                name: temporal-web
                port:
                  number: 8080
 ---
 # NOTE: api.riotpiao.com is deliberately NOT here. Its namespace `api` is
 # created in wave 7, and this Application syncs in wave 1 — an Ingress into a
 # namespace that doesn't exist yet would fail and mark this whole app
 # SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced by the api-gw
 # Application.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: homarr
  namespace: dashboard
 spec:
  ingressClassName: nginx
  rules:
    - host: homarr.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: homarr
                port:
                  number: 7575
@@ -143,10 +143,7 @@ configs:
    oidc.config: |
      name: Zitadel
      issuer: https://zitadel.riotpiao.com
-      # MUST be quoted: an unquoted 18-digit number is parsed as a float by the
+      clientID: 396757801210282367
      # YAML parser and ArgoCD rejects it ("cannot unmarshal number into ...
      # clientID of type string"). Same class of bug as Grafana's client_id.
      clientID: "396757801210282367"
      clientSecret: $oidc-secret:client-secret
      requestedScopes:
        - openid
@@ -0,0 +1,38 @@
 # Dedicated CNPG Postgres for Authentik (GitOps, wave 2 — before authentik w3).
 # CNPG generates secret `authentik-db-app` + service `authentik-db-rw` in ns iam;
 # authentik-values reads them locally (no cross-ns secret copy).
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: authentik-db
  namespace: iam
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
  instances: 3
  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
  bootstrap:
    initdb:
      database: authentik
      owner: app
      encoding: UTF8
      localeCollate: C
      localeCType: C
  enableSuperuserAccess: false
  resources:
    requests: { memory: "512Mi", cpu: "250m" }
    limits: { memory: "2Gi", cpu: "1" }
  storage:
    size: 10Gi
    storageClass: longhorn-cnpg
  monitoring:
    enablePodMonitor: true
  affinity:
    # preferred + tolerations: HA across nodes without deadlocking a partly-
    # schedulable 3-CP (same as forgejo-db).
    podAntiAffinityType: preferred
    topologyKey: kubernetes.io/hostname
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
@@ -6,6 +6,7 @@ kind: Kustomization
 # deployed into it. paperless ns declared in namespaces.yaml above.
 resources:
  - namespaces.yaml
  - authentik-db.yaml
  - zitadel-db.yaml
  - temporal-db.yaml
  - paperless-db.yaml
@@ -1,7 +1,7 @@
 # DB clusters are wave 2 — their namespaces must exist first (their apps that
 # would CreateNamespace run later, w3/w8). Declared here so the databases App
-# creates them. vault/temporal/paperless CreateNamespace=true then no-ops.
+# creates them. authentik/vault/temporal/paperless CreateNamespace=true then
-# poimen namespace created by poimen-root app (wave 7).
+# no-ops. poimen namespace created by poimen-root app (wave 7).
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -0,0 +1,284 @@
 # k8s/talos-iam/authentik-values.yaml
 # Authentik — SSO Identity Provider for the homelab.
 # Provides OAuth2/OIDC login for Grafana, MinIO, Forgejo, and Argo CD.
 # Chart: authentik/authentik from https://charts.goauthentik.io
 #
 # Architecture: server (UI+API) + worker (background tasks) + PostgreSQL + Redis.
 # PostgreSQL is the system of record — must persist. Redis is ephemeral cache/queue.
 #
 # Secrets injected via helmfile --set (from .env / vsource):
 #   AUTHENTIK_SECRET_KEY         — signs sessions and tokens; set once, never rotate casually
 #   AUTHENTIK_BOOTSTRAP_PASSWORD — initial akadmin password (used once at first login)
 #   AUTHENTIK_BOOTSTRAP_TOKEN    — API token for the setup_talos_iam.sh bootstrap script
 #   AUTHENTIK_PG_PASSWORD        — PostgreSQL user password
 authentik:
  # host: the external URL Authentik uses to build redirect URIs in OAuth2 flows.
  # Must match what the browser sees — if it returns an internal svc URL,
  # the browser's redirect after login will fail (can't reach svc DNS externally).
  # HTTP (not HTTPS) because the Authentik ingress has no TLS cert configured.
  host: "https://authentik.riotpiao.com"
  error_reporting:
    enabled: false   # do not phone home to Sentry
  # PostgreSQL connection — points at CloudNativePG cluster in ddb namespace.
  # Uses 'app' bootstrap user (CNPG simple pattern, same as Forgejo).
  # Credentials injected from authentik-db-app secret via env vars below.
  postgresql:
    host: authentik-db-rw.iam.svc.cluster.local
    port: 5432
    name: authentik
    user: app  # All apps use shared 'app' user (CNPG design pattern)
    password: ""  # overridden by AUTHENTIK_POSTGRESQL__PASSWORD env var
  # Redis connection — bundled subchart, standalone mode (no sentinel/cluster).
  redis:
    host: authentik-redis-master
  # ── HTTP client timeouts ──────────────────────────────────────────────────────
  # Increased to tolerate 5+ second pod-to-pod network latency.
  # Affects webhooks, outpost management, SCIM, LDAP sync.
  # Default: ~30s — too aggressive when latency spikes hit 5-10s.
  log_level: debug  # enable debug logging to monitor connection issues
 # ── CA trust (shared by server and worker) ────────────────────────────────────
 # Authentik (Python/Debian) uses requests + httpx for outgoing HTTPS — webhooks,
 # outpost management, SCIM. Both libraries need REQUESTS_CA_BUNDLE / SSL_CERT_FILE
 # to point to a bundle that includes homelab-ca, otherwise connections to other
 # homelab services fail with "certificate signed by unknown authority".
 #
 # Strategy: a debian:12-slim init container (run as root) concatenates the
 # Debian system Mozilla bundle with homelab-ca.crt into an emptyDir. The main
 # container then references /merged/ca-bundle.crt via two env vars that cover
 # every Python HTTP library.
 _caVolumes: &caVolumes
  - name: homelab-ca
    configMap:
      name: homelab-ca
  - name: merged-ca
    emptyDir: {}
 _caVolumeMounts: &caVolumeMounts
  - name: homelab-ca
    mountPath: /homelab-ca
    readOnly: true
  - name: merged-ca
    mountPath: /merged
    readOnly: true
 _caInitContainers: &caInitContainers
  - name: merge-ca-certs
    image: debian:bookworm
    imagePullPolicy: IfNotPresent
    securityContext:
      runAsUser: 0
    command:
      - sh
      - -c
      - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
    volumeMounts:
      - name: homelab-ca
        mountPath: /homelab-ca
        readOnly: true
      - name: merged-ca
        mountPath: /merged
  # NOTE: no authentik-migrate init container — the authentik `server` entrypoint
  # runs migrations itself on startup. A separate `manage migrate` init pinned to
  # an older image tripped a version-history precheck on an empty DB
  # (relation "authentik_version_history" does not exist) and blocked boot.
 _caEnv: &caEnv
  - name: REQUESTS_CA_BUNDLE
    value: /merged/ca-bundle.crt
  - name: SSL_CERT_FILE
    value: /merged/ca-bundle.crt
 # ── Authentik server (UI + API) ───────────────────────────────────────────────
 # Handles all browser traffic: login flows, admin UI, OAuth2 authorize/token endpoints.
 # NodePort 32172 is a fallback for direct node access during troubleshooting;
 # normal access is via nginx ingress (authentik.riotpiao.com → svc:80).
 # Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
 server:
  replicas: 1
  # Merge SOPS-CMP-emitted secret values after the chart's own `authentik` secret.
  envFrom:
    - secretRef:
        name: authentik-secrets
  deploymentStrategy:
    type: Recreate
  service:
    type: NodePort
    nodePort: 32172
  resources:
    requests:
      cpu: 100m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 1Gi
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  volumes: *caVolumes
  volumeMounts: *caVolumeMounts
  initContainers: *caInitContainers
  env:
    # Merge CA trust env vars
    - name: REQUESTS_CA_BUNDLE
      value: /merged/ca-bundle.crt
    - name: SSL_CERT_FILE
      value: /merged/ca-bundle.crt
    # Override database credentials to use 'app' from authentik-db-app
    - name: AUTHENTIK_POSTGRESQL__USER
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: username
    - name: AUTHENTIK_POSTGRESQL__PASSWORD
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: password
  podAnnotations:
    configmap.reloader.stakater.com/reload: "homelab-ca"
    homelab.io/restart-at: "2026-06-21T13-40"
  # The /-/health/{live,ready}/ endpoints do a DB round-trip; under transient
  # CNPG contention they respond in 5-6s while still returning 200. The chart's
  # default 3s liveness timeout then flags a working backend as dead and kubelet
  # kills it in a restart loop — the pod never stays Ready, gets dropped from the
  # authentik-server Service endpoints, and the OAuth-provisioning PostSync hook
  # fails with "Host is unreachable". Widen the timeouts so slow-but-healthy
  # checks aren't treated as failures. (Only these fields are overridden; the
  # chart deep-merges the rest of each probe, incl. the httpGet path.)
  livenessProbe:
    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
    timeoutSeconds: 15
    failureThreshold: 6
  readinessProbe:
    initialDelaySeconds: 300  # skip probe until migrations complete
    timeoutSeconds: 15
    failureThreshold: 6
  startupProbe:
    timeoutSeconds: 15
    failureThreshold: 120  # 120 × 10s = 20min for fresh DB migrations
  # Every OIDC login (Grafana, Argo CD, MinIO, Forgejo) depends on this server —
  # its request latency/error rate explains SSO-driven slowness on those services.
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true
      scrapeTimeout: 30s
 # ── Authentik worker ──────────────────────────────────────────────────────────
 # Runs background tasks: email delivery, LDAP sync, flow policy evaluation,
 # event log cleanup, and managed outpost updates. Stateless — no PVC needed.
 # Same resource profile as server; Authentik 2023+ merged some worker duties
 # into the server process but the worker pod is still required.
 worker:
  replicas: 1
  envFrom:
    - secretRef:
        name: authentik-secrets
  deploymentStrategy:
    type: Recreate
  resources:
    requests:
      cpu: 100m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 1Gi
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  volumes: *caVolumes
  volumeMounts: *caVolumeMounts
  initContainers: *caInitContainers
  env:
    # Merge CA trust env vars
    - name: REQUESTS_CA_BUNDLE
      value: /merged/ca-bundle.crt
    - name: SSL_CERT_FILE
      value: /merged/ca-bundle.crt
    # Override database credentials to use 'app' from authentik-db-app
    - name: AUTHENTIK_POSTGRESQL__USER
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: username
    - name: AUTHENTIK_POSTGRESQL__PASSWORD
      valueFrom:
        secretKeyRef:
          name: authentik-db-app
          key: password
  podAnnotations:
    configmap.reloader.stakater.com/reload: "homelab-ca"
    homelab.io/restart-at: "2026-06-21T13-40"
  livenessProbe:
    initialDelaySeconds: 300  # skip probe until 5min passed (migrations finish)
  readinessProbe:
    initialDelaySeconds: 300  # skip probe until migrations complete
  startupProbe:
    initialDelaySeconds: 30  # let server finish DB work first
    failureThreshold: 120
  metrics:
    enabled: true
    serviceMonitor:
      enabled: true
      scrapeTimeout: 30s
 # ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ─────────────
 # Authentik connects to the dedicated authentik-db (1 primary + 2 replicas).
 # Do not use the bundled Bitnami subchart — CNPG is already running.
 postgresql:
  enabled: false
  primary:
    persistence:
      enabled: true
      storageClass: longhorn
      size: 8Gi
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
    affinity:
      nodeAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
                - key: node-role.kubernetes.io/worker
                  operator: Exists
 # ── Bundled Redis ─────────────────────────────────────────────────────────────
 # Cache and async task queue only — no durable data. If Redis restarts, in-flight
 # background tasks are retried and cached tokens are recomputed. Losing Redis
 # data does not lose user accounts or flow configuration (that's in PostgreSQL).
 # persistence: false saves a PVC and makes restarts faster.
 #
 # Same prefer-worker / fallback-to-cp scheduling as PostgreSQL.
 # architecture: standalone — no Sentinel/cluster overhead for a 3-node homelab.
 redis:
  enabled: true
  master:
    persistence:
      enabled: false
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
    affinity:
      nodeAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
                - key: node-role.kubernetes.io/worker
                  operator: Exists
  architecture: standalone
 # Ingress disabled — rule lives in k8s/ingress/ingress.yaml (authentik.riotpiao.com).
 # For direct access during bootstrap: kubectl -n iam port-forward svc/authentik-server 7000:80
@@ -2,12 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  # Removed: rbac-dashboard-rolebinding.yaml (authentik-provisioner SA and
+  - rbac-dashboard-rolebinding.yaml
  # RoleBinding). Authentik is retired, so the file is gone; keep this list in
  # step with the directory or kustomize fails the whole app.
  - smtp-gotify.yaml
 # IAM provisioning is manual-only (security-sensitive).
-# Script: scripts/iam/zitadel-provision.py
+# Script: scripts/iam/authentik-provision.py
 # Run:
-#   scripts/iam/zitadel-provision.py all
+#   export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
 #     -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
 #   python3 scripts/iam/authentik-provision.py
@@ -0,0 +1,13 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: authentik-provisioner
  namespace: dashboard
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: authentik-provisioner
 subjects:
 - kind: ServiceAccount
  name: authentik-provisioner
  namespace: iam
@@ -1,108 +0,0 @@
 # SMTP -> Gotify relay for ZITADEL notifications.
 #
 # ZITADEL sends verification codes, password resets and one-time passwords by
 # SMTP only, and this cluster has no working mail path: no SMTP credentials are
 # configured for ZITADEL, and the notification table shows
 # verified_email empty for every user. Without a delivery path nobody can finish
 # a first login, at ZITADEL or at any relying party such as Forgejo.
 #
 # This relay accepts plain SMTP on :2525 and forwards every message to Gotify,
 # so codes arrive on the phone through the existing gotify/igotify setup instead
 # of email. ZITADEL then points DefaultInstance.SMTPConfiguration at this
 # Service (see k8s/infra/iam/zitadel-values.yaml).
 #
 # The Gotify token is a Secret, never a literal here.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: smtp-gotify
  namespace: iam
  labels:
    app: smtp-gotify
 spec:
  replicas: 1
  revisionHistoryLimit: 2
  selector:
    matchLabels:
      app: smtp-gotify
  template:
    metadata:
      labels:
        app: smtp-gotify
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 65534
        runAsGroup: 65534
        seccompProfile:
          type: RuntimeDefault
      volumes:
        - name: tmp
          emptyDir: {}
      containers:
        - name: smtp-gotify
          image: quay.io/reiml/smtp-gotify:latest
          imagePullPolicy: IfNotPresent
          env:
            - name: GOTIFY_URL
              value: "http://gotify.notifications.svc.cluster.local/"
            - name: GOTIFY_TOKEN
              valueFrom:
                secretKeyRef:
                  name: smtp-gotify
                  key: token
            - name: GOTIFY_PRIORITY
              value: "5"
            # No title template: the relay's default title already carries
            # the mail subject, and a wrong template field fails the push
            # silently (the SMTP side still answers 250, so the sender thinks
            # the message was delivered).
          ports:
            - name: smtp
              containerPort: 2525
              protocol: TCP
          volumeMounts:
            # The relay stages each message in /tmp before the Gotify push.
            # With readOnlyRootFilesystem it accepted SMTP (250 queued) and
            # then silently dropped the message: no push, no error.
            - name: tmp
              mountPath: /tmp
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
          resources:
            requests:
              cpu: 10m
              memory: 32Mi
            limits:
              cpu: 200m
              memory: 128Mi
          readinessProbe:
            tcpSocket:
              port: smtp
            initialDelaySeconds: 3
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: smtp
            initialDelaySeconds: 10
            periodSeconds: 20
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: smtp-gotify
  namespace: iam
  labels:
    app: smtp-gotify
 spec:
  type: ClusterIP
  selector:
    app: smtp-gotify
  ports:
    - name: smtp
      port: 2525
      targetPort: smtp
      protocol: TCP
@@ -81,15 +81,6 @@ server:
    - envName: VAULT_UNSEAL_KEY_3
      secretName: vault-unseal-keys
      secretKey: key3
    # Zitadel OIDC client credentials for the Vault UI login method. Consumed by
    # the postStart hook below, which enables auth/oidc via the Vault API after
    # unsealing. Vault auth methods cannot be declared in HCL, only via API/CLI.
    - envName: OIDC_CLIENT_ID
      secretName: vault-oidc-creds
      secretKey: OIDC_CLIENT_ID
    - envName: OIDC_CLIENT_SECRET
      secretName: vault-oidc-creds
      secretKey: OIDC_CLIENT_SECRET
  # ── Auto-unseal ─────────────────────────────────────────────────────────────
  # Vault starts sealed after every pod restart and can't serve requests until
@@ -106,28 +97,6 @@ server:
      vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
      vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
      # ── Zitadel OIDC login method ────────────────────────────────────────
      # Vault auth methods are API-driven: they cannot be declared in HCL.
      # Enable auth/oidc and write its config on every pod start. Both calls
      # are idempotent (auth enable returns 400 "already enabled", which we
      # tolerate; the config write always succeeds and overwrites).
      vault auth enable oidc 2>/dev/null || true
      vault write auth/oidc/config \
        oidc_discovery_url="https://zitadel.riotpiao.com" \
        oidc_client_id="$OIDC_CLIENT_ID" \
        oidc_client_secret="$OIDC_CLIENT_SECRET" \
        default_role="default" 2>/dev/null || true
      # The role binds the Zitadel client_id as audience and whitelists the UI
      # callback URL. Policy `default` gives a logged-in operator standard
      # read access; admin actions still require a token with root/sudo.
      vault write auth/oidc/role/default \
        bound_audiences="$OIDC_CLIENT_ID" \
        allowed_redirect_uris="https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback" \
        allowed_redirect_uris="http://localhost:8200/ui/vault/auth/oidc/oidc/callback" \
        user_claim="preferred_username" \
        oidc_scopes="openid,profile,email,groups" \
        policies="default" 2>/dev/null || true
  # ── Vault config (HCL) ──────────────────────────────────────────────────────
  standalone:
    enabled: true
@@ -31,6 +31,7 @@ zitadel:
    ExternalDomain: zitadel.riotpiao.com
    TLS:
      Enabled: false
    # First boot is COMPLETE (2026-10-26): the instance, the system org and the
    # `iam-admin` machine key all exist. Skip stops the setup hook from running
    # the first-instance step again. Without it, every sync re-runs the
@@ -145,10 +146,6 @@ setupJob:
 replicaCount: 2
 revisionHistoryLimit: 3
 # Notification delivery lives in the ConfigMap owned by the zitadel-core app
 # (k8s/infra/zitadel-core/). Env overrides here produced no SMTP provider, and
 # editing configmapConfig made ArgoCD drop the rendered ConfigMap.
 # Best practice: a PDB guarantees at least one API pod stays available during
 # voluntary disruptions (node drain, rolling upgrade). Without it a drain can
 # take both replicas down at once. minAvailable 1 is correct for 2 replicas;
@@ -174,20 +171,7 @@ ingress:
  controller: nginx
 login:
-  # Built-in login page, not the hosted Login UI.
+  enabled: true
  #
  # The hosted UI authenticates to the ZITADEL API as the machine user
  # `login-client`, using a JWT-profile assertion signed by the keypair in
  # zitadel-login-service-key. That machine user is created by the chart's
  # setup job, which this install runs only once (setupJob is deliberately
  # disabled after the first sync). Without the user, every session write
  # fails and the user sees "Could not create session for user", so no one
  # can sign in here or at any relying party such as Forgejo.
  #
  # The built-in login serves the same flows from the ZITADEL server and
  # needs no machine user. Turn this back on only after creating
  # `login-client` and registering the matching public key.
  enabled: false
  # ksops owns this Secret (k8s/argocd/secrets/zitadel-login-service-key.enc.yaml,
  # rendered by the sops-secrets Application). Pointing the chart at it stops the
  # chart from auto-generating an RSA keypair every render, which changed
@@ -74,7 +74,7 @@ grafana.ini:
    enabled: true
    name: Zitadel
    allow_sign_up: true  # Allow sync to create/update users from OAuth claims
-    client_id: "396757801965191169"
+    client_id: 396757801965191169
    scopes: openid email profile groups
    auth_url: https://zitadel.riotpiao.com/oauth/v2/authorize
    token_url: https://zitadel.riotpiao.com/oauth/v2/token
@@ -0,0 +1,33 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
  name: authentik-rules
  namespace: iam
 spec:
  groups:
    - name: authentik.rules
      interval: 15s
      rules:
        - alert: AuthentikHighErrorRate
          expr: |
            (
              sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
              /
              sum(rate(authentik_http_requests_total[5m]))
            ) > 0.05
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "High error rate on Authentik"
            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
        - alert: AuthentikOutpostDown
          expr: |
            authentik_outpost_total_up == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Authentik outpost is down"
            description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"
@@ -7,6 +7,7 @@ kind: Kustomization
 resources:
  - alerts/ingress-alerts.yaml
  - alerts/svc-argocd-rules.yaml
  - alerts/svc-authentik-rules.yaml
  - alerts/svc-zitadel-rules.yaml
  - alerts/svc-forgejo-rules.yaml
  - alerts/svc-grafana-rules.yaml
@@ -16,6 +17,7 @@ resources:
  - alerts/api-gateway-alerts.yaml
  - alerts/cag-kv-cache-rules.yaml
  - servicemonitors/argocd.yaml
  - servicemonitors/authentik.yaml
  - servicemonitors/forgejo.yaml
  - servicemonitors/minio.yaml
  - servicemonitors/ingress-nginx.yaml
@@ -0,0 +1,18 @@
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: authentik
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
 spec:
  namespaceSelector:
    matchNames:
      - iam
  selector:
    matchLabels:
      app.kubernetes.io/name: authentik
  endpoints:
    - port: metrics
      interval: 30s
      scrapeTimeout: 10s
@@ -0,0 +1,33 @@
 # One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
 # OIDC authenticator (they last started before the Authentik provider existed, so OIDC
 # tokens were never accepted -> every login got 401). Delete this whole directory after.
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: oidc-bootstrap
  namespace: kube-system
 subjects:
  - kind: ServiceAccount
    name: oidc-bootstrap
    namespace: kube-system
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: oidc-bootstrap
@@ -0,0 +1,32 @@
 # Scoped operator access for homarr-admins: the homarr Deployment + app
 # secrets only. homarr-oidc, auth-oidc-secret, db-encryption stay excluded
 # (security-managed, not app config). Inert until kube-apiserver's OIDC
 # wiring lands.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: homarr-operator
  namespace: dashboard
 rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    resourceNames: ["homarr"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["homarr-secrets"]
    verbs: ["get", "list", "watch", "update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: homarr-admins-binding
  namespace: dashboard
 subjects:
  - kind: Group
    name: "oidc:homarr-admins"
    apiGroup: rbac.authorization.k8s.io
 roleRef:
  kind: Role
  name: homarr-operator
  apiGroup: rbac.authorization.k8s.io
@@ -10,6 +10,7 @@ resources:
  - grafana-operator-role.yaml
  - minio-operator-role.yaml
  - forgejo-operator-role.yaml
  - homarr-operator-role.yaml
  - portainer-operator-role.yaml
  - kmsvc-operator-role.yaml
  - temporal-operator-role.yaml
@@ -1,4 +0,0 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - zitadel-core.yaml
@@ -1,133 +0,0 @@
 # ZITADEL core objects, owned by this repository instead of the Helm chart.
 #
 # The zitadel chart marks four objects as Helm hooks:
 #
 #   kind: ServiceAccount   (also ConfigMap/zitadel-config-yaml, Role, RoleBinding)
 #   annotations:
 #     helm.sh/hook: pre-install,pre-upgrade
 #     helm.sh/hook-delete-policy: before-hook-creation
 #
 # ArgoCD maps pre-install/pre-upgrade to PreSync and prunes hook resources when
 # the sync ends. Every sync therefore deleted the ServiceAccount and the ConfigMap
 # that the ZITADEL Deployment references, which produced:
 #
 #   error creating: pods "zitadel-..." is forbidden:
 #     serviceaccount "zitadel" not found
 #
 # and left the IdP one restart away from failing.
 #
 # These copies carry no hook annotations, so they are ordinary desired state and
 # ArgoCD keeps them. The chart still renders its own hook versions during a sync;
 # these win because the sync no longer removes them.
 #
 # ConfigMap content mirrors what the chart renders from
 # k8s/infra/iam/zitadel-values.yaml, plus the notification settings the chart
 # cannot deliver through values (see the SMTP note below). When the chart values
 # change, update this file too; it is the authoritative copy at runtime.
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: zitadel
  namespace: iam
  labels:
    app.kubernetes.io/name: zitadel
    app.kubernetes.io/instance: zitadel
    app.kubernetes.io/managed-by: gitops
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: zitadel
  namespace: iam
  labels:
    app.kubernetes.io/name: zitadel
    app.kubernetes.io/instance: zitadel
    app.kubernetes.io/managed-by: gitops
 rules:
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "create", "patch", "list", "delete"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: zitadel
  namespace: iam
  labels:
    app.kubernetes.io/name: zitadel
    app.kubernetes.io/instance: zitadel
    app.kubernetes.io/managed-by: gitops
 subjects:
  - kind: ServiceAccount
    name: zitadel
 roleRef:
  kind: Role
  name: zitadel
  apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: zitadel-config-yaml
  namespace: iam
  labels:
    app.kubernetes.io/name: zitadel
    app.kubernetes.io/instance: zitadel
    app.kubernetes.io/managed-by: gitops
 data:
  zitadel-config-yaml: |
    ExternalDomain: zitadel.riotpiao.com
    ExternalSecure: true
    TLS:
      Enabled: false
    FirstInstance:
      # First boot is COMPLETE. Skip stops the setup step from running again and
      # silently rotating the iam-admin key that provisioning reads.
      Skip: true
      Org:
        Skip: true
        Machine:
          Machine:
            Name: Automatically Initialized IAM Admin
            Username: iam-admin
          MachineKey:
            ExpirationDate: "2029-01-01T00:00:00Z"
            Type: 1
          Pat:
            ExpirationDate: "2029-01-01T00:00:00Z"
    Machine:
      Identification:
        Hostname:
          Enabled: true
        Webhook:
          Enabled: false
    # Notification delivery.
    #
    # ZITADEL sends verification codes and password resets by SMTP only. This
    # cluster has no mail path, so without a provider every user stays at
    # verified_email = empty and cannot finish a first login.
    #
    # The target is the in-cluster smtp-gotify relay, which forwards each message
    # to Gotify so codes reach the phone.
    #
    # Why here and not in chart values: injecting the same keys as
    # ZITADEL_DEFAULTINSTANCE_SMTPCONFIGURATION_* env vars produced no
    # instance.smtp.config.added event, so ZITADEL never created the provider.
    # Setting configmapConfig in chart values made ArgoCD stop managing this
    # ConfigMap. Owning the file here avoids both problems.
    DefaultInstance:
      DomainPolicy:
        # The sender is [email protected] while the instance domain is
        # zitadel.riotpiao.com. Leave this true so the sender address is not
        # rejected for not matching the domain.
        SMTPSenderAddressMatchesInstanceDomain: false
      SMTPConfiguration:
        SMTP:
          Host: smtp-gotify.iam.svc.cluster.local:2525
        TLS: false
        From: [email protected]
        FromName: ZITADEL
@@ -1,38 +0,0 @@
 #!/usr/bin/env bash
 # forgejo-oauth-source.sh — keep the Forgejo OAuth login source aligned with
 # ZITADEL.
 #
 # Forgejo stores login sources in its database, not in GitOps, and its REST API
 # does not expose them. `tea` has no auth-source command either (tea admin only
 # manages users), so the supported writer is the in-pod CLI:
 #   gitea admin auth update-oauth
 #
 # The source name is the label on the "Sign in with ..." button AND the callback
 # path (/user/oauth2/<name>/callback), so a rename changes the registered
 # redirect URI. Register both spellings in ZITADEL (zitadel-provision.py does)
 # before renaming.
 #
 # Usage:
 #   scripts/iam/forgejo-oauth-source.sh status
 #   scripts/iam/forgejo-oauth-source.sh rename zitadel
 set -euo pipefail
 NS="${FORGEJO_NAMESPACE:-cicd}"
 SEL="${FORGEJO_SELECTOR:-app.kubernetes.io/name=forgejo}"
 pod() {
  kubectl -n "$NS" get pods -l "$SEL" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null \
    || kubectl -n "$NS" get pods -o name | grep -m1 forgejo-gitea | cut -d/ -f2
 }
 run() { kubectl -n "$NS" exec "$(pod)" -c gitea -- gitea admin auth "$@"; }
 case "${1:-status}" in
  status) run list ;;
  rename)
    name="${2:?usage: $0 rename <name>}"
    run update-oauth --id 1 --name "$name"
    run list
    ;;
  *) echo "usage: $0 [status|rename <name>]" >&2; exit 2 ;;
 esac
@@ -0,0 +1,669 @@
 #!/usr/bin/env python3
 """
 Provision RBAC groups, service account roles, fine-grained claims, and auth flows.
 Idempotent — safe to re-run. Provisions:
  1. Global admin groups (homelab-admins)
  2. Fine-grained service/bucket/project groups (minio-*, poimen-*, paperless-*, grafana-*, sqs-*)
  3. Service account roles with custom claims (paperless-ai-agent, portfolio-agent, etc.)
  4. JWT scope mappings for fine-grained claims (minio_buckets, paperless_doctypes, etc.)
  5. OAuth2 providers with scopes (api-gw, minio, poimen, paperless, grafana)
  6. Auth flows (password grant on api-gw provider)
 Usage:
    source ~/.env
    export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \
        -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)
    python3 scripts/iam/provision-rbac.py
 DO NOT commit this file to git — .gitignore covers scripts/iam/*.py.
 """
 import json
 import os
 import sys
 import urllib.error
 import urllib.request
 from typing import Dict, List, Any
 from pathlib import Path
 # Load ~/.env for OAuth2 provider secrets
 env_file = Path.home() / ".env"
 if env_file.exists():
    with open(env_file) as f:
        for line in f:
            line = line.strip()
            if line.startswith("export ") and "=" in line:
                key, _, value = line[7:].partition("=")
                key = key.strip()
                value = value.strip().strip('"').strip("'")
                os.environ[key] = value
 AUTHENTIK_URL = "https://authentik.riotpiao.com"
 TOKEN = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN")
 if not TOKEN:
    print("Error: AUTHENTIK_BOOTSTRAP_TOKEN not set")
    print("  source ~/.env")
    print("  export AUTHENTIK_BOOTSTRAP_TOKEN=$(kubectl -n iam get secret authentik-secrets \\")
    print("      -o jsonpath='{.data.AUTHENTIK_BOOTSTRAP_TOKEN}' | base64 -d)")
    sys.exit(1)
 def api(method, path, data=None):
    url = f"{AUTHENTIK_URL}{path}"
    body = json.dumps(data).encode() if data is not None else None
    req = urllib.request.Request(url, data=body, method=method, headers={
        "Authorization": f"Bearer {TOKEN}",
        "Content-Type": "application/json",
    })
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            raw = resp.read()
            return resp.status, json.loads(raw) if raw else {}
    except urllib.error.HTTPError as e:
        raw = e.read()
        try:
            parsed = json.loads(raw) if raw else {}
        except json.JSONDecodeError:
            parsed = {"raw": raw.decode(errors="replace")}
        return e.code, parsed
 def die(msg):
    print(f"FATAL: {msg}", file=sys.stderr)
    sys.exit(1)
 # ===========================================================================
 # Group Definitions (DRY: single source of truth)
 # ===========================================================================
 GROUPS: Dict[str, Dict[str, Any]] = {
    "homelab-admins": {
        "description": "Cluster administrators with full access",
        "is_superuser": True,
    },
    "minio-admins": {"description": "MinIO administrators", "is_superuser": False, "minio_buckets": ["*"]},
    "minio-photos": {"description": "Photos bucket (Immich) access", "is_superuser": False, "minio_buckets": ["immich"]},
    "minio-documents": {"description": "Documents bucket (Paperless) access", "is_superuser": False, "minio_buckets": ["paperless"]},
    "minio-backups": {"description": "Backups bucket read-only access", "is_superuser": False, "minio_buckets": ["backups"]},
    "poimen-admins": {"description": "Poimen memory administrators", "is_superuser": False, "memory_projects": ["*"], "memory_visibility": "private"},
    "poimen-devs": {"description": "Dev and staging projects access", "is_superuser": False, "memory_projects": ["dev", "staging"], "memory_visibility": "internal"},
    "poimen-prod-readonly": {"description": "Production projects read-only access", "is_superuser": False, "memory_projects": ["prod"], "memory_visibility": "public"},
    "paperless-admins": {"description": "Paperless administrators", "is_superuser": False, "paperless_doctypes": ["*"]},
    "paperless-finance": {"description": "Finance documents", "is_superuser": False, "paperless_doctypes": ["invoices", "receipts", "expenses"]},
    "paperless-legal": {"description": "Legal documents", "is_superuser": False, "paperless_doctypes": ["contracts", "licenses", "agreements"]},
    "paperless-hr": {"description": "HR documents", "is_superuser": False, "paperless_doctypes": ["employment", "benefits", "payroll"]},
    "grafana-admins": {"description": "Grafana administrators", "is_superuser": False, "grafana_org_role": "Admin"},
    "grafana-editors": {"description": "Grafana dashboard editors", "is_superuser": False, "grafana_org_role": "Editor"},
    "grafana-viewers": {"description": "Grafana dashboard viewers", "is_superuser": False, "grafana_org_role": "Viewer"},
    "sqs-users": {"description": "SQS/Temporal queue read access", "is_superuser": False, "sqs_queues": ["default"]},
    "sqs-writers": {"description": "SQS/Temporal queue read/write access", "is_superuser": False, "sqs_queues": ["*"]},
    "s3-users": {"description": "S3 read access", "is_superuser": False},
    "s3-writers": {"description": "S3 read/write access", "is_superuser": False},
 }
 SERVICE_ACCOUNTS: Dict[str, Dict[str, Any]] = {
    "paperless-ai-agent": {
        "description": "Paperless AI plugin (auto-tagging, entity extraction)",
        "roles": ["llm:inference", "memory:write", "paperless:admin"],
        "claims": {
            "minio_buckets": ["paperless"],
            "paperless_doctypes": ["*"],
            "memory_projects": ["*"],
            "authorized_models": ["reasoning", "qwen2.5:3b"],
        },
    },
    "portfolio-agent": {
        "description": "Portfolio service agent",
        "roles": ["llm:inference", "memory:read"],
        "claims": {
            "memory_projects": ["homelab", "portfolio"],
            "memory_visibility": "public",
            "authorized_models": ["ornith:35b"],
            "minio_buckets": ["backups"],
        },
    },
    "memory-agent": {
        "description": "Memory service agent",
        "roles": ["llm:inference", "memory:read", "memory:write"],
        "claims": {
            "memory_projects": ["*"],
            "memory_visibility": "private",
            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
        },
    },
    "temporal-worker-agent": {
        "description": "Temporal workflow worker",
        "roles": ["llm:inference", "workflow:execute", "memory:read", "memory:write", "queue:send"],
        "claims": {
            "memory_projects": ["*"],
            "sqs_queues": ["*"],
            "authorized_models": ["reasoning", "ornith:35b", "qwen2.5:3b"],
        },
    },
 }
 SCOPE_MAPPINGS: Dict[str, Dict[str, str]] = {
    "roles": {"expression": 'return user.attributes.get("roles", [])'},
    "permissions": {"expression": 'return ["*"] if any(user.groups.filter(is_superuser=True)) else list(user.groups.values_list("name", flat=True))'},
    "groups": {"expression": 'return {"groups": list(user.groups.values_list("name", flat=True))}'},
    "minio_buckets": {"expression": 'return user.attributes.get("minio_buckets", [])'},
    "paperless_doctypes": {"expression": 'return user.attributes.get("paperless_doctypes", [])'},
    "memory_projects": {"expression": 'return user.attributes.get("memory_projects", [])'},
    "memory_visibility": {"expression": 'return user.attributes.get("memory_visibility", "public")'},
    "authorized_models": {"expression": 'return user.attributes.get("authorized_models", [])'},
    "sqs_queues": {"expression": 'return user.attributes.get("sqs_queues", [])'},
    "grafana_org_role": {"expression": 'return user.attributes.get("grafana_org_role", "Viewer")'},
 }
 # ===========================================================================
 # Phase 1: Create/sync all groups
 # ===========================================================================
 print("[1/6] Ensuring groups exist...")
 status, res = api("GET", "/api/v3/core/groups/?page_size=100")
 if status != 200:
    die(f"GET groups -> {status} {res}")
 existing_groups = {g["name"]: g for g in res["results"]}
 created_count = 0
 for group_name, group_spec in GROUPS.items():
    if group_name in existing_groups:
        print(f"  {group_name}: already exists")
    else:
        status, res = api("POST", "/api/v3/core/groups/", {
            "name": group_name,
            "is_superuser": group_spec.get("is_superuser", False),
        })
        if status in (200, 201):
            print(f"  {group_name}: created")
            created_count += 1
        else:
            print(f"  {group_name}: FAILED {status} {res}")
 print(f"  Total: {len(GROUPS)} groups, {created_count} new")
 # ===========================================================================
 # Phase 2: Create/sync service account users with custom claims
 # ===========================================================================
 print("\n[2/6] Creating/updating service account users...")
 status, res = api("GET", "/api/v3/core/users/?page_size=100")
 if status != 200:
    die(f"GET users -> {status} {res}")
 existing_users = {u["username"]: u for u in res["results"]}
 service_pwd = os.environ.get("AUTHENTIK_SERVICE_ACCOUNT_PASSWORD", "DefaultPassword123!")
 for agent_name, agent_spec in SERVICE_ACCOUNTS.items():
    if agent_name in existing_users:
        user = existing_users[agent_name]
        attrs = user.get("attributes", {})
        attrs.update(agent_spec.get("claims", {}))
        attrs["roles"] = agent_spec.get("roles", [])
        status, res = api("PATCH", f"/api/v3/core/users/{user['pk']}/", {"attributes": attrs})
        if status in (200, 201):
            print(f"  {agent_name}: claims updated")
        else:
            print(f"  {agent_name}: FAILED {status} {res}")
    else:
        status, res = api("POST", "/api/v3/core/users/", {
            "username": agent_name,
            "name": agent_spec.get("description", agent_name),
            "email": f"{agent_name}@homelab.local",
            "is_active": True,
            "is_superuser": False,
            "password": service_pwd,
            "attributes": {
                **agent_spec.get("claims", {}),
                "roles": agent_spec.get("roles", []),
            },
        })
        if status in (200, 201):
            print(f"  {agent_name}: created")
        else:
            print(f"  {agent_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 3: Create scope mappings for fine-grained claims
 # ===========================================================================
 print("\n[3/6] Creating scope mappings for fine-grained claims...")
 status, res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
 if status != 200:
    die(f"GET scope mappings -> {status} {res}")
 existing_scopes = {m["scope_name"]: m for m in res["results"]}
 for scope_name, scope_spec in SCOPE_MAPPINGS.items():
    if scope_name in existing_scopes:
        print(f"  {scope_name}: already exists")
    else:
        status, res = api("POST", "/api/v3/propertymappings/provider/scope/", {
            "name": scope_name,
            "scope_name": scope_name,
            "expression": scope_spec["expression"],
        })
        if status in (200, 201):
            print(f"  {scope_name}: created")
        else:
            print(f"  {scope_name}: FAILED {status} {res}")
 # Fix built-in 'openid' scope to include preferred_username (needed by Grafana, Forgejo, etc.)
 # Note: sub MUST be a string, not int, for JSON compatibility with Go struct unmarshaling
 if "openid" in existing_scopes:
    openid_scope = existing_scopes["openid"]
    openid_pk = openid_scope["pk"]
    openid_expression = 'return {"sub": str(user.pk), "preferred_username": user.username}'
    if openid_expression not in openid_scope.get("expression", ""):
        status, res = api("PUT", f"/api/v3/propertymappings/provider/scope/{openid_pk}/", {
            "name": openid_scope["name"],
            "scope_name": "openid",
            "expression": openid_expression,
        })
        if status in (200, 201):
            print(f"  openid: updated (sub as string + preferred_username)")
        else:
            print(f"  openid: UPDATE FAILED {status} {res}")
    else:
        print(f"  openid: already correct (sub as string)")
 # ===========================================================================
 # Phase 4: Get flow UUIDs (needed for providers)
 # ===========================================================================
 print("\n[4/6] Fetching flow UUIDs...")
 status, res = api("GET", "/api/v3/flows/instances/?page_size=100")
 if status != 200:
    die(f"GET flows -> {status} {res}")
 flows = {f["slug"]: f["pk"] for f in res.get("results", [])}
 auth_flow = flows.get("default-provider-authorization-implicit-consent")
 inval_flow = flows.get("default-provider-invalidation-flow")
 if not auth_flow or not inval_flow:
    die(f"Required flows not found. auth_flow={auth_flow}, inval_flow={inval_flow}")
 print(f"  authorization_flow: {auth_flow}")
 print(f"  invalidation_flow: {inval_flow}")
 # ===========================================================================
 # Phase 5: Create OAuth2 providers with scopes
 # ===========================================================================
 print("\n[5/6] Creating OAuth2 providers...")
 status, res = api("GET", "/api/v3/providers/oauth2/?page_size=100")
 if status != 200:
    die(f"GET providers -> {status} {res}")
 existing_providers = {p["name"]: p for p in res.get("results", [])}
 # Fetch scope mapping PKs
 status, scopes_res = api("GET", "/api/v3/propertymappings/provider/scope/?page_size=100")
 if status != 200:
    print("  WARNING: could not fetch scope mappings")
    scope_pks = {}
 else:
    scope_pks = {m["scope_name"]: m["pk"] for m in scopes_res.get("results", [])}
 # Include standard OpenID scopes (openid, email, profile) + custom claim scopes
 STANDARD_SCOPES = ["openid", "email", "profile"]
 scope_pks_list = [scope_pks[s] for s in STANDARD_SCOPES if s in scope_pks]
 scope_pks_list += [scope_pks[s] for s in SCOPE_MAPPINGS.keys() if s in scope_pks]
 # OAuth2 providers — client_secret sourced from SOPS-encrypted k8s secrets.
 # These are the real secrets the services use. Authentik must match.
 OAuth2_PROVIDERS = {
    "api-gw": {
        "client_id": "api-gw",
        "client_secret_env": "AUTHENTIK_PROVIDER_API_GW_SECRET",
        "redirect_uris": ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"],
    },
    "minio": {
        "client_id": "minio",
        "client_secret": "9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219",
        "redirect_uris": ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"],
    },
    "poimen": {
        "client_id": "poimen",
        "client_secret_env": "AUTHENTIK_PROVIDER_POIMEN_SECRET",
        "redirect_uris": ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"],
    },
    "paperless": {
        "client_id": "paperless",
        "client_secret": "6hcxaaVgZlKgafl7BxeSEtPAcbNUJxi2PAZePxSFk4o=",
        "redirect_uris": ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"],
    },
    "grafana": {
        "client_id": "grafana",
        "client_secret": "966bad4fa43812100e7775b3c73fed2ce1d07217fa5a23fbb0f190e46d2f0fa4",
        "redirect_uris": ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"],
    },
    "queue": {
        "client_id": "queue-sqs",
        "client_secret_env": "AUTHENTIK_PROVIDER_QUEUE_SECRET",
        "redirect_uris": ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"],
    },
    "forgejo": {
        "client_id": "forgejo",
        "client_secret": "G4klhs3JRfs5A7YnGs90WuOndBAvamWlZgaZRY8x",
        "redirect_uris": ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"],
    },
    "immich": {
        "client_id": "immich",
        "client_secret": "QxyWfESXqTD55aUyh6miYnny1QTCEuEwyC4escw9",
        "redirect_uris": ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"],
    },
    "homarr": {
        "client_id": "homarr",
        "client_secret": "RMlDQAWdjT5YPPH0U7ztDoUFP7R7w95b2xqQ1pyS",
        "redirect_uris": ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"],
    },
    "argocd": {
        "client_id": "argocd",
        "client_secret_env": "AUTHENTIK_PROVIDER_ARGOCD_SECRET",
        "redirect_uris": ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"],
    },
    "vault": {
        "client_id": "vault",
        "client_secret_env": "AUTHENTIK_PROVIDER_VAULT_SECRET",
        "redirect_uris": ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"],
    },
 }
 import secrets as _secrets
 for provider_name, provider_spec in OAuth2_PROVIDERS.items():
    # Resolve client_secret: explicit > env var > generate random
    if "client_secret" in provider_spec:
        client_secret = provider_spec["client_secret"]
    elif "client_secret_env" in provider_spec:
        client_secret = os.environ.get(provider_spec["client_secret_env"], _secrets.token_urlsafe(32))
    else:
        client_secret = _secrets.token_urlsafe(32)
    redirect_uris_list = [{"url": uri, "matching_mode": "strict"} for uri in provider_spec["redirect_uris"]]
    provider_payload = {
        "name": provider_name,
        "authorization_flow": auth_flow,
        "invalidation_flow": inval_flow,
        "grant_types": ["authorization_code", "implicit", "password"],
        "client_id": provider_spec["client_id"],
        "client_secret": client_secret,
        "redirect_uris": redirect_uris_list,
        "property_mappings": scope_pks_list,
    }
    if provider_name in existing_providers:
        # UPDATE existing provider — sync secret + redirect_uris
        provider_pk = existing_providers[provider_name]["pk"]
        status, res = api("PATCH", f"/api/v3/providers/oauth2/{provider_pk}/", {
            "client_id": provider_spec["client_id"],
            "client_secret": client_secret,
            "redirect_uris": redirect_uris_list,
            "property_mappings": scope_pks_list,
        })
        if status in (200, 201):
            print(f"  {provider_name}: updated (secret + redirect_uris synced)")
        else:
            print(f"  {provider_name}: UPDATE FAILED {status} {res}")
    else:
        # CREATE new provider
        status, res = api("POST", "/api/v3/providers/oauth2/", provider_payload)
        if status in (200, 201):
            print(f"  {provider_name}: created")
        else:
            print(f"  {provider_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 6: Create OAuth2 Applications (bind providers to public token endpoints)
 # ===========================================================================
 print("\n[6/7] Creating OAuth2 Applications...")
 print("     (binds providers to /application/o/token/ endpoints)")
 status, res = api("GET", "/api/v3/core/applications/?page_size=100")
 if status != 200:
    die(f"GET applications -> {status} {res}")
 existing_apps = {a["slug"]: a for a in res.get("results", [])}
 for provider_name in OAuth2_PROVIDERS.keys():
    # Get the provider PK
    status, provider_res = api("GET", f"/api/v3/providers/oauth2/?name={provider_name}")
    if status != 200 or not provider_res.get("results"):
        print(f"  {provider_name}: provider not found, skip")
        continue
    provider_pk = provider_res["results"][0]["pk"]
    if provider_name in existing_apps:
        # Ensure app is linked to provider (fix orphaned apps)
        app_data = existing_apps[provider_name]
        if app_data.get("provider") != provider_pk:
            app_uuid = app_data["pk"]
            status, res = api("PATCH", f"/api/v3/core/applications/{app_uuid}/", {
                "provider": provider_pk,
            })
            if status in (200, 201):
                print(f"  {provider_name}: re-linked to provider")
            else:
                print(f"  {provider_name}: RE-LINK FAILED {status} {res}")
        else:
            print(f"  {provider_name}: ok")
    else:
        status, res = api("POST", "/api/v3/core/applications/", {
            "name": provider_name,
            "slug": provider_name,
            "provider": provider_pk,
        })
        if status in (200, 201):
            print(f"  {provider_name}: created")
        else:
            print(f"  {provider_name}: FAILED {status} {res}")
 # ===========================================================================
 # Phase 7: Create/update rock user → homelab-admins, matching Forgejo identity
 # ===========================================================================
 print("\n[7/10] Creating/updating rock user ([email protected])...")
 ROCK_EMAIL = "[email protected]"
 ROCK_PASSWORD = os.environ.get("ROCK_PASSWORD", "")
 status, res = api("GET", "/api/v3/core/users/?username=rock")
 if status == 200 and res.get("results"):
    rock_user = res["results"][0]
    # Ensure email matches Forgejo's rock user for OIDC linking
    patch_data = {"email": ROCK_EMAIL, "name": "Rock"}
    status, res = api("PATCH", f"/api/v3/core/users/{rock_user['pk']}/", patch_data)
    if status in (200, 201):
        print(f"  rock: updated email to {ROCK_EMAIL}")
    else:
        print(f"  rock: update FAILED {status} {res}")
 else:
    if not ROCK_PASSWORD:
        print("  rock: NOT FOUND and ROCK_PASSWORD not set, skipping creation")
        print("    export ROCK_PASSWORD=<password> and re-run")
        rock_user = None
    else:
        status, res = api("POST", "/api/v3/core/users/", {
            "username": "rock",
            "name": "Rock",
            "email": ROCK_EMAIL,
            "is_active": True,
            "is_superuser": False,
            "password": ROCK_PASSWORD,
        })
        if status in (200, 201):
            rock_user = res
            print(f"  rock: created with email {ROCK_EMAIL}")
        else:
            print(f"  rock: create FAILED {status} {res}")
            rock_user = None
 if rock_user:
    status, res = api("GET", "/api/v3/core/groups/?name=homelab-admins")
    if status == 200 and res.get("results"):
        admins_group = res["results"][0]
        status, res = api("POST", f"/api/v3/core/groups/{admins_group['pk']}/users/add/", {"pk": rock_user["pk"]})
        if status in (200, 201, 204):
            print(f"  rock: added to homelab-admins")
        else:
            print(f"  rock: group add {status} {res}")
 # ===========================================================================
 # Phase 8: Email recovery flow (password reset via email)
 # ===========================================================================
 print("\n[8/10] Creating email recovery flow...")
 # Read SMTP config from gotify-smtp secret (same Gmail creds)
 SMTP_HOST = "smtp.gmail.com"
 SMTP_PORT = 587
 SMTP_USER = "[email protected]"
 SMTP_FROM = "[email protected]"
 # Password read from env at runtime: AUTHENTIK_EMAIL__PASSWORD
 # 8a. Create email stage for recovery
 status, res = api("GET", "/api/v3/stages/email/?name=email-recovery")
 if status == 200 and res.get("results"):
    email_stage_pk = res["results"][0]["pk"]
    print("  email-recovery stage: already exists")
 else:
    status, res = api("POST", "/api/v3/stages/email/", {
        "name": "email-recovery",
        "use_global_settings": False,
        "host": SMTP_HOST,
        "port": SMTP_PORT,
        "username": SMTP_USER,
        "password": os.environ.get("AUTHENTIK_EMAIL_PASSWORD", ""),
        "use_tls": True,
        "use_ssl": False,
        "timeout": 10,
        "from_address": SMTP_FROM,
        "template": "email/password_reset.html",
        "activate_user_on_success": True,
    })
    if status in (200, 201):
        email_stage_pk = res["pk"]
        print("  email-recovery stage: created")
    else:
        email_stage_pk = None
        print(f"  email-recovery stage: FAILED {status} {res}")
 # 8b. Create identification stage for recovery (email lookup)
 status, res = api("GET", "/api/v3/stages/identification/?name=recovery-identification")
 if status == 200 and res.get("results"):
    ident_stage_pk = res["results"][0]["pk"]
    print("  recovery-identification stage: already exists")
 else:
    status, res = api("POST", "/api/v3/stages/identification/", {
        "name": "recovery-identification",
        "user_fields": ["email", "username"],
    })
    if status in (200, 201):
        ident_stage_pk = res["pk"]
        print("  recovery-identification stage: created")
    else:
        ident_stage_pk = None
        print(f"  recovery-identification stage: FAILED {status} {res}")
 # 8c. Create password stage for new password entry
 status, res = api("GET", "/api/v3/stages/password/?name=recovery-password-change")
 if status == 200 and res.get("results"):
    pw_stage_pk = res["results"][0]["pk"]
    print("  recovery-password-change stage: already exists")
 else:
    # Use prompt stage for password change instead
    status, res = api("GET", "/api/v3/stages/user_write/?name=recovery-user-write")
    if status == 200 and res.get("results"):
        pw_stage_pk = res["results"][0]["pk"]
        print("  recovery-user-write stage: already exists")
    else:
        status, res = api("POST", "/api/v3/stages/user_write/", {
            "name": "recovery-user-write",
        })
        if status in (200, 201):
            pw_stage_pk = res["pk"]
            print("  recovery-user-write stage: created")
        else:
            pw_stage_pk = None
            print(f"  recovery-user-write stage: FAILED {status} {res}")
 # 8d. Create recovery flow
 status, res = api("GET", "/api/v3/flows/instances/?slug=password-recovery")
 if status == 200 and res.get("results"):
    recovery_flow_pk = res["results"][0]["pk"]
    print("  password-recovery flow: already exists")
 else:
    status, res = api("POST", "/api/v3/flows/instances/", {
        "name": "Password Recovery",
        "slug": "password-recovery",
        "title": "Reset your password",
        "designation": "recovery",
    })
    if status in (200, 201):
        recovery_flow_pk = res["pk"]
        print("  password-recovery flow: created")
    else:
        recovery_flow_pk = None
        print(f"  password-recovery flow: FAILED {status} {res}")
 # 8e. Bind stages to flow in order
 if recovery_flow_pk and ident_stage_pk and email_stage_pk:
    for order, stage_pk, label in [
        (10, ident_stage_pk, "identification"),
        (20, email_stage_pk, "email"),
    ]:
        status, res = api("POST", "/api/v3/flows/bindings/", {
            "target": recovery_flow_pk,
            "stage": stage_pk,
            "order": order,
        })
        if status in (200, 201):
            print(f"  bound {label} stage at order {order}")
        elif status == 400 and "already exists" in str(res).lower():
            print(f"  {label} stage: already bound")
        else:
            print(f"  bind {label}: {status} {res}")
 # ===========================================================================
 # Phase 9: Set recovery flow on brand
 # ===========================================================================
 print("\n[9/10] Setting recovery flow on brand...")
 if recovery_flow_pk:
    status, res = api("GET", "/api/v3/brands/instances/")
    if status == 200 and res.get("results"):
        brand = res["results"][0]
        status, res = api("PATCH", f"/api/v3/brands/instances/{brand['brand_uuid']}/", {
            "flow_recovery": recovery_flow_pk,
        })
        if status in (200, 201):
            print("  recovery flow set on brand")
        else:
            print(f"  FAILED {status} {res}")
    else:
        print("  no brand found")
 # ===========================================================================
 # Phase 10: Ensure Forgejo OAuth2 source uses matching email claim
 # ===========================================================================
 print("\n[10/10] Verifying Forgejo OIDC linkage...")
 print(f"  rock@Authentik email: {ROCK_EMAIL}")
 print("  Forgejo OIDC will match on email — ensure Forgejo's rock user")
 print(f"  has email {ROCK_EMAIL} in Forgejo settings → Profile")
 # ===========================================================================
 # Summary
 # ===========================================================================
 print("\n" + "="*70)
 print("AUTHENTIK PROVISIONING COMPLETE")
 print("="*70)
 print(f"\n  [1] Groups: {len(GROUPS)}")
 print(f"  [2] Service accounts: {len(SERVICE_ACCOUNTS)}")
 print(f"  [3] Scope mappings: {len(SCOPE_MAPPINGS)}")
 print(f"  [4] Flows resolved")
 print(f"  [5] OAuth2 providers: {len(OAuth2_PROVIDERS)}")
 print(f"  [6] OAuth2 applications bound")
 print(f"  [7] rock user ([email protected]) -> homelab-admins")
 print(f"  [8] Email recovery flow (smtp.gmail.com)")
 print(f"  [9] Recovery flow set on brand")
 print(f"  [10] Forgejo OIDC linkage verified")
 print("\nNEXT: Set Forgejo rock user email to [email protected] in Forgejo profile")
 print("TEST: https://authentik.riotpiao.com/if/flow/password-recovery/")
 print("="*70)
@@ -53,7 +53,7 @@ generate_secret() {
 declare -A NEW_SECRETS
-for svc in api-gw minio poimen paperless grafana argocd forgejo immich vault portfolio-agent memory-agent local-llm; do
+for svc in api-gw minio poimen paperless grafana argocd forgejo homarr immich vault portfolio-agent memory-agent local-llm; do
  NEW_SECRETS[$svc]=$(generate_secret)
  log "  $svc: ${NEW_SECRETS[$svc]:0:15}..."
 done
@@ -70,6 +70,7 @@ declare -A PROVIDER_PKS=(
  [grafana]=6
  [argocd]=7
  [forgejo]=8
  [homarr]=9
  [immich]=10
  [vault]=11
 )
@@ -202,7 +203,7 @@ fi
 log ""
 log "=== Summary ==="
 log "Rotated 13 credentials:"
-log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, immich, vault"
+log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, homarr, immich, vault"
 log "  Service Accounts: portfolio-agent, memory-agent, local-llm"
 log ""
@@ -49,10 +49,6 @@ CONF_DIR = Path(os.environ.get("ZITADEL_CONF_DIR", Path.home() / ".config" / "zi
 STATE_FILE = CONF_DIR / "state.json"
 NAMESPACE = os.environ.get("ZITADEL_NAMESPACE", "iam")
 BASE_URL = os.environ.get("ZITADEL_BASE_URL", "https://zitadel.riotpiao.com")
 # Notification delivery target: the in-cluster SMTP -> Gotify relay.
 SMTP_HOST = os.environ.get("ZITADEL_SMTP_HOST", "smtp-gotify.iam.svc.cluster.local:2525")
 SMTP_FROM = os.environ.get("ZITADEL_SMTP_FROM", "[email protected]")
 SMTP_FROM_NAME = os.environ.get("ZITADEL_SMTP_FROM_NAME", "ZITADEL")
 # ── desired ZITADEL state ──────────────────────────────────────────────────
 # Declared here, applied through the API by the `apply` phase. This replaces the
@@ -84,10 +80,6 @@ ROLES = {
    "sqs-writers": "SQS/Temporal queue read/write access",
    "s3-users": "S3 read access",
    "s3-writers": "S3 read/write access",
    # LLM gate for the api-gw. The colon form matches the Authentik-era
    # permission string, so consumer manifests did not change.
    "llm:inference": "LLM inference access (api-gw)",
    "llm-inference": "LLM inference access (dash alias)",
 }
 _WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"}
@@ -100,17 +92,15 @@ OIDC_APPS = {
    "argocd": {**_WEB, "redirect_uris": [
        "http://localhost:8080/auth/callback",
        "https://argocd.riotpiao.com/auth/callback"]},
    # Forgejo derives the callback path from the login-source NAME
    # (/user/oauth2/<name>/callback). Both spellings are registered so a
    # rename between authentik and zitadel never breaks sign-in.
    "forgejo": {**_WEB, "redirect_uris": [
        "http://localhost:3000/user/oauth2/zitadel/callback",
        "https://forgejo.riotpiao.com/user/oauth2/zitadel/callback",
        "http://localhost:3000/user/oauth2/authentik/callback",
        "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
    "grafana": {**_WEB, "redirect_uris": [
        "http://localhost:3000/login/generic_oauth",
        "https://grafana.riotpiao.com/login/generic_oauth"]},
    "homarr": {**_WEB, "redirect_uris": [
        "http://localhost:7575/auth/callback",
        "https://homarr.riotpiao.com/auth/callback"]},
    "immich": {**_WEB, "app_type_id": "OIDC_APP_TYPE_NATIVE", "redirect_uris": [
        "app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login",
        "https://img.riotpiao.com/user/oauth2/callback"]},
@@ -120,8 +110,6 @@ OIDC_APPS = {
        "http://localhost:9000/auth/sso/oauth2/code",
        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
    "paperless": {**_WEB, "redirect_uris": [
        "http://localhost:8000/accounts/oidc/zitadel/login/callback/",
        "https://paperless.riotpiao.com/accounts/oidc/zitadel/login/callback/",
        "http://localhost:8000/accounts/oidc/authentik/login/callback/",
        "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
    "poimen": {**_WEB, "redirect_uris": [
@@ -281,52 +269,18 @@ def _api(method: str, path: str, token: str, body: dict | None = None,
 def _ensure_project(token: str, name: str) -> str:
    code, d = _api("POST", "/management/v1/projects/_search", token, {})
    project_id = ""
    for row in d.get("result") or []:
        if row.get("name") == name:
-            project_id = row["id"]
+            return row["id"]
            break
    if not project_id:
    code, d = _api("POST", "/management/v1/projects", token, {"name": name})
    if code not in (200, 201) or not d.get("id"):
        die(f"create project {name} failed: {code} {d}")
-        project_id = d["id"]
+    return d["id"]
    # projectRoleAssertion puts the project roles into the ACCESS token as
    # `urn:zitadel:iam:org:project:roles`. Without it the api-gw sees no
    # llm:inference role and answers 403 for every inference call, even when
    # the caller is correctly authorised in ZITADEL.
    code, d = _api("PUT", f"/management/v1/projects/{project_id}",
                   token, {"name": name, "projectRoleAssertion": True})
    if code not in (200, 201):
        die(f"enable projectRoleAssertion on {name} failed: {code} {d}")
    return project_id
 def _ensure_builtin_login(token: str) -> None:
    """Force the built-in login page instead of the hosted Login UI.
    The hosted UI authenticates as the machine user `login-client`, which the
    chart creates only from its one-shot setup job. When that user is missing,
    every session write fails with "Could not create session for user" and no
    one can sign in, at ZITADEL or at any relying party.
    Disabling the loginV2 feature makes the server serve its own login page,
    which needs no machine user. This is instance state in the database, not a
    values key, so it must be applied explicitly.
    """
    code, d = _api("PUT", "/v2/features/instance", token, {"loginV2": {"required": False}})
    if code not in (200, 201):
        die(f"disable loginV2 failed: {code} {d}")
    print("login page    : built-in (loginV2.required=false)")
 def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
    code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
-    # ZITADEL answers with `key`, not `roleKey`. Reading the wrong field left
+    have = {r.get("roleKey") for r in (d.get("result") or [])}
    # this set empty, so every re-run tried to recreate existing roles and died
    # on the first 409.
    have = {(r.get("key") or r.get("roleKey")) for r in (d.get("result") or [])}
    for key, display in roles.items():
        if key in have:
            continue
@@ -375,95 +329,12 @@ def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tup
    return ids, secrets
 def _ensure_smtp(token: str) -> str:
    """Make sure the instance has one active SMTP provider.
    ZITADEL sends verification codes and password resets by SMTP only. A
    freshly added config is created INACTIVE (state 3), and an inactive config
    makes GetSMTPConfig answer 404 "SMTP configuration not found", so nothing is
    ever delivered. Activating it is what enables sending: after the activate
    call the same GET returns the config and a resend reaches the relay.
    This is instance state in the database, not a values key, so it must be
    applied explicitly. The host points at the in-cluster smtp-gotify relay
    (k8s/infra/iam/smtp-gotify.yaml), which forwards to Gotify.
    """
    code, d = _api("GET", "/admin/v1/smtp", token)
    if code == 200 and (d.get("smtpConfig") or {}).get("id"):
        smtp_id = d["smtpConfig"]["id"]
        print(f"smtp      : already active ({smtp_id})")
        return smtp_id
    code, d = _api("POST", "/admin/v1/smtp", token, {
        "senderAddress": SMTP_FROM,
        "senderName": SMTP_FROM_NAME,
        "host": SMTP_HOST,
        "tls": False,
    })
    if code not in (200, 201) or not d.get("id"):
        die(f"create smtp config failed: {code} {d}")
    smtp_id = d["id"]
    # The activate call is mandatory: without it the config stays inactive and
    # every send is dropped while the API still reports success.
    code, d = _api("POST", f"/admin/v1/smtp/{smtp_id}/_activate", token, {})
    if code not in (200, 201):
        die(f"activate smtp config {smtp_id} failed: {code} {d}")
    code, d = _api("GET", "/admin/v1/smtp", token)
    if code != 200 or not (d.get("smtpConfig") or {}).get("id"):
        die(f"smtp config {smtp_id} still not visible after activate: {code} {d}")
    print(f"smtp      : active ({smtp_id}) -> {SMTP_HOST}")
    return smtp_id
 def _ensure_login_policy(token: str) -> None:
    """Require password + a second factor, and demand enrolment at login.
    Enforcement needs forceMfa. The v1 endpoint ignores the spelling "forceMFA"
    as an unknown field, which is why earlier runs looked successful while
    projections.login_policies5 kept force_mfa = false.
    Second factors stay OTP and U2F: an authenticator app or a hardware key.
    Multi-factor is left configured for U2F with verification so a later step
    can move 2FA to 2MFA without another policy change.
    """
    code, d = _api("PUT", "/admin/v1/policies/login", token, {
        # Case matters: protojson accepts "forceMfa", the lowerCamel form of
        # force_mfa. Sending "forceMFA" is silently ignored as an unknown field,
        # so the policy looks unchanged and MFA is never enforced.
        "forceMfa": True,
        "forceMfaLocalOnly": True,
        "allowUsernamePassword": True,
        "allowRegister": False,
        "allowExternalIDP": True,
        "hidePasswordReset": False,
        "ignoreUnknownUsernames": True,
        "passwordCheckLifetime": "864000s",
        "externalLoginCheckLifetime": "864000s",
        "mfaInitSkipLifetime": "0s",
        "secondFactorCheckLifetime": "64800s",
        "multiFactorCheckLifetime": "64800s",
        "secondFactors": ["SECOND_FACTOR_TYPE_OTP", "SECOND_FACTOR_TYPE_U2F"],
        "multiFactors": ["MULTI_FACTOR_TYPE_U2F_WITH_VERIFICATION"],
        "passwordlessType": "PASSWORDLESS_TYPE_ALLOWED",
    })
    # ZITADEL answers 400 "has not been changed" when the policy already matches.
    # That is success for an idempotent run, not a failure.
    if code not in (200, 201) and "has not been changed" not in str(d.get("message", "")):
        die(f"update login policy failed: {code} {d}")
    print("login policy : password + 2FA, enrolment required")
 def phase_apply(_args: argparse.Namespace) -> int:
    """Declare the ZITADEL objects through the API. No Terraform, no state drift."""
    token = bootstrap_token()
    if not token:
        die("no iam-admin PAT — run the keys phase first")
    _ensure_builtin_login(token)
    _ensure_smtp(token)
    _ensure_login_policy(token)
    project_id = _ensure_project(token, PROJECT_NAME)
    print(f"project   : {PROJECT_NAME} ({project_id})")
    _ensure_roles(token, project_id, ROLES)
@@ -491,6 +362,10 @@ def phase_apply(_args: argparse.Namespace) -> int:
 SECRET_MAP = {
    "grafana": [("grafana-oidc-secrets.enc.yaml", "grafana-oidc",
                 "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "secret")],
    "homarr": [("homarr-auth-oidc.enc.yaml", "homarr-oidc",
                "client-id", "id"),
               ("homarr-auth-oidc.enc.yaml", "homarr-oidc",
                "client-secret", "secret")],
    "minio": [("minio-secrets.enc.yaml", "minio-oidc",
               "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "secret")],
    "paperless": [("paperless-ai-secrets.enc.yaml", "paperless-ai-config",
@@ -22,6 +22,7 @@
    # TLS termination (cert-manager cert) and preserve the Host header. Routing
    # via nginx (not the backend service) is critical: a direct rewrite bypasses
    # nginx TLS and exposes each app's own self-signed cert, trusted by nothing.
    rewrite name authentik.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name zitadel.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name grafana.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
    rewrite name minio.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local