refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)

2026-07-23 20:54:02 -07:00
parent 1c7395d9e1
commit e6f2ab1423
241 changed files with 467 additions and 10416 deletions
@@ -0,0 +1,167 @@
+# Authentik OAuth provisioning — PostSync hook, reruns on every ArgoCD sync
+# (hook-delete-policy: BeforeHookCreation deletes the previous run's Job before
+# creating a new one, so this stays reconciled the same way the rest of the
+# cluster does — no separate manual bootstrap step like setup_talos_iam.sh /
+# provision_oidc.py, which never got migrated off the old helmfile workflow).
+#
+# What it does (see scripts/authentik-provision.py docstring): creates the
+# "groups" scope mapping, homelab-admins / grafana-admins groups, the "rock"
+# admin user, OAuth2 providers + Applications for grafana/minio/forgejo/argocd,
+# and binds homelab-admins to all of them. The script is generated into the
+# authentik-provision-script ConfigMap by kustomize configMapGenerator (see
+# kustomization.yaml), not embedded here.
+#
+# RBAC: this Job only touches Secrets (get existing client secrets, create new
+# ones for forgejo/argocd/rock) across the namespaces those services live in.
+# It never touches any other resource type.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: authentik-provisioner
+  namespace: iam
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: authentik-provisioner
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "create", "update", "patch"]
+---
+# One RoleBinding per namespace the script touches (least-privilege: Secrets
+# only, and only in these 5 namespaces — not a cluster-wide ClusterRoleBinding).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: iam
+subjects:
+  - kind: ServiceAccount
+    name: authentik-provisioner
+    namespace: iam
+roleRef:
+  kind: ClusterRole
+  name: authentik-provisioner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: cicd
+subjects:
+  - kind: ServiceAccount
+    name: authentik-provisioner
+    namespace: iam
+roleRef:
+  kind: ClusterRole
+  name: authentik-provisioner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: argocd
+subjects:
+  - kind: ServiceAccount
+    name: authentik-provisioner
+    namespace: iam
+roleRef:
+  kind: ClusterRole
+  name: authentik-provisioner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: logging
+subjects:
+  - kind: ServiceAccount
+    name: authentik-provisioner
+    namespace: iam
+roleRef:
+  kind: ClusterRole
+  name: authentik-provisioner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: storage
+subjects:
+  - kind: ServiceAccount
+    name: authentik-provisioner
+    namespace: iam
+roleRef:
+  kind: ClusterRole
+  name: authentik-provisioner
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: authentik-provision
+  namespace: iam
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  ttlSecondsAfterFinished: 600
+  backoffLimit: 3
+  template:
+    spec:
+      serviceAccountName: authentik-provisioner
+      restartPolicy: Never
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: provision
+          image: python:3.12-alpine
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - name: AUTHENTIK_BOOTSTRAP_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: authentik-secrets
+                  key: AUTHENTIK_BOOTSTRAP_TOKEN
+          volumeMounts:
+            - name: script
+              mountPath: /script
+          command:
+            - /bin/sh
+            - -c
+            - |
+              set -e
+              echo "waiting for authentik-server..."
+              until wget -q -O /dev/null http://authentik-server.iam.svc.cluster.local/-/health/ready/ 2>/dev/null; do
+                sleep 5
+              done
+              echo "installing kubectl (via python urllib - no apk/curl: this"
+              echo "container runs as non-root UID 1000 and can't write to"
+              echo "apk's directories or /usr/local/bin, both root-owned in"
+              echo "the python:3.12-alpine image; /tmp is world-writable)..."
+              python3 -c "
+              import urllib.request, os, stat
+              kver = urllib.request.urlopen('https://dl.k8s.io/release/stable.txt').read().decode().strip()
+              url = f'https://dl.k8s.io/release/{kver}/bin/linux/amd64/kubectl'
+              urllib.request.urlretrieve(url, '/tmp/kubectl')
+              st = os.stat('/tmp/kubectl')
+              os.chmod('/tmp/kubectl', st.st_mode | stat.S_IEXEC)
+              "
+              export PATH="/tmp:$PATH"
+              echo "running provisioning script..."
+              python3 /script/authentik-provision.py
+      volumes:
+        - name: script
+          configMap:
+            name: authentik-provision-script
@@ -0,0 +1,20 @@
+authentik:
+    secret_key: ENC[AES256_GCM,data:55ne/khf01ZD3FP2Zek+0Ar9C+GP/GPf33PwOwgXdnOXF4kLrEwwJ/E7yrH0nkBKGO221ClGBjlo7kb5vGMmmmg8TVBa1upQbk8b0s723n8=,iv:ZkDp4gOHY1eH44cDhGaZILi8dFDTNuYf1CVE+qrapzE=,tag:JTE4tfMJSowsn9YZ0xKFng==,type:str]
+    bootstrap_password: ENC[AES256_GCM,data:vBk/ivCG4x2TlvURrDxHAiaZlxrF+PeHBAuoZR6muWw=,iv:ccA703HMomR58cD2/6wK1W0IKJb7U1hNws6fmwG15E8=,tag:MdhTx9ebXLjRCnDpUxpJ0g==,type:str]
+    bootstrap_token: ENC[AES256_GCM,data:NwGZzaL8JufXYp6sjeeN1etdgaHT8IuaZFLZJegzs+fPXMvVCGPKKlStythIOa3gwgCXkhA+kV8+jyxYKr1UMw==,iv:EJCetXnkNe+UuhWk4f3vr3kIEbCuVplVKHPne3aVIQ8=,tag:kwlE+9O6DczO+mn9Rqx+Rw==,type:str]
+    postgresql_password: ENC[AES256_GCM,data:+WeZKo+24awwWfTAt5q6KJRkqaHD7YmlnKm4oGaUNvw=,iv:VSJ3d8p4f5SQM2lVdhSLY5dqdQTKwcfzOCATIM5M/cw=,tag:6K8mQmyLEg7KJi3bg+zPig==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJSDJSYmRQNDdjWjlORmJX
+            Zm9yd2xRNi9XKy95Y2NQWUMyTGxjb1NjRTNzCnlNbko5NVJVSitiaE1DQ2QvMVoy
+            czRya01EaFZmZzAzcjRESG5NY1JIVTAKLS0tIFZ1UkREaGc4VUczMjFITWR6N0Ez
+            NStJZmpuWEw1dlRFcExZUlFLWkpBSFkK9r0NG3IKV7+AU00VXVCuHu+aBOOLydD0
+            ncioyDrJWgkyDxn9+BNZh4vX8vEERANYh+1/3P99Ubz28tifhmi2vg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1smu533f803gmd0jq60s2zaj9zlznajy0ca6rtewd4r37mr2hs3uqsrldfh
+    lastmodified: "2026-07-15T23:22:55Z"
+    mac: ENC[AES256_GCM,data:+MqbhlVnhriigDEj/AMJj4yLjcIjxkQc4QANcVeAXefcljHS4yOVtrwh2js3l9WVyIgVdl0MdlW+ycykBU4Pi3u4sdJoiJd945wZDsHvFCREYZIZdsVW/EHmKjIroqxQfeEtbWDStkaE+WTF/yGrZ2HLLztrXIafxZq7KhbXlo0=,iv:PCzhd8j539BxvpQQCds1vMXm3gfmlvSnrbtfOpFaKrY=,tag:NiHwSfQRvf//lpjFAKA1Jg==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2
@@ -0,0 +1,275 @@
+# k8s/talos-iam/authentik-values.yaml
+# Authentik — SSO Identity Provider for the homelab.
+# Provides OAuth2/OIDC login for Grafana, MinIO, Forgejo, and Argo CD.
+# Chart: authentik/authentik from https://charts.goauthentik.io
+#
+# Architecture: server (UI+API) + worker (background tasks) + PostgreSQL + Redis.
+# PostgreSQL is the system of record — must persist. Redis is ephemeral cache/queue.
+#
+# Secrets injected via helmfile --set (from .env / vsource):
+#   AUTHENTIK_SECRET_KEY         — signs sessions and tokens; set once, never rotate casually
+#   AUTHENTIK_BOOTSTRAP_PASSWORD — initial akadmin password (used once at first login)
+#   AUTHENTIK_BOOTSTRAP_TOKEN    — API token for the setup_talos_iam.sh bootstrap script
+#   AUTHENTIK_PG_PASSWORD        — PostgreSQL user password
+
+authentik:
+  # host: the external URL Authentik uses to build redirect URIs in OAuth2 flows.
+  # Must match what the browser sees — if it returns an internal svc URL,
+  # the browser's redirect after login will fail (can't reach svc DNS externally).
+  # HTTP (not HTTPS) because the Authentik ingress has no TLS cert configured.
+  host: "https://authentik.riotpiao.com"
+
+  error_reporting:
+    enabled: false   # do not phone home to Sentry
+
+  # PostgreSQL connection — points at CloudNativePG cluster in ddb namespace.
+  # Uses 'app' bootstrap user (CNPG simple pattern, same as Forgejo).
+  # Credentials injected from authentik-db-app secret via env vars below.
+  postgresql:
+    host: authentik-db-rw.iam.svc.cluster.local
+    port: 5432
+    name: authentik
+    user: app  # All apps use shared 'app' user (CNPG design pattern)
+    password: ""  # overridden by AUTHENTIK_POSTGRESQL__PASSWORD env var
+
+  # Redis connection — bundled subchart, standalone mode (no sentinel/cluster).
+  redis:
+    host: authentik-redis-master
+
+  # ── HTTP client timeouts ──────────────────────────────────────────────────────
+  # Increased to tolerate 5+ second pod-to-pod network latency.
+  # Affects webhooks, outpost management, SCIM, LDAP sync.
+  # Default: ~30s — too aggressive when latency spikes hit 5-10s.
+  log_level: debug  # enable debug logging to monitor connection issues
+
+# ── CA trust (shared by server and worker) ────────────────────────────────────
+# Authentik (Python/Debian) uses requests + httpx for outgoing HTTPS — webhooks,
+# outpost management, SCIM. Both libraries need REQUESTS_CA_BUNDLE / SSL_CERT_FILE
+# to point to a bundle that includes homelab-ca, otherwise connections to other
+# homelab services fail with "certificate signed by unknown authority".
+#
+# Strategy: a debian:12-slim init container (run as root) concatenates the
+# Debian system Mozilla bundle with homelab-ca.crt into an emptyDir. The main
+# container then references /merged/ca-bundle.crt via two env vars that cover
+# every Python HTTP library.
+_caVolumes: &caVolumes
+  - name: homelab-ca
+    configMap:
+      name: homelab-ca
+  - name: merged-ca
+    emptyDir: {}
+
+_caVolumeMounts: &caVolumeMounts
+  - name: homelab-ca
+    mountPath: /homelab-ca
+    readOnly: true
+  - name: merged-ca
+    mountPath: /merged
+    readOnly: true
+
+_caInitContainers: &caInitContainers
+  - name: merge-ca-certs
+    image: debian:bookworm
+    imagePullPolicy: IfNotPresent
+    securityContext:
+      runAsUser: 0
+    command:
+      - sh
+      - -c
+      - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
+    volumeMounts:
+      - name: homelab-ca
+        mountPath: /homelab-ca
+        readOnly: true
+      - name: merged-ca
+        mountPath: /merged
+  # NOTE: no authentik-migrate init container — the authentik `server` entrypoint
+  # runs migrations itself on startup. A separate `manage migrate` init pinned to
+  # an older image tripped a version-history precheck on an empty DB
+  # (relation "authentik_version_history" does not exist) and blocked boot.
+
+_caEnv: &caEnv
+  - name: REQUESTS_CA_BUNDLE
+    value: /merged/ca-bundle.crt
+  - name: SSL_CERT_FILE
+    value: /merged/ca-bundle.crt
+
+# ── Authentik server (UI + API) ───────────────────────────────────────────────
+# Handles all browser traffic: login flows, admin UI, OAuth2 authorize/token endpoints.
+# NodePort 32172 is a fallback for direct node access during troubleshooting;
+# normal access is via nginx ingress (authentik.riotpiao.com → svc:80).
+# Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
+server:
+  replicas: 1
+  # Merge SOPS-CMP-emitted secret values after the chart's own `authentik` secret.
+  envFrom:
+    - secretRef:
+        name: authentik-secrets
+  deploymentStrategy:
+    type: Recreate
+  service:
+    type: NodePort
+    nodePort: 32172
+  resources:
+    requests:
+      cpu: 100m
+      memory: 512Mi
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  volumes: *caVolumes
+  volumeMounts: *caVolumeMounts
+  initContainers: *caInitContainers
+  env:
+    # Merge CA trust env vars
+    - name: REQUESTS_CA_BUNDLE
+      value: /merged/ca-bundle.crt
+    - name: SSL_CERT_FILE
+      value: /merged/ca-bundle.crt
+    # Override database credentials to use 'app' from authentik-db-app
+    - name: AUTHENTIK_POSTGRESQL__USER
+      valueFrom:
+        secretKeyRef:
+          name: authentik-db-app
+          key: username
+    - name: AUTHENTIK_POSTGRESQL__PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: authentik-db-app
+          key: password
+  podAnnotations:
+    configmap.reloader.stakater.com/reload: "homelab-ca"
+    homelab.io/restart-at: "2026-06-21T13-40"
+  # The /-/health/{live,ready}/ endpoints do a DB round-trip; under transient
+  # CNPG contention they respond in 5-6s while still returning 200. The chart's
+  # default 3s liveness timeout then flags a working backend as dead and kubelet
+  # kills it in a restart loop — the pod never stays Ready, gets dropped from the
+  # authentik-server Service endpoints, and the OAuth-provisioning PostSync hook
+  # fails with "Host is unreachable". Widen the timeouts so slow-but-healthy
+  # checks aren't treated as failures. (Only these fields are overridden; the
+  # chart deep-merges the rest of each probe, incl. the httpGet path.)
+  livenessProbe:
+    timeoutSeconds: 15
+    failureThreshold: 6
+  readinessProbe:
+    timeoutSeconds: 15
+    failureThreshold: 6
+  startupProbe:
+    timeoutSeconds: 15
+    failureThreshold: 120  # 120 × 10s = 20min for fresh DB migrations
+  # Every OIDC login (Grafana, Argo CD, MinIO, Forgejo) depends on this server —
+  # its request latency/error rate explains SSO-driven slowness on those services.
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      scrapeTimeout: 30s
+
+# ── Authentik worker ──────────────────────────────────────────────────────────
+# Runs background tasks: email delivery, LDAP sync, flow policy evaluation,
+# event log cleanup, and managed outpost updates. Stateless — no PVC needed.
+# Same resource profile as server; Authentik 2023+ merged some worker duties
+# into the server process but the worker pod is still required.
+worker:
+  replicas: 1
+  envFrom:
+    - secretRef:
+        name: authentik-secrets
+  deploymentStrategy:
+    type: Recreate
+  resources:
+    requests:
+      cpu: 100m
+      memory: 512Mi
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  volumes: *caVolumes
+  volumeMounts: *caVolumeMounts
+  initContainers: *caInitContainers
+  env:
+    # Merge CA trust env vars
+    - name: REQUESTS_CA_BUNDLE
+      value: /merged/ca-bundle.crt
+    - name: SSL_CERT_FILE
+      value: /merged/ca-bundle.crt
+    # Override database credentials to use 'app' from authentik-db-app
+    - name: AUTHENTIK_POSTGRESQL__USER
+      valueFrom:
+        secretKeyRef:
+          name: authentik-db-app
+          key: username
+    - name: AUTHENTIK_POSTGRESQL__PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: authentik-db-app
+          key: password
+  podAnnotations:
+    configmap.reloader.stakater.com/reload: "homelab-ca"
+    homelab.io/restart-at: "2026-06-21T13-40"
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      scrapeTimeout: 30s
+
+# ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ─────────────
+# Authentik connects to the dedicated authentik-db (1 primary + 2 replicas).
+# Do not use the bundled Bitnami subchart — CNPG is already running.
+postgresql:
+  enabled: false
+  primary:
+    persistence:
+      enabled: true
+      storageClass: longhorn
+      size: 8Gi
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: node-role.kubernetes.io/worker
+                  operator: Exists
+
+# ── Bundled Redis ─────────────────────────────────────────────────────────────
+# Cache and async task queue only — no durable data. If Redis restarts, in-flight
+# background tasks are retried and cached tokens are recomputed. Losing Redis
+# data does not lose user accounts or flow configuration (that's in PostgreSQL).
+# persistence: false saves a PVC and makes restarts faster.
+#
+# Same prefer-worker / fallback-to-cp scheduling as PostgreSQL.
+# architecture: standalone — no Sentinel/cluster overhead for a 3-node homelab.
+redis:
+  enabled: true
+  master:
+    persistence:
+      enabled: false
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: node-role.kubernetes.io/worker
+                  operator: Exists
+  architecture: standalone
+
+# Ingress disabled — rule lives in k8s/ingress/ingress.yaml (authentik.riotpiao.com).
+# For direct access during bootstrap: kubectl -n iam port-forward svc/authentik-server 7000:80
@@ -0,0 +1,35 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# NOTE: no top-level `namespace:` transformer here (removed) - it used to
+# force-rewrite metadata.namespace to "iam" on every resource in this
+# kustomization, which was harmless while every manifest here only ever
+# targeted the iam namespace itself. authentik-provision-job.yaml's
+# RoleBindings deliberately target cicd/argocd/logging/storage (least-
+# privilege access for the authentik-provisioner ServiceAccount to touch
+# Secrets in those namespaces) - the namespace transformer would have
+# silently rewritten all of them back to iam, breaking the RBAC. Every
+# manifest in this directory already sets its own explicit
+# metadata.namespace, so dropping the transformer changes nothing for the
+# existing resources/.
+resources:
+  - authentik-provision-job.yaml
+  - rbac-dashboard-rolebinding.yaml
+
+# Provisioning/verification python lives in scripts/*.py (real files, linted +
+# diff-friendly) and is generated into ConfigMaps here rather than embedded in
+# the job YAML. disableNameSuffixHash keeps the names stable so the Jobs'
+# configMap volume refs and PostSync hook-delete semantics keep working; each
+# hook Job is recreated per sync so it always mounts the latest script.
+configMapGenerator:
+  - name: authentik-provision-script
+    namespace: iam
+    files:
+      - authentik-provision.py=scripts/authentik-provision.py
+
+generatorOptions:
+  disableNameSuffixHash: true
+# authentik-migrations-job.yaml removed — redundant + broken. The authentik
+# `server` entrypoint runs migrations itself; this standalone job lacked the
+# authentik-secrets envFrom (Secret key missing) and always failed.
+# SOPS secrets (*.enc.yaml) handled by ArgoCD SOPS plugin at sync time
+# authentik/vault deployed via ArgoCD Helm source
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: authentik-provisioner
+  namespace: dashboard
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: authentik-provisioner
+subjects:
+- kind: ServiceAccount
+  name: authentik-provisioner
+  namespace: iam
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""
+Authentik OAuth provisioning - idempotent, safe to re-run (ArgoCD PostSync hook).
+
+Creates/updates, in order:
+  1. A custom "groups" OAuth2 scope mapping (Authentik ships openid/email/profile
+     by default but NOT groups - required for ArgoCD RBAC group mapping and
+     Grafana's role_attribute_path, both of which read a `groups` claim).
+  2. Groups: homelab-admins (is_superuser=true), grafana-admins.
+  3. User "rock": created if missing, always (re-)synced into both groups above.
+     Password is generated once and only written to the k8s Secret
+     rock-credentials (iam ns) the first time the user is created - re-runs
+     never rotate an existing password.
+  4. OAuth2/OIDC providers + Applications for: grafana, minio, forgejo, argocd.
+     Client secrets are read from existing k8s Secrets (grafana-oidc, minio-oidc)
+     if present, or generated once and written out (forgejo-oidc, oidc-secret)
+     the first time.
+  5. PolicyBinding of homelab-admins -> every Application above, so "rock" (and
+     anyone else in that group) has guaranteed access regardless of each app's
+     default visibility.
+
+Talks to Authentik over the in-cluster Service (authentik-server.iam.svc:80),
+authenticating with the bootstrap token. Everything is done with GET-then-
+create-or-patch so this can be re-run on every ArgoCD sync without duplicating
+or clobbering objects (PostSync hook, not a one-shot Job with hook-delete).
+
+kubectl is used only to read/write the small set of Secrets this script
+touches - it shells out rather than using the Python k8s client to keep the
+container image to stdlib Python + the kubectl binary, no pip installs.
+"""
+import json
+import os
+import secrets
+import string
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+
+AUTHENTIK_URL = "http://authentik-server.iam.svc.cluster.local"
+TOKEN = os.environ["AUTHENTIK_BOOTSTRAP_TOKEN"]
+
+
+def api(method, path, data=None):
+    url = f"{AUTHENTIK_URL}{path}"
+    body = json.dumps(data).encode() if data is not None else None
+    req = urllib.request.Request(
+        url,
+        data=body,
+        method=method,
+        headers={
+            "Authorization": f"Bearer {TOKEN}",
+            "Content-Type": "application/json",
+        },
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            raw = resp.read()
+            return resp.status, (json.loads(raw) if raw else {})
+    except urllib.error.HTTPError as e:
+        raw = e.read()
+        try:
+            parsed = json.loads(raw) if raw else {}
+        except json.JSONDecodeError:
+            parsed = {"raw": raw.decode(errors="replace")}
+        return e.code, parsed
+
+
+def die(msg):
+    print(f"FATAL: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def gen_secret(n=40):
+    alphabet = string.ascii_letters + string.digits
+    return "".join(secrets.choice(alphabet) for _ in range(n))
+
+
+def kubectl_get_secret_key(namespace, name, key):
+    """Returns decoded value, or None if the secret/key doesn't exist."""
+    p = subprocess.run(
+        ["kubectl", "-n", namespace, "get", "secret", name, "-o", f"jsonpath={{.data.{key}}}"],
+        capture_output=True, text=True,
+    )
+    if p.returncode != 0 or not p.stdout.strip():
+        return None
+    import base64
+    return base64.b64decode(p.stdout).decode()
+
+
+def kubectl_create_secret(namespace, name, literals: dict):
+    """Idempotent: create-or-update via dry-run|apply, same pattern used
+    elsewhere in this repo (setup_vault.sh, apply-vault-secrets.sh)."""
+    args = ["kubectl", "-n", namespace, "create", "secret", "generic", name]
+    for k, v in literals.items():
+        args += [f"--from-literal={k}={v}"]
+    args += ["--dry-run=client", "-o", "yaml"]
+    render = subprocess.run(args, capture_output=True, text=True)
+    if render.returncode != 0:
+        die(f"rendering secret {namespace}/{name}: {render.stderr}")
+    apply = subprocess.run(["kubectl", "apply", "-f", "-"], input=render.stdout,
+                            capture_output=True, text=True)
+    if apply.returncode != 0:
+        die(f"applying secret {namespace}/{name}: {apply.stderr}")
+    print(f"  secret {namespace}/{name}: {apply.stdout.strip()}")
+
+
+def get_or_create(list_path, create_path, query, payload, patch_existing=None):
+    status, res = api("GET", f"{list_path}?{query}")
+    if status != 200:
+        die(f"GET {list_path}?{query} -> {status} {res}")
+    results = res.get("results", [])
+    if results:
+        obj = results[0]
+        if patch_existing:
+            status, obj2 = api("PATCH", f"{create_path}{obj['pk']}/", patch_existing)
+            if status not in (200, 201):
+                die(f"PATCH {create_path}{obj['pk']}/ -> {status} {obj2}")
+            return obj2
+        return obj
+    status, obj = api("POST", create_path, payload)
+    if status not in (200, 201):
+        die(f"POST {create_path} -> {status} {obj}")
+    return obj
+
+
+# -----------------------------------------------------------------------------
+print("[1/5] Ensuring custom 'groups' scope mapping exists...")
+groups_mapping = get_or_create(
+    "/api/v3/propertymappings/provider/scope/",
+    "/api/v3/propertymappings/provider/scope/",
+    "scope_name=groups",
+    {
+        "name": "homelab: groups claim",
+        "scope_name": "groups",
+        # request.user.ak_groups is deprecated in authentik 2026.x (logs a
+        # deprecation warning on every token issue) -> use request.user.groups.
+        "expression": (
+            "return {\"groups\": [group.name for group in request.user.groups.all()]}"
+        ),
+    },
+    # Force the expression onto the already-created mapping on re-run.
+    patch_existing={
+        "expression": (
+            "return {\"groups\": [group.name for group in request.user.groups.all()]}"
+        ),
+    },
+)
+GROUPS_MAPPING_PK = groups_mapping["pk"]
+
+# Fetch the standard openid/email/profile mapping pks (shipped by default).
+status, res = api("GET", "/api/v3/propertymappings/provider/scope/")
+by_scope = {m["scope_name"]: m["pk"] for m in res["results"]}
+SCOPE_PKS = [by_scope["openid"], by_scope["email"], by_scope["profile"], GROUPS_MAPPING_PK]
+
+status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-authorization-implicit-consent")
+AUTHORIZATION_FLOW_PK = res["results"][0]["pk"]
+status, res = api("GET", "/api/v3/flows/instances/?slug=default-provider-invalidation-flow")
+INVALIDATION_FLOW_PK = res["results"][0]["pk"]
+status, res = api("GET", "/api/v3/crypto/certificatekeypairs/?has_key=true")
+SIGNING_KEY_PK = res["results"][0]["pk"]
+
+# -----------------------------------------------------------------------------
+print("[2/5] Ensuring groups homelab-admins / grafana-admins exist...")
+homelab_admins = get_or_create(
+    "/api/v3/core/groups/", "/api/v3/core/groups/",
+    "name=homelab-admins",
+    {"name": "homelab-admins", "is_superuser": True},
+)
+grafana_admins = get_or_create(
+    "/api/v3/core/groups/", "/api/v3/core/groups/",
+    "name=grafana-admins",
+    {"name": "grafana-admins", "is_superuser": False},
+)
+
+# -----------------------------------------------------------------------------
+print("[3/5] Ensuring user 'rock' exists with admin group membership...")
+status, res = api("GET", "/api/v3/core/users/?username=rock")
+rock_password = None
+if res.get("results"):
+    rock = res["results"][0]
+    status, rock = api("PATCH", f"/api/v3/core/users/{rock['pk']}/", {
+        "groups": [homelab_admins["pk"], grafana_admins["pk"]],
+        "is_active": True,
+    })
+    if status not in (200, 201):
+        die(f"PATCH user rock -> {status} {rock}")
+    print("  rock already exists, group membership synced (password unchanged)")
+else:
+    rock_password = gen_secret(24)
+    status, rock = api("POST", "/api/v3/core/users/", {
+        "username": "rock",
+        "name": "Rock",
+        "is_active": True,
+        "groups": [homelab_admins["pk"], grafana_admins["pk"]],
+        "path": "users",
+        "type": "internal",
+    })
+    if status not in (200, 201):
+        die(f"POST user rock -> {status} {rock}")
+    status, pw_res = api("POST", f"/api/v3/core/users/{rock['pk']}/set_password/",
+                          {"password": rock_password})
+    if status not in (200, 204):
+        die(f"set_password for rock -> {status} {pw_res}")
+    kubectl_create_secret("iam", "rock-credentials", {
+        "username": "rock",
+        "password": rock_password,
+    })
+    print("  rock created, credentials stored in iam/rock-credentials")
+
+# -----------------------------------------------------------------------------
+print("[4/5] Ensuring OAuth2 providers + applications for grafana/minio/forgejo/argocd...")
+
+SERVICES = {
+    "grafana": {
+        "client_secret_source": ("logging", "grafana-oidc", "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET"),
+        "redirect_uris": ["https://grafana.riotpiao.com/login/generic_oauth"],
+        "launch_url": "https://grafana.riotpiao.com",
+        "display_name": "Grafana",
+    },
+    "minio": {
+        "client_secret_source": ("storage", "minio-oidc", "MINIO_IDENTITY_OPENID_CLIENT_SECRET"),
+        "redirect_uris": ["https://minio.riotpiao.com/oauth_callback"],
+        "launch_url": "https://minio.riotpiao.com",
+        "display_name": "MinIO",
+    },
+    "forgejo": {
+        # No secret exists yet for forgejo - generate + store on first run.
+        "client_secret_source": ("cicd", "forgejo-oidc", "CLIENT_SECRET"),
+        "generate_if_missing": True,
+        "redirect_uris": [
+            "https://forgejo.riotpiao.com/user/oauth2/authentik/callback",
+            "https://forgejo.riotpiao.com/user/oauth2/openidconnect/callback",
+        ],
+        "launch_url": "https://forgejo.riotpiao.com",
+        "display_name": "Forgejo",
+    },
+    "argocd": {
+        # oidc-secret uses hyphenated keys (client-id/client-secret) per
+        # argocd-values.yaml's `$oidc-secret:client-id` / `:client-secret` refs.
+        "client_secret_source": ("argocd", "oidc-secret", "client-secret"),
+        "generate_if_missing": True,
+        "extra_secret_literals": {"client-id": "argocd"},
+        "redirect_uris": ["https://argocd.riotpiao.com/auth/callback"],
+        "launch_url": "https://argocd.riotpiao.com",
+        "display_name": "Argo CD",
+    },
+    "homarr": {
+        "client_secret_source": ("dashboard", "homarr-oidc", "client-secret"),
+        "generate_if_missing": True,
+        "extra_secret_literals": {"client-id": "homarr"},
+        "redirect_uris": ["https://homarr.riotpiao.com/api/auth/callback/oidc"],
+        "launch_url": "https://homarr.riotpiao.com",
+        "display_name": "Homarr",
+    },
+}
+
+app_pks_for_binding = []
+
+for name, cfg in SERVICES.items():
+    ns, secret_name, key = cfg["client_secret_source"]
+    client_secret = kubectl_get_secret_key(ns, secret_name, key)
+    if client_secret is None:
+        if not cfg.get("generate_if_missing"):
+            print(f"  WARNING: {ns}/{secret_name} key {key} not found and "
+                  f"generate_if_missing not set for '{name}' - skipping provider/app")
+            continue
+        client_secret = gen_secret(40)
+        literals = {key: client_secret}
+        literals.update(cfg.get("extra_secret_literals", {}))
+        kubectl_create_secret(ns, secret_name, literals)
+        print(f"  {name}: generated new client secret -> {ns}/{secret_name}")
+    else:
+        print(f"  {name}: using existing client secret from {ns}/{secret_name}")
+
+    provider = get_or_create(
+        "/api/v3/providers/oauth2/", "/api/v3/providers/oauth2/",
+        f"name={name}",
+        {
+            "name": name,
+            "client_id": name,
+            "client_secret": client_secret,
+            "client_type": "confidential",
+            "authorization_flow": AUTHORIZATION_FLOW_PK,
+            "invalidation_flow": INVALIDATION_FLOW_PK,
+            "signing_key": SIGNING_KEY_PK,
+            "property_mappings": SCOPE_PKS,
+            "sub_mode": "hashed_user_id",
+            "include_claims_in_id_token": True,
+            # authentik 2026.x requires grant_types to be set explicitly; the
+            # API defaults it to [] when omitted, which makes /authorize reject
+            # every login with "Invalid grant_type for provider" ->
+            # invalid_request. authorization_code = the web SSO flow all these
+            # apps use; refresh_token = long-lived sessions (offline_access).
+            "grant_types": ["authorization_code", "refresh_token"],
+            "redirect_uris": [
+                {"matching_mode": "strict", "url": u} for u in cfg["redirect_uris"]
+            ],
+        },
+        # Keep the redirect_uris/mappings/grant_types in sync on re-run, but
+        # never touch client_secret again once created (that's the source of
+        # truth in the k8s Secret, and re-sending it here is harmless anyway).
+        patch_existing={
+            "property_mappings": SCOPE_PKS,
+            "grant_types": ["authorization_code", "refresh_token"],
+            "redirect_uris": [
+                {"matching_mode": "strict", "url": u} for u in cfg["redirect_uris"]
+            ],
+        },
+    )
+
+    # superuser_full_list=true is REQUIRED on the LIST: the applications list
+    # applies access-policy filtering to the results array (these apps are bound
+    # to homelab-admins, and the bootstrap-token user akadmin is not a member),
+    # so without it the GET returns an empty results list even though the app
+    # exists -> fall through to POST -> 400 "already exists".
+    #
+    # We deliberately do NOT patch_existing here: the application DETAIL endpoint
+    # (PATCH /applications/{pk}/) enforces the same access policy and does NOT
+    # honor superuser_full_list, so PATCH-by-pk returns 404 for akadmin once the
+    # homelab-admins binding exists. That 404 aborted the loop before later
+    # providers got their grant_types. slug/provider/launch_url are set at
+    # creation and are stable (provider is get_or_create'd by name, stable pk),
+    # so find-or-create is sufficient.
+    application = get_or_create(
+        "/api/v3/core/applications/", "/api/v3/core/applications/",
+        f"slug={name}&superuser_full_list=true",
+        {
+            "name": cfg["display_name"],
+            "slug": name,
+            "provider": provider["pk"],
+            "meta_launch_url": cfg["launch_url"],
+        },
+    )
+    app_pks_for_binding.append((name, application["pk"]))
+    print(f"  {name}: provider pk={provider['pk']} application pk={application['pk']}")
+
+# -----------------------------------------------------------------------------
+print("[5/5] Binding homelab-admins to every application (guaranteed access for rock)...")
+for name, app_pk in app_pks_for_binding:
+    get_or_create(
+        "/api/v3/policies/bindings/", "/api/v3/policies/bindings/",
+        f"target={app_pk}&group={homelab_admins['pk']}",
+        {
+            "target": app_pk,
+            "group": homelab_admins["pk"],
+            "order": 0,
+            "enabled": True,
+        },
+    )
+    print(f"  {name}: homelab-admins bound")
+
+print("\nDone. Summary:")
+print("  groups:  homelab-admins (superuser), grafana-admins")
+print("  user:    rock -> homelab-admins + grafana-admins")
+print(f"  apps:    {', '.join(n for n, _ in app_pks_for_binding)}")
+if rock_password:
+    print("  NOTE: rock's password was generated this run - see")
+    print("  kubectl -n iam get secret rock-credentials -o jsonpath='{.data.password}' | base64 -d")
@@ -0,0 +1,174 @@
+# k8s/talos-iam/vault-values.yaml
+# HashiCorp Vault — secrets backend for the homelab.
+# Stores OIDC client secrets, TLS certs, and any other sensitive values.
+# Accessed via the `talos` CLI (talos-cli/) which wraps `vault kv get/put`.
+#
+# Storage backend: MinIO S3 (minio.storage.svc.cluster.local) — no extra PVC.
+# Auto-unseal: postStart hook reads unseal keys from vault-unseal-keys Secret
+# (written by setup_vault.sh after operator init; operator must run that script
+# once after first install to initialize and store the keys).
+
+# ── Global ────────────────────────────────────────────────────────────────────
+# tlsDisable: true — TLS terminated at the nginx ingress (vault.riotpiao.com)
+# or at port-forward. In-cluster traffic to Vault is plain HTTP; this is acceptable
+# because all clients are on the pod network (not crossing node boundaries).
+global:
+  enabled: true
+  tlsDisable: true
+
+# ── Agent Injector ────────────────────────────────────────────────────────────
+# The injector mutates pods to sidecar Vault Agent for automatic secret injection.
+# Not used here — secrets are fetched explicitly via the talos CLI.
+# Enabling it would add a webhook that intercepts all pod creates cluster-wide,
+# which is unnecessary overhead for a homelab with manual secret management.
+injector:
+  enabled: false
+
+server:
+  replicas: 1
+
+  annotations:
+    secret.reloader.stakater.com/reload: "vault-unseal-keys"
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+  # ── Scheduling ─────────────────────────────────────────────────────────────
+  # Tolerate cp-1 so Vault can run there if worker-1 is down.
+  # Prefer worker-1 under normal conditions (keeps Vault off the same node as etcd).
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: Exists
+
+  # ── Environment variables ───────────────────────────────────────────────────
+  # extraEnvironmentVars: non-secret config passed directly.
+  extraEnvironmentVars:
+    VAULT_LOG_LEVEL: info
+
+  # extraSecretEnvironmentVars: pulls values from K8s Secrets into env vars.
+  # vault-minio-creds is created by the helmfile presync hook from MINIO_ROOT_USER/PASSWORD.
+  # vault-unseal-keys is a placeholder created at first deploy; setup_vault.sh
+  # overwrites it with real unseal keys after `vault operator init`.
+  # Vault reads the keys from env on every pod start and the postStart hook unseals.
+  extraSecretEnvironmentVars:
+    - envName: AWS_ACCESS_KEY_ID
+      secretName: vault-minio-creds
+      secretKey: access_key
+    - envName: AWS_SECRET_ACCESS_KEY
+      secretName: vault-minio-creds
+      secretKey: secret_key
+    - envName: VAULT_UNSEAL_KEY_1
+      secretName: vault-unseal-keys
+      secretKey: key1
+    - envName: VAULT_UNSEAL_KEY_2
+      secretName: vault-unseal-keys
+      secretKey: key2
+    - envName: VAULT_UNSEAL_KEY_3
+      secretName: vault-unseal-keys
+      secretKey: key3
+
+  # ── Auto-unseal ─────────────────────────────────────────────────────────────
+  # Vault starts sealed after every pod restart and can't serve requests until
+  # unsealed. postStart runs immediately after the container starts, sleeps 5s
+  # to let the Vault process bind its port, then feeds the unseal keys one by one.
+  # `|| true` prevents the hook from failing if a key was already used (idempotent).
+  # 3-of-5 Shamir unseal is the default — we stored all 3 used keys in the Secret.
+  postStart:
+    - /bin/sh
+    - -c
+    - |
+      sleep 5
+      vault operator unseal "$VAULT_UNSEAL_KEY_1" || true
+      vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
+      vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
+
+  # ── Vault config (HCL) ──────────────────────────────────────────────────────
+  standalone:
+    enabled: true
+    config: |
+      ui = false   # UI served via Vault's own HTTP; enabled below via ui: enabled: true
+
+      listener "tcp" {
+        address     = "0.0.0.0:8200"
+        tls_disable = "true"
+
+        # No auth in front of Vault's metrics endpoint — acceptable since all
+        # Prometheus scrape traffic stays on the pod network (not exposed via ingress).
+        telemetry {
+          unauthenticated_metrics_access = "true"
+        }
+      }
+
+      telemetry {
+        prometheus_retention_time = "30s"
+        disable_hostname          = true
+      }
+
+      # S3 storage backend pointing at the in-cluster MinIO service.
+      # AWS_ env vars (from vault-minio-creds Secret) supply the credentials.
+      # s3_force_path_style: MinIO uses path-style URLs (not virtual-hosted).
+      # disable_ssl: MinIO in this cluster has no TLS.
+      storage "s3" {
+        endpoint            = "http://minio-cluster-hl.storage.svc.cluster.local:9000"
+        bucket              = "vault"
+        region              = "us-east-1"
+        s3_force_path_style = "true"
+        disable_ssl         = "true"
+        max_parallel        = 128
+      }
+
+      # api_addr: the address other Vault nodes (or HA standbys) use to reach
+      # this node. Single-node standalone, but Vault requires it to be set.
+      api_addr = "http://vault.iam.svc.cluster.local:8200"
+      cluster_addr = "https://vault-0.vault-internal.iam.svc.cluster.local:8201"
+
+  # ── Service ─────────────────────────────────────────────────────────────────
+  # NodePort 32171 — fallback for direct node access during bootstrap before
+  # the ingress is up. Normal access is via nginx ingress (vault.riotpiao.com).
+  service:
+    type: NodePort
+    port: 8200
+    nodePort: 32171
+
+  # ── Persistence ─────────────────────────────────────────────────────────────
+  # No PVC — all Vault state (secrets, policies, tokens) is stored in MinIO S3.
+  # This means Vault survives node loss as long as MinIO is healthy.
+  dataStorage:
+    enabled: false
+
+  auditStorage:
+    enabled: false
+
+# ── UI ────────────────────────────────────────────────────────────────────────
+# Vault's web UI is used for the OIDC browser login flow (Vault as an OIDC
+# provider, if configured) and for manual operator inspection.
+# Accessible at http://vault.riotpiao.com or via port-forward.
+ui:
+  enabled: true
+
+# ── Metrics ───────────────────────────────────────────────────────────────────
+# vault_core_unsealed is the availability signal (0 after a restart until the
+# postStart hook above finishes unsealing). Pairs with the telemetry{} stanzas
+# in standalone.config above, which actually turn the /v1/sys/metrics endpoint on.
+serverTelemetry:
+  serviceMonitor:
+    enabled: true
+    selectors: {}
+    interval: 30s
+    scrapeTimeout: 10s
+