k8s/iam: add cloudnativepg postgres and vault + authentik

- PostgreSQL 3-replica HA with pgvector
- Vault S3 storage backend (MinIO)
- Authentik federated OIDC provider
- Vault auto-unseal via postStart hook

k8s/ddb/cnpg-values.yaml

@@ -0,0 +1,51 @@
+# CloudNativePG Operator Configuration
+# Handles PostgreSQL cluster management with HA replication
+#
+# Timeout settings tuned for clusters with 5+ second network latency spikes
+
+# Operator deployment
+replicaCount: 1
+
+# Operator Pod configuration
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/port: "8080"
+
+# Resource limits
+resources:
+  limits:
+    memory: 512Mi
+  requests:
+    memory: 256Mi
+
+# Monitoring
+monitoring:
+  enabled: true
+  serviceMonitor:
+    enabled: false  # disable until we have prometheus-operator CRDs
+
+# ── Configuration for operator behavior ────────────────────────────────────────
+# Applied to the cnpg-controller-manager-config ConfigMap
+config:
+  # HTTP client timeout for communicating with Postgres instances
+  # Default: 30s. Increased to 120s to tolerate 5+ second network latency spikes
+  instanceManagerHTTPClientTimeout: 120s
+
+  # TLS verification for instance manager
+  instanceManagerTLSInsecureSkipVerify: false
+
+  # Allow debug logging to diagnose HTTP communication issues
+  enableDebugLogging: true
+
+  # Pod debugging (disabled; security risk in production)
+  enablePodDebugging: false
+
+  # In-place updates for instance manager (disabled; safer for HA)
+  enableInstanceManagerInplaceUpdates: false
+
+  # Azure PVC updates (not applicable for Longhorn)
+  enableAzurePVCUpdates: false
+
+  # Certificate lifetime (90 days) and renewal threshold (7 days before expiry)
+  certificateDuration: 90
+  expiringCheckThreshold: 7

k8s/ddb/init-users.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Safe database initialization script
+# Applies init-users.sql with environment variable substitution
+# Exit on any error
+set -euo pipefail
+
+# Get superuser password from CNPG secret
+PG_PASSWORD=$(kubectl get secret -n ddb ddb-cluster-app -o jsonpath='{.data.password}' | base64 -d)
+export PGPASSWORD="$PG_PASSWORD"
+
+# Apply SQL with safe variable substitution (psql -v prevents injection)
+psql \
+  -h ddb-cluster-rw.ddb.svc.cluster.local \
+  -U postgres \
+  -d postgres \
+  -v authentik_password="$AUTHENTIK_PG_PASSWORD" \
+  -v story_crater_password="$STORY_CRATER_PG_PASSWORD" \
+  -f k8s/ddb/init-users.sql
+
+unset PGPASSWORD
+echo "✓ Database initialization complete"

k8s/ddb/init-users.sql

@@ -0,0 +1,11 @@
+-- Database initialization for homelab applications
+-- Idempotent: safe to re-run
+-- Required environment variables:
+--   AUTHENTIK_PG_PASSWORD
+--   STORY_CRATER_PG_PASSWORD
+
+CREATE ROLE IF NOT EXISTS authentik WITH LOGIN PASSWORD :'authentik_password';
+CREATE DATABASE IF NOT EXISTS authentik OWNER authentik;
+
+CREATE ROLE IF NOT EXISTS story_crater WITH LOGIN PASSWORD :'story_crater_password';
+CREATE DATABASE IF NOT EXISTS story_crater OWNER story_crater;

k8s/talos-iam/.env.example

@@ -0,0 +1,27 @@
+# talos-iam/.env.example
+# Copy to talos-iam/.env and fill in. The real .env is gitignored — never commit it.
+# These are ADMIN credentials for the homelab identity provider; use strong values.
+
+# Authentik's signing/encryption key. SET ONCE — rotating it invalidates all
+# existing sessions, tokens, and encrypted fields. Generate:
+#   openssl rand -base64 60 | tr -d '\n'
+AUTHENTIK_SECRET_KEY=
+
+# Initial password for the built-in admin user 'akadmin'. Change after first login.
+#   openssl rand -base64 24
+AUTHENTIK_BOOTSTRAP_PASSWORD=
+
+# Initial API token for 'akadmin' (used for automation / blueprints).
+#   openssl rand -hex 32
+AUTHENTIK_BOOTSTRAP_TOKEN=
+
+# Password for the bundled PostgreSQL 'authentik' user (system of record).
+#   openssl rand -base64 24
+PG_PASSWORD=
+
+# OAuth2 client secrets for downstream OIDC integrations.
+# These are registered with Authentik and injected as K8s secrets
+# into the logging and storage namespaces by this bootstrap script.
+#   openssl rand -hex 32
+GRAFANA_OIDC_CLIENT_SECRET=
+MINIO_OIDC_CLIENT_SECRET=

k8s/talos-iam/OAUTH_APP_SETUP.md

@@ -0,0 +1,479 @@
+# OAuth App Setup & Management Guide
+
+This guide provides a standardized workflow for registering new services with the homelab OIDC provider (Authentik) and wiring them for Vault JWT authentication.
+
+> **TL;DR:** Run `register_oauth_app.py` with your app's details; it handles Authentik + Vault + K8s secrets automatically.
+
+---
+
+## Architecture Overview
+
+The homelab uses a three-tier authentication stack:
+
+```
+User / Service
+  ↓
+Authentik (OIDC IdP)
+  ├─ Grafana
+  ├─ MinIO
+  ├─ Forgejo
+  ├─ Argo CD
+  ├─ Your New Service
+  └─ ...
+  ↓ (device code / authorization code flow)
+Vault (KV + JWT auth)
+  ├─ JWT role mapping (sub/groups → policies)
+  ├─ OIDC browser login (vault-browser provider)
+  └─ KV secrets (secret/cluster/*, secret/mcp/*)
+```
+
+**Key concepts:**
+- **Authentik** is the OIDC identity provider — it issues tokens and manages users/groups.
+- **Vault** validates Authentik's JWT tokens and maps them to policies & secret access.
+- **Kubernetes secrets** store OAuth credentials that services read at startup (ConfigMap-based).
+- **Groups** in Authentik control access policies and Vault role assignment.
+
+---
+
+## Prerequisites
+
+Before registering a new OAuth service, ensure:
+
+1. **Authentik is running** (in `iam` namespace)
+   ```bash
+   kubectl get pods -n iam | grep authentik-server
+   ```
+
+2. **Vault is running** (in `storage` namespace)
+   ```bash
+   kubectl get pods -n storage | grep vault-0
+   ```
+
+3. **SSH/API access to Authentik** — port-forward available
+   ```bash
+   kubectl port-forward -n iam svc/authentik-server 7000:80
+   ```
+
+4. **Vault bootstrap is complete** (run `setup_vault.sh` if not already done)
+
+5. **Required environment variables** — populate these in `~/.authentik/.env`:
+   ```bash
+   # Token for Authentik API (from bootstrap)
+   AUTHENTIK_BOOTSTRAP_TOKEN=<your-token>
+   
+   # Service-specific client secret (generate via openssl rand -hex 32)
+   # Example: MY_SERVICE_OIDC_CLIENT_SECRET=<generated-secret>
+   ```
+
+---
+
+## One-Line Setup (Recommended)
+
+For most services, use the automated registration script:
+
+```bash
+cd /Users/rockliang/workplace/homelab
+
+# Register an OAuth app with automatic Vault JWT wiring
+python3 k8s/talos-iam/register_oauth_app.py \
+  --service-name my-app \
+  --namespace my-namespace \
+  --redirect-uri "https://my-app.riotpiao.homelab.com/oauth2/callback" \
+  --service-name-in-vault "MY_SERVICE" \
+  --vault-jwt-policy "shell-secrets"
+```
+
+The script will:
+1. ✅ Create an OAuth2 provider in Authentik (from credentials in Vault)
+2. ✅ Create an Application in Authentik
+3. ✅ Create/bind groups for RBAC (optional)
+4. ✅ Create a Kubernetes secret with the client credentials
+5. ✅ Create a Vault JWT role for the service's JWT auth method
+6. ✅ Write policies for the service's scope in Vault
+
+---
+
+## Manual Setup (Step-by-Step)
+
+If you prefer manual control, or if the script doesn't fit your use case:
+
+### Step 1: Generate Credentials
+
+```bash
+# Generate a new client secret
+CLIENT_SECRET=$(openssl rand -hex 32)
+echo "CLIENT_SECRET=$CLIENT_SECRET"
+
+# Store in Vault (required for service initialization)
+# Convention: YOURSERVICE_OIDC_CLIENT_SECRET
+talos put cluster/YOURSERVICE_OIDC_CLIENT_SECRET YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET"
+
+# Or set in .env temporarily
+export YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET"
+```
+
+### Step 2: Create Authentik OAuth2 Provider
+
+Access Authentik UI via port-forward:
+
+```bash
+kubectl port-forward -n iam svc/authentik-server 7000:80 &
+# http://localhost:7000/if/admin/ → login with akadmin
+```
+
+**In Authentik UI:**
+1. Navigate to **Applications → Providers → Create → OpenID Connect (OAuth2) Provider**
+2. Fill in:
+   - **Name:** `yourservice`
+   - **Client ID:** `yourservice` (or custom)
+   - **Client Secret:** (paste from `$CLIENT_SECRET` above)
+   - **Redirect URIs:** `https://your-app.riotpiao.homelab.com/oauth2/callback` (or your app's callback URL)
+   - **Sub Mode:** `Hashed User ID`
+   - **Include claims in ID Token:** ✓ Enabled
+   - **Grant Types:** `Authorization Code`, `Refresh Token`
+   - **Signing Key:** (select the homelab-oidc key)
+
+3. **Save** and note the **provider slug** (usually auto-generated from Name).
+
+### Step 3: Create Authentik Application
+
+**In Authentik UI:**
+1. Navigate to **Applications → Applications → Create**
+2. Fill in:
+   - **Name:** `Your App Display Name`
+   - **Slug:** `yourservice` (must match provider slug)
+   - **Provider:** (select the provider created above)
+   - **Meta Launch URL:** `https://your-app.riotpiao.homelab.com` (optional, for app launcher)
+
+3. **Save**
+
+### Step 4: Configure OIDC in Your Service
+
+Pass the following environment variables to your service:
+
+```bash
+# OIDC endpoint (in-cluster: authentik-server.iam.svc.cluster.local)
+OIDC_ISSUER_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/
+
+# OAuth2 credentials (from step 1-2)
+OIDC_CLIENT_ID=yourservice
+OIDC_CLIENT_SECRET=$CLIENT_SECRET
+
+# Redirect URI (must match what you configured in step 2)
+OIDC_REDIRECT_URI=https://your-app.riotpiao.homelab.com/oauth2/callback
+
+# Optionally, token validation endpoint
+OIDC_TOKEN_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/token/
+OIDC_USERINFO_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/userinfo/
+
+# Optional: JWKS endpoint for offline token validation
+OIDC_JWKS_URL=http://authentik-server.iam.svc.cluster.local/application/o/yourservice/jwks/
+```
+
+**Store secrets in Vault:**
+```bash
+# Store the client secret for runtime retrieval
+talos put cluster/YOURSERVICE_OIDC_CLIENT_SECRET YOURSERVICE_OIDC_CLIENT_SECRET="$CLIENT_SECRET"
+
+# Store other config if needed
+talos put cluster/yourservice-oidc \
+  client_id=yourservice \
+  issuer_url="http://authentik-server.iam.svc.cluster.local/application/o/yourservice/" \
+  callback_uri="https://your-app.riotpiao.homelab.com/oauth2/callback"
+```
+
+### Step 5: Mount Credentials in Your Service
+
+**Option A: Kubernetes Secret (ConfigMap-based)**
+
+Create a Kubernetes secret with the credentials:
+
+```bash
+kubectl create secret generic yourservice-oidc \
+  --from-literal=client_id=yourservice \
+  --from-literal=client_secret="$CLIENT_SECRET" \
+  -n your-namespace
+```
+
+Reference in your Helm values or Pod spec:
+
+```yaml
+# In Helm values
+env:
+  - name: OIDC_CLIENT_ID
+    valueFrom:
+      secretKeyRef:
+        name: yourservice-oidc
+        key: client_id
+  - name: OIDC_CLIENT_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: yourservice-oidc
+        key: client_secret
+```
+
+**Option B: Vault KV Secret (Runtime)**
+
+Store in Vault and retrieve at startup:
+
+```bash
+# Store full config
+vault kv put secret/services/yourservice \
+  client_id=yourservice \
+  client_secret="$CLIENT_SECRET" \
+  issuer_url="http://authentik-server.iam.svc.cluster.local/application/o/yourservice/"
+
+# Service reads at startup:
+# curl -H "Authorization: Bearer $VAULT_TOKEN" \
+#   http://vault.storage.svc.cluster.local:8200/v1/secret/data/services/yourservice
+```
+
+---
+
+## Vault JWT Authentication (For Services)
+
+If your service needs to authenticate to Vault directly (beyond just reading secrets), create a JWT role:
+
+### 1. Create Vault JWT Role
+
+```bash
+# Authenticate to Vault (as cluster-admin or root)
+kubectl port-forward -n storage svc/vault 8200:8200 &
+export VAULT_ADDR=http://127.0.0.1:8200
+vault login -method=oidc role=homelab
+
+# Create a service-specific JWT role
+vault write auth/jwt/role/yourservice \
+  role_type=jwt \
+  bound_audiences="vault" \
+  user_claim="sub" \
+  bound_claims='{"client_id":["yourservice"]}' \
+  policies="shell-secrets,service-read-secrets" \
+  ttl=4h \
+  max_ttl=8h
+```
+
+### 2. Create Service Policy in Vault
+
+```bash
+# Policy that allows the service to read its own secrets
+vault policy write service-read-yourservice - <<'EOF'
+path "secret/data/services/yourservice" {
+  capabilities = ["read"]
+}
+path "secret/data/cluster/minio" {
+  capabilities = ["read"]
+}
+EOF
+
+# Update the role to use this policy
+vault write auth/jwt/role/yourservice \
+  policies="service-read-secrets,service-read-yourservice"
+```
+
+### 3. Service JWT Authentication Flow
+
+Your service obtains a Vault token from Authentik's JWT:
+
+```python
+# Example service code (Python)
+import requests
+import json
+
+# 1. Authenticate to Authentik OIDC (get ID token)
+oidc_response = requests.post(
+    "http://authentik-server.iam.svc.cluster.local/application/o/yourservice/token/",
+    data={
+        "grant_type": "client_credentials",
+        "client_id": "yourservice",
+        "client_secret": os.environ["OIDC_CLIENT_SECRET"],
+        "audience": "vault"  # Important: request JWT for Vault
+    }
+)
+id_token = oidc_response.json()["access_token"]
+
+# 2. Authenticate to Vault using the JWT
+vault_response = requests.post(
+    "http://vault.storage.svc.cluster.local:8200/v1/auth/jwt/login",
+    data=json.dumps({
+        "role": "yourservice",
+        "jwt": id_token
+    }),
+    headers={"Content-Type": "application/json"}
+)
+vault_token = vault_response.json()["auth"]["client_token"]
+
+# 3. Use Vault token to read secrets
+secret_response = requests.get(
+    "http://vault.storage.svc.cluster.local:8200/v1/secret/data/services/yourservice",
+    headers={"X-Vault-Token": vault_token}
+)
+secrets = secret_response.json()["data"]["data"]
+```
+
+---
+
+## RBAC & Groups
+
+### Adding Group-Based Access Control
+
+Groups in Authentik map to Vault policies and app bindings:
+
+```bash
+# Create a group for your service's administrators
+python3 k8s/talos-iam/provision_oidc.py --add-group "yourservice-admins"
+
+# Add users to the group (Authentik UI)
+# Applications → Groups → yourservice-admins → Users
+
+# Bind group to app (Authentik UI)
+# Applications → Applications → yourservice → Policies → Binding
+# Select the group and enable the binding
+```
+
+### Group-Based Vault Policy Mapping
+
+Create a Vault policy that checks Authentik group membership:
+
+```bash
+vault policy write service-admin-yourservice - <<'EOF'
+# Full access for admins
+path "secret/data/services/yourservice/*" {
+  capabilities = ["read", "create", "update", "delete"]
+}
+path "secret/data/mcp/*" {
+  capabilities = ["read"]
+}
+EOF
+
+# Create a separate JWT role for admins
+vault write auth/jwt/role/yourservice-admin \
+  role_type=jwt \
+  bound_audiences="vault" \
+  user_claim="sub" \
+  bound_claims='{"groups":["yourservice-admins"]}' \
+  policies="shell-secrets,service-admin-yourservice" \
+  ttl=4h
+```
+
+---
+
+## Verification Checklist
+
+After registration, verify everything works:
+
+```bash
+# 1. Verify Authentik provider exists
+curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \
+  http://localhost:7000/api/v3/providers/oauth2/?name=yourservice
+
+# 2. Verify Authentik application exists
+curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \
+  http://localhost:7000/api/v3/core/applications/?slug=yourservice
+
+# 3. Verify K8s secret is mounted
+kubectl get secret yourservice-oidc -n your-namespace -o jsonpath='{.data}' | base64 -d
+
+# 4. Verify Vault JWT role
+vault read auth/jwt/role/yourservice
+
+# 5. Test OAuth login (requires port-forward to your app)
+# Open http://your-app.riotpiao.homelab.com/login
+# Should redirect to Authentik, then back to your app
+
+# 6. Test JWT auth to Vault (if applicable)
+# Service obtains ID token and authenticates to Vault
+vault login -method=jwt role=yourservice jwt=$ID_TOKEN
+```
+
+---
+
+## Troubleshooting
+
+### "JWKS URL not found" error in Vault
+
+**Cause:** Authentik is not reachable from Vault pod at the configured URL.
+
+**Fix:**
+1. Check Authentik is running: `kubectl get pods -n iam`
+2. Verify URL uses in-cluster address: `http://authentik-server.iam.svc.cluster.local`
+3. Test DNS from Vault pod:
+   ```bash
+   kubectl exec -n storage vault-0 -- nslookup authentik-server.iam.svc.cluster.local
+   ```
+
+### "OAuth callback failed" or redirect loop
+
+**Cause:** Redirect URI in Authentik does not match what the service sends.
+
+**Fix:**
+1. Check Authentik provider's Redirect URIs: UI → Applications → Providers → Edit
+2. Must exactly match the URI your service redirects to (protocol, domain, port, path)
+3. Common mistake: `https://` in service but `http://` in Authentik
+
+### Service can't read K8s secret
+
+**Cause:** Secret is in wrong namespace or service account lacks permissions.
+
+**Fix:**
+1. Verify secret exists: `kubectl get secret yourservice-oidc -n your-namespace`
+2. Check RBAC for service account:
+   ```bash
+   kubectl describe sa yourservice -n your-namespace
+   ```
+
+### "client_id mismatch" or "client authentication failed"
+
+**Cause:** Client secret in K8s secret does not match what's in Authentik.
+
+**Fix:**
+1. Rotate the secret in Authentik (delete and recreate)
+2. Update the K8s secret with the new value
+3. Restart the service pod(s)
+
+---
+
+## Advanced: Custom Property Mappings
+
+For services that need custom JWT claims (e.g., MinIO's `policy` claim), use property mappings:
+
+```bash
+# Create a custom scope with expression
+curl -X POST http://localhost:7000/api/v3/propertymappings/provider/scope/ \
+  -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "yourservice-claims",
+    "scope_name": "yourservice-custom",
+    "expression": "return {\"department\": request.user.attributes.get(\"department\", \"unknown\")}"
+  }'
+
+# Add to provider's property_mappings:
+# Authentik UI → Applications → Providers → Edit yourservice
+# Select the custom scope in the Property Mappings list
+```
+
+---
+
+## Integration Examples
+
+### Integrate with Grafana
+
+See `k8s/logging/grafana-values.yaml` for an example of configuring generic OAuth in Grafana.
+
+### Integrate with MinIO
+
+See `provision_oidc.py` for MinIO's group-based policy claim injection.
+
+### Integrate with Custom Go App
+
+See `k8s/talos-iam/go-example-oidc/` for a minimal Authorization Code flow example.
+
+---
+
+## Related Documentation
+
+- [`README.md`](README.md) — Authentik deployment & key rotation
+- [`provision_oidc.py`](provision_oidc.py) — Automated OIDC provisioning for core apps
+- [`setup_vault.sh`](setup_vault.sh) — Vault initialization & JWT auth wiring
+- [`CLAUDE.md`](../CLAUDE.md) — Homelab architecture & secrets management

k8s/talos-iam/README.md

@@ -0,0 +1,302 @@
+# Talos IAM — Authentik
+
+Standalone **SSO / Identity Provider** for the homelab.
+Authentik gives every homelab app one login (OIDC / OAuth2 / SAML / forward-auth). It is an
+**identity provider, not a secret vault** — its system of record is PostgreSQL (users, apps,
+tokens, policies) with Redis for cache/queue. There is no MinIO/S3 involvement.
+
+> Want a Vault-style secret store instead? That's a different tool — OpenBao / HashiCorp Vault /
+> Infisical — and a separate setup. This folder is SSO only.
+
+## Architecture
+
+```
+                        ┌─────────────────────────────────────┐
+                        │           namespace: iam             │
+                        │                                      │
+  Browser / CLI ───────▶│  authentik-server  (UI + API :80)   │
+                        │  authentik-worker  (tasks / flows)  │
+                        │  postgresql        (Longhorn 8Gi)   │
+                        │  redis             (ephemeral)       │
+                        └───────────────┬─────────────────────┘
+                                        │ OIDC / OAuth2
+                      ┌─────────────────┼─────────────────┐
+                      ▼                 ▼                  ▼
+              ns: logging          ns: storage        ns: iam
+              Grafana              MinIO              (CronJob)
+              client_id=grafana    client_id=minio    key-rotation
+```
+
+**OIDC clients provisioned:**
+
+| App | Namespace | Client ID | Redirect URI |
+|---|---|---|---|
+| Grafana | `logging` | `grafana` | `http://localhost:3000/login/generic_oauth` |
+| MinIO | `storage` | `minio` | `http://localhost:9001/oauth_callback` |
+| Portainer | `portainer` | — | placeholder (CE has no OIDC) |
+
+**Groups:**
+
+| Group | Maps to |
+|---|---|
+| `grafana-admins` | Grafana Admin role |
+| `grafana-viewers` | Grafana Viewer role |
+| `minio-admins` | MinIO `readwrite` policy |
+| `minio-readonly` | MinIO `readonly` policy |
+
+---
+
+## Layout
+
+```
+talos-iam/
+├── setup_talos_iam.sh                    # one-shot deploy (namespace → helm → verify → provision)
+├── authentik-values.yaml                 # Helm values (Postgres on Longhorn, Redis ephemeral, tolerations)
+├── provision_oidc.py                     # idempotent OIDC provisioner (providers, groups, K8s secrets)
+├── register_oauth_app.py                 # 🆕 register new services with OAuth + Vault JWT (recommended)
+├── example_register_dashboard.py         # 🆕 example: register 'dashboard-service' with full RBAC
+├── verify_existing_oauth_integrations.sh # 🆕 verify Grafana/MinIO/Forgejo/Argo CD OIDC still work
+├── OAUTH_APP_SETUP.md                    # 🆕 comprehensive guide: manual & automated OAuth setup
+├── key_rotate.rs                         # stdlib-only Rust script — rotates OIDC signing key
+├── key-rotation-cronjob.yaml             # K8s CronJob running key_rotate.rs quarterly
+├── setup_vault.sh                        # Vault initialization & JWT auth wiring (run after Authentik)
+├── go-example-oidc/                      # Go Authorization Code flow example against Authentik
+│   ├── main.go
+│   ├── go.mod
+│   └── .env.example
+├── .env.example                          # required secrets — copy to ~/.authentik/.env and fill in
+└── README.md                              # you are here
+```
+
+---
+
+## Quickstart
+
+```bash
+cp k8s/talos-iam/.env.example ~/.authentik/.env
+# fill in the secrets (generators are in .env.example)
+
+bash k8s/talos-iam/setup_talos_iam.sh
+```
+
+The script creates the `iam` namespace, installs the Authentik Helm chart (server, worker, bundled
+PostgreSQL + Redis), waits for rollout, probes the readiness endpoint, then calls
+`provision_oidc.py` to wire up Grafana and MinIO as OIDC clients.
+
+---
+
+## Access
+
+```bash
+kubectl port-forward svc/authentik-server -n iam 7000:80
+# Admin UI:  http://localhost:7000/if/admin/
+# Login:     akadmin / <AUTHENTIK_BOOTSTRAP_PASSWORD>
+```
+
+---
+
+## OIDC Provisioning
+
+`provision_oidc.py` idempotently creates all Authentik resources from the API — safe to re-run.
+
+```bash
+# port-forward must be active (localhost:7000)
+source ~/.authentik/.env
+python k8s/talos-iam/provision_oidc.py
+```
+
+What it provisions:
+- RSA-4096 signing certificate `homelab-oidc`
+- OAuth2 providers for Grafana and MinIO (with the signing cert attached)
+- Property mapping that injects a `policy` JWT claim for MinIO access control
+- Groups: `grafana-admins`, `grafana-viewers`, `minio-admins`, `minio-readonly`
+- K8s secrets `grafana-oidc` (ns: `logging`) and `minio-oidc` (ns: `storage`)
+
+---
+
+## Key Rotation
+
+OIDC signing keys should be rotated periodically. The `key_rotate.rs` script generates a new
+RSA-4096 cert in Authentik and patches all providers to use it. The old cert stays in the JWKS
+endpoint until you delete it — existing tokens remain valid through their TTL (default 5 min).
+
+**Manual rotation** (port-forward must be active):
+```bash
+source ~/.authentik/.env
+python k8s/talos-iam/provision_oidc.py --rotate
+```
+
+**Automated rotation** (quarterly CronJob in-cluster):
+```bash
+# One-time setup
+kubectl create configmap key-rotation-script \
+  --from-file=rotate_key.rs=k8s/talos-iam/key_rotate.rs \
+  -n iam --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl create secret generic authentik-key-rotation-token \
+  --from-literal=AUTHENTIK_BOOTSTRAP_TOKEN="${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  -n iam --dry-run=client -o yaml | kubectl apply -f -
+
+kubectl apply -f k8s/talos-iam/key-rotation-cronjob.yaml
+
+# Test the job immediately
+kubectl create job --from=cronjob/authentik-key-rotation test-rotation -n iam
+kubectl logs -n iam -l job-name=test-rotation -f
+```
+
+**Validate rotation:**
+```bash
+# Confirm signing_key is set and changed
+curl -s -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \
+  "http://localhost:7000/api/v3/providers/oauth2/?name=grafana" \
+  | python3 -c "import json,sys; p=json.load(sys.stdin)['results'][0]; print('signing_key:', p['signing_key'])"
+
+# Confirm JWKS shows both old and new key during transition
+curl -s http://localhost:7000/application/o/grafana/.well-known/jwks.json \
+  | python3 -c "import json,sys; [print('kid:', k['kid']) for k in json.load(sys.stdin)['keys']]"
+```
+
+> After rotation, nothing in `~/.authentik/.env` changes. Client secrets, the bootstrap token,
+> and `AUTHENTIK_SECRET_KEY` are all separate from the OIDC signing keypair.
+
+---
+
+## Go OIDC Example
+
+A minimal Authorization Code flow demo against Authentik — useful for verifying the IdP
+end-to-end or as a starting point for a new OIDC client.
+
+```bash
+cp k8s/talos-iam/go-example-oidc/.env.example ~/.authentik/.env
+# add OIDC_CLIENT_ID and OIDC_CLIENT_SECRET for an app you register in Authentik
+
+cd k8s/talos-iam/go-example-oidc
+go mod tidy && go run .
+# open http://localhost:8080/login
+```
+
+The callback prints the verified ID token claims as JSON — `email`, `name`, `sub`, and any
+custom claims (e.g. the MinIO `policy` claim).
+
+---
+
+## Node Resilience — Auto-Start on Reboot
+
+All four components (server, worker, PostgreSQL, Redis) carry the control-plane toleration:
+
+```yaml
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+```
+
+- **Any node goes down** → Kubernetes reschedules all Authentik pods onto the surviving node.
+- **PostgreSQL PVC** (8Gi Longhorn `ReadWriteOnce`) → Longhorn reattaches automatically (~2 min). All user/app/token data is preserved.
+- **Redis** is ephemeral (no PVC) — restarts clean, which is correct (cache/queue only).
+- **Full cluster reboot** → cp-1 comes up first; Kubernetes reconciles Deployments; Longhorn reattaches. Zero manual action needed.
+
+**Startup order after reboot:**
+1. `kubelet` starts on both nodes
+2. `etcd` + API server on cp-1
+3. Controllers reconcile Authentik Deployments and PostgreSQL StatefulSet
+4. PostgreSQL starts (Authentik server/worker wait via init probes)
+5. Redis starts
+6. Authentik server + worker become ready
+
+---
+
+## Verify
+
+```bash
+kubectl get pods -n iam
+# authentik-server, authentik-worker, authentik-postgresql-0, authentik-redis-master-0 → Running
+
+curl -fsS -o /dev/null -w '%{http_code}\n' http://localhost:7000/-/health/ready/
+# 204
+```
+
+---
+
+## Registering New Services (OAuth App Workflow)
+
+The homelab provides **automated OAuth registration** for new services via `register_oauth_app.py`:
+
+```bash
+# Register a new service with OAuth + Vault JWT auth (recommended)
+python3 k8s/talos-iam/register_oauth_app.py \
+  --service-name my-service \
+  --namespace my-ns \
+  --redirect-uri "https://my-service.riotpiao.homelab.com/oauth2/callback" \
+  --service-name-in-vault MY_SERVICE \
+  --vault-jwt-policy shell-secrets \
+  --add-group my-service-admins
+```
+
+This automates:
+1. ✅ Authentik OAuth2 provider creation (credentials from Vault)
+2. ✅ Authentik application binding
+3. ✅ Kubernetes secret provisioning (client ID/secret)
+4. ✅ Vault JWT role creation (for service → Vault auth)
+5. ✅ Group-based RBAC setup (optional)
+
+**For manual control or step-by-step guidance,** see [`OAUTH_APP_SETUP.md`](OAUTH_APP_SETUP.md) which covers both automated and manual workflows.
+
+### Example: Register "dashboard-service"
+
+```bash
+# 1. Generate client secret and store in Vault
+DASHBOARD_OIDC_CLIENT_SECRET=$(openssl rand -hex 32)
+talos put cluster/DASHBOARD_OIDC_CLIENT_SECRET DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET"
+
+# 2. Register with automation
+export DASHBOARD_OIDC_CLIENT_SECRET
+python3 k8s/talos-iam/register_oauth_app.py \
+  --service-name dashboard-service \
+  --namespace apps \
+  --redirect-uri "https://dashboard.riotpiao.homelab.com/oauth2/callback" \
+  --service-name-in-vault DASHBOARD \
+  --vault-jwt-policy shell-secrets \
+  --add-group dashboard-admins \
+  --vault-jwt-bound-claims '{"groups":["dashboard-admins"]}'
+
+# 3. Service now has:
+#   - Authentik provider (dashboard-service)
+#   - K8s secret (dashboard-service-oidc) in 'apps' namespace
+#   - Vault JWT role (dashboard-service) with group-based access
+```
+
+### Verification
+
+After registration, verify the integration:
+
+```bash
+# Check Authentik provider and app
+curl -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \
+  http://localhost:7000/api/v3/core/applications/?slug=my-service | jq .
+
+# Check K8s secret
+kubectl get secret my-service-oidc -n my-ns -o yaml
+
+# Check Vault JWT role
+vault read auth/jwt/role/my-service
+
+# Test OAuth login
+# Browser: https://my-service.riotpiao.homelab.com/login
+# Should redirect to Authentik → back to service with session
+```
+
+**Full verification script:**
+```bash
+bash k8s/talos-iam/verify_existing_oauth_integrations.sh
+```
+
+---
+
+## Notes
+
+- **`AUTHENTIK_SECRET_KEY` is set-once.** Rotating it invalidates all sessions, tokens, and encrypted fields in the database. Keep `~/.authentik/.env` safe and backed up.
+- **PostgreSQL holds everything that matters** — users, providers, groups, tokens, certificates. It lives on an 8Gi Longhorn PVC. Redis is ephemeral by design.
+- **No ingress.** Access is via port-forward, matching the rest of the homelab.
+- **OIDC signing keys** (rotated by `provision_oidc.py --rotate`) are separate from all `.env` credentials. Rotation requires no client reconfiguration — Grafana and MinIO pick up the new public key from the JWKS endpoint automatically.
+- **New app registration** is automated via `register_oauth_app.py` and fully documented in `OAUTH_APP_SETUP.md`. Both manual and automated workflows are supported.

k8s/talos-iam/authentik-migrations-job.yaml

@@ -0,0 +1,24 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: authentik-migrations
+  namespace: iam
+spec:
+  ttlSecondsAfterFinished: 600
+  backoffLimit: 3
+  template:
+    spec:
+      serviceAccountName: default
+      restartPolicy: Never
+      containers:
+      - name: migrate
+        image: ghcr.io/goauthentik/server:2026.5.4
+        imagePullPolicy: IfNotPresent
+        envFrom:
+        - secretRef:
+            name: authentik
+        command:
+        - /bin/sh
+        - -c
+        - |
+          python -m manage migrate --noinput && echo "✓ Migrations complete"

k8s/talos-iam/authentik-values.yaml

@@ -0,0 +1,225 @@
+# k8s/talos-iam/authentik-values.yaml
+# Authentik — SSO Identity Provider for the homelab.
+# Provides OAuth2/OIDC login for Grafana, MinIO, Forgejo, and Argo CD.
+# Chart: authentik/authentik from https://charts.goauthentik.io
+#
+# Architecture: server (UI+API) + worker (background tasks) + PostgreSQL + Redis.
+# PostgreSQL is the system of record — must persist. Redis is ephemeral cache/queue.
+#
+# Secrets injected via helmfile --set (from .env / vsource):
+#   AUTHENTIK_SECRET_KEY         — signs sessions and tokens; set once, never rotate casually
+#   AUTHENTIK_BOOTSTRAP_PASSWORD — initial akadmin password (used once at first login)
+#   AUTHENTIK_BOOTSTRAP_TOKEN    — API token for the setup_talos_iam.sh bootstrap script
+#   AUTHENTIK_PG_PASSWORD        — PostgreSQL user password
+
+authentik:
+  # host: the external URL Authentik uses to build redirect URIs in OAuth2 flows.
+  # Must match what the browser sees — if it returns an internal svc URL,
+  # the browser's redirect after login will fail (can't reach svc DNS externally).
+  # HTTP (not HTTPS) because the Authentik ingress has no TLS cert configured.
+  host: "https://authentik.riotpiao.homelab.com"
+
+  error_reporting:
+    enabled: false   # do not phone home to Sentry
+
+  # PostgreSQL connection — points at CloudNativePG cluster in ddb namespace.
+  # password is injected via helmfile --set at deploy time.
+  postgresql:
+    host: ddb-cluster-rw.ddb.svc.cluster.local
+    port: 5432
+    name: authentik
+    user: authentik
+    password: ""  # injected via helmfile --set authentik.postgresql.password
+
+  # Redis connection — bundled subchart, standalone mode (no sentinel/cluster).
+  redis:
+    host: authentik-redis-master
+
+  # ── HTTP client timeouts ──────────────────────────────────────────────────────
+  # Increased to tolerate 5+ second pod-to-pod network latency.
+  # Affects webhooks, outpost management, SCIM, LDAP sync.
+  # Default: ~30s — too aggressive when latency spikes hit 5-10s.
+  log_level: debug  # enable debug logging to monitor connection issues
+
+# ── CA trust (shared by server and worker) ────────────────────────────────────
+# Authentik (Python/Debian) uses requests + httpx for outgoing HTTPS — webhooks,
+# outpost management, SCIM. Both libraries need REQUESTS_CA_BUNDLE / SSL_CERT_FILE
+# to point to a bundle that includes homelab-ca, otherwise connections to other
+# homelab services fail with "certificate signed by unknown authority".
+#
+# Strategy: a debian:12-slim init container (run as root) concatenates the
+# Debian system Mozilla bundle with homelab-ca.crt into an emptyDir. The main
+# container then references /merged/ca-bundle.crt via two env vars that cover
+# every Python HTTP library.
+_caVolumes: &caVolumes
+  - name: homelab-ca
+    configMap:
+      name: homelab-ca
+  - name: merged-ca
+    emptyDir: {}
+
+_caVolumeMounts: &caVolumeMounts
+  - name: homelab-ca
+    mountPath: /homelab-ca
+    readOnly: true
+  - name: merged-ca
+    mountPath: /merged
+    readOnly: true
+
+_caInitContainers: &caInitContainers
+  - name: merge-ca-certs
+    image: debian:bookworm
+    imagePullPolicy: IfNotPresent
+    securityContext:
+      runAsUser: 0
+    command:
+      - sh
+      - -c
+      - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/homelab-ca.crt) > /merged/ca-bundle.crt
+    volumeMounts:
+      - name: homelab-ca
+        mountPath: /homelab-ca
+        readOnly: true
+      - name: merged-ca
+        mountPath: /merged
+  - name: authentik-migrate
+    image: ghcr.io/goauthentik/server:2026.5.4
+    imagePullPolicy: IfNotPresent
+    command:
+      - sh
+      - -c
+      - python -m manage migrate --noinput
+    envFrom:
+      - secretRef:
+          name: authentik
+    volumeMounts: *caVolumeMounts
+
+_caEnv: &caEnv
+  - name: REQUESTS_CA_BUNDLE
+    value: /merged/ca-bundle.crt
+  - name: SSL_CERT_FILE
+    value: /merged/ca-bundle.crt
+
+# ── Authentik server (UI + API) ───────────────────────────────────────────────
+# Handles all browser traffic: login flows, admin UI, OAuth2 authorize/token endpoints.
+# NodePort 32172 is a fallback for direct node access during troubleshooting;
+# normal access is via nginx ingress (authentik.riotpiao.homelab.com → svc:80).
+# Recreate: single replica + RWO-adjacent state — avoids split-brain on redeploy.
+server:
+  replicas: 1
+  deploymentStrategy:
+    type: Recreate
+  service:
+    type: NodePort
+    nodePort: 32172
+  resources:
+    requests:
+      cpu: 100m
+      memory: 512Mi
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  volumes: *caVolumes
+  volumeMounts: *caVolumeMounts
+  initContainers: *caInitContainers
+  env: *caEnv
+  podAnnotations:
+    configmap.reloader.stakater.com/reload: "homelab-ca"
+    homelab.io/restart-at: "2026-06-21T13-40"
+  # Every OIDC login (Grafana, Argo CD, MinIO, Forgejo) depends on this server —
+  # its request latency/error rate explains SSO-driven slowness on those services.
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      scrapeTimeout: 60s
+
+# ── Authentik worker ──────────────────────────────────────────────────────────
+# Runs background tasks: email delivery, LDAP sync, flow policy evaluation,
+# event log cleanup, and managed outpost updates. Stateless — no PVC needed.
+# Same resource profile as server; Authentik 2023+ merged some worker duties
+# into the server process but the worker pod is still required.
+worker:
+  replicas: 1
+  deploymentStrategy:
+    type: Recreate
+  resources:
+    requests:
+      cpu: 100m
+      memory: 512Mi
+    limits:
+      cpu: 1000m
+      memory: 1Gi
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  volumes: *caVolumes
+  volumeMounts: *caVolumeMounts
+  initContainers: *caInitContainers
+  env: *caEnv
+  podAnnotations:
+    configmap.reloader.stakater.com/reload: "homelab-ca"
+    homelab.io/restart-at: "2026-06-21T13-40"
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true
+      scrapeTimeout: 60s
+
+# ── PostgreSQL (external: CloudNativePG cluster in ddb namespace) ─────────────
+# Authentik connects to the centralized ddb-cluster (1 primary + 2 replicas with pgvector).
+# Do not use the bundled Bitnami subchart — CNPG is already running.
+postgresql:
+  enabled: false
+  primary:
+    persistence:
+      enabled: true
+      storageClass: longhorn
+      size: 8Gi
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: node-role.kubernetes.io/worker
+                  operator: Exists
+
+# ── Bundled Redis ─────────────────────────────────────────────────────────────
+# Cache and async task queue only — no durable data. If Redis restarts, in-flight
+# background tasks are retried and cached tokens are recomputed. Losing Redis
+# data does not lose user accounts or flow configuration (that's in PostgreSQL).
+# persistence: false saves a PVC and makes restarts faster.
+#
+# Same prefer-worker / fallback-to-cp scheduling as PostgreSQL.
+# architecture: standalone — no Sentinel/cluster overhead for a 3-node homelab.
+redis:
+  enabled: true
+  master:
+    persistence:
+      enabled: false
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: node-role.kubernetes.io/worker
+                  operator: Exists
+  architecture: standalone
+
+# Ingress disabled — rule lives in k8s/ingress/ingress.yaml (authentik.riotpiao.homelab.com).
+# For direct access during bootstrap: kubectl -n iam port-forward svc/authentik-server 7000:80

k8s/talos-iam/example_register_dashboard.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Example: Register 'dashboard-service' with OAuth + Vault JWT auth.
+
+This is a complete working example that demonstrates:
+1. Creating an Authentik OAuth2 provider and application
+2. Creating Kubernetes secrets for OIDC credentials
+3. Creating a Vault JWT role for service authentication
+4. Group-based RBAC
+
+To use this example:
+
+  # 1. Ensure prerequisites
+  kubectl port-forward -n iam svc/authentik-server 7000:80 &
+  kubectl port-forward -n storage svc/vault 8200:8200 &
+  
+  # 2. Generate and store credentials
+  DASHBOARD_OIDC_CLIENT_SECRET=$(openssl rand -hex 32)
+  export DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET"
+  talos put cluster/DASHBOARD_OIDC_CLIENT_SECRET DASHBOARD_OIDC_CLIENT_SECRET="$DASHBOARD_OIDC_CLIENT_SECRET"
+  
+  # 3. Run registration
+  python3 register_oauth_app.py \\
+    --service-name dashboard-service \\
+    --namespace apps \\
+    --redirect-uri "https://dashboard.riotpiao.homelab.com/oauth2/callback" \\
+    --service-name-in-vault DASHBOARD \\
+    --vault-jwt-policy shell-secrets \\
+    --add-group dashboard-admins \\
+    --vault-jwt-bound-claims '{"groups":["dashboard-admins"]}'
+  
+  # 4. Verify registration
+  curl -s -H "Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN" \\
+    http://localhost:7000/api/v3/core/applications/?slug=dashboard-service | jq .
+  
+  # 5. Test OAuth callback (requires app running)
+  # Browser: http://dashboard.riotpiao.homelab.com/login
+  # Should redirect to Authentik → back to dashboard with session
+"""
+
+import subprocess
+import sys
+import os
+
+
+def example_dashboard_service():
+    """Register the example 'dashboard-service' with full RBAC."""
+    
+    # Check prerequisites
+    print("Checking prerequisites...")
+    
+    required_env = [
+        "AUTHENTIK_BOOTSTRAP_TOKEN",
+        "DASHBOARD_OIDC_CLIENT_SECRET",
+    ]
+    
+    missing = [v for v in required_env if not os.environ.get(v)]
+    if missing:
+        print(f"❌ Missing environment variables: {', '.join(missing)}")
+        print("\nSet them:")
+        print("  talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN | source")
+        print("  talos get cluster/DASHBOARD_OIDC_CLIENT_SECRET --key DASHBOARD_OIDC_CLIENT_SECRET | source")
+        return False
+    
+    # Call the registration script
+    cmd = [
+        "python3",
+        "register_oauth_app.py",
+        "--service-name", "dashboard-service",
+        "--namespace", "apps",
+        "--redirect-uri", "https://dashboard.riotpiao.homelab.com/oauth2/callback",
+        "--service-name-in-vault", "DASHBOARD",
+        "--vault-jwt-policy", "shell-secrets",
+        "--add-group", "dashboard-admins",
+        "--vault-jwt-bound-claims", '{"groups":["dashboard-admins"]}',
+    ]
+    
+    print(f"\nRunning: {' '.join(cmd)}\n")
+    
+    result = subprocess.run(cmd, check=False)
+    return result.returncode == 0
+
+
+def verify_example():
+    """Verify that the dashboard-service was registered correctly."""
+    print("\n" + "="*70)
+    print("Verification Steps")
+    print("="*70 + "\n")
+    
+    print("✅ Check Authentik provider:")
+    print("  curl -s -H 'Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN' \\")
+    print("    http://localhost:7000/api/v3/providers/oauth2/?name=dashboard-service \\")
+    print("    | jq '.results[0] | {name, client_id, redirect_uris}'")
+    print()
+    
+    print("✅ Check Authentik application:")
+    print("  curl -s -H 'Authorization: Bearer $AUTHENTIK_BOOTSTRAP_TOKEN' \\")
+    print("    http://localhost:7000/api/v3/core/applications/?slug=dashboard-service \\")
+    print("    | jq '.results[0] | {name, slug, provider}'")
+    print()
+    
+    print("✅ Check Kubernetes secret:")
+    print("  kubectl get secret dashboard-service-oidc -n apps -o jsonpath='{.data}' \\")
+    print("    | base64 -d | jq .")
+    print()
+    
+    print("✅ Check Vault JWT role:")
+    print("  vault read auth/jwt/role/dashboard-service")
+    print()
+    
+    print("✅ Test Vault JWT authentication:")
+    print("  # Get ID token from Authentik")
+    print("  ID_TOKEN=$(curl -s -X POST \\")
+    print("    http://localhost:7000/application/o/dashboard-service/token/ \\")
+    print("    -d 'grant_type=client_credentials&client_id=dashboard-service&client_secret=...' \\")
+    print("    | jq -r '.access_token')")
+    print()
+    print("  # Authenticate to Vault with JWT")
+    print("  vault write auth/jwt/login \\")
+    print("    role=dashboard-service \\")
+    print("    jwt=\"$ID_TOKEN\"")
+    print()
+    
+    print("✅ Test OAuth redirect (requires app running):")
+    print("  curl -L https://dashboard.riotpiao.homelab.com/login")
+    print("  # Should redirect to Authentik, then back to dashboard")
+    print()
+
+
+if __name__ == "__main__":
+    if not example_dashboard_service():
+        sys.exit(1)
+    
+    verify_example()

k8s/talos-iam/go-example-oidc/.env.example

@@ -0,0 +1,8 @@
+# Copy to ~/.authentik/.env and fill in your values.
+# Port-forward Authentik first: kubectl port-forward svc/authentik-server -n iam 7000:80
+
+AUTHENTIK_BASE_URL=http://localhost:7000
+APP_SLUG=go-example
+OIDC_CLIENT_ID=go-example
+OIDC_CLIENT_SECRET=your-client-secret-here
+REDIRECT_URL=http://localhost:8080/callback

k8s/talos-iam/go-example-oidc/go.mod

@@ -0,0 +1,14 @@
+module homelab/go-example-oidc
+
+go 1.22
+
+require (
+	github.com/coreos/go-oidc/v3 v3.11.0
+	github.com/joho/godotenv v1.5.1
+	golang.org/x/oauth2 v0.24.0
+)
+
+require (
+	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+)

k8s/talos-iam/go-example-oidc/go.sum

@@ -0,0 +1,20 @@
+github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI=
+github.com/coreos/go-oidc/v3 v3.11.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk=
+github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
+golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
+golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
+golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

k8s/talos-iam/go-example-oidc/main.go

@@ -0,0 +1,90 @@
+// Minimal OIDC Authorization Code flow against Authentik.
+//
+// Setup:
+//   1. In Authentik create a provider + app (slug "go-example", redirect URI http://localhost:8080/callback).
+//   2. Fill in ~/.authentik/.env (see .env.example).
+//   3. go mod tidy && go run .
+//   4. Open http://localhost:8080/login
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"github.com/joho/godotenv"
+	"golang.org/x/oauth2"
+)
+
+// oauthState is a fixed random value for this process — good enough for a local demo.
+// In production, generate a per-request random state and store it in a cookie.
+var oauthState = "homelab-oidc-example"
+
+func main() {
+	// Load ~/.authentik/.env; shell env vars already set take precedence.
+	home, _ := os.UserHomeDir()
+	godotenv.Load(filepath.Join(home, ".authentik", ".env"))
+
+	ctx := context.Background()
+
+	// go-oidc discovers the token endpoint, auth endpoint, and JWKS URI automatically
+	// from Authentik's /.well-known/openid-configuration.
+	issuer := os.Getenv("AUTHENTIK_BASE_URL") + "/application/o/" + os.Getenv("APP_SLUG")
+	provider, err := oidc.NewProvider(ctx, issuer)
+	if err != nil {
+		log.Fatalf("OIDC discovery failed (%s): %v", issuer, err)
+	}
+
+	cfg := &oauth2.Config{
+		ClientID:     os.Getenv("OIDC_CLIENT_ID"),
+		ClientSecret: os.Getenv("OIDC_CLIENT_SECRET"),
+		RedirectURL:  os.Getenv("REDIRECT_URL"),
+		Endpoint:     provider.Endpoint(),
+		Scopes:       []string{oidc.ScopeOpenID, "email", "profile"},
+	}
+	verifier := provider.Verifier(&oidc.Config{ClientID: cfg.ClientID})
+
+	// /login — redirect the browser to Authentik's authorization endpoint
+	http.HandleFunc("/login", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, cfg.AuthCodeURL(oauthState), http.StatusFound)
+	})
+
+	// /callback — Authentik redirects here with ?code=...&state=...
+	http.HandleFunc("/callback", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Query().Get("state") != oauthState {
+			http.Error(w, "state mismatch", http.StatusBadRequest)
+			return
+		}
+
+		// Exchange the authorization code for tokens
+		token, err := cfg.Exchange(ctx, r.URL.Query().Get("code"))
+		if err != nil {
+			http.Error(w, "token exchange: "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		// Verify the ID token signature against Authentik's JWKS, then extract claims
+		rawID, _ := token.Extra("id_token").(string)
+		idToken, err := verifier.Verify(ctx, rawID)
+		if err != nil {
+			http.Error(w, "id_token verify: "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var claims map[string]any
+		idToken.Claims(&claims)
+
+		w.Header().Set("Content-Type", "application/json")
+		enc := json.NewEncoder(w)
+		enc.SetIndent("", "  ")
+		enc.Encode(claims)
+	})
+
+	fmt.Println("open http://localhost:8080/login")
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

k8s/talos-iam/key-rotation-cronjob.yaml

@@ -0,0 +1,52 @@
+# Rotates the Authentik OIDC signing key quarterly for all homelab providers.
+#
+# Prerequisites (run once):
+#   # 1. ConfigMap from the Rust source file
+#   kubectl create configmap key-rotation-script \
+#     --from-file=rotate_key.rs=key_rotate.rs \
+#     -n iam --dry-run=client -o yaml | kubectl apply -f -
+#
+#   # 2. Token secret — fill in your value, never commit it
+#   kubectl create secret generic authentik-key-rotation-token \
+#     --from-literal=AUTHENTIK_BOOTSTRAP_TOKEN="${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+#     -n iam --dry-run=client -o yaml | kubectl apply -f -
+#
+# Apply:  kubectl apply -f key-rotation-cronjob.yaml
+# Test:   kubectl create job --from=cronjob/authentik-key-rotation test-rotation -n iam
+# Logs:   kubectl logs -n iam -l job-name=test-rotation -f
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: authentik-key-rotation
+  namespace: iam
+spec:
+  schedule: "0 0 1 */3 *"   # 00:00 UTC on the 1st of Jan, Apr, Jul, Oct
+  concurrencyPolicy: Forbid  # skip if a previous job is still running
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          volumes:
+            - name: script
+              configMap:
+                name: key-rotation-script
+          containers:
+            - name: rotate
+              image: rust:1.82-slim
+              command:
+                - sh
+                - -c
+                - rustc /scripts/rotate_key.rs -o /tmp/rotate_key && /tmp/rotate_key
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+              env:
+                - name: AUTHENTIK_BASE_URL
+                  value: "http://authentik-server.iam.svc.cluster.local"
+                - name: AUTHENTIK_BOOTSTRAP_TOKEN
+                  valueFrom:
+                    secretKeyRef:
+                      name: authentik-key-rotation-token
+                      key: AUTHENTIK_BOOTSTRAP_TOKEN

k8s/talos-iam/key_rotate.rs

@@ -0,0 +1,114 @@
+use std::env;
+use std::io::{Read, Write};
+use std::net::TcpStream;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+const SIGNING_CERT_NAME: &str = "homelab-oidc";
+const PROVIDERS: &[&str] = &["grafana", "minio"];
+
+fn main() {
+    let base_url = env::var("AUTHENTIK_BASE_URL")
+        .unwrap_or_else(|_| "http://localhost:7000".into());
+    let token = env::var("AUTHENTIK_BOOTSTRAP_TOKEN")
+        .expect("AUTHENTIK_BOOTSTRAP_TOKEN must be set");
+
+    let host = parse_host(&base_url);
+
+    let ts = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    let new_name = format!("{}-{}", SIGNING_CERT_NAME, ts);
+
+    let body = format!(
+        r#"{{"common_name":"{}","validity_days":365,"key_size":4096}}"#,
+        new_name
+    );
+    let resp = http(&host, &token, "POST", "/api/v3/crypto/certificatekeypairs/generate/", Some(&body));
+    let new_pk = extract_str(&resp, "pk").expect("no pk in generate response");
+    println!("created  '{}' pk={}", new_name, new_pk);
+
+    for provider in PROVIDERS {
+        let list = http(
+            &host, &token, "GET",
+            &format!("/api/v3/providers/oauth2/?name={}", provider),
+            None,
+        );
+        let provider_pk = match extract_str(&list, "pk") {
+            Some(pk) => pk,
+            None => {
+                eprintln!("WARN: provider '{}' not found — skipping", provider);
+                continue;
+            }
+        };
+        let patch = format!(r#"{{"signing_key":"{}"}}"#, new_pk);
+        http(&host, &token, "PATCH",
+            &format!("/api/v3/providers/oauth2/{}/", provider_pk),
+            Some(&patch),
+        );
+        println!("rotated  '{}' → signing_key={}", provider, new_pk);
+    }
+
+    println!("rotation complete");
+}
+
+fn parse_host(base_url: &str) -> String {
+    let stripped = base_url
+        .trim_start_matches("http://")
+        .trim_start_matches("https://");
+    let host = stripped.split('/').next().unwrap_or(stripped);
+    if host.contains(':') {
+        host.to_string()
+    } else {
+        format!("{}:80", host)
+    }
+}
+
+fn http(host: &str, token: &str, method: &str, path: &str, body: Option<&str>) -> String {
+    let mut stream = TcpStream::connect(host)
+        .unwrap_or_else(|e| panic!("connect {}: {}", host, e));
+
+    let body_str = body.unwrap_or("");
+    let hostname = host.split(':').next().unwrap_or(host);
+    let req = format!(
+        "{method} {path} HTTP/1.1\r\n\
+         Host: {hostname}\r\n\
+         Authorization: Bearer {token}\r\n\
+         Content-Type: application/json\r\n\
+         Content-Length: {len}\r\n\
+         Connection: close\r\n\
+         \r\n\
+         {body_str}",
+        len = body_str.len(),
+    );
+    stream.write_all(req.as_bytes()).unwrap();
+
+    let mut raw = String::new();
+    stream.read_to_string(&mut raw).unwrap();
+
+    let (head, resp_body) = raw.split_once("\r\n\r\n").unwrap_or((&raw, ""));
+
+    let status: u16 = head.lines().next()
+        .and_then(|l| l.split_whitespace().nth(1))
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(0);
+    if status >= 400 {
+        panic!("{} {} → HTTP {} — {}", method, path, status, resp_body.trim());
+    }
+
+    resp_body.to_string()
+}
+
+// Extracts the value of the first `"key": <value>` match in raw JSON.
+// Handles both quoted strings ("pk": "uuid") and bare numbers ("pk": 5).
+fn extract_str(json: &str, key: &str) -> Option<String> {
+    let needle = format!("\"{}\":", key);
+    let after_colon = json.find(&needle)? + needle.len();
+    let rest = json[after_colon..].trim_start();
+    if let Some(inner) = rest.strip_prefix('"') {
+        Some(inner[..inner.find('"')?].to_string())
+    } else {
+        let end = rest.find(|c: char| c == ',' || c == '}' || c.is_ascii_whitespace())?;
+        Some(rest[..end].to_string())
+    }
+}

k8s/talos-iam/register_oauth_app.py

@@ -0,0 +1,512 @@
+#!/usr/bin/env python3
+"""
+register_oauth_app.py
+Automated OAuth app registration & Vault JWT wiring for homelab services.
+
+This script automates the full workflow of registering a new service with
+Authentik OIDC and wiring it for Vault JWT authentication.
+
+Usage:
+  python3 register_oauth_app.py \\
+    --service-name my-service \\
+    --namespace my-ns \\
+    --redirect-uri "https://my-service.riotpiao.homelab.com/oauth2/callback" \\
+    [--service-name-in-vault MY_SERVICE] \\
+    [--vault-jwt-policy shell-secrets]
+
+Requirements:
+  - Authentik running in iam namespace (port-forward 7000:80 active)
+  - Vault running in storage namespace
+  - kubectl configured (KUBECONFIG → cluster-config/kubeconfig)
+  - Environment variables set:
+      AUTHENTIK_BOOTSTRAP_TOKEN       — Authentik API token
+      {SERVICE_NAME}_OIDC_CLIENT_SECRET  — OAuth client secret (generated or from Vault)
+
+Examples:
+
+  # Simple: register a web app with default settings
+  python3 register_oauth_app.py \\
+    --service-name myapp \\
+    --namespace apps \\
+    --redirect-uri "https://myapp.riotpiao.homelab.com/callback"
+
+  # Advanced: full service with Vault JWT auth
+  python3 register_oauth_app.py \\
+    --service-name myservice \\
+    --namespace my-namespace \\
+    --redirect-uri "https://myservice.riotpiao.homelab.com/oauth2/callback" \\
+    --service-name-in-vault MYSERVICE \\
+    --vault-jwt-policy shell-secrets \\
+    --add-group myservice-admins \\
+    --vault-jwt-bound-claims '{"groups":["myservice-admins"]}'
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from typing import Optional
+
+
+BASE_AUTHENTIK_URL = "http://localhost:7000/api/v3"
+VAULT_NAMESPACE = "storage"
+IAM_NAMESPACE = "iam"
+
+
+def _auth_headers() -> dict:
+    """Return HTTP headers for Authentik API requests."""
+    token = os.environ.get("AUTHENTIK_BOOTSTRAP_TOKEN", "")
+    if not token:
+        sys.exit(
+            "ERROR: AUTHENTIK_BOOTSTRAP_TOKEN not set\n"
+            "  Set it in ~/.authentik/.env or export it:\n"
+            "    talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN"
+        )
+    return {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+    }
+
+
+def _request(method: str, path: str, body: Optional[dict] = None, **params) -> dict:
+    """HTTP request helper for Authentik API."""
+    url = BASE_AUTHENTIK_URL + path
+    if params:
+        url += "?" + urllib.parse.urlencode(params)
+    
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode() if body else None,
+        headers=_auth_headers(),
+        method=method,
+    )
+    
+    try:
+        with urllib.request.urlopen(req) as resp:
+            raw = resp.read()
+            return json.loads(raw) if raw else {}
+    except urllib.error.HTTPError as exc:
+        detail = exc.read().decode(errors="replace")
+        print(f"ERROR {method} {path}: HTTP {exc.code} — {detail}", file=sys.stderr)
+        raise
+
+
+def get_pk(path: str, **filters) -> Optional[str]:
+    """Fetch the PK of the first result matching filters."""
+    results = _request("GET", path, **filters).get("results", [])
+    return results[0]["pk"] if results else None
+
+
+def get_or_create(path: str, filters: dict, data: dict) -> str:
+    """Create or fetch a resource; returns PK."""
+    pk = get_pk(path, **filters)
+    if pk is None:
+        pk = _request("POST", path, data)["pk"]
+        print(f"✓ created  {path.rstrip('/')} {filters}")
+    else:
+        print(f"✓ exists   {path.rstrip('/')} {filters}")
+    return pk
+
+
+def fetch_flows_and_signing_key() -> tuple[str, str, str]:
+    """Fetch required Authentik resource PKs."""
+    auth_pk = get_pk(
+        "/flows/instances/",
+        slug="default-provider-authorization-implicit-consent"
+    )
+    inval_pk = get_pk(
+        "/flows/instances/",
+        slug="default-provider-invalidation-flow"
+    )
+    
+    if not auth_pk or not inval_pk:
+        sys.exit(
+            "ERROR: required Authentik flows not found\n"
+            "  Run provision_oidc.py first to initialize defaults"
+        )
+    
+    results = _request("GET", "/crypto/certificatekeypairs/", has_key=True).get("results", [])
+    signing_pk = results[0]["pk"] if results else None
+    
+    if not signing_pk:
+        sys.exit(
+            "ERROR: no signing key found in Authentik\n"
+            "  Create one: Authentik UI → System → Certificates"
+        )
+    
+    return auth_pk, inval_pk, signing_pk
+
+
+def apply_k8s_secret(name: str, namespace: str, literals: dict) -> None:
+    """Create or update a Kubernetes secret."""
+    env = {**os.environ}
+    manifest = subprocess.run(
+        [
+            "kubectl", "create", "secret", "generic", name,
+            "-n", namespace, "--dry-run=client", "-o", "yaml",
+            *[f"--from-literal={k}={v}" for k, v in literals.items()],
+        ],
+        capture_output=True, text=True, check=True, env=env,
+    ).stdout
+    
+    subprocess.run(
+        ["kubectl", "apply", "-f", "-"],
+        input=manifest, text=True, check=True, env=env
+    )
+    print(f"✓ secret   {name} → ns/{namespace}")
+
+
+def create_authentik_provider(
+    service_name: str,
+    client_id: str,
+    client_secret: str,
+    redirect_uris: list[str],
+    auth_flow_pk: str,
+    inval_flow_pk: str,
+    signing_key_pk: str,
+) -> str:
+    """Create OAuth2 provider in Authentik."""
+    data = {
+        "name": service_name,
+        "client_id": client_id,
+        "client_secret": client_secret,
+        "authorization_flow": auth_flow_pk,
+        "invalidation_flow": inval_flow_pk,
+        "redirect_uris": [{"matching_mode": "strict", "url": u} for u in redirect_uris],
+        "sub_mode": "hashed_user_id",
+        "include_claims_in_id_token": True,
+        "property_mappings": [],  # Will be populated by caller if needed
+        "grant_types": ["authorization_code", "refresh_token"],
+        "signing_key": signing_key_pk,
+    }
+    
+    pk = get_pk("/providers/oauth2/", name=service_name)
+    if pk is None:
+        pk = _request("POST", "/providers/oauth2/", data)["pk"]
+        print(f"✓ created  provider={service_name}  client_id={client_id}")
+    else:
+        _request("PATCH", f"/providers/oauth2/{pk}/", data)
+        print(f"✓ synced   provider={service_name}  (credentials from Vault)")
+    
+    return pk
+
+
+def create_authentik_application(
+    display_name: str,
+    slug: str,
+    provider_pk: str,
+    launch_url: Optional[str] = None,
+) -> str:
+    """Create application in Authentik."""
+    data = {"name": display_name, "slug": slug, "provider": provider_pk}
+    if launch_url:
+        data["meta_launch_url"] = launch_url
+    
+    return get_or_create(
+        "/core/applications/",
+        filters={"slug": slug},
+        data=data
+    )
+
+
+def create_authentik_group(group_name: str) -> str:
+    """Create a group in Authentik."""
+    return get_or_create(
+        "/core/groups/",
+        filters={"name": group_name},
+        data={"name": group_name, "is_superuser": False}
+    )
+
+
+def bind_group_to_app(app_pk: str, group_pk: str) -> None:
+    """Bind a group to an application (allow-list policy)."""
+    existing = _request(
+        "GET", "/policies/bindings/",
+        target=app_pk, group=group_pk
+    ).get("results", [])
+    
+    if existing:
+        print(f"✓ exists   binding group={group_pk} → app={app_pk}")
+    else:
+        _request("POST", "/policies/bindings/", {
+            "target": app_pk,
+            "group": group_pk,
+            "enabled": True,
+            "order": 0,
+        })
+        print(f"✓ created  binding group={group_pk} → app={app_pk}")
+
+
+def create_vault_jwt_policy(
+    service_name: str,
+    vault_addr: str,
+) -> str:
+    """Create a Vault policy for the service."""
+    policy_name = f"service-read-{service_name}"
+    
+    # Policy definition: service can read its own secrets
+    policy_rules = f"""
+path "secret/data/services/{service_name}/*" {{
+  capabilities = ["read"]
+}}
+path "secret/data/cluster/*" {{
+  capabilities = ["read"]
+}}
+"""
+    
+    # Write policy via vault CLI (requires auth)
+    try:
+        subprocess.run(
+            ["vault", "policy", "write", policy_name, "-"],
+            input=policy_rules,
+            text=True,
+            check=True,
+            env={**os.environ, "VAULT_ADDR": vault_addr}
+        )
+        print(f"✓ created  Vault policy={policy_name}")
+    except subprocess.CalledProcessError as e:
+        print(f"⚠ warning  Failed to create Vault policy: {e}", file=sys.stderr)
+        return ""
+    
+    return policy_name
+
+
+def create_vault_jwt_role(
+    service_name: str,
+    vault_addr: str,
+    policies: list[str],
+    bound_claims: Optional[dict] = None,
+) -> None:
+    """Create a Vault JWT auth role for the service."""
+    bound_claims_json = json.dumps(bound_claims) if bound_claims else "{}"
+    
+    role_config = {
+        "role_type": "jwt",
+        "bound_audiences": ["vault"],
+        "user_claim": "sub",
+        "bound_claims": bound_claims,
+        "token_policies": policies,
+        "token_ttl": "4h",
+        "token_max_ttl": "8h",
+    }
+    
+    try:
+        subprocess.run(
+            ["vault", "write", f"auth/jwt/role/{service_name}", "-"],
+            input=json.dumps(role_config),
+            text=True,
+            check=True,
+            env={**os.environ, "VAULT_ADDR": vault_addr}
+        )
+        print(f"✓ created  Vault JWT role={service_name}  policies={policies}")
+    except subprocess.CalledProcessError as e:
+        print(f"⚠ warning  Failed to create Vault JWT role: {e}", file=sys.stderr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Register a new OAuth service with Authentik + Vault",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    
+    parser.add_argument(
+        "--service-name",
+        required=True,
+        help="Service name (used as Authentik provider/app slug)"
+    )
+    parser.add_argument(
+        "--namespace",
+        required=True,
+        help="Kubernetes namespace where service runs"
+    )
+    parser.add_argument(
+        "--redirect-uri",
+        required=True,
+        help="OAuth2 redirect URI (e.g., https://myapp.riotpiao.homelab.com/callback)"
+    )
+    parser.add_argument(
+        "--service-name-in-vault",
+        default=None,
+        help="Service name for Vault (defaults to uppercase service-name)"
+    )
+    parser.add_argument(
+        "--vault-jwt-policy",
+        default="shell-secrets",
+        help="Vault policy to attach to JWT role (comma-separated for multiple)"
+    )
+    parser.add_argument(
+        "--add-group",
+        default=None,
+        help="Create and bind a group to the app (e.g., 'myservice-admins')"
+    )
+    parser.add_argument(
+        "--vault-jwt-bound-claims",
+        default=None,
+        help='JSON string of claims to bind JWT role (e.g., \'{"groups":["myservice-admins"]}\')'
+    )
+    parser.add_argument(
+        "--no-vault",
+        action="store_true",
+        help="Skip Vault JWT role creation (just register with Authentik)"
+    )
+    parser.add_argument(
+        "--vault-addr",
+        default=None,
+        help="Vault address (defaults to localhost:8200 via port-forward)"
+    )
+    
+    args = parser.parse_args()
+    
+    service_name = args.service_name.lower()
+    namespace = args.namespace
+    redirect_uri = args.redirect_uri
+    vault_service_name = (args.service_name_in_vault or service_name).upper()
+    vault_addr = args.vault_addr or "http://127.0.0.1:8200"
+    vault_policies = [p.strip() for p in args.vault_jwt_policy.split(",")]
+    
+    # Fetch or generate client secret from environment / Vault
+    client_secret_env = f"{vault_service_name}_OIDC_CLIENT_SECRET"
+    client_secret = os.environ.get(client_secret_env, "")
+    
+    if not client_secret:
+        sys.exit(
+            f"ERROR: {client_secret_env} not set\n"
+            f"  Generate and store in Vault:\n"
+            f"    talos put cluster/{client_secret_env} "
+            f"{client_secret_env}=$(openssl rand -hex 32)"
+        )
+    
+    print(f"\n{'='*70}")
+    print(f"OAuth App Registration: {service_name}")
+    print(f"{'='*70}\n")
+    
+    print(f"📋 Configuration:")
+    print(f"  Service Name:     {service_name}")
+    print(f"  Namespace:        {namespace}")
+    print(f"  Redirect URI:     {redirect_uri}")
+    print(f"  Vault Service:    {vault_service_name}")
+    print(f"  Vault Policies:   {', '.join(vault_policies)}")
+    if args.add_group:
+        print(f"  Group:            {args.add_group}")
+    print()
+    
+    # Fetch Authentik resources
+    print("🔍 Fetching Authentik configuration...")
+    auth_flow_pk, inval_flow_pk, signing_key_pk = fetch_flows_and_signing_key()
+    print(f"✓ flows & signing key found\n")
+    
+    # Create Authentik provider
+    print("🔐 Creating Authentik OAuth2 Provider...")
+    provider_pk = create_authentik_provider(
+        service_name=service_name,
+        client_id=service_name,
+        client_secret=client_secret,
+        redirect_uris=[redirect_uri],
+        auth_flow_pk=auth_flow_pk,
+        inval_flow_pk=inval_flow_pk,
+        signing_key_pk=signing_key_pk,
+    )
+    print()
+    
+    # Create Authentik application
+    print("📱 Creating Authentik Application...")
+    app_pk = create_authentik_application(
+        display_name=service_name.replace("-", " ").title(),
+        slug=service_name,
+        provider_pk=provider_pk,
+        launch_url=f"https://{service_name}.riotpiao.homelab.com"
+    )
+    print()
+    
+    # Create and bind group (if requested)
+    if args.add_group:
+        print(f"👥 Creating Group: {args.add_group}")
+        group_pk = create_authentik_group(args.add_group)
+        bind_group_to_app(app_pk, group_pk)
+        print()
+    
+    # Create K8s secret
+    print("🔑 Creating Kubernetes Secret...")
+    apply_k8s_secret(
+        f"{service_name}-oidc",
+        namespace,
+        {
+            "client_id": service_name,
+            "client_secret": client_secret,
+            "issuer_url": f"http://authentik-server.iam.svc.cluster.local/application/o/{service_name}/",
+            "redirect_uri": redirect_uri,
+        }
+    )
+    print()
+    
+    # Create Vault resources (if not disabled)
+    if not args.no_vault:
+        print("🔐 Configuring Vault JWT Authentication...")
+        
+        # Parse bound claims if provided
+        bound_claims = None
+        if args.vault_jwt_bound_claims:
+            try:
+                bound_claims = json.loads(args.vault_jwt_bound_claims)
+            except json.JSONDecodeError as e:
+                print(f"⚠ warning  Invalid JSON for --vault-jwt-bound-claims: {e}", file=sys.stderr)
+        
+        # Create Vault policy
+        policy_name = create_vault_jwt_policy(service_name, vault_addr)
+        
+        # Create Vault JWT role
+        if policy_name:
+            all_policies = list(vault_policies) + [policy_name]
+            create_vault_jwt_role(
+                service_name,
+                vault_addr,
+                all_policies,
+                bound_claims
+            )
+        print()
+    
+    # Summary
+    print(f"{'='*70}")
+    print(f"✅ Registration Complete!")
+    print(f"{'='*70}\n")
+    
+    print("📝 Next Steps:\n")
+    
+    print(f"1. Configure your service with these environment variables:")
+    print(f"   export OIDC_ISSUER_URL='http://authentik-server.iam.svc.cluster.local/application/o/{service_name}/'")
+    print(f"   export OIDC_CLIENT_ID='{service_name}'")
+    print(f"   export OIDC_CLIENT_SECRET='$({vault_service_name}_OIDC_CLIENT_SECRET)'")
+    print(f"   export OIDC_REDIRECT_URI='{redirect_uri}'")
+    print()
+    
+    print(f"2. Mount the Kubernetes secret in your Helm values:")
+    print(f"   env:")
+    print(f"     - name: OIDC_CLIENT_SECRET")
+    print(f"       valueFrom:")
+    print(f"         secretKeyRef:")
+    print(f"           name: {service_name}-oidc")
+    print(f"           key: client_secret")
+    print()
+    
+    if not args.no_vault:
+        print(f"3. Test Vault JWT authentication:")
+        print(f"   kubectl port-forward -n {VAULT_NAMESPACE} svc/vault 8200:8200 &")
+        print(f"   vault login -method=oidc role=homelab")
+        print(f"   vault read auth/jwt/role/{service_name}")
+        print()
+    
+    print(f"4. Verify OAuth flow:")
+    print(f"   kubectl port-forward -n {IAM_NAMESPACE} svc/authentik-server 7000:80 &")
+    print(f"   # Open Authentik UI: http://localhost:7000/if/admin/")
+    print(f"   # Check: Applications → {service_name}")
+    print()
+
+
+if __name__ == "__main__":
+    main()

k8s/talos-iam/setup_talos_iam.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# k8s/talos-iam/setup_talos_iam.sh
+# Validates required env vars are set, then delegates to helmfile.
+# All provisioning logic lives in helmfile hooks + provision_oidc.py.
+#
+# Usage:
+#   vsource .env && bash k8s/talos-iam/setup_talos_iam.sh
+#
+# To re-provision OIDC without redeploying Authentik:
+#   vsource .env && helmfile apply -l name=authentik
+#
+# To patch existing providers (update redirect URIs, scopes):
+#   vsource .env && helmfile apply -l name=authentik  # postsync runs provision_oidc.py --patch implicitly
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Validate all required secrets are loaded from Vault ──────────────────────
+MISSING=()
+for var in \
+  AUTHENTIK_SECRET_KEY \
+  AUTHENTIK_BOOTSTRAP_PASSWORD \
+  AUTHENTIK_BOOTSTRAP_TOKEN \
+  AUTHENTIK_PG_PASSWORD \
+  GRAFANA_OIDC_CLIENT_SECRET \
+  MINIO_OIDC_CLIENT_SECRET \
+  AUTHENTIK_FORGEJO_CLIENT_SECRET \
+  AUTHENTIK_ARGOCD_CLIENT_SECRET; do
+  [[ -z "${!var:-}" ]] && MISSING+=("$var")
+done
+
+if [[ ${#MISSING[@]} -gt 0 ]]; then
+  echo "ERROR: the following vars are not set — run 'vsource .env' first:" >&2
+  printf '  %s\n' "${MISSING[@]}" >&2
+  echo "" >&2
+  echo "If a var is missing from Vault, store it first:" >&2
+  echo "  talos put cluster/VAR_NAME VAR_NAME=\"\$(openssl rand -hex 32)\"" >&2
+  exit 1
+fi
+
+echo "All required secrets present."
+echo ""
+echo "Provider credential mapping (client_id → client_secret):"
+printf '  %-10s  client_id=%-40s  secret=%s\n' \
+  "grafana"  "${GRAFANA_OIDC_CLIENT_ID:-"grafana (default)"}"  "GRAFANA_OIDC_CLIENT_SECRET" \
+  "minio"    "${MINIO_OIDC_CLIENT_ID:-"minio (default)"}"      "MINIO_OIDC_CLIENT_SECRET" \
+  "forgejo"  "${AUTHENTIK_FORGEJO_CLIENT_ID:-"forgejo (default)"}" "AUTHENTIK_FORGEJO_CLIENT_SECRET" \
+  "argocd"   "${AUTHENTIK_ARGOCD_CLIENT_ID:-"argocd (default)"}"   "AUTHENTIK_ARGOCD_CLIENT_SECRET"
+echo ""
+echo "Delegating to helmfile..."
+cd "${SCRIPT_DIR}/../.."
+helmfile apply -l name=authentik

k8s/talos-iam/setup_vault.sh

@@ -0,0 +1,527 @@
+#!/usr/bin/env bash
+# k8s/talos-iam/setup_vault.sh
+# Deploys HashiCorp Vault into the iam namespace, initialises it, and wires
+# the JWT auth backend to Authentik so the talos-cli secrets subcommand works.
+#
+# Prerequisites:
+#   - Authentik already running in the iam namespace (run setup_talos_iam.sh first)
+#   - kubectl configured (KUBECONFIG → cluster-config/kubeconfig)
+#   - helm >= 3.x
+#   - vault CLI installed locally (https://developer.hashicorp.com/vault/downloads)
+#   - talos-iam/.env containing (see .env.example):
+#       MINIO_ROOT_USER=...
+#       MINIO_ROOT_PASSWORD=...
+#     The script seeds initial secrets and needs read access to MinIO credentials.
+#
+# What this script does (in order):
+#   1.  Create vault-minio-creds K8s Secret (MinIO creds for the S3 backend)
+#   2.  Helm install hashicorp/vault
+#   3.  Wait for vault pod to be Running
+#   4.  vault operator init → capture unseal keys + root token
+#   5.  vault operator unseal (3 of 5 key shares)
+#   6.  Login with root token
+#   7.  Enable KV v2 secret engine at secret/
+#   8.  Enable JWT auth backend, configure with Authentik JWKS
+#   9.  Write policies (shell-secrets, mcp-readonly, cluster-admin)
+#   10. Create JWT roles (shell, mcp, cluster)
+#   11. Seed initial secrets from .env
+#   12. Register talos-cli-shell OIDC app in Authentik via API
+#   13. Print next steps
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
+export KUBECONFIG
+
+NAMESPACE=storage
+RELEASE=vault
+AUTHENTIK_URL="http://authentik-server.iam.svc.cluster.local"
+VAULT_ADDR_CLUSTER="http://vault.storage.svc.cluster.local:8200"
+# Local access via port-forward during setup
+VAULT_PORT_FWD_ADDR="http://127.0.0.1:8200"
+UNSEAL_KEYS_FILE="$HOME/.vault-data/.vault-init.json"   # KEEP OFFLINE after setup
+
+# ── Failure surfacing ─────────────────────────────────────────────────────────
+CURRENT_STEP="init"
+on_err() {
+  local rc=$?
+  echo "[ERROR] step '${CURRENT_STEP}' failed (exit ${rc} at line ${BASH_LINENO[0]})" >&2
+  exit "${rc}"
+}
+trap on_err ERR
+
+step() { CURRENT_STEP="$1"; echo ""; echo "==> $2"; }
+
+# ── Load credentials ──────────────────────────────────────────────────────────
+step load_env "Loading credentials from talos-iam/.env..."
+if [[ -f "$HOME/.authentik/.env" ]]; then
+  # shellcheck source=/dev/null
+  source "$HOME/.authentik/.env"
+fi
+
+for var in MINIO_ROOT_USER MINIO_ROOT_PASSWORD AUTHENTIK_BOOTSTRAP_TOKEN; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "ERROR: ${var} is not set — export it or add it to talos-iam/.env" >&2
+    exit 1
+  fi
+done
+
+# ── 1. MinIO credentials secret for Vault S3 backend ─────────────────────────
+step minio_secret "Creating vault-minio-creds secret..."
+kubectl create secret generic vault-minio-creds \
+  --namespace="${NAMESPACE}" \
+  --from-literal=access_key="${MINIO_ROOT_USER}" \
+  --from-literal=secret_key="${MINIO_ROOT_PASSWORD}" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# ── 1b. Placeholder vault-unseal-keys secret (required before Helm install) ───
+# extraSecretEnvironmentVars references vault-unseal-keys at pod start time,
+# but the real keys only exist after vault operator init. Create empty placeholders
+# so the pod starts; step vault_unseal_secret overwrites them with real values.
+step vault_unseal_placeholder "Creating placeholder vault-unseal-keys secret..."
+kubectl create secret generic vault-unseal-keys \
+  --namespace="${NAMESPACE}" \
+  --from-literal=key1="" \
+  --from-literal=key2="" \
+  --from-literal=key3="" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# ── 1c. Register talos-cli-shell OIDC app in Authentik ───────────────────────
+# Must exist before Helm install so Vault can validate the JWKS URL at jwt/config time.
+step authentik_app "Registering talos-cli-shell OIDC provider in Authentik..."
+AUTHENTIK_API="http://127.0.0.1:7000/api/v3"
+
+kubectl port-forward -n "${NAMESPACE}" svc/authentik-server 7000:80 &
+AK_PF_PID=$!
+trap 'kill ${AK_PF_PID} 2>/dev/null' EXIT
+sleep 3
+
+# Fetch required flow PKs from Authentik
+_auth_flow_pk=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API}/flows/instances/?slug=default-provider-authorization-implicit-consent" \
+  | jq -r '.results[0].pk // empty')
+_inval_flow_pk=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API}/flows/instances/?slug=default-provider-invalidation-flow" \
+  | jq -r '.results[0].pk // empty')
+
+if [[ -z "${_auth_flow_pk}" || -z "${_inval_flow_pk}" ]]; then
+  echo "ERROR: required Authentik flows not found — is provision_oidc.py already run?" >&2
+  exit 1
+fi
+
+_signing_key_pk=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API}/crypto/certificatekeypairs/?has_key=true&page_size=1" \
+  | jq -r '.results[0].pk // empty')
+
+if [[ -z "${_signing_key_pk}" ]]; then
+  echo "ERROR: no signing key found in Authentik — create one under System → Certificates" >&2
+  exit 1
+fi
+
+_provider_payload=$(jq -n \
+  --arg auth_flow    "${_auth_flow_pk}" \
+  --arg inval_flow   "${_inval_flow_pk}" \
+  --arg signing_key  "${_signing_key_pk}" \
+  '{
+    name: "talos-cli-shell",
+    client_type: "confidential",
+    grant_types: ["client_credentials"],
+    token_validity: "minutes=5",
+    sub_mode: "hashed_user_id",
+    include_claims_in_id_token: true,
+    audience: "vault",
+    authorization_flow: $auth_flow,
+    invalidation_flow: $inval_flow,
+    signing_key: $signing_key,
+    redirect_uris: []
+  }')
+
+_existing_provider=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API}/providers/oauth2/?name=talos-cli-shell" | jq -r '.results[0].pk // empty')
+
+if [[ -n "${_existing_provider}" ]]; then
+  echo "Provider talos-cli-shell exists (pk=${_existing_provider}) — patching config..."
+  PROVIDER_PK="${_existing_provider}"
+  curl -sf -X PATCH "${AUTHENTIK_API}/providers/oauth2/${PROVIDER_PK}/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "${_provider_payload}" > /dev/null
+  CLIENT_SECRET="(unchanged — retrieve from Authentik admin UI if needed)"
+else
+  PROVIDER_PK=$(curl -sf -X POST "${AUTHENTIK_API}/providers/oauth2/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "${_provider_payload}" | jq -r '.pk')
+
+  CLIENT_SECRET=$(curl -sf -X POST \
+    "${AUTHENTIK_API}/providers/oauth2/${PROVIDER_PK}/set_secret/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d '{}' | jq -r '.client_secret')
+
+  echo "Created provider pk=${PROVIDER_PK}"
+  echo ""
+  echo "┌─────────────────────────────────────────────────────────────────┐"
+  echo "│  Authentik OIDC app registered.                                  │"
+  echo "│  Client ID:     talos-cli-shell                                  │"
+  echo "│  Client Secret: ${CLIENT_SECRET}  │"
+  echo "│  Store this secret — it will not be shown again.                 │"
+  echo "└─────────────────────────────────────────────────────────────────┘"
+fi
+
+# Idempotent: ensure the application exists and is bound to the provider.
+# Runs on both create and patch paths so re-runs always produce a consistent state.
+_existing_app=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API}/core/applications/?slug=talos-cli-shell" | jq -r '.results[0].pk // empty')
+
+if [[ -n "${_existing_app}" ]]; then
+  echo "Application talos-cli-shell exists (pk=${_existing_app}) — ensuring provider binding..."
+  curl -sf -X PATCH "${AUTHENTIK_API}/core/applications/${_existing_app}/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "{\"provider\": ${PROVIDER_PK}}" > /dev/null
+else
+  curl -sf -X POST "${AUTHENTIK_API}/core/applications/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "$(jq -n --argjson pk "${PROVIDER_PK}" \
+      '{"name":"talos-cli-shell","slug":"talos-cli-shell","provider":$pk}')" > /dev/null
+  echo "Application talos-cli-shell created and bound to provider pk=${PROVIDER_PK}"
+fi
+
+kill "${AK_PF_PID}" 2>/dev/null || true
+unset AK_PF_PID
+
+# ── 1d. Register vault-browser OIDC app in Authentik (browser / UI login) ────
+# Separate from talos-cli-shell: this uses authorization_code grant so humans
+# can log in via the Vault UI or `vault login -method=oidc`.
+step authentik_vault_browser "Registering vault-browser OIDC provider in Authentik..."
+AUTHENTIK_API_BROWSER="http://127.0.0.1:7000/api/v3"
+
+kubectl port-forward -n iam svc/authentik-server 7000:80 &
+AK_PF2_PID=$!
+trap 'kill ${AK_PF2_PID} 2>/dev/null' EXIT
+sleep 3
+
+_auth_flow_pk2=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API_BROWSER}/flows/instances/?slug=default-provider-authorization-implicit-consent" \
+  | jq -r '.results[0].pk // empty')
+_inval_flow_pk2=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API_BROWSER}/flows/instances/?slug=default-provider-invalidation-flow" \
+  | jq -r '.results[0].pk // empty')
+_signing_key_pk2=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API_BROWSER}/crypto/certificatekeypairs/?has_key=true&page_size=1" \
+  | jq -r '.results[0].pk // empty')
+
+_vault_browser_payload=$(jq -n \
+  --arg auth_flow   "${_auth_flow_pk2}" \
+  --arg inval_flow  "${_inval_flow_pk2}" \
+  --arg signing_key "${_signing_key_pk2}" \
+  '{
+    name: "vault-browser",
+    client_type: "confidential",
+    client_id: "vault-browser",
+    grant_types: ["authorization_code", "refresh"],
+    token_validity: "hours=8",
+    sub_mode: "hashed_user_id",
+    include_claims_in_id_token: true,
+    authorization_flow: $auth_flow,
+    invalidation_flow: $inval_flow,
+    signing_key: $signing_key,
+    redirect_uris: [
+      {"matching_mode": "strict", "url": "http://vault.riotpiao.homelab.com/ui/vault/auth/oidc/oidc/callback"},
+      {"matching_mode": "strict", "url": "http://localhost:8250/oidc/callback"}
+    ]
+  }')
+
+_existing_vault_browser=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API_BROWSER}/providers/oauth2/?name=vault-browser" | jq -r '.results[0].pk // empty')
+
+if [[ -n "${_existing_vault_browser}" ]]; then
+  echo "Provider vault-browser exists (pk=${_existing_vault_browser}) — patching..."
+  VAULT_BROWSER_PK="${_existing_vault_browser}"
+  curl -sf -X PATCH "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "${_vault_browser_payload}" > /dev/null
+  # Re-fetch the client secret from the detail endpoint (list responses mask it)
+  VAULT_BROWSER_CLIENT_SECRET=$(curl -sf \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/" \
+    | jq -r '.client_secret')
+else
+  VAULT_BROWSER_PK=$(curl -sf -X POST "${AUTHENTIK_API_BROWSER}/providers/oauth2/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "${_vault_browser_payload}" | jq -r '.pk')
+
+  VAULT_BROWSER_CLIENT_SECRET=$(curl -sf -X POST \
+    "${AUTHENTIK_API_BROWSER}/providers/oauth2/${VAULT_BROWSER_PK}/set_secret/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d '{}' | jq -r '.client_secret')
+
+  echo "Created vault-browser provider pk=${VAULT_BROWSER_PK}"
+fi
+
+# Bind application
+_existing_vault_app=$(curl -sf \
+  -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+  "${AUTHENTIK_API_BROWSER}/core/applications/?slug=vault-browser" | jq -r '.results[0].pk // empty')
+
+if [[ -n "${_existing_vault_app}" ]]; then
+  curl -sf -X PATCH "${AUTHENTIK_API_BROWSER}/core/applications/${_existing_vault_app}/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "{\"provider\": ${VAULT_BROWSER_PK}}" > /dev/null
+  echo "Application vault-browser updated."
+else
+  curl -sf -X POST "${AUTHENTIK_API_BROWSER}/core/applications/" \
+    -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -d "$(jq -n --argjson pk "${VAULT_BROWSER_PK}" \
+      '{"name":"vault-browser","slug":"vault-browser","provider":$pk}')" > /dev/null
+  echo "Application vault-browser created."
+fi
+
+kill "${AK_PF2_PID}" 2>/dev/null || true
+unset AK_PF2_PID
+
+# ── 2. Ensure vault bucket exists in MinIO ────────────────────────────────────
+step minio_bucket "Ensuring 'vault' bucket exists in MinIO..."
+kubectl run vault-bucket-init --rm -i --restart=Never \
+  --namespace=storage \
+  --image=minio/mc:latest \
+  --env="MC_HOST_local=http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio.storage.svc.cluster.local:9000" \
+  --command -- mc mb --ignore-existing local/vault
+
+# ── 3. Helm install Vault ─────────────────────────────────────────────────────
+step helm_install "Installing HashiCorp Vault via Helm..."
+helm repo add hashicorp https://helm.releases.hashicorp.com
+helm repo update hashicorp
+
+helm upgrade --install "${RELEASE}" hashicorp/vault \
+  --namespace "${NAMESPACE}" \
+  --values "${SCRIPT_DIR}/vault-values.yaml" \
+  --wait --timeout=120s
+
+# ── 4. Wait for vault pod ─────────────────────────────────────────────────────
+step wait_pod "Waiting for vault-0 pod to be Running..."
+kubectl wait pod/vault-0 \
+  --namespace="${NAMESPACE}" \
+  --for=condition=Ready=false \
+  --timeout=60s 2>/dev/null || true   # pod starts uninitialized (not Ready) — that's expected
+
+# Give it a moment to bind the port
+sleep 5
+
+# ── 5. Port-forward for init/unseal ──────────────────────────────────────────
+step port_forward "Starting port-forward to vault-0 on 127.0.0.1:8200..."
+kubectl port-forward -n "${NAMESPACE}" pod/vault-0 8200:8200 &
+PF_PID=$!
+trap 'kill ${PF_PID} 2>/dev/null; on_err' ERR
+trap 'kill ${PF_PID} 2>/dev/null' EXIT
+sleep 3
+
+export VAULT_ADDR="${VAULT_PORT_FWD_ADDR}"
+
+# ── 6. Init ───────────────────────────────────────────────────────────────────
+step vault_init "Initialising Vault (5 key shares, threshold 3)..."
+if vault status 2>/dev/null | grep -q "Initialized.*true"; then
+  echo "Vault is already initialised — skipping init."
+else
+  vault operator init \
+    -key-shares=5 \
+    -key-threshold=3 \
+    -format=json > "${UNSEAL_KEYS_FILE}"
+
+  chmod 600 "${UNSEAL_KEYS_FILE}"
+  echo ""
+  echo "┌─────────────────────────────────────────────────────────────────┐"
+  echo "│  IMPORTANT: unseal keys + root token saved to:                  │"
+  echo "│  ${UNSEAL_KEYS_FILE}"
+  echo "│  Move this file OFFLINE (USB / password manager) immediately.   │"
+  echo "└─────────────────────────────────────────────────────────────────┘"
+fi
+
+# ── 6b. Store unseal keys in K8s Secret for auto-unseal on restart ───────────
+step vault_unseal_secret "Storing 3 unseal keys in vault-unseal-keys Secret..."
+kubectl create secret generic vault-unseal-keys \
+  --namespace="${NAMESPACE}" \
+  --from-literal=key1="$(jq -r '.unseal_keys_b64[0]' "${UNSEAL_KEYS_FILE}")" \
+  --from-literal=key2="$(jq -r '.unseal_keys_b64[1]' "${UNSEAL_KEYS_FILE}")" \
+  --from-literal=key3="$(jq -r '.unseal_keys_b64[2]' "${UNSEAL_KEYS_FILE}")" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+# ── 7. Unseal ─────────────────────────────────────────────────────────────────
+step vault_unseal "Unsealing Vault (3 of 5 shares)..."
+if vault status 2>/dev/null | grep -q "Sealed.*false"; then
+  echo "Vault is already unsealed — skipping."
+else
+  for i in 0 1 2; do
+    KEY=$(jq -r ".unseal_keys_b64[${i}]" "${UNSEAL_KEYS_FILE}")
+    vault operator unseal "${KEY}"
+  done
+fi
+
+# ── 8. Login with root token ──────────────────────────────────────────────────
+step vault_login "Logging in with root token..."
+ROOT_TOKEN=$(jq -r ".root_token" "${UNSEAL_KEYS_FILE}")
+vault login "${ROOT_TOKEN}"
+
+# ── 9. Enable KV v2 ──────────────────────────────────────────────────────────
+step kv_enable "Enabling KV v2 at secret/..."
+vault secrets enable -path=secret kv-v2 2>/dev/null || echo "KV v2 already enabled."
+
+# ── 10. Enable JWT auth backend ───────────────────────────────────────────────
+step jwt_enable "Enabling JWT auth backend..."
+vault auth enable jwt 2>/dev/null || echo "JWT auth already enabled."
+
+JWKS_URL="${AUTHENTIK_URL}/application/o/talos-cli-shell/jwks/"
+vault write auth/jwt/config \
+  jwks_url="${JWKS_URL}" \
+  default_role="shell"
+
+echo "JWT auth configured with JWKS at: ${JWKS_URL}"
+
+# ── 11. Write policies ────────────────────────────────────────────────────────
+step policies "Writing Vault policies..."
+
+vault policy write shell-secrets - <<'EOF'
+# shell-secrets: read MCP keys and cloud tokens for interactive shell sessions
+path "secret/data/mcp/*" {
+  capabilities = ["read"]
+}
+path "secret/data/cloud/*" {
+  capabilities = ["read"]
+}
+EOF
+
+vault policy write mcp-readonly - <<'EOF'
+# mcp-readonly: each MCP server reads only its own path
+path "secret/data/mcp/{{identity.entity.aliases.*.metadata.client_id}}" {
+  capabilities = ["read"]
+}
+EOF
+
+vault policy write cluster-admin - <<'EOF'
+# cluster-admin: read cluster service credentials (MinIO, Grafana, etc.)
+path "secret/data/cluster/*" {
+  capabilities = ["read", "update"]
+}
+path "secret/data/mcp/*" {
+  capabilities = ["read", "create", "update", "delete"]
+}
+path "secret/data/cloud/*" {
+  capabilities = ["read", "create", "update", "delete"]
+}
+EOF
+
+# ── 12. Create JWT roles ──────────────────────────────────────────────────────
+step jwt_roles "Creating JWT roles..."
+
+# shell role: interactive shell sessions via talos-cli
+vault write auth/jwt/role/shell \
+  role_type=jwt \
+  bound_audiences="vault" \
+  user_claim="sub" \
+  policies="shell-secrets" \
+  ttl=4h \
+  max_ttl=8h
+
+# mcp role: MCP server processes (short TTL, non-renewable)
+vault write auth/jwt/role/mcp \
+  role_type=jwt \
+  bound_audiences="vault" \
+  user_claim="sub" \
+  policies="mcp-readonly" \
+  ttl=1h \
+  max_ttl=1h
+
+# cluster role: admin operations (bootstrap scripts, rotation jobs)
+vault write auth/jwt/role/cluster - <<'EOF'
+{
+  "role_type": "jwt",
+  "bound_audiences": ["vault"],
+  "user_claim": "sub",
+  "bound_claims": {"groups": ["homelab-admins"]},
+  "token_policies": ["cluster-admin"],
+  "token_ttl": "1h"
+}
+EOF
+
+# ── 13. Enable OIDC auth (browser / UI login via Authentik) ──────────────────
+step oidc_enable "Enabling OIDC auth method for browser login..."
+vault auth enable oidc 2>/dev/null || echo "OIDC auth already enabled."
+
+vault write auth/oidc/config \
+  oidc_discovery_url="http://authentik.riotpiao.homelab.com/application/o/vault-browser/" \
+  oidc_client_id="vault-browser" \
+  oidc_client_secret="${VAULT_BROWSER_CLIENT_SECRET}" \
+  default_role="homelab"
+
+# homelab role: all authenticated Authentik users get shell-secrets + cluster-admin.
+# Restrict further by adding bound_claims once group-based mapping is configured.
+vault write auth/oidc/role/homelab \
+  role_type=oidc \
+  bound_audiences="vault-browser" \
+  allowed_redirect_uris="http://vault.riotpiao.homelab.com/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
+  user_claim="sub" \
+  oidc_scopes="openid,profile,email" \
+  token_policies="shell-secrets,cluster-admin" \
+  token_ttl=8h \
+  token_max_ttl=12h
+
+echo "OIDC auth configured."
+echo "  Browser login: http://vault.riotpiao.homelab.com  → sign in with Authentik"
+echo "  CLI login:     VAULT_ADDR=http://vault.riotpiao.homelab.com vault login -method=oidc"
+
+# ── 14. Seed initial secrets ──────────────────────────────────────────────────
+step seed_secrets "Seeding initial secrets from environment (accessible after OIDC login)..."
+
+# Cluster service credentials
+vault kv put secret/cluster/minio \
+  user="${MINIO_ROOT_USER}" \
+  password="${MINIO_ROOT_PASSWORD}"
+
+if [[ -n "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
+  vault kv put secret/cluster/grafana \
+    password="${GRAFANA_ADMIN_PASSWORD}"
+fi
+
+if [[ -n "${DUCKDNS_TOKEN:-}" ]]; then
+  vault kv put secret/cloud/duckdns \
+    token="${DUCKDNS_TOKEN}"
+fi
+
+# MCP / AI tooling keys (optional — add when available)
+if [[ -n "${ANTHROPIC_API_KEY:-}" ]]; then
+  vault kv put secret/mcp/anthropic api_key="${ANTHROPIC_API_KEY}"
+fi
+
+if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+  vault kv put secret/mcp/github token="${GITHUB_TOKEN}"
+fi
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> Vault bootstrap complete."
+echo ""
+echo "Next steps:"
+echo "  1. Move ${UNSEAL_KEYS_FILE} OFFLINE (USB / password manager)."
+echo "  2. Browser login: open http://vault.riotpiao.homelab.com → choose OIDC → sign in with Authentik"
+echo "  3. CLI login:     export VAULT_ADDR=http://vault.riotpiao.homelab.com"
+echo "                    vault login -method=oidc"
+echo "  4. Read MinIO credentials from Vault after login:"
+echo "                    vault kv get secret/cluster/minio"
+echo "  5. talos-cli (JWT):  talos secrets login && talos secrets status"

k8s/talos-iam/vault-values.yaml

@@ -0,0 +1,171 @@
+# k8s/talos-iam/vault-values.yaml
+# HashiCorp Vault — secrets backend for the homelab.
+# Stores OIDC client secrets, TLS certs, and any other sensitive values.
+# Accessed via the `talos` CLI (talos-cli/) which wraps `vault kv get/put`.
+#
+# Storage backend: MinIO S3 (minio.storage.svc.cluster.local) — no extra PVC.
+# Auto-unseal: postStart hook reads unseal keys from vault-unseal-keys Secret
+# (written by setup_vault.sh after operator init; operator must run that script
+# once after first install to initialize and store the keys).
+
+# ── Global ────────────────────────────────────────────────────────────────────
+# tlsDisable: true — TLS terminated at the nginx ingress (vault.riotpiao.homelab.com)
+# or at port-forward. In-cluster traffic to Vault is plain HTTP; this is acceptable
+# because all clients are on the pod network (not crossing node boundaries).
+global:
+  enabled: true
+  tlsDisable: true
+
+# ── Agent Injector ────────────────────────────────────────────────────────────
+# The injector mutates pods to sidecar Vault Agent for automatic secret injection.
+# Not used here — secrets are fetched explicitly via the talos CLI.
+# Enabling it would add a webhook that intercepts all pod creates cluster-wide,
+# which is unnecessary overhead for a homelab with manual secret management.
+injector:
+  enabled: false
+
+server:
+  replicas: 1
+
+  annotations:
+    secret.reloader.stakater.com/reload: "vault-unseal-keys"
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+  # ── Scheduling ─────────────────────────────────────────────────────────────
+  # Tolerate cp-1 so Vault can run there if worker-1 is down.
+  # Prefer worker-1 under normal conditions (keeps Vault off the same node as etcd).
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: node-role.kubernetes.io/worker
+                operator: Exists
+
+  # ── Environment variables ───────────────────────────────────────────────────
+  # extraEnvironmentVars: non-secret config passed directly.
+  extraEnvironmentVars:
+    VAULT_LOG_LEVEL: info
+
+  # extraSecretEnvironmentVars: pulls values from K8s Secrets into env vars.
+  # vault-minio-creds is created by the helmfile presync hook from MINIO_ROOT_USER/PASSWORD.
+  # vault-unseal-keys is a placeholder created at first deploy; setup_vault.sh
+  # overwrites it with real unseal keys after `vault operator init`.
+  # Vault reads the keys from env on every pod start and the postStart hook unseals.
+  extraSecretEnvironmentVars:
+    - envName: AWS_ACCESS_KEY_ID
+      secretName: vault-minio-creds
+      secretKey: access_key
+    - envName: AWS_SECRET_ACCESS_KEY
+      secretName: vault-minio-creds
+      secretKey: secret_key
+    - envName: VAULT_UNSEAL_KEY_1
+      secretName: vault-unseal-keys
+      secretKey: key1
+    - envName: VAULT_UNSEAL_KEY_2
+      secretName: vault-unseal-keys
+      secretKey: key2
+    - envName: VAULT_UNSEAL_KEY_3
+      secretName: vault-unseal-keys
+      secretKey: key3
+
+  # ── Auto-unseal ─────────────────────────────────────────────────────────────
+  # Vault starts sealed after every pod restart and can't serve requests until
+  # unsealed. postStart runs immediately after the container starts, sleeps 5s
+  # to let the Vault process bind its port, then feeds the unseal keys one by one.
+  # `|| true` prevents the hook from failing if a key was already used (idempotent).
+  # 3-of-5 Shamir unseal is the default — we stored all 3 used keys in the Secret.
+  postStart:
+    - /bin/sh
+    - -c
+    - |
+      sleep 5
+      vault operator unseal "$VAULT_UNSEAL_KEY_1" || true
+      vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
+      vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
+
+  # ── Vault config (HCL) ──────────────────────────────────────────────────────
+  standalone:
+    enabled: true
+    config: |
+      ui = false   # UI served via Vault's own HTTP; enabled below via ui: enabled: true
+
+      listener "tcp" {
+        address     = "0.0.0.0:8200"
+        tls_disable = "true"
+
+        # No auth in front of Vault's metrics endpoint — acceptable since all
+        # Prometheus scrape traffic stays on the pod network (not exposed via ingress).
+        telemetry {
+          unauthenticated_metrics_access = "true"
+        }
+      }
+
+      telemetry {
+        prometheus_retention_time = "30s"
+        disable_hostname          = true
+      }
+
+      # S3 storage backend pointing at the in-cluster MinIO service.
+      # AWS_ env vars (from vault-minio-creds Secret) supply the credentials.
+      # s3_force_path_style: MinIO uses path-style URLs (not virtual-hosted).
+      # disable_ssl: MinIO in this cluster has no TLS.
+      storage "s3" {
+        endpoint            = "http://minio.storage.svc.cluster.local:9000"
+        bucket              = "vault"
+        region              = "us-east-1"
+        s3_force_path_style = "true"
+        disable_ssl         = "true"
+      }
+
+      # api_addr: the address other Vault nodes (or HA standbys) use to reach
+      # this node. Single-node standalone, but Vault requires it to be set.
+      api_addr = "http://vault.storage.svc.cluster.local:8200"
+
+  # ── Service ─────────────────────────────────────────────────────────────────
+  # NodePort 32171 — fallback for direct node access during bootstrap before
+  # the ingress is up. Normal access is via nginx ingress (vault.riotpiao.homelab.com).
+  service:
+    type: NodePort
+    port: 8200
+    nodePort: 32171
+
+  # ── Persistence ─────────────────────────────────────────────────────────────
+  # No PVC — all Vault state (secrets, policies, tokens) is stored in MinIO S3.
+  # This means Vault survives node loss as long as MinIO is healthy.
+  dataStorage:
+    enabled: false
+
+  auditStorage:
+    enabled: false
+
+# ── UI ────────────────────────────────────────────────────────────────────────
+# Vault's web UI is used for the OIDC browser login flow (Vault as an OIDC
+# provider, if configured) and for manual operator inspection.
+# Accessible at http://vault.riotpiao.homelab.com or via port-forward.
+ui:
+  enabled: true
+
+# ── Metrics ───────────────────────────────────────────────────────────────────
+# vault_core_unsealed is the availability signal (0 after a restart until the
+# postStart hook above finishes unsealing). Pairs with the telemetry{} stanzas
+# in standalone.config above, which actually turn the /v1/sys/metrics endpoint on.
+serverTelemetry:
+  serviceMonitor:
+    enabled: true
+    selectors: {}
+    interval: 30s
+    scrapeTimeout: 10s

k8s/talos-iam/verify_existing_oauth_integrations.sh

@@ -0,0 +1,211 @@
+#!/usr/bin/env bash
+# verify_existing_oauth_integrations.sh
+# Verification script that checks existing OIDC integrations
+# (Grafana, MinIO, Forgejo, Argo CD) are still working after updates.
+#
+# Usage:
+#   bash k8s/talos-iam/verify_existing_oauth_integrations.sh
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
+export KUBECONFIG
+
+echo "╔════════════════════════════════════════════════════════════════════╗"
+echo "║  OAuth Integrations Verification                                   ║"
+echo "╚════════════════════════════════════════════════════════════════════╝"
+echo ""
+
+# Check prerequisites
+echo "📋 Checking prerequisites..."
+required_cmds=("kubectl" "curl" "jq")
+for cmd in "${required_cmds[@]}"; do
+  if ! command -v "$cmd" &> /dev/null; then
+    echo "❌ $cmd not found in PATH"
+    exit 1
+  fi
+done
+
+# Check namespaces exist
+for ns in iam logging storage; do
+  if ! kubectl get ns "$ns" &> /dev/null; then
+    echo "❌ Namespace $ns not found"
+    exit 1
+  fi
+done
+
+echo "✓ All prerequisites met"
+echo ""
+
+# Function to check pod status
+check_pod_status() {
+  local namespace=$1
+  local label=$2
+  local component=$3
+  
+  echo "Checking $component..."
+  if kubectl get pods -n "$namespace" -l "$label" -o wide 2>/dev/null | grep -q Running; then
+    echo "  ✓ Running"
+    return 0
+  else
+    echo "  ❌ Not running"
+    return 1
+  fi
+}
+
+# Function to test Authentik API endpoint
+test_authentik_api() {
+  echo "Testing Authentik API..."
+  if ! kubectl port-forward -n iam svc/authentik-server 7000:80 &> /dev/null & then
+    sleep 2
+    if curl -sf http://localhost:7000/-/health/ready/ &> /dev/null; then
+      echo "  ✓ API healthy (HTTP 204)"
+    else
+      echo "  ❌ API not responding"
+      return 1
+    fi
+  fi
+}
+
+# Check core services
+echo "🔍 Service Status"
+echo "─────────────────────────────────────────────────────────────────────"
+
+check_pod_status iam "app=authentik,component=server" "Authentik Server" || true
+check_pod_status iam "app=authentik,component=worker" "Authentik Worker" || true
+check_pod_status iam "app=authentik,component=postgresql" "Authentik PostgreSQL" || true
+check_pod_status storage "app.kubernetes.io/name=vault" "Vault" || true
+check_pod_status logging "app.kubernetes.io/name=grafana" "Grafana" || true
+check_pod_status storage "app.kubernetes.io/name=minio" "MinIO" || true
+
+echo ""
+
+# Verify Authentik has expected providers
+echo "🔐 Authentik OAuth Providers"
+echo "─────────────────────────────────────────────────────────────────────"
+
+if [[ -z "${AUTHENTIK_BOOTSTRAP_TOKEN:-}" ]]; then
+  echo "⚠ AUTHENTIK_BOOTSTRAP_TOKEN not set — skipping provider verification"
+  echo "  Set it: export AUTHENTIK_BOOTSTRAP_TOKEN=\"$(talos get cluster/AUTHENTIK_BOOTSTRAP_TOKEN --key AUTHENTIK_BOOTSTRAP_TOKEN 2>/dev/null)\""
+  echo ""
+else
+  # Port-forward to Authentik
+  if ! pgrep -f "kubectl port-forward.*7000:80" > /dev/null; then
+    kubectl port-forward -n iam svc/authentik-server 7000:80 > /dev/null 2>&1 &
+    sleep 2
+  fi
+  
+  api_url="http://localhost:7000/api/v3"
+  
+  # Check for expected providers
+  for provider in grafana minio forgejo argocd talos-cli-shell; do
+    response=$(curl -sf -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+      "${api_url}/providers/oauth2/?name=${provider}" 2>/dev/null || echo "{}")
+    
+    if echo "$response" | jq -e '.results[0]' &> /dev/null; then
+      pk=$(echo "$response" | jq -r '.results[0].pk')
+      client_id=$(echo "$response" | jq -r '.results[0].client_id // "N/A"')
+      echo "  ✓ ${provider}  (pk=${pk}, client_id=${client_id})"
+    else
+      echo "  ✗ ${provider}  NOT FOUND"
+    fi
+  done
+  
+  echo ""
+  
+  # Check for expected groups
+  echo "👥 Authentik Groups"
+  echo "─────────────────────────────────────────────────────────────────────"
+  
+  for group in homelab-admins grafana-admins grafana-viewers minio-admins minio-readonly; do
+    response=$(curl -sf -H "Authorization: Bearer ${AUTHENTIK_BOOTSTRAP_TOKEN}" \
+      "${api_url}/core/groups/?name=${group}" 2>/dev/null || echo "{}")
+    
+    if echo "$response" | jq -e '.results[0]' &> /dev/null; then
+      echo "  ✓ ${group}"
+    else
+      echo "  ✗ ${group}  NOT FOUND"
+    fi
+  done
+  
+  echo ""
+fi
+
+# Verify K8s secrets for apps
+echo "🔑 Kubernetes Secrets"
+echo "─────────────────────────────────────────────────────────────────────"
+
+for secret_spec in "grafana-oidc:logging" "minio-oidc:storage" "authentik-oidc-forgejo:cicd" "oidc-secret:cicd"; do
+  IFS=':' read -r secret_name ns <<< "$secret_spec"
+  if kubectl get secret "$secret_name" -n "$ns" &> /dev/null 2>&1; then
+    size=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath='{.data}' 2>/dev/null | wc -c)
+    echo "  ✓ ${secret_name} → ns/${ns} (${size} bytes)"
+  else
+    echo "  ✗ ${secret_name} → ns/${ns}  NOT FOUND"
+  fi
+done
+
+echo ""
+
+# Verify Vault JWT auth
+echo "🔐 Vault JWT Authentication"
+echo "─────────────────────────────────────────────────────────────────────"
+
+if command -v vault &> /dev/null; then
+  # Check if Vault auth method is enabled
+  vault_status=$(kubectl exec -n storage vault-0 -- vault auth list -format=json 2>/dev/null | jq 'keys' || echo "[]")
+  
+  if echo "$vault_status" | jq -e '.[] | select(. == "jwt/")' &> /dev/null; then
+    echo "  ✓ JWT auth method enabled"
+    
+    # List JWT roles
+    roles=$(kubectl exec -n storage vault-0 -- vault list auth/jwt/role -format=json 2>/dev/null | jq '.[]' || echo "")
+    if [[ -n "$roles" ]]; then
+      echo "  ✓ JWT roles found:"
+      echo "$roles" | while read role; do
+        echo "    - ${role}"
+      done
+    else
+      echo "  ✗ No JWT roles found"
+    fi
+  else
+    echo "  ⚠ JWT auth method not enabled"
+  fi
+else
+  echo "  ⚠ vault CLI not found — skipping Vault checks"
+fi
+
+echo ""
+
+# Summary
+echo "╔════════════════════════════════════════════════════════════════════╗"
+echo "║  Verification Complete                                             ║"
+echo "╚════════════════════════════════════════════════════════════════════╝"
+echo ""
+
+echo "Next steps:"
+echo ""
+echo "1. Verify OIDC login flow (Grafana):"
+echo "   kubectl port-forward -n logging svc/grafana 3000:80"
+echo "   # Open http://localhost:3000/login"
+echo "   # Should show 'Sign in with ...' option"
+echo ""
+
+echo "2. Verify MinIO OIDC (if configured):"
+echo "   kubectl port-forward -n storage svc/minio 9001:9001"
+echo "   # Open http://localhost:9001"
+echo "   # Should show identity provider option"
+echo ""
+
+echo "3. Verify Vault JWT role:"
+echo "   kubectl port-forward -n storage svc/vault 8200:8200"
+echo "   export VAULT_ADDR=http://127.0.0.1:8200"
+echo "   vault read auth/jwt/role/shell"
+echo ""
+
+echo "4. Test JWT authentication to Vault:"
+echo "   # Get ID token from Authentik (requires app integration)"
+echo "   # Then authenticate: vault login -method=jwt role=shell jwt=\$ID_TOKEN"
+echo ""