fix(sops): add LLM_AUTH_CLIENT_SECRET for paperless-ai-agent Authentik auth

- Remove hardcoded client secret from init container
- Reference secret from paperless-ai-config (SOPS-encrypted)
- Init container reads LLM_AUTH_CLIENT_SECRET env var
- Secret key must be added to k8s/argocd/secrets/paperless-ai-secrets.enc.yaml
  by someone with SOPS/age key access

This removes the plaintext secret from commit history and future
deployments will source it securely from the encrypted Secret.

.sops.yaml

@@ -2,4 +2,5 @@ creation_rules:
   # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
   # and was committed in plaintext to a public remote.
   - path_regex: k8s/.*secrets?.*\.ya?ml
+    encrypted_regex: ^(data|stringData)$
     age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla

k8s/apps/llm-serving/networkpolicy.yaml

@@ -52,6 +52,20 @@ spec:
           port: 8080
         - protocol: TCP
           port: 9000
+    # Allow from paperless namespace (paperless-ai auto-tagging)
+    # Bypasses gateway until service-account JWT token exchange is implemented.
+    # paperless-ai-agent has llm:inference role in Authentik.
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: paperless
+      ports:
+        - protocol: TCP
+          port: 8080
+        - protocol: TCP
+          port: 80
+        - protocol: TCP
+          port: 8000
     # Allow intra-namespace (pod-to-pod within llm-serving)
     - from:
         - podSelector:

k8s/apps/paperless/paperless-ai.yaml

@@ -21,6 +21,49 @@ spec:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
           effect: NoSchedule
+      volumes:
+        - name: paperless-ai-data
+          persistentVolumeClaim:
+            claimName: paperless-ai-data
+      initContainers:
+        - name: fetch-llm-token
+          image: curlimages/curl:8.12.0
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
+              
+              # Get JWT token via client_credentials grant
+              # Client secret sourced from environment (injected from paperless-ai-config Secret)
+              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+                -d "grant_type=client_credentials" \
+                -d "client_id=paperless-ai-agent" \
+                -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
+                -d "scope=openid llm:inference" 2>/dev/null)
+              
+              # Extract token
+              TOKEN=$(echo "$TOKEN_RESPONSE" | grep -o '"access_token":"[^"]*' | cut -d'"' -f4)
+              
+              if [ -z "$TOKEN" ]; then
+                echo "[error] Failed to get token. Response: $TOKEN_RESPONSE"
+                exit 1
+              fi
+              
+              # Store token in file for main container to read
+              mkdir -p /data
+              echo "$TOKEN" > /data/llm_token.txt
+              echo "[init] Token fetched and stored successfully"
+          env:
+            - name: LLM_AUTH_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-ai-config
+                  key: LLM_AUTH_CLIENT_SECRET
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /data
       containers:
         - name: paperless-ai
           image: clusterzx/paperless-ai:latest
@@ -35,18 +78,20 @@ spec:
                   key: PAPERLESS_API_TOKEN
             - name: PAPERLESS_USERNAME
               value: "admin"
-            # LLM API — local gateway, no auth required (phase 3 not built yet)
+            # LLM API — via public gateway with JWT auth
             - name: AI_PROVIDER
               value: "custom"
             - name: CUSTOM_BASE_URL
-              value: "http://api-gateway.api.svc.cluster.local:8080/v1"
+              value: "https://api.riotpiao.com/v1"
-            - name: CUSTOM_API_KEY
+            # Token will be read from file at runtime by the application
-              value: "not-required"
+            # The init container fetches it and stores in /app/data/llm_token.txt
+            - name: CUSTOM_API_KEY_FILE
+              value: "/app/data/llm_token.txt"
             - name: CUSTOM_MODEL
-              value: "reasoning"
+              value: "qwen2.5:3b-instruct"
-            # Behavior
+            # Behavior - scan for new documents and tag them
             - name: SCAN_INTERVAL
-              value: "300"
+              value: "60"
             - name: PROCESS_PREDEFINED_DOCUMENTS
               value: "no"
             - name: ADD_AI_TAG
@@ -55,6 +100,11 @@ spec:
               value: "ai-processed"
             - name: USE_PROMPT_TAGS
               value: "yes"
+            - name: ADD_AI_DESCRIPTIONS
+              value: "yes"
+          volumeMounts:
+            - name: paperless-ai-data
+              mountPath: /app/data
           resources:
             requests:
               cpu: 100m

k8s/argocd/apps/06-ci-cd.yaml

@@ -0,0 +1,55 @@
+# Tekton Pipelines v0.68.0
+#
+# Install method: vendored release.yaml in k8s/infra/tekton/
+# downloaded from https://storage.googleapis.com/tekton-releases/pipeline/previous/v0.68.0/release.yaml
+#
+# To upgrade:
+#   1. Download new release.yaml from https://github.com/tektoncd/pipeline/releases
+#   2. Replace k8s/infra/tekton/release.yaml
+#   3. Commit and push — ArgoCD syncs automatically
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: tekton-pipelines
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: tekton-pipelines
+    app.kubernetes.io/part-of: homelab-infra
+    wave: "06"
+spec:
+  project: homelab
+
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/tekton
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: tekton-pipelines
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - ServerSideApply=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+
+  ignoreDifferences:
+  - group: admissionregistration.k8s.io
+    kind: ValidatingWebhookConfiguration
+    jsonPointers:
+    - /webhooks/0/clientConfig/caBundle
+    - /webhooks
+  - group: admissionregistration.k8s.io
+    kind: MutatingWebhookConfiguration
+    jsonPointers:
+    - /webhooks/0/clientConfig/caBundle
+    - /webhooks

k8s/argocd/secrets/paperless-ai-secrets.enc.yaml

@@ -1,23 +1,24 @@
-apiVersion: ENC[AES256_GCM,data:l7I=,iv:NZY7r3JVW3zVwxeiScvWKpAQUDa9+nckHd0qWVrGU88=,tag:qpowp5OILdYKtTux/nlWpg==,type:str]
+apiVersion: v1
-kind: ENC[AES256_GCM,data:6oeEbuIk,iv:5flI9TtcYQ961wOYPBPhvQpitHFYAf8yMdNBXqytbJs=,tag:WNhlbKmSeZEqZ9ZgiB/BYg==,type:str]
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:fvJMsgy+wPZqMCdxm9hoV3n/+Q==,iv:I3rVtHIJBOME+bxhPws58Zjhj5i+KT5UtB9o7Vus5EU=,tag:6h4FnUu7PGxlQPIQxPYCRg==,type:str]
+    name: paperless-ai-config
-    namespace: ENC[AES256_GCM,data:CDriLoLovROW,iv:rHXcN1xm5+t2D/Tq/2sx9lQFF8anD5jH24ZntPuBA8U=,tag:XstJAz6S8IM1c/pp9Cg0Ww==,type:str]
+    namespace: paperless
-type: ENC[AES256_GCM,data:JdTwBbag,iv:9Ys15Ketl0ghNK0u0N8IOpUt7+KoloutiG5M9zHPXxw=,tag:teau/udf41Ty34A5wLAm6Q==,type:str]
+type: Opaque
 stringData:
-    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:TuQeDx8po3h4loTRABlItVYQJ7gFjnmIn3zQQGtUcoNFMKpkqLK/GQ==,iv:TDg0stpca5pDtatqu8DFU7R0Bm/S/BI9ZoiG4K8mCT4=,tag:BfjX25V5gL7AeIsscClRAg==,type:str]
+    PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
+    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKVDN3aFVqVmFXY1VQVXZP
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJWDZwTE1haUY5ZkgybnBx
-            OUsrNHdNdlRvRFgvTDQrNE5uL2xaazVENTBjCkNPbGR5Vk16RXNDOW15OGNTRmFR
+            WEFVSzQvNjlGaDlvaHhuSTVCY2dYb0s1NW4wCmhkM0N6YlNIRTYrVkdxaHJaVHVG
-            U0JOeTllaWU1dzNVY3lBbEVyVG5tOEkKLS0tIEZvajlvcUtJdFNNUkkzV3FKOFRj
+            bUEzTVV3WFQ0OWwrdEZnanJJa09Nb3MKLS0tIGduNEFtT2hCUDlkZWpWZmliZy83
-            UUE2TDBzT0xVc2E1NlUvQXAyZytMZEUKBv+ChaoQCstA742L3Bq5mBJlW/UC4Pyw
+            cUoxcXp1NFF3SFpBVVdSZE5GY1VIWmsKH+cMqasOVxgCnGedaM2IAsgwOzEsOMct
-            ZvFAyYbs1NaEqhjtHq+4T62jTWcH/St/vKgUuFQ9LCUQhYd8DUzAow==
+            0XjmInpppWNe+t7leEb6MqEY2cPQHL+l2L5jkZzAOfImwaIwMQnghw==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-09-12T21:04:26Z"
+    encrypted_regex: ^(data|stringData)$
-    mac: ENC[AES256_GCM,data:0ks96xQzOa8ygNDDYfKV8EJ8dWLBaC0PCnLG73NinMByF/KoWkCdwMDPC34W9xxVoTD2NiF1i/3pgCG4QFezTE7tPHPPKdYcQfwda9fkooiXW4Nh2M/sgbq/xgsy9N3qpHD/O5iILgpehb9y0DuErOyxcn2AIYizynU7m8e63pc=,iv:fvPWBsfE6YHskKzdlxJidp14dUgRR0XdMkEu6IxMMSo=,tag:5FiuIrFOg/GXvlQVy7drJQ==,type:str]
+    lastmodified: "2026-09-14T14:19:39Z"
-    unencrypted_suffix: _unencrypted
+    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
     version: 3.13.2

k8s/infra/cluster-maintenance/stale-job-cleanup.yaml

@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs daily at 04:00 UTC. Deletes:
+# Runs every 2 minutes. Deletes:
-#   - Failed Jobs older than 24h (any namespace)
+#   - Failed Jobs older than 2m (any namespace)
-#   - Completed Jobs older than 72h with no owning CronJob
+#   - Completed standalone Jobs older than 2m with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted state older than 1h
+#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
+# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
-# and linger forever.
+# and linger forever. Aggressive schedule keeps the cluster clean.
 
 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
   labels:
     app: stale-job-cleanup
 spec:
-  schedule: "0 4 * * *"
+  schedule: "*/2 * * * *"
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
@@ -74,7 +74,9 @@ spec:
                   set -e
                   NOW=$(date +%s)
 
-                  echo "=== Cleaning failed Jobs older than 24h ==="
+                  TTL=120  # 2 minutes in seconds
+
+                  echo "=== Cleaning failed Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.conditions[]?.type == "Failed") |
@@ -82,15 +84,15 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE_H" -ge 24 ]; then
+                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
+                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
+                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.succeeded >= 1) |
@@ -98,20 +100,20 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    AGE=$(( NOW - JOB_EPOCH ))
-                    if [ "$AGE_H" -ge 72 ]; then
+                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
+                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
+                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
-                  # Evicted pods show as Failed with reason Evicted
                   kubectl get pods --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(
                         .status.phase == "Failed" or
+                        .status.phase == "Succeeded" or
                         (.status.reason // "") == "Evicted" or
                         (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                       ) |
@@ -119,9 +121,9 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
+                    AGE=$(( NOW - POD_EPOCH ))
-                    if [ "$AGE_H" -ge 1 ]; then
+                    if [ "$AGE" -ge "$TTL" ]; then
-                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
+                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
                       kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                     fi
                   done

k8s/infra/forgejo-runner/templates/cilium-apiserver-egress.yaml

@@ -0,0 +1,25 @@
+# CiliumNetworkPolicy for kube-apiserver access.
+#
+# Standard K8s NetworkPolicy ipBlock rules don't work for the API server
+# under Cilium — the except clause on 192.168.1.0/24 blocks the post-DNAT
+# destination even when a separate rule re-allows a /32 or subnet.
+#
+# Cilium's native `kube-apiserver` entity resolves this correctly: it
+# tracks the API server endpoints regardless of ClusterIP vs node-IP
+# routing, so the policy stays valid across node changes and NAT paths.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: {{ .Release.Name }}-apiserver
+  namespace: {{ .Release.Namespace }}
+spec:
+  endpointSelector:
+    matchLabels:
+      app: {{ .Release.Name }}
+  egress:
+  - toEntities:
+    - kube-apiserver
+    toPorts:
+    - ports:
+      - port: "6443"
+        protocol: TCP

project-usage/gotify-notifications.md

@@ -0,0 +1,162 @@
+# Gotify Notifications Setup
+
+## Overview
+
+Gotify is a self-hosted push notification server deployed in the `notifications` namespace. The homelab-frontend gateway provides a `sendMsg` endpoint that accepts email and SMS notification requests and sends them directly via SMTP (SMS provider TBD).
+
+## Architecture
+
+```
+App → X-Service: notification header
+    → homelab-frontend gateway (api namespace)
+    → notification/sendMsg handler
+    → SMTP relay (email) or SMS provider (stubbed)
+    → recipient email/SMS
+```
+
+**Not used:** Gotify's native message store is available for UI/push notifications, but the sendMsg flow bypasses it (direct send, no storage).
+
+## Usage
+
+### Send Email
+
+```bash
+curl -X POST https://api.riotpiao.com \
+  -H 'X-Service: notification' \
+  -H 'X-Resource: sendMsg' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "format": "smtp",
+    "title": "Alert",
+    "message": "System CPU high",
+    "priority": 5,
+    "extras": {
+      "to_email": "[email protected]",
+      "cc": "[email protected]"
+    }
+  }'
+```
+
+**Response (success):**
+```json
+{
+  "status": "success",
+  "messageId": "[email protected]"
+}
+```
+
+**Response (error):**
+```json
+{
+  "status": "error",
+  "error": "failed to send email: connection refused"
+}
+```
+
+### Send SMS (Stubbed)
+
+SMS support is stubbed. Currently returns "not implemented" error. To enable:
+
+1. Choose SMS provider (Twilio, AWS SNS, Vonage, etc.)
+2. Set `SMS_API_URL` and `SMS_API_KEY` environment vars in gateway Deployment
+3. Implement provider integration in `internal/notification/handler.go` sendSMS() method
+
+```bash
+curl -X POST https://api.riotpiao.com \
+  -H 'X-Service: notification' \
+  -H 'X-Resource: sendMsg' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "format": "sms",
+    "message": "System CPU high",
+    "extras": {
+      "phone": "+12025551234"
+    }
+  }'
+```
+
+## Configuration
+
+### SMTP Settings
+
+Gateway reads SMTP config from environment variables (pulled from `smtp-credentials` Secret in `api` namespace):
+- `SMTP_HOST` — SMTP server hostname
+- `SMTP_PORT` — SMTP server port (587 TLS or 465 SSL)
+- `SMTP_FROM` — Sender email address
+- `SMTP_USER` — SMTP auth username
+- `SMTP_PASS` — SMTP auth password
+
+Secret is SOPS-encrypted in git. Create via:
+```bash
+kubectl create secret generic smtp-credentials \
+  --from-literal=host=mail.riotpiao.com \
+  --from-literal=port=587 \
+  --from-literal=from=[email protected] \
+  --from-literal=user=smtp-user \
+  --from-literal=password=smtp-password \
+  -n api \
+  -o yaml | sops -e /dev/stdin > k8s/smtp-secrets.enc.yaml
+```
+
+Then add to `k8s/kustomization.yaml`:
+```yaml
+resources:
+- smtp-secrets.enc.yaml
+```
+
+### Gotify Server
+
+Gotify runs in `notifications` namespace with:
+- PostgreSQL backend (CNPG)
+- SMTP emailer sidecar (unused by sendMsg, but available for UI notifications)
+- Health check on `:80/health`
+
+Config: `k8s/apps/gotify/`
+
+## Testing
+
+```bash
+cd homelab-frontend
+bash examples/sendmsg-email.sh https://api.riotpiao.com
+```
+
+## Roadmap
+
+- [ ] SMS provider integration (pick: Twilio/SNS/Vonage)
+- [ ] Request rate limiting per source
+- [ ] Message queuing for retries (via SQS if high volume expected)
+- [ ] Audit logging (who sent what, to whom, when)
+- [ ] Template support (subject + body with placeholders)
+
+## Troubleshooting
+
+### "SMTP_HOST not set"
+Gateway env vars not loaded. Check:
+```bash
+kubectl -n api describe pod api-gateway-xyz
+kubectl -n api logs api-gateway-xyz | grep SMTP
+```
+
+### "connection refused" on SMTP
+SMTP server unreachable. Verify:
+```bash
+kubectl -n api exec -it api-gateway-xyz -- \
+  nc -zv $SMTP_HOST $SMTP_PORT
+```
+
+### "authentication failed"
+Wrong SMTP username/password. Verify credentials:
+```bash
+kubectl -n api get secret smtp-credentials -o yaml | grep password | base64 -d
+```
+
+### "X-Resource: sendMsg not found"
+Notification handler not registered. Check `internal/server/router.go`:
+- Verify `X-Service: notification` case exists
+- Confirm `notification.NewHandler()` called in `NewRouter()`
+
+## References
+
+- [API Documentation](../homelab-frontend/API.md#notification-services)
+- [Gotify Server Docs](https://gotify.net)
+- [SMTP Configuration Best Practices](https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol)

scripts/iam/provision-rbac.py

@@ -283,36 +283,111 @@ if status != 200:
 else:
     scope_pks = {m["scope_name"]: m["pk"] for m in scopes_res.get("results", [])}
 
-scope_pks_list = [scope_pks[s] for s in SCOPE_MAPPINGS.keys() if s in scope_pks]
+# Include standard OpenID scopes (openid, email, profile) + custom claim scopes
+STANDARD_SCOPES = ["openid", "email", "profile"]
+scope_pks_list = [scope_pks[s] for s in STANDARD_SCOPES if s in scope_pks]
+scope_pks_list += [scope_pks[s] for s in SCOPE_MAPPINGS.keys() if s in scope_pks]
 
+# OAuth2 providers — client_secret sourced from SOPS-encrypted k8s secrets.
+# These are the real secrets the services use. Authentik must match.
 OAuth2_PROVIDERS = {
-    "api-gw": {"client_id": "api-gw", "redirect_uris": ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"]},
+    "api-gw": {
-    "minio": {"client_id": "minio", "redirect_uris": ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+        "client_id": "api-gw",
-    "poimen": {"client_id": "poimen", "redirect_uris": ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+        "client_secret_env": "AUTHENTIK_PROVIDER_API_GW_SECRET",
-    "paperless": {"client_id": "paperless", "redirect_uris": ["http://localhost:8000/auth/complete", "https://paperless.riotpiao.com/auth/complete"]},
+        "redirect_uris": ["http://localhost:3000/callback", "https://api.riotpiao.com/callback"],
-    "grafana": {"client_id": "grafana", "redirect_uris": ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"]},
+    },
-    "queue": {"client_id": "queue-sqs", "redirect_uris": ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
+    "minio": {
+        "client_id": "minio",
+        "client_secret": "9d2867fe08c3bf7fedd7e32bbaf4456fce3b0aaf788966d7559e1955947b0219",
+        "redirect_uris": ["http://localhost:9000/auth/sso/oauth2/code", "https://minio.riotpiao.com/auth/sso/oauth2/code"],
+    },
+    "poimen": {
+        "client_id": "poimen",
+        "client_secret_env": "AUTHENTIK_PROVIDER_POIMEN_SECRET",
+        "redirect_uris": ["http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"],
+    },
+    "paperless": {
+        "client_id": "paperless",
+        "client_secret": "6hcxaaVgZlKgafl7BxeSEtPAcbNUJxi2PAZePxSFk4o=",
+        "redirect_uris": ["http://localhost:8000/accounts/oidc/authentik/login/callback/", "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"],
+    },
+    "grafana": {
+        "client_id": "grafana",
+        "client_secret": "966bad4fa43812100e7775b3c73fed2ce1d07217fa5a23fbb0f190e46d2f0fa4",
+        "redirect_uris": ["http://localhost:3000/login/generic_oauth", "https://grafana.riotpiao.com/login/generic_oauth"],
+    },
+    "queue": {
+        "client_id": "queue-sqs",
+        "client_secret_env": "AUTHENTIK_PROVIDER_QUEUE_SECRET",
+        "redirect_uris": ["http://localhost:8080/callback", "https://queue.riotpiao.com/callback"],
+    },
+    "forgejo": {
+        "client_id": "forgejo",
+        "client_secret": "G4klhs3JRfs5A7YnGs90WuOndBAvamWlZgaZRY8x",
+        "redirect_uris": ["http://localhost:3000/user/oauth2/authentik/callback", "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"],
+    },
+    "immich": {
+        "client_id": "immich",
+        "client_secret": "QxyWfESXqTD55aUyh6miYnny1QTCEuEwyC4escw9",
+        "redirect_uris": ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"],
+    },
+    "homarr": {
+        "client_id": "homarr",
+        "client_secret": "RMlDQAWdjT5YPPH0U7ztDoUFP7R7w95b2xqQ1pyS",
+        "redirect_uris": ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"],
+    },
+    "argocd": {
+        "client_id": "argocd",
+        "client_secret_env": "AUTHENTIK_PROVIDER_ARGOCD_SECRET",
+        "redirect_uris": ["http://localhost:8080/auth/callback", "https://argocd.riotpiao.com/auth/callback"],
+    },
+    "vault": {
+        "client_id": "vault",
+        "client_secret_env": "AUTHENTIK_PROVIDER_VAULT_SECRET",
+        "redirect_uris": ["http://localhost:8200/ui/vault/auth/oidc/oidc/callback", "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"],
+    },
 }
 
+import secrets as _secrets
+
 for provider_name, provider_spec in OAuth2_PROVIDERS.items():
-    if provider_name in existing_providers:
+    # Resolve client_secret: explicit > env var > generate random
-        print(f"  {provider_name}: already exists")
+    if "client_secret" in provider_spec:
+        client_secret = provider_spec["client_secret"]
+    elif "client_secret_env" in provider_spec:
+        client_secret = os.environ.get(provider_spec["client_secret_env"], _secrets.token_urlsafe(32))
     else:
-        # Build redirect_uris list with proper schema
+        client_secret = _secrets.token_urlsafe(32)
-        redirect_uris_list = [{"url": uri, "matching_mode": "strict"} for uri in provider_spec["redirect_uris"]]
+
-        
+    redirect_uris_list = [{"url": uri, "matching_mode": "strict"} for uri in provider_spec["redirect_uris"]]
-        client_secret = os.environ.get(f"AUTHENTIK_PROVIDER_{provider_name.upper()}_SECRET", f"{provider_name}-secret-placeholder")
+
-        
+    provider_payload = {
-        status, res = api("POST", "/api/v3/providers/oauth2/", {
+        "name": provider_name,
-            "name": provider_name,
+        "authorization_flow": auth_flow,
-            "authorization_flow": auth_flow,
+        "invalidation_flow": inval_flow,
-            "invalidation_flow": inval_flow,
+        "grant_types": ["authorization_code", "implicit", "password"],
-            "grant_types": ["authorization_code", "implicit", "password"],
+        "client_id": provider_spec["client_id"],
+        "client_secret": client_secret,
+        "redirect_uris": redirect_uris_list,
+        "property_mappings": scope_pks_list,
+    }
+
+    if provider_name in existing_providers:
+        # UPDATE existing provider — sync secret + redirect_uris
+        provider_pk = existing_providers[provider_name]["pk"]
+        status, res = api("PATCH", f"/api/v3/providers/oauth2/{provider_pk}/", {
             "client_id": provider_spec["client_id"],
             "client_secret": client_secret,
             "redirect_uris": redirect_uris_list,
             "property_mappings": scope_pks_list,
         })
+        if status in (200, 201):
+            print(f"  {provider_name}: updated (secret + redirect_uris synced)")
+        else:
+            print(f"  {provider_name}: UPDATE FAILED {status} {res}")
+    else:
+        # CREATE new provider
+        status, res = api("POST", "/api/v3/providers/oauth2/", provider_payload)
         if status in (200, 201):
             print(f"  {provider_name}: created")
         else:
@@ -330,17 +405,28 @@ if status != 200:
 existing_apps = {a["slug"]: a for a in res.get("results", [])}
 
 for provider_name in OAuth2_PROVIDERS.keys():
+    # Get the provider PK
+    status, provider_res = api("GET", f"/api/v3/providers/oauth2/?name={provider_name}")
+    if status != 200 or not provider_res.get("results"):
+        print(f"  {provider_name}: provider not found, skip")
+        continue
+    provider_pk = provider_res["results"][0]["pk"]
+
     if provider_name in existing_apps:
-        print(f"  {provider_name}: already exists")
+        # Ensure app is linked to provider (fix orphaned apps)
+        app_data = existing_apps[provider_name]
+        if app_data.get("provider") != provider_pk:
+            app_uuid = app_data["pk"]
+            status, res = api("PATCH", f"/api/v3/core/applications/{app_uuid}/", {
+                "provider": provider_pk,
+            })
+            if status in (200, 201):
+                print(f"  {provider_name}: re-linked to provider")
+            else:
+                print(f"  {provider_name}: RE-LINK FAILED {status} {res}")
+        else:
+            print(f"  {provider_name}: ok")
     else:
-        # Get the provider PK to link
-        status, provider_res = api("GET", f"/api/v3/providers/oauth2/?name={provider_name}")
-        if status != 200 or not provider_res.get("results"):
-            print(f"  {provider_name}: provider not found")
-            continue
-        
-        provider_pk = provider_res["results"][0]["pk"]
-        
         status, res = api("POST", "/api/v3/core/applications/", {
             "name": provider_name,
             "slug": provider_name,