fix(ingress): restore the authentik.riotpiao.com route

Commit a02bee5 had both the Ingress rule and the cloudflared tunnel
hostname for authentik.riotpiao.com. Both were lost, so the tunnel
answered 404 for the hostname and the ingress had no matching rule.

The CoreDNS rewrite and the authentik-server Service were still intact;
only the two routing entries were missing. Re-add the Ingress (namespace
iam, service authentik-server port 80, TLS terminated by the wildcard
default-ssl-certificate) and the tunnel hostname.

.env.example

@@ -31,6 +31,17 @@ AUTHENTIK_BOOTSTRAP_TOKEN=
 #   openssl rand -base64 24
 AUTHENTIK_PG_PASSWORD=
 
+# ── ZITADEL (Authentik replacement) ─────────────────────────────────────────────
+# Instance URL, same hostname as the OIDC issuer.
+ZITADEL_URL="https://zitadel.riotpiao.com"
+
+# PAT of the 'iam-admin' machine user, created by the chart's setup job on the
+# first sync. Leave empty until then. Read it with:
+#   kubectl -n iam get secret iam-admin-pat -o jsonpath='{.data.pat}' | base64 -d
+# scripts/iam/zitadel-provision.py reads this variable first, then the cluster
+# Secret. The masterkey and the client secrets never live in this file.
+ZITADEL_BOOTSTRAP_TOKEN=
+
 # ── MinIO ──────────────────────────────────────────────────────────────────────
 MINIO_ROOT_USER=
 MINIO_ROOT_PASSWORD=

.gitignore

@@ -47,6 +47,15 @@ k8s/storage/test/test
 terraform/.terraform/
 terraform/*.tfstate
 terraform/*.tfstate.*
+# The Zitadel root module keeps its own state (see terraform/zitadel/providers.tf).
+# That state carries generated client secrets and machine keys.
+terraform/zitadel/*.tfstate
+terraform/zitadel/*.tfstate.*
+terraform/zitadel/*.tfvars
+terraform/edge/*.tfstate
+terraform/edge/*.tfstate.*
+terraform/edge/*.tfvars
+terraform/edge/terraform.tfvars.local
 terraform.tfvars.local
 skills-lock.json
 secrets-plaintext.yaml
@@ -69,3 +78,14 @@ k8s/**/*-secret.yaml
 
 # IAM provisioning scripts contain credential references — never commit
 scripts/iam/*.py
+# Exception: this one reads every credential from the cluster at run time and
+# embeds none. Keep it reviewable in git.
+!scripts/iam/zitadel-provision.py
+
+# Runtime credential export from the old Authentik provisioning run. Holds live
+# M2M tokens; never commit it.
+/authentik-m2m-tokens.env
+
+# Never commit backup copies of plaintext tfvars (they carry cluster PKI)
+terraform/*.bak
+terraform/edge/.secrets/

.pi/loops/loops-01a16cb7-780d-72af-bac3-487935898ff3.json

@@ -0,0 +1,22 @@
+{
+  "nextId": 2,
+  "loops": [
+    {
+      "id": "1",
+      "prompt": "Check Longhorn and CNPG recovery progress in the homelab cluster and report terse. Run: kubectl -n longhorn-system get volumes.longhorn.io -o json (count by state/robustness), kubectl -n longhorn-system get replicas.longhorn.io -o json (count running/stopped/failed), kubectl get clusters.postgresql.cnpg.io -A (ready instances per cluster), kubectl -n cicd get pods,pvc | grep forgejo-db. Report: volumes healthy vs degraded, replicas running vs stopped, and each CNPG cluster ready count. If every volume is attached/healthy and every CNPG cluster shows 3/3 ready, say RECOVERED, list any leftover anomalies, and stop this loop with LoopDelete. Otherwise name what is still rebuilding and keep going.",
+      "trigger": {
+        "type": "cron",
+        "schedule": "*/5 * * * *"
+      },
+      "status": "expired",
+      "recurring": true,
+      "createdAt": 1792898370946,
+      "updatedAt": 1792901371403,
+      "expiresAt": 1793503170946,
+      "source": "tool",
+      "readOnly": true,
+      "maxFires": 6,
+      "fireCount": 6
+    }
+  ]
+}

diagram/poimen-schema-relations.architecture.json

@@ -0,0 +1,37 @@
+{
+  "schema_version": 1,
+  "diagram_type": "architecture",
+  "meta": {
+    "title": "Poimen Memory — Schema Relations",
+    "quality_profile": "showcase",
+    "viewBox": [900, 480]
+  },
+  "components": [
+    { "id": "node", "type": "database", "label": "memory_node", "sublabel": "Episodes (raw messages)", "pos": [40, 60], "size": [150, 55], "tag": "BIGSERIAL" },
+    { "id": "entity", "type": "database", "label": "memory_entity", "sublabel": "Extracted entities", "pos": [260, 60], "size": [150, 55], "tag": "UUID" },
+    { "id": "edge", "type": "database", "label": "memory_edge", "sublabel": "Facts / relationships", "pos": [480, 60], "size": [150, 55], "tag": "UUID" },
+    { "id": "community", "type": "database", "label": "memory_community", "sublabel": "Entity clusters", "pos": [700, 60], "size": [150, 55], "tag": "UUID" },
+    { "id": "pgvector", "type": "cloud", "label": "pgvector HNSW", "sublabel": "768-dim cosine search", "pos": [140, 260], "size": [160, 55], "tag": "nomic-embed-v2" },
+    { "id": "tsvector", "type": "cloud", "label": "pg ts_rank", "sublabel": "Full-text search", "pos": [400, 260], "size": [160, 55], "tag": "GIN index" },
+    { "id": "tenant", "type": "security", "label": "tenant_id", "sublabel": "Multi-tenant isolation", "pos": [660, 260], "size": [140, 55], "tag": "W7 / mig 014" }
+  ],
+  "boundaries": [
+    { "kind": "region", "label": "Temporal Knowledge Graph (Zep §2)", "wraps": ["node", "entity", "edge", "community"] },
+    { "kind": "security-group", "label": "Search & Isolation", "wraps": ["pgvector", "tsvector", "tenant"] }
+  ],
+  "connections": [
+    { "id": "node-entity", "from": "node", "to": "entity", "variant": "emphasis" },
+    { "id": "entity-edge", "from": "entity", "to": "edge", "variant": "emphasis" },
+    { "id": "entity-community", "from": "community", "to": "entity", "variant": "dashed" },
+    { "id": "edge-node", "from": "edge", "to": "node", "variant": "dashed" },
+    { "id": "entity-vec", "from": "entity", "to": "pgvector", "variant": "default" },
+    { "id": "entity-fts", "from": "entity", "to": "tsvector", "variant": "dashed" },
+    { "id": "edge-fts", "from": "edge", "to": "tsvector", "variant": "dashed" },
+    { "id": "tenant-all", "from": "tenant", "to": "entity", "variant": "security" }
+  ],
+  "cards": [
+    { "dot": "emerald", "title": "Three-Tier Graph", "items": ["memory_node → episode subgraph", "memory_entity → entity subgraph", "memory_edge → fact relationships", "memory_community → clusters (Phase 4)"] },
+    { "dot": "rose", "title": "Bi-Temporal", "items": ["t_valid / t_invalid: event time", "t_created / t_expired: transaction time", "Contradiction: active → candidate → confirmed"] },
+    { "dot": "cyan", "title": "Search", "items": ["HNSW: name, summary, fact embeddings", "ts_rank: BM25 fallback (always available)", "tenant_id: per-table isolation (mig 014)"] }
+  ]
+}

diagram/poimen-schema-relations.dataflow.json

@@ -0,0 +1,158 @@
+{
+  "schema_version": 1,
+  "diagram_type": "dataflow",
+  "meta": {
+    "title": "Poimen Memory \u2014 Schema Relations",
+    "viewBox": [
+      960,
+      480
+    ],
+    "quality_profile": "showcase"
+  },
+  "stages": [
+    {
+      "label": "Episodes"
+    },
+    {
+      "label": "Entities"
+    },
+    {
+      "label": "Relationships"
+    },
+    {
+      "label": "Indexes"
+    }
+  ],
+  "nodes": [
+    {
+      "id": "node",
+      "type": "database",
+      "label": "memory_node",
+      "sublabel": "Raw messages",
+      "stage": 0,
+      "row": 0,
+      "tag": "BIGSERIAL"
+    },
+    {
+      "id": "entity",
+      "type": "database",
+      "label": "memory_entity",
+      "sublabel": "Named entities",
+      "stage": 1,
+      "row": 0,
+      "tag": "UUID + tenant_id"
+    },
+    {
+      "id": "community",
+      "type": "database",
+      "label": "memory_community",
+      "sublabel": "Clusters (Phase 4)",
+      "stage": 1,
+      "row": 1,
+      "tag": "UUID"
+    },
+    {
+      "id": "edge",
+      "type": "database",
+      "label": "memory_edge",
+      "sublabel": "Facts + bi-temporal",
+      "stage": 2,
+      "row": 0,
+      "tag": "UUID + tenant_id"
+    },
+    {
+      "id": "hnsw",
+      "type": "cloud",
+      "label": "HNSW Index",
+      "sublabel": "768-dim cosine",
+      "stage": 3,
+      "row": 0,
+      "tag": "pgvector"
+    },
+    {
+      "id": "fts",
+      "type": "cloud",
+      "label": "GIN Index",
+      "sublabel": "ts_rank BM25",
+      "stage": 3,
+      "row": 1,
+      "tag": "full-text"
+    }
+  ],
+  "flows": [
+    {
+      "from": "node",
+      "to": "entity",
+      "label": "LLM extraction",
+      "classification": "ingest",
+      "variant": "emphasis",
+      "route": "straight"
+    },
+    {
+      "from": "entity",
+      "to": "edge",
+      "label": "source + target FK",
+      "classification": "foreign key",
+      "variant": "emphasis",
+      "route": "straight"
+    },
+    {
+      "from": "entity",
+      "to": "community",
+      "label": "community_id",
+      "classification": "membership",
+      "variant": "dashed",
+      "route": "straight",
+      "labelDy": 30
+    },
+    {
+      "from": "edge",
+      "to": "hnsw",
+      "classification": "vector",
+      "variant": "default",
+      "route": "straight",
+      "label": "embedding",
+      "labelDy": -30
+    },
+    {
+      "from": "edge",
+      "to": "fts",
+      "label": "fact text",
+      "classification": "text",
+      "variant": "dashed",
+      "route": "vertical-channel",
+      "labelDy": 30
+    }
+  ],
+  "cards": [
+    {
+      "dot": "emerald",
+      "title": "Three-Tier Graph",
+      "items": [
+        "memory_node \u2192 episodes (raw messages)",
+        "memory_entity \u2192 extracted entities",
+        "memory_edge \u2192 facts between entities",
+        "memory_community \u2192 clusters (Phase 4)"
+      ]
+    },
+    {
+      "dot": "rose",
+      "title": "Bi-Temporal + Tenant",
+      "items": [
+        "t_valid/t_invalid: when true in reality",
+        "t_created/t_expired: system transaction time",
+        "tenant_id: row-level multi-tenant isolation",
+        "Contradiction: active \u2192 candidate \u2192 confirmed"
+      ]
+    },
+    {
+      "dot": "cyan",
+      "title": "Search Indexes",
+      "items": [
+        "HNSW: name, summary, fact embeddings",
+        "GIN: ts_rank on name+summary and fact",
+        "RRF fusion: sem_weight + lex_weight"
+      ]
+    }
+  ]
+}

diagram/rbac-delegation-flow.sequence.json

@@ -0,0 +1,63 @@
+{
+  "schema_version": 1,
+  "diagram_type": "sequence",
+  "meta": {
+    "title": "Service Acting on Behalf of User",
+    "viewBox": [1000, 600],
+    "column_fit": "spread",
+    "quality_profile": "showcase"
+  },
+  "participants": [
+    { "id": "user", "type": "external", "label": "User", "sublabel": "browser" },
+    { "id": "app", "type": "frontend", "label": "Web App", "sublabel": "poimen UI" },
+    { "id": "authentik", "type": "security", "label": "Authentik", "sublabel": "OIDC" },
+    { "id": "worker", "type": "backend", "label": "Backend Worker", "sublabel": "temporal/queue" },
+    { "id": "memory", "type": "backend", "label": "Memory API", "sublabel": "poimen-memory" },
+    { "id": "jwks", "type": "security", "label": "JWKS", "sublabel": "RS256" }
+  ],
+  "segments": [
+    { "from": 150, "to": 260, "label": "User Auth" },
+    { "from": 270, "to": 370, "label": "Delegation" },
+    { "from": 380, "to": 510, "label": "Downstream" }
+  ],
+  "messages": [
+    { "id": "login", "from": "user", "to": "app", "y": 165, "label": "login (OIDC)", "variant": "default" },
+    { "id": "oidc-flow", "from": "app", "to": "authentik", "y": 200, "label": "authorization_code flow", "variant": "emphasis" },
+    { "id": "user-jwt", "from": "authentik", "to": "app", "y": 235, "label": "user JWT {groups,roles}", "variant": "return" },
+    { "id": "trigger-job", "from": "user", "to": "app", "y": 285, "label": "trigger async job", "variant": "default" },
+    { "id": "queue-job", "from": "app", "to": "worker", "y": 320, "label": "enqueue job + user JWT", "variant": "emphasis", "note": "JWT passed as job metadata" },
+    { "id": "job-ack", "from": "worker", "to": "app", "y": 355, "label": "job queued", "variant": "return" },
+    { "id": "call-memory", "from": "worker", "to": "memory", "y": 400, "label": "POST /memory/ingest", "variant": "emphasis", "note": "Bearer <user JWT>" },
+    { "id": "fetch-jwks", "from": "memory", "to": "jwks", "y": 435, "label": "GET /jwks", "variant": "dashed" },
+    { "id": "jwks-keys", "from": "jwks", "to": "memory", "y": 470, "label": "RSA pubkey", "variant": "return" },
+    { "id": "ingest-ok", "from": "memory", "to": "worker", "y": 505, "label": "202 Accepted", "variant": "return", "note": "uses USER's groups/roles" }
+  ],
+  "activations": [
+    { "participant": "user", "from": 160, "to": 291, "type": "external" },
+    { "participant": "app", "from": 165, "to": 361, "type": "frontend" },
+    { "participant": "authentik", "from": 195, "to": 241, "type": "security" },
+    { "participant": "worker", "from": 315, "to": 511, "type": "backend" },
+    { "participant": "memory", "from": 395, "to": 511, "type": "backend" },
+    { "participant": "jwks", "from": 430, "to": 476, "type": "security" }
+  ],
+  "cards": [
+    {
+      "dot": "emerald",
+      "title": "Delegation Pattern",
+      "items": [
+        "User JWT travels with the job payload",
+        "Worker uses user's token for downstream",
+        "Memory sees USER as the principal"
+      ]
+    },
+    {
+      "dot": "rose",
+      "title": "Security Model",
+      "items": [
+        "No service-account escalation",
+        "User's permissions bound the action",
+        "Audit trail traces to user.sub"
+      ]
+    }
+  ]
+}

diagram/rbac-fine-grained.architecture.json

@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "diagram_type": "architecture",
+  "meta": {
+    "title": "Memory Service Fine-Grained RBAC",
+    "quality_profile": "showcase",
+    "viewBox": [900, 520]
+  },
+  "components": [
+    { "id": "jwt", "type": "security", "label": "JWT Token", "sublabel": "groups[] + roles[]", "pos": [40, 60], "size": [140, 60], "tag": "Authentik" },
+    { "id": "cap-check", "type": "backend", "label": "Capability Check", "sublabel": "has_capability()", "pos": [260, 60], "size": [160, 60], "tag": "Phase 1" },
+    { "id": "scope-checker", "type": "backend", "label": "Scope Checker", "sublabel": "filter_resources()", "pos": [500, 60], "size": [160, 60], "tag": "Phase 2" },
+    { "id": "decision", "type": "backend", "label": "AccessDecision", "sublabel": "Allow / Deny", "pos": [740, 60], "size": [130, 60] },
+    { "id": "project-scope", "type": "database", "label": "ProjectScope", "sublabel": "dev/staging/prod/*", "pos": [40, 200], "size": [140, 55] },
+    { "id": "visibility-scope", "type": "database", "label": "VisibilityScope", "sublabel": "public/private", "pos": [220, 200], "size": [140, 55] },
+    { "id": "owner-scope", "type": "database", "label": "OwnerScope", "sublabel": "self/any/user", "pos": [400, 200], "size": [140, 55] },
+    { "id": "group-scope", "type": "database", "label": "GroupScope", "sublabel": "in_any_group()", "pos": [580, 200], "size": [140, 55] },
+    { "id": "entities", "type": "database", "label": "Entities", "sublabel": "memory_entity", "pos": [40, 340], "size": [120, 50] },
+    { "id": "edges", "type": "database", "label": "Edges", "sublabel": "memory_edge", "pos": [200, 340], "size": [120, 50] },
+    { "id": "projects", "type": "database", "label": "Projects", "sublabel": "project_id", "pos": [360, 340], "size": [120, 50] },
+    { "id": "convos", "type": "database", "label": "Conversations", "sublabel": "owner_id", "pos": [520, 340], "size": [120, 50] }
+  ],
+  "boundaries": [
+    { "kind": "security-group", "label": "Access Control Pipeline", "wraps": ["cap-check", "scope-checker", "decision"] },
+    { "kind": "region", "label": "Scope Checkers", "wraps": ["project-scope", "visibility-scope", "owner-scope", "group-scope"] },
+    { "kind": "region", "label": "Protected Resources", "wraps": ["entities", "edges", "projects", "convos"] }
+  ],
+  "connections": [
+    { "id": "jwt-to-cap", "from": "jwt", "to": "cap-check", "label": "claims", "variant": "emphasis" },
+    { "id": "cap-to-scope", "from": "cap-check", "to": "scope-checker", "variant": "emphasis" },
+    { "id": "scope-to-decision", "from": "scope-checker", "to": "decision", "variant": "emphasis" },
+    { "id": "scope-to-proj", "from": "scope-checker", "to": "project-scope", "variant": "dashed", "fromSide": "bottom", "toSide": "top" },
+    { "id": "scope-to-vis", "from": "scope-checker", "to": "visibility-scope", "variant": "dashed", "fromSide": "bottom", "toSide": "top" },
+    { "id": "scope-to-owner", "from": "scope-checker", "to": "owner-scope", "variant": "dashed", "fromSide": "bottom", "toSide": "top" },
+    { "id": "scope-to-group", "from": "scope-checker", "to": "group-scope", "variant": "dashed", "fromSide": "bottom", "toSide": "top" }
+  ],
+  "cards": [
+    { "dot": "emerald", "title": "Phase 1: Capability", "items": ["has_capability(memory:read)", "Checks roles[] claim", "Binary allow/deny"] },
+    { "dot": "rose", "title": "Phase 2: Scope", "items": ["Project: dev/staging/prod", "Visibility: public/private", "Owner: self-owned only", "Group: poimen-admins/devs"] },
+    { "dot": "cyan", "title": "JWT Claims", "items": ["groups: user.groups", "roles: attributes.roles", "permissions: group names"] }
+  ]
+}

diagram/rbac-m2m-flow.sequence.json

@@ -0,0 +1,54 @@
+{
+  "schema_version": 1,
+  "diagram_type": "sequence",
+  "meta": {
+    "title": "Service-to-Service (M2M) Authentication",
+    "viewBox": [1080, 520],
+    "column_fit": "spread",
+    "quality_profile": "showcase"
+  },
+  "participants": [
+    { "id": "worker", "type": "backend", "label": "Ingest Worker", "sublabel": "temporal/cron" },
+    { "id": "authentik", "type": "security", "label": "Authentik", "sublabel": "OIDC provider" },
+    { "id": "memory", "type": "backend", "label": "Memory Service", "sublabel": "poimen-memory" },
+    { "id": "jwks", "type": "security", "label": "JWKS", "sublabel": "RS256 keys" }
+  ],
+  "segments": [
+    { "from": 150, "to": 230, "label": "Token Mint" },
+    { "from": 240, "to": 380, "label": "API + RBAC" }
+  ],
+  "messages": [
+    { "id": "token-req", "from": "worker", "to": "authentik", "y": 165, "label": "POST /token grant_type=password", "variant": "emphasis", "note": "client_id + username + app_password + scope" },
+    { "id": "jwt-resp", "from": "authentik", "to": "worker", "y": 205, "label": "JWT {groups,roles,aud,iss}", "variant": "return", "note": "groups=[poimen-admins] roles=[memory:write]" },
+    { "id": "api-call", "from": "worker", "to": "memory", "y": 260, "label": "POST /memory/ingest", "variant": "emphasis", "note": "Authorization: Bearer <JWT>" },
+    { "id": "fetch-jwks", "from": "memory", "to": "jwks", "y": 295, "label": "GET /jwks (cached)", "variant": "dashed" },
+    { "id": "jwks-resp", "from": "jwks", "to": "memory", "y": 330, "label": "RSA pubkey (kid)", "variant": "return" },
+    { "id": "verify-ok", "from": "memory", "to": "worker", "y": 365, "label": "202 Accepted", "variant": "return", "note": "sig ✓ iss ✓ aud ✓ has_capability ✓" }
+  ],
+  "activations": [
+    { "participant": "worker", "from": 160, "to": 371, "type": "backend" },
+    { "participant": "authentik", "from": 165, "to": 211, "type": "security" },
+    { "participant": "memory", "from": 255, "to": 371, "type": "backend" },
+    { "participant": "jwks", "from": 290, "to": 336, "type": "security" }
+  ],
+  "cards": [
+    {
+      "dot": "emerald",
+      "title": "M2M Pattern",
+      "items": [
+        "grant_type=password (Authentik aliases client_credentials)",
+        "App-password token authenticates the service account",
+        "NOT client_secret — that creates nameless SA"
+      ]
+    },
+    {
+      "dot": "rose",
+      "title": "JWT Claims",
+      "items": [
+        "groups: from user.groups (poimen-admins)",
+        "roles: from user.attributes.roles",
+        "iss/aud: per-provider issuer"
+      ]
+    }
+  ]
+}

diagram/rbac-user-flow.sequence.json

@@ -0,0 +1,61 @@
+{
+  "schema_version": 1,
+  "diagram_type": "sequence",
+  "meta": {
+    "title": "User Direct Access with JWT",
+    "viewBox": [1000, 640],
+    "column_fit": "spread",
+    "quality_profile": "showcase"
+  },
+  "participants": [
+    { "id": "user", "type": "external", "label": "User", "sublabel": "browser" },
+    { "id": "app", "type": "frontend", "label": "Web App", "sublabel": "poimen UI" },
+    { "id": "authentik", "type": "security", "label": "Authentik", "sublabel": "OIDC provider" },
+    { "id": "memory", "type": "backend", "label": "Memory API", "sublabel": "poimen-memory" },
+    { "id": "jwks", "type": "security", "label": "JWKS", "sublabel": "RS256 keys" }
+  ],
+  "segments": [
+    { "from": 150, "to": 370, "label": "OIDC Login" },
+    { "from": 380, "to": 540, "label": "API + RBAC" }
+  ],
+  "messages": [
+    { "id": "click-login", "from": "user", "to": "app", "y": 165, "label": "click Login", "variant": "default" },
+    { "id": "redirect-auth", "from": "app", "to": "authentik", "y": 200, "label": "redirect /authorize", "variant": "emphasis", "note": "scope=openid groups roles" },
+    { "id": "show-login", "from": "authentik", "to": "user", "y": 235, "label": "login form", "variant": "default" },
+    { "id": "submit-creds", "from": "user", "to": "authentik", "y": 270, "label": "username + password", "variant": "security" },
+    { "id": "auth-code", "from": "authentik", "to": "app", "y": 305, "label": "redirect ?code=xxx", "variant": "return" },
+    { "id": "exchange-token", "from": "app", "to": "authentik", "y": 340, "label": "POST /token code=xxx", "variant": "emphasis" },
+    { "id": "jwt-resp", "from": "authentik", "to": "app", "y": 375, "label": "JWT {groups,roles,sub}", "variant": "return" },
+    { "id": "query-api", "from": "app", "to": "memory", "y": 420, "label": "GET /memory/query", "variant": "emphasis", "note": "Authorization: Bearer <JWT>" },
+    { "id": "fetch-jwks", "from": "memory", "to": "jwks", "y": 455, "label": "GET /jwks (cached)", "variant": "dashed" },
+    { "id": "jwks-keys", "from": "jwks", "to": "memory", "y": 490, "label": "RSA pubkey", "variant": "return" },
+    { "id": "query-resp", "from": "memory", "to": "app", "y": 525, "label": "200 JSON results", "variant": "return", "note": "has_capability(memory:read) ✓" }
+  ],
+  "activations": [
+    { "participant": "user", "from": 160, "to": 276, "type": "external" },
+    { "participant": "app", "from": 165, "to": 531, "type": "frontend" },
+    { "participant": "authentik", "from": 195, "to": 381, "type": "security" },
+    { "participant": "memory", "from": 415, "to": 531, "type": "backend" },
+    { "participant": "jwks", "from": 450, "to": 496, "type": "security" }
+  ],
+  "cards": [
+    {
+      "dot": "emerald",
+      "title": "Authorization Code Flow",
+      "items": [
+        "User authenticates directly with Authentik",
+        "App exchanges code for JWT",
+        "Scopes: openid groups roles permissions"
+      ]
+    },
+    {
+      "dot": "rose",
+      "title": "JWT Contents",
+      "items": [
+        "iss: .../application/o/poimen/",
+        "aud: poimen (provider client_id)",
+        "groups: user's group memberships"
+      ]
+    }
+  ]
+}

k8s/apps/cloudflared/configmap.yaml

@@ -0,0 +1,60 @@
+# cloudflared ingress configuration.
+#
+# This list is the tunnel's routing table: for each incoming hostname the
+# connector dials the matching origin. The final rule is a catch-all, so a
+# hostname absent from this list gets 404 from the connector — DNS alone does
+# not route a request into a service.
+#
+# Management model: this file makes the tunnel LOCALLY managed. cloudflared is
+# given a credentials file instead of a token, so Cloudflare stops pushing the
+# remotely-managed ingess list and this ConfigMap wins. Revert the Deployment to
+# `run --token` to return to the dashboard-managed list.
+#
+# Origin services self-reference the public hostname. That resolves because
+# CoreDNS rewrites each of these names to ingress-nginx
+# (terraform/files/coredns/Corefile), so the connector reaches the in-cluster
+# ingress instead of looping back out through Cloudflare.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cloudflared-config
+  namespace: cloudflared
+data:
+  config.yml: |
+    tunnel: 32d0bf93-e1bd-45b8-93a9-71ff9f83de80
+    credentials-file: /etc/cloudflared/creds/credentials.json
+
+    # WARP-to-Tunnel routing. The remotely-managed config had this enabled;
+    # dropping the key would break WARP access to private addresses. cloudflared
+    # 2026.x has no `enabled` field — the key's presence is what enables it, and
+    # `enabled: true` here makes the binary warn about an unused key.
+    warp-routing: {}
+
+    ingress:
+      - hostname: paperless.riotpiao.com
+        service: https://paperless.riotpiao.com
+      - hostname: img.riotpiao.com
+        service: https://img.riotpiao.com
+      - hostname: riotpiao.com
+        service: https://riotpiao.com
+      - hostname: grafana.riotpiao.com
+        service: https://grafana.riotpiao.com
+      - hostname: gotify.riotpiao.com
+        service: https://gotify.riotpiao.com
+      - hostname: igotify.riotpiao.com
+        service: https://igotify.riotpiao.com
+      - hostname: poimen.riotpiao.com
+        service: https://poimen.riotpiao.com
+      - hostname: comfyui.riotpiao.com
+        service: https://comfyui.riotpiao.com
+      - hostname: forgejo.riotpiao.com
+        service: https://forgejo.riotpiao.com
+      # ZITADEL. Without this rule the connector answers 404 for the hostname
+      # even though the DNS CNAME points at the tunnel.
+      - hostname: zitadel.riotpiao.com
+        service: https://zitadel.riotpiao.com
+      # AUTHENTIK. Same reasoning as ZITADEL: a hostname absent from this table
+      # gets 404 from the connector. CoreDNS rewrites the name to ingress-nginx.
+      - hostname: authentik.riotpiao.com
+        service: https://authentik.riotpiao.com
+      - service: http_status:404

k8s/apps/cloudflared/deployment.yaml

@@ -13,21 +13,63 @@ spec:
       labels:
         app: cloudflared
     spec:
+      # Ingress-nginx forces HTTPS (force-ssl-redirect), so cloudflared must
+      # dial https://<hostname> and trust homelab-ca or the TLS handshake fails
+      # with "certificate signed by unknown authority". Same pattern as
+      # authentik-values.yaml: merge the system bundle with homelab-ca.crt.
+      initContainers:
+        - name: merge-ca-certs
+          image: debian:bookworm
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            runAsUser: 0
+          command:
+            - sh
+            - -c
+            - (cat /etc/ssl/certs/ca-certificates.crt 2>/dev/null; cat /homelab-ca/ca.crt) > /merged/ca-bundle.crt
+          volumeMounts:
+            - name: homelab-ca
+              mountPath: /homelab-ca
+              readOnly: true
+            - name: merged-ca
+              mountPath: /merged
+      volumes:
+        - name: config
+          configMap:
+            name: cloudflared-config
+        - name: credentials
+          secret:
+            secretName: cloudflared-credentials
+        - name: homelab-ca
+          configMap:
+            name: homelab-ca
+        - name: merged-ca
+          emptyDir: {}
       containers:
         - name: cloudflared
           image: cloudflare/cloudflared:latest
           args:
             - tunnel
             - --no-autoupdate
+            - --protocol
+            - http2
+            - --config
+            - /etc/cloudflared/config.yml
             - run
-            - --token
-            - $(TUNNEL_TOKEN)
           env:
-            - name: TUNNEL_TOKEN
+            - name: SSL_CERT_FILE
-              valueFrom:
+              value: /merged/ca-bundle.crt
-                secretKeyRef:
+          volumeMounts:
-                  name: cloudflared-token
+            - name: config
-                  key: token
+              mountPath: /etc/cloudflared/config.yml
+              subPath: config.yml
+              readOnly: true
+            - name: credentials
+              mountPath: /etc/cloudflared/creds
+              readOnly: true
+            - name: merged-ca
+              mountPath: /merged
+              readOnly: true
           resources:
             requests:
               cpu: 10m

k8s/apps/cloudflared/kustomization.yaml

@@ -3,3 +3,4 @@ kind: Kustomization
 namespace: cloudflared
 resources:
   - deployment.yaml
+  - configmap.yaml

k8s/apps/comfyui/ingress.yaml

@@ -4,7 +4,9 @@ metadata:
   name: comfyui
   namespace: comfyui
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation and no tls: block. This host is covered by
+    # the *.riotpiao.com wildcard, which nginx serves as its
+    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
@@ -16,10 +18,6 @@ metadata:
       proxy_set_header Connection "upgrade";
 spec:
   ingressClassName: nginx
-  tls:
-    - secretName: comfyui-tls
-      hosts:
-        - comfyui.riotpiao.com
   rules:
     - host: comfyui.riotpiao.com
       http:

k8s/apps/homarr/homarr-values.yaml

@@ -1,78 +0,0 @@
-# Homarr landing page with Authentik SSO
-
-# Probe tuning (chart DOES expose these — the old PostSync patch-job was
-# unnecessary and fragile: it only patched one Deployment revision, so any later
-# rollout reverted to the chart's aggressive defaults). Homarr's first-boot icon
-# updater blocks the event loop for ~50s ("icons updater took 49553ms"), during
-# which /api/health/live can't answer within the default 10s×3 window → kubelet
-# SIGTERMs the pod → CrashLoopBackOff (247 restarts, 503 at the ingress). Give
-# liveness a wide window so the icon import can finish without a kill.
-livenessProbe:
-  initialDelaySeconds: 60
-  periodSeconds: 30
-  timeoutSeconds: 5
-  failureThreshold: 10
-readinessProbe:
-  initialDelaySeconds: 30
-  periodSeconds: 15
-  timeoutSeconds: 5
-  failureThreshold: 6
-
-image:
-  repository: ghcr.io/homarr-labs/homarr
-  tag: "latest"
-  pullPolicy: Always
-
-replicaCount: 1
-
-# Configure SSO via environment variables
-# Chart supports these via top-level env dict (not array)
-env:
-  AUTH_PROVIDERS: "oidc,credentials"
-  AUTH_OIDC_ISSUER: "https://authentik.riotpiao.com/application/o/homarr/"
-  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
-  # hides the "Sign in with Authentik" button entirely when it's absent (per the
-  # authentik Homarr integration + homarr SSO docs). This was the missing var.
-  AUTH_OIDC_URI: "https://authentik.riotpiao.com/application/o/authorize/"
-  AUTH_OIDC_CLIENT_NAME: "Authentik"
-  AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
-  AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
-  AUTH_OIDC_AUTO_LOGIN: "false"
-  # Link the OIDC identity to an existing homarr account with the same email.
-  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
-  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
-  # per run ("callback took longer than expected"), compounding CPU pressure.
-  DISABLE_ANALYTICS: "true"
-  BASE_URL: "https://homarr.riotpiao.com"
-  NEXTAUTH_URL: "https://homarr.riotpiao.com"
-
-# Client credentials from homarr-oidc secret
-# Chart doesn't support envFrom, so we add via extraEnv
-extraEnv:
-  - name: AUTH_OIDC_CLIENT_ID
-    valueFrom:
-      secretKeyRef:
-        name: homarr-oidc
-        key: client-id
-  - name: AUTH_OIDC_CLIENT_SECRET
-    valueFrom:
-      secretKeyRef:
-        name: homarr-oidc
-        key: client-secret
-
-tolerations:
-  - key: node-role.kubernetes.io/control-plane
-    operator: Exists
-    effect: NoSchedule
-
-resources:
-  requests:
-    cpu: 250m
-    memory: 384Mi
-  limits:
-    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
-    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
-    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
-    # real CPU headroom.
-    cpu: "2"
-    memory: 1Gi

k8s/apps/homarr/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: dashboard
-# Probes are now tuned via homarr-values.yaml (chart-native); the old
-# fix-probes-job PostSync hook is removed. homarr-secrets/auth-oidc/db-encryption
-# Secrets are delivered by the sops-secrets (ksops) Application.
-resources: []

k8s/apps/llm-serving/reasoning.yaml

@@ -28,9 +28,14 @@ spec:
       # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
       # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
       # each GPU carries ~9.67GB weights + ~128KB/token KV. At
-      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
+      # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves
-      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
+      # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a
-      # above the 131072 target with room to spare -- the old
+      # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x
+      # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that
+      # metric after the rollout -- max_num_batched_tokens=8192 also raises
+      # activation memory out of the same budget, so the post-boot
+      # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is
+      # only valid if it still covers 2x65k + 3x8k. The old
       # OffloadingConnector CPU-DRAM spillover (tuned for the previous
       # model's much smaller 16384 context) is no longer needed and is
       # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
@@ -54,10 +59,37 @@ spec:
       - --tensor-parallel-size=1
       - --pipeline-parallel-size=2
       - --max-model-len=131072
-      - --gpu-memory-utilization=0.90
+      - --gpu-memory-utilization=0.95
-      - --max-num-seqs=4
+      - --max-num-seqs=5
+      # Prefill scheduling -- see the measured starvation this fixes.
+      # V1's scheduler has no prefill/decode phase: it walks self.running and
+      # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py:
+      # "There's no decoding phase nor prefill phase in the scheduler"). The cap
+      # per request is applied BEFORE the budget clamp:
+      #     if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold
+      #     take = min(take, token_budget)
+      # The loop is `while ... and token_budget > 0`. With the threshold at its
+      # default of 0 the cap is disabled, so one long prompt takes the ENTIRE
+      # budget, token_budget hits 0, the loop exits and every other request is
+      # skipped for that step -- measured as 262 tok/s contended prefill and
+      # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs.
+      # Hence threshold must be strictly BELOW the budget or it never binds:
+      # at 8192/8192 a single request is served per step (still starved), while
+      # 8192/2048 serves four. Verified by replaying the loop.
+      - --max-num-batched-tokens=8192
+      - --long-prefill-token-threshold=2048
       - --enable-chunked-prefill
       - --enable-prefix-caching
+      #
+      # NOT set, deliberately: --max-num-partial-prefills and
+      # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships
+      # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle
+      # rejects any value other than the default for either one, calling
+      # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0"
+      # -> AssertionError, because there is no V0 engine to fall back to.
+      # The Deployment uses strategy Recreate, so setting them is an outage.
+      # V1 does concurrent chunked prefill natively; it needs no flag, only the
+      # budget/threshold ratio above.
       # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
       # blocks.
       - --reasoning-parser=qwen3

k8s/apps/messaging/queue-crd/templates/rbac.yaml

@@ -12,6 +12,19 @@ rules:
   - apiGroups: ["kmsvc.io"]
     resources: ["queues"]
     verbs: ["get", "list", "watch", "update", "patch"]
+  # The operator watches TemporalWorker objects and creates Deployments for
+  # them. Without these two rules the controller logs
+  # "cannot list resource temporalworkers/deployments ... at the cluster scope"
+  # and never reconciles.
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
   - apiGroups: ["kmsvc.io"]
     resources: ["queues/status"]
     verbs: ["get", "update", "patch"]

k8s/apps/paperless/backup-cronjob.yaml

@@ -48,7 +48,8 @@ spec:
                   mountPath: /backup
           containers:
             - name: mc-mirror
-              image: quay.io/minio/mc:latest
+              # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
+              image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
               env:
                 - name: ACCESS_KEY
                   valueFrom:

k8s/apps/paperless/paperless-ai-new.yaml

@@ -49,9 +49,9 @@ spec:
               echo "[init] Fetching JWT token from Authentik for paperless-ai-agent..."
               
               # Get JWT token via client_credentials grant
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+              TOKEN_RESPONSE=$(curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
                 -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
+                -d "client_id=396757803559027071" \
                 -d "client_secret=29skAhE0tBzWn0qrPvmPTQMva0joFjcCr7msDQr_VXQ" \
                 -d "scope=openid llm:inference" 2>/dev/null)
               

k8s/apps/paperless/paperless-ai.yaml

@@ -9,6 +9,14 @@ metadata:
     app.kubernetes.io/name: paperless-ai
 spec:
   replicas: 1
+  # Recreate, not RollingUpdate: paperless-ai-data is a single RWO PVC. With
+  # RollingUpdate the new pod tries to mount the PVC while the old pod still
+  # holds it -> "Multi-Attach error" / stuck Init:0/1 forever, rollout wedged
+  # at "1 desired, 1 ready, 1 updated, 1 unavailable". Recreate terminates the
+  # old pod first so the PVC detaches before the new one starts. Same pattern
+  # as the Forgejo gitea Deployment.
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app.kubernetes.io/name: paperless-ai
@@ -43,9 +51,9 @@ spec:
               
               # Get JWT token via client_credentials grant
               # Client secret sourced from environment (injected from paperless-ai-config Secret)
-              TOKEN_RESPONSE=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \
+              TOKEN_RESPONSE=$(curl -s -X POST https://zitadel.riotpiao.com/oauth/v2/token \
                 -d "grant_type=client_credentials" \
-                -d "client_id=paperless-ai-agent" \
+                -d "client_id=396757803559027071" \
                 -d "client_secret=${LLM_AUTH_CLIENT_SECRET}" \
                 -d "scope=openid llm:inference" 2>/dev/null)
               

k8s/apps/sms/deployment-macos.yaml

@@ -1,115 +0,0 @@
-# macOS VM (Docker-OSX) hosting the BlueBubbles server.
-#
-# ── Why a VM and not a container ────────────────────────────────────────────
-# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
-# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
-# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
-# container — a VM in a box, not a macOS container.
-#
-# ── Why this works on worker-2 ──────────────────────────────────────────────
-# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
-# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
-# module). Bare metal, so no nested virtualisation needed.
-#
-# ── Read this before relying on it ──────────────────────────────────────────
-# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
-#    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
-#    into iMessage, THEN install BlueBubbles inside the guest. This manifest
-#    only provides the machine; it does not provision macOS.
-# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
-#    own guidance: "test sending an iMessage to yourself. If it does not
-#    succeed, it's likely best to restart from the beginning."
-# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
-#    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
-# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
-#    warranty".
-#
-# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
-# inside the guest and is NOT required for plain send/receive. Skip it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: macos-bluebubbles
-  labels:
-    app.kubernetes.io/name: macos-bluebubbles
-    app.kubernetes.io/part-of: sms
-spec:
-  replicas: 1
-  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
-  # concurrently — two QEMU processes on one image corrupts it.
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: macos-bluebubbles
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: macos-bluebubbles
-        app.kubernetes.io/part-of: sms
-    spec:
-      # Dedicated node. The taint keeps everything else off worker-2; this
-      # toleration is what lets the VM on. Both halves are required.
-      nodeSelector:
-        workload: imessage
-      tolerations:
-        - key: workload
-          operator: Equal
-          value: imessage
-          effect: NoSchedule
-      containers:
-        - name: macos
-          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
-          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
-          # only schedules because the sms namespace is labelled privileged.
-          securityContext:
-            privileged: true
-          env:
-            # Generates a unique serial / board-serial / UUID / MAC and persists
-            # them to bootdisk.qcow2. This synthetic identity is what iMessage
-            # activates against — it must stay stable across restarts, which is
-            # why the PVC matters.
-            - name: GENERATE_UNIQUE
-              value: "true"
-            # Identity is only plausible if it matches a real product line.
-            - name: DEVICE_MODEL
-              value: "iMacPro1,1"
-            - name: RAM
-              value: "12"
-            - name: CORES
-              value: "6"
-            - name: EXTRA
-              # Expose the BlueBubbles server port from the guest to the pod.
-              # Guest :1234 (BlueBubbles default) -> pod :1234.
-              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
-          ports:
-            - name: vnc
-              containerPort: 5999
-              protocol: TCP
-            - name: bluebubbles
-              containerPort: 1234
-              protocol: TCP
-          resources:
-            requests:
-              cpu: "6"
-              memory: 14Gi
-            limits:
-              cpu: "12"
-              memory: 20Gi
-          volumeMounts:
-            - name: macos-disk
-              mountPath: /home/arch/OSX-KVM/disk
-            - name: kvm
-              mountPath: /dev/kvm
-          # No readiness probe on purpose. The guest takes many minutes to boot,
-          # and until macOS + BlueBubbles are installed BY HAND there is nothing
-          # listening on 1234. A probe here would crash-loop the pod through the
-          # entire interactive install.
-      volumes:
-        - name: macos-disk
-          persistentVolumeClaim:
-            claimName: macos-disk
-        - name: kvm
-          hostPath:
-            path: /dev/kvm
-            type: CharDevice

k8s/apps/sms/kustomization.yaml

@@ -1,10 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: sms
-resources:
-  - namespace.yaml
-  - storageclass.yaml
-  - pvc-macos.yaml
-  - deployment-macos.yaml
-  - service.yaml
-  - networkpolicy.yaml

k8s/apps/sms/namespace.yaml

@@ -1,19 +0,0 @@
-# iMessage delivery for the cluster.
-#
-# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
-# that hooks Messages.app private APIs — it cannot be containerised on Linux,
-# because containers share the host kernel and macOS needs XNU + Cocoa. The only
-# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
-# needs a privileged pod with /dev/kvm.
-#
-# Hence privileged PodSecurity: the cluster default from the Talos controlplane
-# is `enforce: baseline`, which forbids privileged containers and host devices.
-# Scope is limited to this namespace.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: sms
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged

k8s/apps/sms/networkpolicy.yaml

@@ -1,22 +0,0 @@
-# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
-# ID and exposes an unauthenticated VNC console; nothing should reach it except
-# opted-in clients.
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
-  name: sms-default-deny
-spec:
-  podSelector:
-    matchLabels:
-      app.kubernetes.io/part-of: sms
-  policyTypes:
-    - Ingress
-  ingress:
-    - from:
-        - namespaceSelector: {}
-          podSelector:
-            matchLabels:
-              sms-client: "true"
-      ports:
-        - protocol: TCP
-          port: 1234

k8s/apps/sms/pvc-macos.yaml

@@ -1,25 +0,0 @@
-# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
-#
-# This volume is NOT disposable: it holds the VM's serial number, board serial,
-# UUID and MAC, which together form the identity iMessage was activated against.
-# Losing it means re-running activation, which is the least reliable step of the
-# whole setup.
-#
-# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
-# the installer, the base system, and qcow2 growth.
-#
-# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
-# Losing worker-2's disk therefore means losing the activated identity and
-# redoing iMessage activation. Once the guest is installed and activated, take
-# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: macos-disk
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: longhorn-imessage-local
-  resources:
-    requests:
-      storage: 200Gi

k8s/apps/sms/service.yaml

@@ -1,31 +0,0 @@
-# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
-# it is an unauthenticated console onto a machine holding a live Apple ID
-# session. Reach it with `kubectl port-forward`, never an Ingress.
-apiVersion: v1
-kind: Service
-metadata:
-  name: macos-vnc
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: macos-bluebubbles
-  ports:
-    - name: vnc
-      port: 5999
-      targetPort: vnc
----
-# The BlueBubbles REST API, once installed inside the guest. This is the stable
-# name cluster services use, so callers never depend on the pod IP or on whether
-# the backend is this VM or a real Mac mini later.
-apiVersion: v1
-kind: Service
-metadata:
-  name: bluebubbles
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: macos-bluebubbles
-  ports:
-    - name: http
-      port: 1234
-      targetPort: bluebubbles

k8s/apps/sms/storageclass.yaml

@@ -1,32 +0,0 @@
-# Dedicated StorageClass for the macOS VM disk.
-#
-# The default `longhorn` class does not work here, for two independent reasons:
-#
-# 1. Replica count. Default is 3, and Longhorn schedules against
-#    storageMaximum - storageScheduled with over-provisioning at 100%. Free
-#    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
-#    volume has only one node that can hold even a single replica — a 3-replica
-#    volume fails outright with ReplicaSchedulingFailure.
-# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
-#    created, before any pod is scheduled. Combined with strict-local that
-#    pins the data to an arbitrary node, not the one the VM runs on.
-#
-# So: one replica, kept local to the VM, bound only once the pod has a node.
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: longhorn-imessage-local
-provisioner: driver.longhorn.io
-allowVolumeExpansion: true
-reclaimPolicy: Delete
-# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
-# the replica is placed on that node and not somewhere else.
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
-  # Serving it over the network from another node's disk would be the single
-  # worst thing for guest responsiveness, so force it local.
-  numberOfReplicas: "1"
-  dataLocality: "strict-local"
-  staleReplicaTimeout: "30"
-  fsType: "ext4"

k8s/argocd/apps/10-storage-observability.yaml

@@ -249,3 +249,26 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: local-path-provisioner
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/local-path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: local-path-storage
+  # No CreateNamespace: the Namespace object (with its privileged PSA label,
+  # which the helper Pods need for their hostPath) is part of the manifest.
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/apps/30-security.yaml

@@ -57,6 +57,55 @@ spec:
       prune: true
       selfHeal: true
 ---
+# ZITADEL — the Authentik replacement. Coexistence, not cutover: Authentik keeps
+# serving in the same `iam` namespace until the last consumer moves, then this
+# becomes the only IdP. Migration plan (outside git):
+# ~/workplace/homelab-doc/ZITADEL_MIGRATION.html
+#
+# Wave 3 ordering: sops-secrets (wave 0) creates the `zitadel-masterkey` Secret,
+# databases (wave 2) creates the `zitadel-db` CNPG cluster and its
+# `zitadel-db-app` DSN secret. Both must exist before the init job starts.
+#
+# automated sync is deliberately absent for the first sync — same reason as
+# forgejo. The chart ships `pre-install,pre-upgrade` Helm hooks (zitadel-init,
+# zitadel-setup), which ArgoCD maps to PreSync and re-runs on every sync, manual
+# or automated. A human reads the first sync's hook output before any repeat can
+# run. The init job runs `init zitadel` only (CNPG owns the database and the app
+# user), and the setup job creates the first org and the `iam-admin` machine key
+# that Terraform consumes. Read the SECOND-SYNC HAZARD block at the top of
+# k8s/infra/iam/zitadel-values.yaml before enabling automated sync.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: zitadel
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://charts.zitadel.com
+      chart: zitadel
+      # Pinned: the chart trails the upstream app release, and the verified
+      # masterkey/passwap facts were read at v4.19.x. A floating tag would
+      # change the app version without a diff anyone reviews.
+      targetRevision: 10.4.0
+      helm:
+        valueFiles:
+          - $values/k8s/infra/iam/zitadel-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: iam
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+---
 # Raw IAM manifests: key-rotation cronjob + authentik migration job.
 apiVersion: argoproj.io/v1alpha1
 kind: Application

k8s/argocd/apps/60-applications.yaml

@@ -110,37 +110,6 @@ spec:
     syncOptions:
       - CreateNamespace=true
 ---
-# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
-# running the BlueBubbles server, plus its dedicated local StorageClass.
-#
-# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching
-# toleration for that node's taint. Until worker-2 is provisioned this app
-# syncs everything except the pod, which stays Pending — that is expected.
-#
-# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which
-# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would
-# not have those labels.
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: sms
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/apps/sms
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: sms
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
----
 # Document management. Raw manifests (no Helm): postgres is the dedicated
 # paperless-db CNPG cluster in k8s/infra/databases (wave 2), redis is
 # in-cluster only (no PVC), media lives on the cp-3 USB HDD (see
@@ -202,40 +171,6 @@ spec:
     syncOptions:
       - CreateNamespace=true
 ---
-# Consolidated: homarr + homarr-patches → homarr
-# Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: homarr
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "8"
-spec:
-  project: homelab
-  sources:
-    - repoURL: https://homarr-labs.github.io/charts
-      chart: homarr
-      targetRevision: "*"
-      helm:
-        valueFiles:
-          - $values/k8s/apps/homarr/homarr-values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      ref: values
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: dashboard
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
----
 # Portfolio site at riotpiao.com - static Next.js site from rock/riotpiao.com repo.
 # Points directly to infra/portfolio/base (bypassing repo's own argocd-apps.yaml
 # which has wrong URLs). Image built by Forgejo Actions on rock/portfolio repo.

k8s/argocd/apps/98-talos-repair.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: talos-repair
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "6"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/talos-repair
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/apps/99-oidc-bootstrap.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: oidc-bootstrap
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/oidc-bootstrap
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/projects/homelab-project.yaml

@@ -35,9 +35,9 @@ spec:
     - https://grafana.github.io/helm-charts
     - https://helm.releases.hashicorp.com
     - https://charts.goauthentik.io
+    - https://charts.zitadel.com
     - https://strimzi.io/charts/
     - https://charts.bitnami.com/bitnami
-    - https://homarr-labs.github.io/charts
     - https://go.temporal.io/helm-charts
     - https://portainer.github.io/k8s/
     # Substrate charts (cert-manager, ingress-nginx, reloader) — app-of-apps owned

k8s/argocd/secrets/cloudflared-credentials.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: cloudflared-credentials
+    namespace: cloudflared
+type: Opaque
+stringData:
+    credentials.json: ENC[AES256_GCM,data:LnJVGuxJAOwDYbLgX00OHICDDldZgwgvGafOIwKoJKnjg3FjFMQYThNCOL/H+6tctuKixJNcAckjg70tmCKWxNq9I35XX/YSqob6Qv4fywv32hsNIAe44BU035MGZ5zmc6bZZ8wAgTJZw6Th4TzY7taFBCRdj2fW4T+qVfq7strn68G4ufVMK6UXlOkEDLjR2Vxj32lqSLdbdMma96XB+265hwL6IVfaugG662mcGhPwgHk=,iv:NFmyHN5m2mJUBgn1qgmJZ2YgkFxBZyp7+gU1Ng6B/7M=,tag:/66JtHK5Yu7pRxB0b0eifQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaDVycUE4RFpoRjdpTlpX
+            dGpzN3hwRFQ2cmtXVXVzWnZJOXJEZXZRWEhNCkJwbm9QV0lGcGtNRTJqbStrc3l1
+            Q2M4bE1WbzJpTXZQWjJyTGpFS0JYcVUKLS0tIDBvU2U5aEtVTTZUaE85eEkyYitG
+            bGtSVkhKNnRuNlZWMWh6Z1M4VXM1WU0KdusvtM85V0xrO+uJFUY+UH0tuwdF0GiW
+            79J4Rlxk8ne25psKsvwjLYLxZ53xjFHCC4FCHTNkYwTKCDEKjw46Lg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-26T02:45:04Z"
+    mac: ENC[AES256_GCM,data:KF7KwXCnNLAg1YI3u0lLI/ho4sITHAq09ac2WO4TK3TCthkuivb89YjhuuJKs0NFhjn6aRLqmZavUqcXH2HujiryqbLNVexOShQM0wSzgCzmWdbi0UzpgJ4KcQA3PufDtsQz4HHLXx396EbZrM3Ie3WSu/PvRm5iWOxzTGCGc9Q=,iv:Eo920oXXD1FLZVJXywZ6C+LRL+qZjmnZXv0+Oa/GMb8=,tag:6LuPVSet4WjvrvGUAxaJlw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/etcd-defrag-creds.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: etcd-defrag-creds
+    namespace: kube-system
+    labels:
+        app.kubernetes.io/name: etcd-defrag
+type: Opaque
+stringData:
+    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
+    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
+    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
+            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
+            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
+            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
+            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T05:37:26Z"
+    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/forgejo-registry-pat.enc.yaml

@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:ECE=,iv:bISz4HovH++X7DW1Qj8Cw0L6+EvPB0+68hh7tfyW5C0=,tag:w+SjD7MsfeIuSf62n+Zl7Q==,type:str]
-kind: ENC[AES256_GCM,data:AxXaV3Mb,iv:xfJb354Rrjz3zLctW2i6hl40yh9EsfIqKxDvmn8jqnU=,tag:1xQ/RR3KhPLKCTWCYjebJg==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:gdQ4OoMYKUPEbsUeA5OI4i1xnh8=,iv:IA2kaCyqQmkYpLolvcTF4aleh+yd/ImXJMhRMvpGCgo=,tag:8qyYYw4EhBKKPzEmpepAeQ==,type:str]
-    namespace: ENC[AES256_GCM,data:/mOSyXWmRg==,iv:cpeHUTlMqlJzJttGtuR3DoiMtvVqFmDS0/5Tl7K7c2M=,tag:q4GxWgWn2wQJxJHqnq4WQA==,type:str]
-type: ENC[AES256_GCM,data:92OcrsuV,iv:Z1XBy6iZ6unGrK4/SSdDa58pbPL3022gU/f1AOM4uvc=,tag:cd4wWRq/ohm6BJ/Eo6HOIw==,type:str]
-stringData:
-    REGISTRY_PAT: ENC[AES256_GCM,data:zDKruOXhzsIcFZTTp4r4r9rwHih9uf1cCp06KZ6eIXSIZDvR6aQY4A==,iv:5Vch5Z7xn1KkRxrgOs2p/M58nd6WhtcuUhoZDCi8LSY=,tag:14SAZIWGOxJMqLBtOorjYQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIaG5QVUYyNjhvaENPcUFy
-            K3RSSVM2N2hTeFhNK0M5YWZmODhzZU1HaHlFCmNGZWdFcFEvanVwcXpGSmVvRHVx
-            aG45c2lBY1RuSTYwbDZTVk1QeHNWOTQKLS0tIDg4TnVMNjNtaU1VQk5zQjUvU2hM
-            R245ZVdqc0ZWWVhJb3dOZ3lpU3JjUlUKawSg09ZPq8FKx5tvOVZZ+K4yh7eTQsUp
-            be8mWUpS0+eEmNqh35BwU3HrETMQFA6a1kjVp30JOMtqa5rbYlzF7w==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-23T23:14:53Z"
-    mac: ENC[AES256_GCM,data:yzi6woUaUCi6w9pG/eKnU7k/VfZgoXg/tW8p9joG8p7iabLWMdlXlx23m/CItw/NE0zeGWA5iZPPFZXOit2vN36VzK3kQNoFW6QbhvYLZi+78C/RWIdcpZkxB/tPIRG0vq8Q1SA+rwGjG/0xeAFh+R7k+YBTMd2XWBH3P7EI5T4=,iv:kf6MDaAVrtDvPIEjHMMLxXDSDRC3I1GpsPeJFUYppiw=,tag:NukVbGLa9EoMYmRsa4nBtA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/forgejo-registry-token.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: forgejo-registry-token
+    namespace: cicd
+data:
+    username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
+    password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
+            ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
+            cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
+            WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
+            SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml

@@ -5,7 +5,7 @@ metadata:
     namespace: ENC[AES256_GCM,data:ZHnVZyd3wA==,iv:3evUnFTQfRz/ZilUQIl/9dsCba3nsX6o/iQsGXMBfgM=,tag:+9kp+VRu1ksroDTbTGmGPA==,type:str]
 type: ENC[AES256_GCM,data:bHX+Bu9o,iv:8qqNddsuw4YoHAuJfUcPIqjcqNySIh5Ln8OhsfZxuk8=,tag:P7QGUXo9Io2Y19YqlWv5sQ==,type:str]
 stringData:
-    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:qW2SQSsGD0YupU2SMzRaYwkLRjXAwV1OJBnhATpCJrOHWbEytw6IphOmPPd3VZKfHncekA4agXi4K3rUPHIx0Q==,iv:NCeW/UfIl0RvHV39ubN2mDfFp7GTQNuFv8KiEF+7KOE=,tag:lisx4h4PqUfLwBX8ca9Siw==,type:str]
+    GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ENC[AES256_GCM,data:nJz9pqa6w9ZGOjanZ32D38Bolg2YrdHs1crhb2uZkKyaaKDpklVb4FOKZ46K+NS1NXjoS6XDUAlV69CSvm1PeA==,iv:g9F/iLuzNC0WogtDJ/TAb3+97N6iGkbOTlb4SzYzwx8=,tag:4F3kUheGbtW2U6jgOvRO1A==,type:str]
 sops:
     age:
         - enc: |
@@ -17,7 +17,7 @@ sops:
             LS0cIDUhjPFbeu7qUATOrat6vMUi0UREbXSFZ1KyAjNAwJZaUusPMA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T18:02:01Z"
+    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:yUBBV7dLEPld9yJacn+YaOT8M1NJ2ploGIxV66aVOH5x4tDAsskcUQJlzApWCQmORuCuMKg6qfpGgLTojfD5msha+ufVVJXHl7ua+STar1/FPJXaHm+z7YyZdsD5j6f3AH8q18DuLy2DYLjXELmPBsp50g7MDJEm3xLhCUXygjQ=,iv:xl0OtA/2umz0YH5b+6QjlAByZWbvfGOmHqomziNjlkw=,tag:xwkQDZx19Lf6QSwU2zPYTQ==,type:str]
+    mac: ENC[AES256_GCM,data:0cgVNtFMwhbEHBxu7T1XfstcQlgUTxijq9Mz36zC60E7kYXRNwGt2/0KoCaqaE0leypwAdrPn2OjeoDPeCN6x+n0vLRVX2tnli6HazBLbRsPHljDAV0ASF07iNhb0PvgtWay432Mm+sGATkel3ITdAaP0lISNLUYfEe1/TbYHkk=,iv:FZ9Bgtk7LOC3aOSOo1+50zwLGhlcUZYJaUQ9r05cAn0=,tag:tsLQ1frZ7li5l717CCtgoQ==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

k8s/argocd/secrets/homarr-auth-oidc.enc.yaml

@@ -1,24 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:uiQ=,iv:vknDc16Zrz/lxS1WG3ksG4SHjhJSTDdklVxxwL8eylA=,tag:1srpVd6QTYQGGGYGXT8Cbw==,type:str]
-kind: ENC[AES256_GCM,data:K6EPYaNE,iv:VWzawfTDNlag+shjBqwSzZGcQQq/TGxeHDqd4fAh5Vc=,tag:ncPqMYjq/73oMzphWhuuFA==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:vaRNQHFYfdUz1hzuYO0JxQ==,iv:W7aUJID5mxoO1NQAcyPDYV1FGtBT8myKVMA3s2WgrPk=,tag:4Ef4GBtTR3dVanscvigRmw==,type:str]
-    namespace: ENC[AES256_GCM,data:HDlmfGwad37/,iv:JFPXxz9xp5vDJRcWy73SYGWDMyHKWQq9Bvy/NJTwTnU=,tag:2ldVXgzKn4xp1rZI/VYgUw==,type:str]
-type: ENC[AES256_GCM,data:UgolDLIV,iv:O4himI90UVwx+OjKJXobu10Zg3a5IoQcjAORsdhaWpg=,tag:xltntRsamhQkPEoBNtMytw==,type:str]
-stringData:
-    oidc-client-id: ENC[AES256_GCM,data:RLdxXY/a,iv:US/fBReK7u8WA4IS6TSzYiHQkte2CuVgugIMpA/pIlA=,tag:QKrHGQrzI9YpfNQD+ts68A==,type:str]
-    oidc-client-secret: ENC[AES256_GCM,data:SS+I92xEkOcJHTfWvZ1AA8hh1bFAtND5adZ6Uh73nIPiqAo958GSag==,iv:9HcmXWrGtflc6WWQe0n/pgB1Xveakofc3hvz5vuE29A=,tag:0emgeyM4GBftVKn/fx7XPg==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCWkR5V0ZYSTBSSU1OTWF1
-            TzFxdnZkUFFIcGJWZ1VUU21ZUnU2cUdob1drCkhVTm02dkZPZ2lNWktOcWZ2c0Vi
-            dG1PSmZBR2M2anA2Z09iYUU0SjVzTlEKLS0tIGF1eVBBUHdhVk9YSzJlQXVwd3RE
-            bUd3UTlwR1NiYjNlcGpweTBhdmk5b0UK8pwUoVCr1uBX9dQ7EGRgFMAPk/z8N9Fg
-            6GRUORFug5h7x5kjXK9sU0yTHjFSekwBi75GLng0IrLEGJxHVM5wVQ==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T21:07:30Z"
-    mac: ENC[AES256_GCM,data:NupjQWBJHMOo3/VX5n5nRx+RFT14N71XkrjXRC/VCP7i7H8yn3tcRdH9g14usgWunmqg2KKpGMUIZ9bsQ9Vds9sBvUKvLyRkfQnPwPh9+PFCF3ZZLPgzqcT19kg4I3QabmO+LPzQXsZWLKIqgF9/B60EVIgYyKhZmP7XblY5AVo=,iv:r0deq8+txE8TwzdtxOMaIAZbZc2fu+Tp4Jd2HZGbkhM=,tag:qMpZp6LcWjJHYMDp9ecBkA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/homarr-db-encryption.enc.yaml

@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:FO4=,iv:kKIuJp4uGeX6vECsXk7w43F9S281jObQchmK8r1+kmA=,tag:x5LYd34dzNDLCma/78M4+w==,type:str]
-kind: ENC[AES256_GCM,data:o4RTN8SK,iv:uyOJWjDmHIPuBl9Wc1S/HxUabV2yytWr1bntHW68sTI=,tag:3On0SnQVJ/ICMMryQe+8JQ==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:a89dHmLdSeMHE7ZP0Q==,iv:OWJvJhFqJ7A0UGVJnZILzCJ/4VuVFib9sUDjXuwyqJQ=,tag:1XrIQn1GDh8RQ4vmQAu7+g==,type:str]
-    namespace: ENC[AES256_GCM,data:4wBVpZJgQGM9,iv:3Tn1u/2TP/+SnELu34PDf4Jauf2L4f8+MtgrPIlyKzk=,tag:L3qbKFBjm16g8KX5vUsK6A==,type:str]
-type: ENC[AES256_GCM,data:mdrF9vUb,iv:7ukOgwXlnRf5mpJ/+6YXtyGRonnqehEgT4pJJIafw7c=,tag:90mt7jpNb3N80kCEi+R/rA==,type:str]
-stringData:
-    db-encryption-key: ENC[AES256_GCM,data:NWJryAEd/eUJxbNiu05RzLF/l7GDxIQM7RQe3xKBz6XklgRAFwQxxjakcVd85pePUBaQYz3zttFglWhfJlWhkA==,iv:eUWbcs0B/mIheyAQ2+DyKgcA98CRKQGAVC3O0WIkfU8=,tag:be06X6AQqlohYlxuYoZiTg==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1Ym5yQ1pmWHZjVnYxbUVO
-            SkZ5dnI2OXR3cmZjZkM0K1ErMElLNktlK1JFCmpleFBrcXkwcDA4cGFuWWN6VDQv
-            dVV4NjB4MjEreENVQnZ2ejVLblRYOVUKLS0tIG9VbXJvUmQ3SzMzN3IrRDlac2sr
-            eGZNYXBoZFZDeisrVCtUMXVPaEpoNFkKjwhUU+Zsqv4Be+m558fET5UhF/2rO73J
-            AhGbJVaA7PaOCswPWiOLso26zD+MBB3VKHNdUFnenpoM16Nu4pOf5w==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T21:07:30Z"
-    mac: ENC[AES256_GCM,data:KrGQY/eOmihYCvQCbfcM6bZ6rfx4zKh62M8NM0VQr1//1dSHpOOh25dYGLuQeECnqLxSpGQNWzr2xjUYWs4NOdSEFGq6FL2iHE3LnqoBZspp/JFJYP6+z/8jSUojwKEW7OuZZdPN6gF2u+IPOeWbGftnjS7X+atoSkYag2ty2vk=,iv:IOPEgBizsxrVoK5oSdWmi0uCBUnReOEnMYXuwSBKTzA=,tag:9k/XQ/AE7wSFq5XMbY0FTA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/homarr-secrets.enc.yaml

@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:4OY=,iv:0Lgui0+X2oTlgCX5HEIIihEsGJD0C2+pDIOTCNi4r+g=,tag:uL7vlreaHjcLJjV7e5I7PQ==,type:str]
-kind: ENC[AES256_GCM,data:x/ltkind,iv:KRBfS7COsTs/l3Fhyk5FvH7IVc+r/6M6FVOo+kPDe1Q=,tag:UuTbe213hDfK/WUtKS/doQ==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:LpOEecTJBuGy5PqNSFM=,iv:Q+DCLGtw3wihhdAgEbzYHyQhFTPWFyE2iJKl9mYaA2E=,tag:E2wrig4lRXGDvSCHec5clQ==,type:str]
-    namespace: ENC[AES256_GCM,data:6kMVCXaF68NO,iv:gnCCkV7GKXA2HWOOeNUGnpAycf8U7pl6F8Vfo6DhLR4=,tag:0VdeySsuhUr2heXyGXZwBg==,type:str]
-type: ENC[AES256_GCM,data:6muC3Zqh,iv:gBaC+6MXvZY/bIpy/cEG/zWwLBaUFMVfxAHa1OnwfnQ=,tag:v6NYyhAKAwovluHAXNaOVA==,type:str]
-stringData:
-    SECRET_ENCRYPTION_KEY: ENC[AES256_GCM,data:jX9ULlaBYTSTFGRMplSder5rMrsz7THU3kgJSGef0r5rZvtoaAG9NVYU2QUEuD0sZgfqRooeMOkDCAqFc8qsuw==,iv:uOmx8w3XPKnoxfasf7gqi/bljxigpLwaAC30c2TroKg=,tag:Dmbu6m3+UZa1w0UR0LRh9g==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6TDVyNGRtYWgzMDYwdjdr
-            NXJWcGJZNUx6YzBGWm1nWlp0SlN3WXhaR0VBCjZ3bDBmUTFCUjg3U21wak90eHZK
-            NlRCTUZQSHhoS0ZSUGZ4UHhST01KNmMKLS0tIGUxcGo0NmJ2bmJ6eGJ3QWxzYVM5
-            c2Z6UkFjQmZ4ejIvK1JQQkZ0VFFIZ3cKA8UzmGsX+s3yOermOHqbnnEtekHCTqmC
-            R3Jyzf6OolZMJpNwbetcsOfgO61aM7LHl/Xb/hFLbLjZ+eECrt8VpA==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T18:02:01Z"
-    mac: ENC[AES256_GCM,data:HTrEEVDm8JqnVBFaY/FJknxma0nI95tRiwVc0K76k+2R4jgKCBA3EUAzPVS+gA4rTAeKHPSHrT2eidlkseT2WQUjqMRrpHcUsbSjNfpTEY2g/U0PIiOXZMRGfwmsQVgmu2fqeMOyzDSyya0scdWJXPraRPNZ+OpEkNVlm4rWcgs=,iv:KY3dnnmooJgsu66wc7gbzeIGUXZNgVrxAhpAwxoG+G8=,tag:7r6d41oTD136N57rtY7auA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/memory-agent-oidc.enc.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: memory-agent-oidc
+    namespace: poimen
+data:
+    CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
+    CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
+    ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
+    TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
+            Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
+            K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
+            cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
+            es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/minio-secrets.enc.yaml

@@ -1,72 +1,72 @@
-apiVersion: ENC[AES256_GCM,data:Mdg=,iv:u/JjNJfdyz7ehk8lHZRh1u9zkNuufHosyXv10Gm+T6E=,tag:pkgZVJid6kpGuy61+oEMlQ==,type:str]
+apiVersion: v1
-kind: ENC[AES256_GCM,data:/kWhhqyz,iv:DymezGpBHmKT3BM/CcUKEGIWZw/JXNutReSKfhfmoD4=,tag:DnA+316Z+0eqBWN8YLyv1A==,type:str]
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:xs1k//nWy1iJ52Q=,iv:NDGvM319FCypxFLWwONcF12osaU/EL0IwD0c2xtqYnU=,tag:6IhfISF08WQbxrUmT49d7g==,type:str]
+    name: minio-creds
-    namespace: ENC[AES256_GCM,data:jpUzI7BmZA==,iv:PQrfm67la6RJCRq0H7qpiY8n2hPUjcaK6gmJ/zPQW30=,tag:bp/3GYbRpYOIub5nT0zz/A==,type:str]
+    namespace: storage
-type: ENC[AES256_GCM,data:fndXB4Y1,iv:EXOe0day1ScxvZ7ozn1QyHgaZgnvlPNmvSQG74aw7lM=,tag:quPmKvFTjRqtZoVUEoQAHQ==,type:str]
+type: Opaque
 stringData:
-    config.env: ENC[AES256_GCM,data:3RIQ1i6NtY1Tv2eJY8JXSgfBEqi9px/9TJVNrPf9+brtGy9Mx3SwjKVnjGXUcA2cYcNZb1JR7temXmznXQdsRfZ9yohe4oc1vL3yPGPY9YfP7AHGYj9E7tghu/rgkHOf93RmfdKv0Rrq6miBcA==,iv:0RQVMi2mMHBRTJ1OX6IDFvq9Qr7zpqgQzt+MPPgBIHA=,tag:HNXLKf5X4P3E+hc7YbDtNQ==,type:str]
+    config.env: ENC[AES256_GCM,data:1yogGyBdwrlg27H8R5Kr9VcPu+xwS5/nPhi7eLcmVYtXmPlNokCsWHA70qnn/+I6i4ODys8GML++VeZ/lmcHiP7RIbY+G6xmw6nfBCXitgE8kQwTPsUUzn9bobOn4Sr1YMBf7FQ/Q0N8aupGJA==,iv:EfCdim4TLkP/FP1u1yoKZCq8YGFzVh7ca4yu1+WzXwE=,tag:rTYe0SIAEsUhh0KHQIEKNw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtUVJFQ1ErNmkzd1N6b29t
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            L0VWR2tlcEIxeG1CR1hXRkw2cEt0Q2crMWlVClg2UlNhUzdJZGpLZjJtYlAvbk16
+            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            eHk0ZHZiNzVRTWhyQzlieTU5Y3QraGsKLS0tIHA5UVhvUXhNSDk0QlV0R2RzR1dP
+            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            azVEQzgvSFNwak9TcW1USFNOb3RvSkUKNLuW3h3cHDSSCZGe9Vyiv5m+wXihtCYX
+            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            oJtbRK+kJF8oOUo4emI+SLGrCsWq5cG2wN/wgP+ChR1zUgRgfq5oOA==
+            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T18:02:01Z"
+    encrypted_regex: ^(data|stringData)$
-    mac: ENC[AES256_GCM,data:hQfimKuhfiIiZQAyNDZQR25HZ5biU+kaWig3ElVPiqMY3B0LrmKyOEs7ittd3j3meUkPEFSmm1yYSXcdCuf6w7cwSbEiL5DG8t7QKPbnx/aRdyHAJW+1VC0e3Lye43gaHya0iUaWS/YFQcrT01oEbckLIDWPKfvA6pg2bZ4kMZ4=,iv:AydyWMVcjTQu+7Fzw5vtqkvTTeQEMt8y6snESoLUDE8=,tag:SoKklNqa9nH9G6d9+PnolQ==,type:str]
+    lastmodified: "2026-10-26T04:03:16Z"
-    unencrypted_suffix: _unencrypted
+    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
     version: 3.13.2
 ---
-apiVersion: ENC[AES256_GCM,data:Mdg=,iv:u/JjNJfdyz7ehk8lHZRh1u9zkNuufHosyXv10Gm+T6E=,tag:pkgZVJid6kpGuy61+oEMlQ==,type:str]
+apiVersion: v1
-kind: ENC[AES256_GCM,data:/kWhhqyz,iv:DymezGpBHmKT3BM/CcUKEGIWZw/JXNutReSKfhfmoD4=,tag:DnA+316Z+0eqBWN8YLyv1A==,type:str]
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:XEQHsQqTgwukYw==,iv:wTw2eoooDA9EfB2df3jzb0IWUdtLSwcuKzx8yDBaihQ=,tag:oncXt2gM1k6Qppafpu/D1w==,type:str]
+    name: minio-oidc
-    namespace: ENC[AES256_GCM,data:jpUzI7BmZA==,iv:PQrfm67la6RJCRq0H7qpiY8n2hPUjcaK6gmJ/zPQW30=,tag:bp/3GYbRpYOIub5nT0zz/A==,type:str]
+    namespace: storage
-type: ENC[AES256_GCM,data:fndXB4Y1,iv:EXOe0day1ScxvZ7ozn1QyHgaZgnvlPNmvSQG74aw7lM=,tag:quPmKvFTjRqtZoVUEoQAHQ==,type:str]
+type: Opaque
 stringData:
-    MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:IYBfNW19cTo8WkCGT4zuCljzi2bkbYWJ8DY/yedR2MJNqEO+bMVvjx0OqSqbPp6rnwGyMQOhBN0gXPbEVRDBGg==,iv:I1aGIikbHTKvh//4Inhp7UoNfhjlIXSpcyP4RqljvqM=,tag:Zh3Vh5DQYAsTtiW59D0/QA==,type:str]
+    MINIO_IDENTITY_OPENID_CLIENT_SECRET: ENC[AES256_GCM,data:eFfwW2ZnV+o3/6XHjUvyFsDbaIAdosY2PXOnRdN4Ie49YDIjfJPe+Mg1FsbgoEHEjuB33hfxesb4TWOJQfW8Og==,iv:+kRr9A3auts1XHgZWa+t5b2m4lF0x9dVhiRpB17fzv4=,tag:tH/jszFzLQdqORGr9vfTGw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtUVJFQ1ErNmkzd1N6b29t
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            L0VWR2tlcEIxeG1CR1hXRkw2cEt0Q2crMWlVClg2UlNhUzdJZGpLZjJtYlAvbk16
+            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            eHk0ZHZiNzVRTWhyQzlieTU5Y3QraGsKLS0tIHA5UVhvUXhNSDk0QlV0R2RzR1dP
+            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            azVEQzgvSFNwak9TcW1USFNOb3RvSkUKNLuW3h3cHDSSCZGe9Vyiv5m+wXihtCYX
+            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            oJtbRK+kJF8oOUo4emI+SLGrCsWq5cG2wN/wgP+ChR1zUgRgfq5oOA==
+            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T18:02:01Z"
+    encrypted_regex: ^(data|stringData)$
-    mac: ENC[AES256_GCM,data:hQfimKuhfiIiZQAyNDZQR25HZ5biU+kaWig3ElVPiqMY3B0LrmKyOEs7ittd3j3meUkPEFSmm1yYSXcdCuf6w7cwSbEiL5DG8t7QKPbnx/aRdyHAJW+1VC0e3Lye43gaHya0iUaWS/YFQcrT01oEbckLIDWPKfvA6pg2bZ4kMZ4=,iv:AydyWMVcjTQu+7Fzw5vtqkvTTeQEMt8y6snESoLUDE8=,tag:SoKklNqa9nH9G6d9+PnolQ==,type:str]
+    lastmodified: "2026-10-26T04:03:16Z"
-    unencrypted_suffix: _unencrypted
+    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
     version: 3.13.2
 ---
-apiVersion: ENC[AES256_GCM,data:Mdg=,iv:u/JjNJfdyz7ehk8lHZRh1u9zkNuufHosyXv10Gm+T6E=,tag:pkgZVJid6kpGuy61+oEMlQ==,type:str]
+apiVersion: v1
-kind: ENC[AES256_GCM,data:/kWhhqyz,iv:DymezGpBHmKT3BM/CcUKEGIWZw/JXNutReSKfhfmoD4=,tag:DnA+316Z+0eqBWN8YLyv1A==,type:str]
+kind: Secret
 metadata:
-    name: ENC[AES256_GCM,data:rYX7nN6SYoG5HycaNNrprDM=,iv:4uX7+Sl1UzgsImBH+qm9p8DlOfJu0i0W3bpoE7cFxXg=,tag:RROCnYBVKVwPBNSGgV30FQ==,type:str]
+    name: minio-user-ollama
-    namespace: ENC[AES256_GCM,data:jpUzI7BmZA==,iv:PQrfm67la6RJCRq0H7qpiY8n2hPUjcaK6gmJ/zPQW30=,tag:bp/3GYbRpYOIub5nT0zz/A==,type:str]
+    namespace: storage
-type: ENC[AES256_GCM,data:fndXB4Y1,iv:EXOe0day1ScxvZ7ozn1QyHgaZgnvlPNmvSQG74aw7lM=,tag:quPmKvFTjRqtZoVUEoQAHQ==,type:str]
+type: Opaque
 stringData:
-    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:FhXl6yqTIzHsbJO6P96KQrA=,iv:gGTaSKxk721GwMQWWyymqUPZlGLidrBzWCWdRtnlOr0=,tag:IaVl8GQqPstBuUjD4N6JTQ==,type:str]
+    CONSOLE_ACCESS_KEY: ENC[AES256_GCM,data:Zxr/Nj0SUKSJbBD1cFjt3GE=,iv:r/B9ULn68hsXdH8uRQkKAoQTJIJIYKMoJuWOZfxS2I8=,tag:Xo4i+IVxuqrp+qDpYQysPw==,type:str]
-    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:s22GOXY4B/m+S7tgDw8P2gU2F15vdn1tqAp53FWzFTI=,iv:GXBhHQ8YiaVjRRMw2yG4FxdulVOuOurkLXXC0u/SFRA=,tag:pvHcD9+4qaxnvhOSkNxoHA==,type:str]
+    CONSOLE_SECRET_KEY: ENC[AES256_GCM,data:PoIDNZ+Xs6BdhGm10y1HdL5jy47EVHfHHIJhGy/KX6Q=,iv:voAv32GoTKi9s9/4MQy2yT6i9VY4/igA5XXzehwUqRQ=,tag:DmPyk5biVvlxHjeyDyrLbQ==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtUVJFQ1ErNmkzd1N6b29t
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2SStVR0lpOWs0VnN6dkdp
-            L0VWR2tlcEIxeG1CR1hXRkw2cEt0Q2crMWlVClg2UlNhUzdJZGpLZjJtYlAvbk16
+            RktMUmNRR2diTUtiVk9tRk1wSjVTMnRtVlJBCkk1S0dwNUNHb25RQlZ4ZFRocisv
-            eHk0ZHZiNzVRTWhyQzlieTU5Y3QraGsKLS0tIHA5UVhvUXhNSDk0QlV0R2RzR1dP
+            aS9kbmQydWVmN1AxU3VvbFZsS2tmTHcKLS0tICtRY0ZKeFhCTUdaNW5PSFRNZ2dG
-            azVEQzgvSFNwak9TcW1USFNOb3RvSkUKNLuW3h3cHDSSCZGe9Vyiv5m+wXihtCYX
+            OERIZ0xMbHBvSVNyaGxCVFpqRzlxa1EKbrjfk/CFlTSyJYgTzQ67W3JyR1fwv8VF
-            oJtbRK+kJF8oOUo4emI+SLGrCsWq5cG2wN/wgP+ChR1zUgRgfq5oOA==
+            dxGeAgLY4EQrf+2PbODtazRFreS+sl7dxq+rKMOVZTsjGHakkqHy9A==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-12T18:02:01Z"
+    encrypted_regex: ^(data|stringData)$
-    mac: ENC[AES256_GCM,data:hQfimKuhfiIiZQAyNDZQR25HZ5biU+kaWig3ElVPiqMY3B0LrmKyOEs7ittd3j3meUkPEFSmm1yYSXcdCuf6w7cwSbEiL5DG8t7QKPbnx/aRdyHAJW+1VC0e3Lye43gaHya0iUaWS/YFQcrT01oEbckLIDWPKfvA6pg2bZ4kMZ4=,iv:AydyWMVcjTQu+7Fzw5vtqkvTTeQEMt8y6snESoLUDE8=,tag:SoKklNqa9nH9G6d9+PnolQ==,type:str]
+    lastmodified: "2026-10-26T04:03:16Z"
-    unencrypted_suffix: _unencrypted
+    mac: ENC[AES256_GCM,data:gxu4WcJEuFgjtWCfAjose8R8a566/OLrXoiJmKfcvVfTFDTdGM7I4gzQVvpjEfra5Hjyt7kvLlaV2K5bKJ0k6NXbbmgrqAqEOEcPg+cnPy1FtQGAYOo3VzgKqrpLOCs/7wh4YR23XHHsjQlHAcP8BmhrqnpHXR9FMZObrKjnQj8=,iv:KJZfw0n2DOtdZHeVDk6TrpO8q57yD+xdEVt9+hM0D4Q=,tag:rHQaMjvQwaB8Am/AlTKq1g==,type:str]
     version: 3.13.2

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: oauth2-credentials
+    namespace: iam
+data:
+    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
+    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
+    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
+    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
+            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
+            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
+            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
+            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/paperless-ai-secrets.enc.yaml

@@ -6,7 +6,7 @@ metadata:
 type: Opaque
 stringData:
     PAPERLESS_API_TOKEN: ENC[AES256_GCM,data:wlBC85ep6zioWLhfhNczkKfTMR8tmciN+4r3of8GpQoJmKyE2r7pOg==,iv:Rem/UJl5wUNi8G+uEV7f2eCLabaY26sZ7+AsKYiOZ4I=,tag:bGS1fu13Mwsrr+5NUEzzLw==,type:str]
-    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:co7FfmRXKgrz9fNgR10KtlHL/iG4F5WKJgjcKem1H5hF5FviVi4+CQNekA==,iv:ADtBHiI6BtrkuH1iIfKV3I5odNLMQe5y/9GGTJrPNN0=,tag:j6wX6zO5VhKoLzdr7BQZCw==,type:str]
+    LLM_AUTH_CLIENT_SECRET: ENC[AES256_GCM,data:OEZqGgJ4cpMsWr/PJH50lsQKvk6sjVNPCFcZP3yQ/7nvlKP3giPlLmW9XgWr1ELq2hwWDs61+f3fAyENt4ln+Q==,iv:W0Tii/nvgm9ZMIEtfzgO5pj4Li+6fpxfd3zFtQhj874=,tag:tsM/fGeduMOEfMZsDZ6lrQ==,type:str]
 sops:
     age:
         - enc: |
@@ -19,6 +19,6 @@ sops:
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-09-14T14:19:39Z"
+    lastmodified: "2026-10-26T04:03:16Z"
-    mac: ENC[AES256_GCM,data:jDdmiFg6uB1za7sAjO1CnGHdRPyUnmWCMJxqHrE3BO82pNbtm8yLOf+W6CfFKAjfZYqEE2mdKhABm7zeJgJlub3eXJbAirAFrp5jrbfHw0qsFo85Gj/BsrSDB401Aky6PxvRomz66FyX5zDDXIGAMEMlyNEadMplTfrZQQ9hQWM=,iv:UTSF72ofUyWs1LXGKQwBP60hK64UxhqTFM8QTVozGoo=,tag:OKivMDAqwWgrX+kNjvQrPg==,type:str]
+    mac: ENC[AES256_GCM,data:gOrqy2fPX+77303L+E1EypaTU8mdUXYjbRZnTIJtoRpZhEmHsVvS/Ho8bVcl5KiTPN7UoFNmGyDz8fpyQ/rLTbeCiUYmyxvH7ym9DT4oWvRxITpOqBxz83pWv1BB++HJvqNQ5FllqpnAz60/bAVyxOdOXk4J3lDnuJibNOpxhW8=,iv:suV68u1Uiejl0NNJ3WlAHshgdrSL+aTwUxUBeMXlfWs=,tag:5wug4NeDZ+qclvFGPiEviQ==,type:str]
     version: 3.13.2

k8s/argocd/secrets/pi-sandbox-secrets.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: pi-code-agent-api-keys
+    namespace: poimen-sandbox
+stringData:
+    ANTHROPIC_API_KEY: ENC[AES256_GCM,data:TBcno/dzj3qykHjeX96UKZekuA/WMQT2E7a1ziPyYo5J6lWLPydgli84KeJZhH62/8CnnFtFOhVIe4FWCGPGjvqKH6M8WbiQHaP9y7FN0qQUpFHKNVAQ2R+s0wInhw0eUmANvd7bRx8aIZ2X,iv:oex19yVtWUevoJ0A3mkMLCfWu85o1gPSNp43x1iV0LE=,tag:BW3zQnwvagPceeNird9CZg==,type:str]
+    DEEPSEEK_API_KEY: ENC[AES256_GCM,data:vq/5gOsMVjlVMd0oXiBtHFHFEr+IF2YR2iP2OBFWx3ZFNoQ=,iv:QhLU/BRIJb/7QKDCRyw+HWPphZifVEhIOLywYDgOb20=,tag:BZdtXw+WsuK+P4E0caTa0Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNm1LNE5nWktnd3c3STg3
+            UlVWbWJxWElxUHMwa0JVMk81Z0c4d0JaQm1vCnpLQWJWV3hxOU00NzJIaG51Mmlr
+            ZWpNTFJWcmdtV0w4YndQeHpvSUx2MTAKLS0tIDVnNkNuVWhPUFZ5V2hFOEt1eitG
+            blVqRy8yNnYzSU1FNzZGNkxrRElvWVEKL3J77INje75+CvKXaJHC3CvONgHv7d0a
+            a10MITvfreFr6M4AN3Z/mno49SDMp15Q4oQRagHHxSTS1K54mWHo8g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T01:19:49Z"
+    mac: ENC[AES256_GCM,data:GpYl8IRefms36GvO3RW41c0US7SOgoOhd4rumxsCdEf5ME9kJ0eOxPx8klMgnlYrkBnwHmad8jj7C+HcA5Eim7mdUaYOZZWPFohvC6N4GLHX85aH78P09CelqG5tdm/Gm+1QfxYu0vANYGCAVjLK7YbBbZjbZmbvVfQvAbs3mOY=,iv:y3lglGu8FhTHQpO1poSIKgGYJFu5J9yjIaOjulda6fY=,tag:5Sgh4TbtNFMw7xOJjYRn0g==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -7,29 +7,37 @@ metadata:
       exec:
         path: ksops
 files:
-  - agent-pod-models.enc.yaml
+    - agent-pod-models.enc.yaml
-  - agent-pod-ssh-key.enc.yaml
+    - agent-pod-ssh-key.enc.yaml
-  - authentik-secrets.enc.yaml
+    - authentik-secrets.enc.yaml
-  - cloudflare-secrets.enc.yaml
+    - cloudflare-secrets.enc.yaml
-  - forgejo-registry-pat.enc.yaml
+    - cloudflared-credentials.enc.yaml
-  - forgejo-registry-pat-api.enc.yaml
+    - forgejo-registry-pat-api.enc.yaml
-  - forgejo-registry-pull.enc.yaml
+    - forgejo-registry-pull.enc.yaml
-  - forgejo-runner-token.enc.yaml
+    - forgejo-runner-token.enc.yaml
-  - forgejo-secrets.enc.yaml
+    - forgejo-secrets.enc.yaml
-  - grafana-oidc-secrets.enc.yaml
+    - grafana-oidc-secrets.enc.yaml
-  - grafana-secrets.enc.yaml
+    - grafana-secrets.enc.yaml
-  - homarr-auth-oidc.enc.yaml
+    - homelab-ca-secrets.enc.yaml
-  - homarr-db-encryption.enc.yaml
+    - loki-secrets.enc.yaml
-  - homarr-secrets.enc.yaml
+    - minio-secrets.enc.yaml
-  - homelab-ca-secrets.enc.yaml
+    - paperless-secrets.enc.yaml
-  - loki-secrets.enc.yaml
+    - vault-secrets.enc.yaml
-  - minio-secrets.enc.yaml
+    - vault-unseal-keys.enc.yaml
-  - paperless-secrets.enc.yaml
+    - portfolio-secrets.enc.yaml
-  - vault-secrets.enc.yaml
+    - gotify-admin-secrets.enc.yaml
-  - vault-unseal-keys.enc.yaml
+    - gotify-tokens-secrets.enc.yaml
-  - portfolio-secrets.enc.yaml
+    - gotify-smtp-secrets.enc.yaml
-  - gotify-admin-secrets.enc.yaml
+    - forgejo-smtp-secrets.enc.yaml
-  - gotify-tokens-secrets.enc.yaml
+    - paperless-ai-secrets.enc.yaml
-  - gotify-smtp-secrets.enc.yaml
+    - memory-agent-oidc.enc.yaml
-  - forgejo-smtp-secrets.enc.yaml
+    - oauth2-credentials.enc.yaml
-  - paperless-ai-secrets.enc.yaml
+    - forgejo-registry-token.enc.yaml
+    - pi-sandbox-secrets.enc.yaml
+    - etcd-defrag-creds.enc.yaml
+    - zitadel-masterkey.enc.yaml
+    # Login UI RSA keypair, adopted from the chart-generated Secret so ArgoCD
+    # owns it. Without this, the chart re-generates the keypair on every render,
+    # which changes checksum/secret-login-service-key and leaves both Zitadel
+    # Deployments permanently OutOfSync.
+    - zitadel-login-service-key.enc.yaml

k8s/argocd/secrets/talos-repair-config.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: talos-repair-config
+    namespace: cicd
+type: Opaque
+stringData:
+    config: ENC[AES256_GCM,data:WO+jinh/2Rq5VeGLoDDJOQL4zfYJINSTYZCwGWndbzfIHw4XRwxDB7BixjCdpPuLOkNIBAXHQpKPBhlQpvRTso70L26njgVdVENUGeNhxvb6/cwl3VYRsfQQxKoQW9Qu6DyCD7ItuyazXMdIbENKBoih7GA2IjKdGeEtpB+a6PO+6rpI0AQRIjM9h65yJ6gvWeLZmb6U0qCFW61SelHmVSsfehxH/R9EKQROKwD0Z/TgYnoS28OuDtixAQVfnrI3gtgUAeXgnPQTJf1Vb9iwbRBvaSWkPSIqa7hOqWEIM2qyrvhTsEb8XDRUHTmp4BCxYktIxoiSWZEbeh0y0HKRzekZIj4rHA+c/JYSPqS0dvn1Ga+8PTc1LNdXDWIcKI58YQ2jzA0oE2vm0YzmOjcDTzTGLDBk49gIql+AMfwoxCop7t1PB7BrJw6M4bj9M5aHIq8ZlGXDvtWZcSEBj0ChZ0q20cB4v4bvMkbuJR/PS25uNkeG+UBCPidBQqCUw37aNf8SDwOaAC+6PeHdLmcJ/dkKntGFlU5MQzWQXfqs/fp95ze3mr7f3et7SqVl7fAlYSdEt6dA2xxnFnqgNwAm3UacV/We6SRnjrFlBAHZ3Q5hIuKkxmszsS4Bh/VuNtZI5YJ4ldnoIZC3aWKDt+rVB6P+Qxp1T1XqZj8GY715DZRP1q+Z7iQ9RcXiJBTD5qWVpnkESOvbTEYv/W3oXTufVRrJ8Ps0on/xWRF5NU04+tqvToC8+MIQDz5rb+eXYCEWjZG9ZhInY+M2HRTLg+c49dRlyqVnnqfNWBejSS/hoQbf5HmkUbtzsCftvu2NwS2nOMhm55xcCsAunB+0uY+Bj4jqyqRvWUjA99yHIdQH5JTJXE4FQPxdBTmBFp8xVdYWvlnCojK4qfv5EurLUunBugdSdYidi2TKYEoW2mX19RdVnkQgBjW9caaZUX9BIi+zNhObKenoWxRORCb0dHw72mwXMaSQM4IDygcuwi/79OtkYik3juECVmI58bu/GFRo2NtmgkGdMD29SMV6rerDtYjGjetgFvWkDy6hrB/Vz4dI37eGt/EH/7G0z/IT9Q2sM8nzJDtCYdicepLQ1WaM0P4BEn2rrqhv81VvDH7M9e7wtz4xly5RYQUr5wkzvoSEIglFekPvPmhIxEsNdaBj6MlVmLTbEbv85oBLwm0ZLRJcgUsrBbZghGKQaI5udAk1dgwOng9v0Ai+ilJ2RSt2VI6WG1rwRnDG+Hv+H1cKT/NwA8ocmM/kxQlUE3BxESrxhTnKq6W0rzPInIj+xeS8v1X4sloGJIiweCCI8YrqUnmYKKQ6Rtd5B4jS5fAa8jp274/R4xvLaR2r70wKp890Iwfb948PwqjNpH/XlOUcfIxPu3jHO65OEyX3OG/+qm+g91AmxlrwZRDYeSioHztz55Du+JKX2Twnb5ttEwVm6I7eQWGbzw50Qgg6I0cHM1YRrXv/h35C8pkXxV7vMlyR0g5ELAoRWYkuveniruKNIIPBw6SY81RtjIIoPvKzNAFssNdiNxfYEsHiDlfK9kUvTLs8qREmLt1rzgD/vHtanmKh7WMtHN42tSUp+NhoHPlEyaaJLUMkFS7suh2S9xPmhisw61npROXrc2PBxLR52i8t0cJYFa1XuZ40KMCZ7kw0UmKiUxeFDTlZkaOTocVqw3qLY906argYnVdkpaoJMUXsnE3JmXch9HG1qpP+hY8LJSOl5ejAiQADHYDm8P/+9VVlNL6eo2O6qFUs9lC+I8hVeSOvJd8+aLeWisFeboKjarhw/52eXwZdTEpPHa7yMDsIQgciqN82yagJYrEL6BevvZ/kGHP5TszYYfcKcN28k9y/R2JYND2TvYeIKb2Nmj9rhwuXBOFnlep4Exw1QS0If80gJG1BCub+u6GKk9FaFHUSH/x3MIRHwVzCHRnZxqbwc1uGZ2I3rRxVq5MHo5p7htS/KYi7+cQjSdRvfdlCofo6YKP38qQwh9y9yAUqMEOABtjUaEJSeWlGq2yh0onm+Uqm54fcCSr7S5MvBTZ1oblLqVUrZBDTIkY5cdbuiyyoj7GPLHrxUo25yiGI+FXHKy136E0Rv7p5N8o3U5O8ycPdk5vOVYgvM3pdLS1zHHsVYIYY/Y/svtDUnMhJONl5AwQ2sDZ2GiWx1nWJ4VumLuwb/Yow/BHtnW6SOeWFSwRkgJdNL5mzrNoexytbrDaSWR0IpuLUbvK/Gi4MJaEkm1acQB1CMOjIymVu1pcih0H1Zl4O+fhQM+bVke6NKrBSUKErHWca9iNH16Du7iXBbaK0Dz73nIhALJKCivr1rWviXXxeQuzTFMRXdK/SalJySHM7ov6qOikQTG/Q97zQ+UYiUnLo3UxlWngXrnzGy8fAb58v9bHhEtnvCrAhiJTer0x4GYQJGwYjQDbN9ucpu00PDnQU0GqYJHqCf4Vm9Y3SYTs92T6Lftvu3nw/ZH6PblcznC6sueEPON5QzzQppepezdolRNyx4g8XYsI+XJlCNrExWxDxErLBTOm3f7aB19RYCS1tG0G/aKEJTXh9RE5VyWSyBzPNMuLZF9HIIHp3yGpqGM7xsYKsMtngBiyXC+i/C/geIgUFJym/sqWn4iPDWtd0YeELg46B5krcolTg87KEeiqDAebieqL5ZiftnVi10kV2L2XPs4ofJjpVI1Q53RsgT89zWXYv97DTZeDwSE8FY2vbADnUpGk/UZ2WMQlX8iwFOTo1woE6AHyqrq6kzGamUGsyoAarqmJk77o2ktzO2M4OqsVqJErSWTcCLZOlxsDWUn/OsQX5+9vaAhHwDHMmrqWo6cJuAtvaLYOHVB/BUOiI/MJLSYeDIrKUML3Ew0GbX89jkR45SPjRdsXQ/oVCJDxm7WjF9jd6bly2wPTmWCzoqeTJU48jPk47tDHWA7QjLEah6PajxbAs/xSN4c2uK9ltR3K/urNpOTYwsFMLgRfzontPe8RMB9A6aZcwQnIhLTpgX8fw/j+aqwqJwVcIk57/KvcOksBjN4rwPQ78kCmpLJ/KEjYh1z8EwIdVX3UDvq8c8YSZnzwsTAVfhWxigvLGzQDTXrr8aNMDXwB2iUVwhNdGw/E1UCSvsKbvw+p9YRaCAzPnnKa8AVAJySAc1Ys/DAYskdKYaTPUwdCXyZfLzfOsFvEzBxqrRgxxK77ij5UQXKINjQpAIP6pkDze+JTwWQsPfcMZ5xtb2tpiUF9FvCVqR1dfblppq3/f3ZHSl7f4s/sOM2/ZRkiI4m8yxhWDMtauxyrnkgiLsOBUcK4MclD5E3H6KtaJODFiCyH+NPtAMmaZ1iduRM51qxmrHlcI5B4ErJum/iZg+zIokXqIi/U6B+tbjj0EKugN+5V03JaoTvdaGQKNvJHnyP0odFN05ZgZnYCKGGh41EGyCZ8/lmrNbAfZRPjRtHw5ypBUiW8Yz+SkumgWA637ASf1AklCNs9MBBuOPmGckDWy8zTvUCgWCwYehIjKSQ+1hYoWpraLNOZy7tHlTQZ7G5zIFR3cJljO4bfmpAGDsSeDQtBsAI90gD4P3ipKFXVQTfl7c3CD1kB8TiOpG0xVGlYQ4dP15z1wTUfHRz82eFxfwn2fc/R/AKmINq9P5PEL31jhLM+gTxctb2BiGn8cCRuorLCg9FtkwC4jy2AluVj6uZZOGvDCYyGqFsdObgwb+FdIoCYDdeZXo+1dxlxhL1258L6h5+yJBgSy6QDN26N6MMNnK2iD7QTQwtB9CfaX3Bk6T0nZ0TT6zSxu7ix7UG+E5AKwoMVcD0EuLlmJ0IhLI+4ZZ8z/QUIzeTZ9ve+r8uyHKmWr2oR8ec5m95PGG5JzWmw30sDelFFl6S3KxsHwzlJsXeXbE5I/Z0RWZQBa8xPbmY54qYXPzp3O+9WYys6A697UzL6fcCORYokB/TR5exBS2saW4OhkwX5ss9aUJGZiMFSgfF1paxqXkGIx3IuDH5xP3QxhGxNMB+T+Yn+meApf2A00dEUng1HRtgBU5Mk/TM19g8l3nVtwTrmAZDm50jIn+G51GpeoBeYPONhGfb1Ngao383TPkvShaNujE3zsJLfJCrNda76rIechIhAC25wWauUMGUU8daL8cdtJnAEeidnolR+lVf5WFKUxvO36X84FCSaR4PqzvlNk7hOvd607h7IGOEVx7/XRLpWDDrrT6hpY653lNf3RCsxdQ7kS9e5hi/RpU1mf9mdTOsf+AtHkkog336jKdyl1/plDV/AlRS8/QjlPfeyvGmA6gEC8cZmBkvyZPbs81tYmif8q2Amd9La3wldh3evYOOkC/b1QQ3fLryoeUl5/9oVprW/aEfmcTxG9CytaFfsx6cAejAb7g6fz/iaWE57fXfth7K7ATi827ZwAs0XX9lLkyGn60ePtqg771isK5/v1jCsbDQDwHwNLehH+BX9zZWWjaQxK85/o09Oaiy0oGY1nivv5kHelmGwj8qqd,iv:a4NHhErZolhOuzNCUU51GQiCfCT8GzZ3GQ24s7fkuyU=,tag:w+GrXmaYWADTrBxMko3xtg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmk5aE1GN2dpVlREZU1s
+            N3V5ZUQweWxMaVo3Y1VjdzZTYW96eFBYZGxvCnNJcjVXKzI4VnliTWFXQy9iSy9k
+            eEZmWlVPQzhZK09NZGowZUVXc1BoZTQKLS0tIHIrMEpTM0lMU0w0eG02aUxJY2VZ
+            aUQ0cmkvK0UwdzhqcDJwM1RDcnpEU0UKQRiTa9/yNrqlkIdlNeaIDzIKa/xvx5nW
+            7wTNfjq2M3go0DMnaVjJA3e15JZBOpOs0QP9z9ydKF/FS4PeCIEFyQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T04:27:33Z"
+    mac: ENC[AES256_GCM,data:CxZdlGJdIFFTwaeeS9LW2XDDgvUmSsBeBKtFYyDM1JliS8BHtgEv48yuCWxP5sR3SaYtcgNSIQ55zUa89jM6dQHqvjK+1jSYu1xzFWPOlxUu8gVSlU0QLS4GU1RChXuPpTUOjxzn4ymZukGgclZk9iuMvI0/U+nGD8nvs8k0zpM=,iv:MGZ7YsOsjelJPjLYWtALOrM6r8Vd1HgbU/PKL4c3r3k=,tag:dIE38TCobESY3j56NVEvIw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/zitadel-login-service-key.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: zitadel-login-service-key
+    namespace: iam
+type: kubernetes.io/tls
+data:
+    tls.crt: ENC[AES256_GCM,data:Ad4omL7pD4QhixW6wSVsVor5FtHbWyOHeYbxOQsEBAaSuvb7JIkvnKqo+Xy6zKuMmRViUIbqhlLQDXCaseAVGgpu3PdtShwJ7LRXJmr0XtQV1M6YtRbRexN6UpFA3XdD+JfifMhtvd29rM3WzLPX69uk4k7voYN5m0pNXSN23jVeIeqm+xB1CpxDW/guZ2hA5+b5XBzVOkMsSOSdWFWH2lK7nInv6ZatCa0/fuIKmB5nZpSV9tHpCTiTnBuNc0rW1X4HOWa8zMiSbjQAqtDitm7lrVHJyImXwBjmuS0wM35VwrwxVUUEjS8kgqnyb02JvNRdNjuwOmIcyKjrdgO1B1ft0CrAbRNkk2dYR8Y7SNhR4F4tYYF5L2szQgj+5tcYrnRGEI2QkXbcmTa0Y9V99+D2bKjt5t59Pi/1r8/jt0CNy0GeeZDvsN5scy0/A4WqtJ2sZDyTnTPwtUpAAwVC2WSqsD11gkzhyRJAgmAmEgfPIL7kTZsZBoSQeuuCJ82Ghqx9kLTpui6kLULxxUpP9oefsKw9PjTwExRbNNcPV5p7PPi9dN00TNe9iYuFqoZFP/lp5SMjDqbsV4Ie1ffcfyZsIHMuOv5Ths7n8ExYdR0zIn+aQePXASt+dE6njhFiblxyBZsn0xON3EIUaE9e3WsdcQKjlPOaAW/DvU6PwBqCeaB2nYzM2NFk6W3lN8BahTE9k7xXrILHIrXuIXezotCK6jclb5yZmdm/fTCX1iejGVDktvvcmX6bbv3YXTRZmsviRhLFHA8oClU0alTsY5Q3nn2Xl0UfXU+rW+10AIpY5WFMzP6A8hXvHx0ISC1S4AIqjmlsDmq2yEZKpbiL+Bs52ymW7t2rQ+KGhWH/bcC3VrJwUWnVYP9b80xyHZdKqubwLc5UcDzCs/Lan23VpLxSJkqb8vZP7SHQ1nc+rlwrqFDkzQ+yMNLZmutqbaAmMyXbrR51Rz3VqofraPXjS+lmLAuEcuTEFbrImsct5Zb57MtKvhHi6CUWt/Vj/NOUJlwy09zPy4JAMMJ81ooyBj6CBHY2f1vfR/UUDN5VZdfg4dUWXS0CQdOb3hx19COr4lR4RcpA46qJEjum+4icm5ckziYT5Nm7NB+ydXJ+Eys2Ba0dGRM6CIeQVhghhCDOZ6uoQ2A6DNql6wjrfUml8UwuJNO0nXVqNS28pLwddsOjlbsysJAUEma6EHXb2Vjb90JZErVswGuhQ2+UdhtpuIkgBYX/MMCZV942C+K41A9478BaCiWuC9KbnAcx81ceR+T3syARctZvQaYaiH7WfF8qyo0VOGzZRYjFbqeyHefiOmPmWTS/XbIti3evyhmxHwjUcidK2njrPB0O/3w6R69UA1p7ph9EgFsj18gPYnJthikgX8vcfVaVnPIAwCqY5DJJmIFTNXVkNDBX+2orrW2qj6PEH+go+uc+aUmu646niqaMAcrUl7gGNGd2OWFHdIRKdtogOvoVpQMDB4eWHPRsL29EVt3rnknHfpq7cIjQXuk9XtlB8zjllYdrh50Q8ZNBOvzWgCiMvAyanfIA+kW9KHQlmb1qJGYp7it2Lrg1K7sDEekrlbuoNsf+2rl8YRQz8pVBuCSzG9DPp4+Rj+ddiqX/Rrs3ufWDKFV9zJoN1EpJ8+z6n8rsQJBe/lSFLW1mLVkaPSNNdIQfdJ6MbsHpMlqqFYP8dOKckxv82uOkHoCLHrgaXiiMd1dBsX1fBiv2NvzDiuLyN81d7R2wNAQbkG9b+/+C6wVr6kDg0n6seseoCyeSI++PJIkCrkTXzd7rLs8Of9CHZQEB15wjemlmYGlJnCiFWPFa6h0jJPatPrfTYv+GpqzWV8u1Uh3aNjd+g1wVQok/ojNOZJahUSI4B67HkJI+I7WOF9C+rRCwNutqshIxYzW7CpyVey5P6Kyv+/cucDEgb0ggH032MBtzTxazpJcE4Y6fTrOiXHp9j2KjzyUt2YeCCL8UMwbwJvW9p1I5gx3ArsOuyHm1kA==,iv:PLsjmrTwjM7IUS7ZBSyVvJfAJzCSUtU+UAvHbSwoaVk=,tag:pUQwbHoZgfHW4CpPKoULAw==,type:str]
+    tls.key: ENC[AES256_GCM,data:rO1NVJZP9TP96C48it4+2lGyRutwqdlX6qiM9R/+0hgYc5ceeeopaCUG/i/LOgltxRQK8qkyed/W3zU2VI6alBpt75XkT3o9ZRy3uxe8mB3ixS7+xQnMc0ci8Mf2bCtGzqwUQj8XPRP4K7Mxw8r4XO5rQLnSmNme0+BxS+XiE6hqifS16UUKfLuFeUtTEoI850sg1GQt0uaHf9kOrpmvsbg3sflO2dQXk8uSFi3B1ghG5DdWImVoc4f5kFK86XG65WVVfYjLaUvm7ZBPT0mwvq3SgPdEXeWq1fp7tMQqCRKuluRZon1e+J5SlWOPUTxwyTq336oK2Nfk3HsiQ3uCG76oCmI5yTe2tCN1f2IXFXBvLNM9QTxc/KFhI4uI6bvHjimOSpXYuRdroH0hc4ohLP44WG0E8Nz7sAb3Ai3PbDTKnqYQemvGEUTdcdbWuMK5/JE4f1IE9DgOOO9RF+s7uXTdzZAmoMcRtVsckQJEJl7AKgqSn7BbK0cjRL/QXclNySQ/qcgSDFiV/pPbGC9JFweJon5F0Qn/AIAnGGzqmVbS6x45UAJieI5oI/30OxjzF+Dov6EGMoE0mys+6l4j8v1EFCtHhhopS7zM4jZRTnNzlZWPtEISU2kopLroGpKESE145VQnB7X66F40KXcOIiXmU1H7Yxw5rJ0SqLAFWQw7gyNdzbnbfCmGASByxnYYwZRy3QDZYrQlthLd5tlGe0cg7j02ZSHmmJNQOx5yOmSQI7z4njDvJlOPncC14T59yDZUIYYLTAYjL8nmCNk2yD55ImVGRIb/f0JGAc2vHMlDN6fkuiSLtbASz11PwWzdln9tS9BTpRx7sfv5+kMhFkzomufftAJbs5XdOxcvGRHkRkdyCn3t8jcRXaNDBpq3o8zWLNh+411I65DMDN1B0c4+7Uw4N17I+KfrVh5Fb8ix0LW9aEL/SUxeIqdEu0ZMTncxGS0QVzZmtJUTdWuT0JjtgfsD65jqpg9l8RdIKX4i0ox2+GeQuXKCQiCaXS47yDxUcICYZB7UDnfwBcXlnGWw1frvkBBqwvwUVa7G7Al1A1a35ob1qb9gpyJZGqmEyV4Qe3PmfUHkp+/bAWarLXnk8P4l2faFjCEnPDIejGMQgDdtkodq7lfqwtHwg02bXNl9X3FbXjyp51/IJshE9dhHLkniWyHwZvHZJ4jjqndf08682gxTnW0bscfv7d6lKznE7rgd/el5ygdlL0PSr0pbxJtcd7MYrACMfZZxRuY3PWZayr1n4iaqh161S8ldeEcLCiHFvDeGHslUK+Sl383AHhs46NHEVgMbhuKJ1thHckQSgpqm9vGE1gBIo0d7z40EulgD+KS4Qm/vdydCgqtzyNmFDg17L+SVCQiyckGxgl/j1YzQKz0o1EWXNssk121m7Nh/e/bYNN8kC6doJPBQBkdhU/+DVZa4+/je+qpqvqnjpheX42k49ubk638QhOkcMmx6ZNnEbL9Ra0mNJHZO27t6++LgN2XkPg6t8VO94yx1c/VFyoF9Qg21O1+BadK3RF9l9s1eUB7n93YwUDhxZpimr8j8oLCn+PvksKwzWEIiAFXUWvuROwh6yC31GghO6veGx7UqXGPM2O8+mkOxXZObbnYAAhteRhXO/k15HL3qf94pudZ6+tETmeej4FKdhZzF9iOsTu/69izQFhl57E+n5hDjRzeVANKQOdq2x7qDG7pGfpJzrDVR95Z1q7Y/d+7FJjBML+YvzGvbxZno+PbPmLGrgxSebAWgO73fUDkBHvosMDjeHjg0NKBXYfR3kImceo4iJwF0bOeFQA09j+6OKW1EId9r1EzYwPGR0DUDt+A9f4DkU/27dbKUmJovBPIAXtYHErwZ7EW0ITd8ebNXyOiArz8JJietDsR++lwetArzrDszsPsiumoSu1/FajtbBfUQqllxJBeoIqNEPHQED2ysSWzqTlHe8tfmY3Vp/PlZh8GbpA/EV5uikWzDSaZbADqXGny/xd+g+8mm4J3DxgPOEEgFqeg+q4C1U4yEy3gYkknT6AcIO8MGOa3daRowEv2zfhVlrNhv2ezLjzsPBQJSK/ph9KZbYiREZVCBQioJJlvfG/dkkycjkVqvTNtn3swhipxUqYc0vKmXFG+R9+fUOP5u9ctqmdIf7Jclew4OBO1ML+sH+l6NojgfX+T+OqQ8EyNMY2Aa7i24ppwS8ySDwyYEk9kjopVZ8L3MSw3CQnWBNMg734qUawCUoFMtGPZ0jC+o6DiV18xpkB1Ffrmjs+72AoKk6wR9zIYyVQG7wr7O4PLWeXEX2NgBDeKaWi0gHInVAb2puJKgGgaZbImuecSaKaqA0y1fXu3+4teozv9eLkBxhYUJRwIBQbA7E3lQ4ocziE3ZMxdAIGAOEJHgDs6UreB4ZMLgdKxqhYzR3GxrmZmspOw/TVBGGjClrefGZh0/gLkXSDZ0uugW1rNM3TkA9t2U0eBCNyRh+8sScUKzja4Ei2ukkfoCNEf0alJkG3udwBtjDBcTRhQJARFD+UWMut5Xqw/dAxs777/VmY00/wD9FaCq0aVc/CJI++oGYgunPDd7ACSTyuY0UST1AEBrCrFBB+T+eKrFcU/4gBXEGiAY4x8lrp7fNZ/nsGFvBvs5sxq9NvLFXGA8Tv3LHU8x8m8oGHVnT0AaFJIKM8vGf2bkdD89EmQkKG9pfnKPCTbyFszqOJlgORDLSn9bNvDYlVC3dS5HQbqdv1a1fn1WrLDEXTNuD1Nj6dJXUBLAZfqzRHaPRi04YNpw9MY+c6f3XmAvNmwrFgdnxHja47ObX4XGigltlCWUSImRboBinCeZ9k9GdQ82Pw36dO433Z3IjtjuFLTWFsUurpAvmgiemzJ3xnhra6pOEr2jy2dw6fUL4gMQtPpSwqB4Qd9TY6MOUpwBqK0MpuRMBisa3zOBphJyvnXOIG9RCYzivzOQM+PKzLmznjTyeql1Ipj/9LwEgQ==,iv:PbRgjgm5ERWlAkkWc7As8eIx8e0rjTUkQ7XlDBrEUlU=,tag:yh2/JuHu5I+dWlxVAmkkXA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByOGNaakZoV3lhemhMSVVF
+            RklKbEsxeVFMWWdwQ3ZqR0lhRHlPM1dWT3h3Cnk2RnZUNnplVEwvNW9qMExsSUlt
+            VVVXejMrcDRRTEdtT1krdThqWThLWWcKLS0tIEVZZEVBWFZyVi84S1puRHdtMWR1
+            cDBHc1J5aHFpQkxSSEx2VDRjZ1RsdXcKG5MgnUBpFmlNGSkxBijXp7m492ujyqsC
+            TbTfU8SWJwXhio7V+zDbXG+JW4R51wr9F9Sj47DCw7b64nbsD3N5TQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-26T02:15:51Z"
+    mac: ENC[AES256_GCM,data:+2eGERwwzeK5JE9H6qhsu+tskR3Y9c7DecVGopfG8Ggq5SHQBfYyvz//IU+MGCOb6nIyP3JPf9+9IDS2DthVcw+RLtviQkN1TeWuUTDbxBxHdCoUv9zepwU7+d0qUMrdUK0N9QQSRZ4TpiwceViDeMuhPF/vTmADP9QWwid/qwk=,iv:nuJo37Dh8MAf+qI/Ty2jsW0IheNhgmZ4l0Va8pJG6+4=,tag:vCpifXvYF1/QU7XExc6raQ==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/zitadel-masterkey.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: zitadel-masterkey
+    namespace: iam
+type: Opaque
+stringData:
+    masterkey: ENC[AES256_GCM,data:Da543Jt48o6p3f6/AbZRsjChz3NexPJoi/mC/LB6jUI=,iv:irS0n76tZ6QzoQMm0dWrvbw+SPfXz4h13jQb/Bjfgwg=,tag:tfbgaAdxmPIjleB1OXLn0Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOZld2VG9NaFhIVWpBcGpO
+            SmVwNm9FT2wrQmFKb0RGQmY4TCtITjFEZjBjCmNuNXNJYTdsZE44cDNzR2hiNTdy
+            QWlVdzBtdjd0cSsxUjQxMzNyblhWTkkKLS0tIHBLT2x1MDBGdjE1eHFOdHVGa1NO
+            ZGM1K2lDTksraFdPeFdSZ1JyU1VtcjAKgTjjhr7u4YRuQVZQ7kAa1clJ0Rzy10hQ
+            Umcd4/YStyXHZjsJJpFSpOFdP5cZscGo3xBgpXtxAscKTS1Nw4derg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-25T01:24:21Z"
+    mac: ENC[AES256_GCM,data:QFVk7KLsMLCAMH/PHVGCRyfU3k2JiO8AE3jJXmhnlUB4C8MYCTyOWQbnTFXozCMOr0kp+hAt3dYn/e23SWhyvJ7BgVLRb5QF1py704Dr46atVoLUvwHpFI7dyGK/LDTxqiM40RXAnDtWUJlTkVCzKdSFiV8CokIoByKWwxFPayM=,iv:t7HWMcpQl/DJ8Rp51Du7b5gURZO+C+FodsZSJuy5jBc=,tag:qEXSS4S0Y5cvn7T6EjakhA==,type:str]
+    version: 3.13.2

k8s/bootstrap/cert-manager/cert-manager-values.yaml

@@ -8,6 +8,19 @@
 # depend on them existing first.
 installCRDs: true
 
+# Point the ACME DNS-01 self-check at the public resolvers directly.
+#
+# Without these flags cert-manager resolves through the pod resolver
+# (CoreDNS at 10.96.0.10). The self-check then never saw the TXT record and
+# reported "Waiting for DNS-01 challenge propagation" forever. The wildcard
+# certificate renewal stalled on 2026-09-23 and the cert expired on
+# 2026-10-23, which broke every Cloudflare Tunnel origin with
+# "x509: certificate has expired".
+#
+extraArgs:
+  - --dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53
+  - --dns01-recursive-nameservers-only
+
 # Tolerate the control-plane taint (talos-cp-1 is the only schedulable node).
 tolerations:
   - key: node-role.kubernetes.io/control-plane

k8s/bootstrap/cert-manager/homelab-ca-configmap.yaml

@@ -271,3 +271,40 @@ data:
     NDKi7/Gxha790QN3TD/U7p9GcEtfXKv8KISKfxnfD9o9Yx6pWOXcgvGefQkpXuRm
     kGzDM+aY/VKs5Et5/RyKavBDihuOcumLsmoRg/bFoB/xpeURTyiehUdu
     -----END CERTIFICATE-----
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: homelab-ca
+  namespace: cloudflared
+data:
+  ca.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIFBjCCAu6gAwIBAgIJAKoBPdKPDL5AMA0GCSqGSIb3DQEBCwUAMDAxEzARBgNV
+    BAMMCmhvbWVsYWItY2ExGTAXBgNVBAoMEHJpb3RwaWFvLmhvbWVsYWIwHhcNMjYw
+    ODEyMjA1NDQzWhcNMzYwODA5MjA1NDQzWjAwMRMwEQYDVQQDDApob21lbGFiLWNh
+    MRkwFwYDVQQKDBByaW90cGlhby5ob21lbGFiMIICIjANBgkqhkiG9w0BAQEFAAOC
+    Ag8AMIICCgKCAgEAzw796vZRFOBc/orUu9J6rv91A3SNLJsqz/VuetwAXCaVSlK5
+    kL/RaBKKkcXToCC5avwix6W21FnwLUW4R8QlQdin8+XEHpx2LY5R4GmSvaCbU7wm
+    YwYgVGQVv8KlgcMbz/pF09vXU29cHUHwlkN+BWVGAxVGWxam94j1pBzV0dOhhRw2
+    q5sZYUUrPO5l775od2FMMnMzoRFugxDvJof0p3fAChNAIiFhP95lrcb2YsTwK+a2
+    6mk3AZc0gMtmgVsH+zryGFcAmWLixq41ye/H23ktj2bjEHCr9mIDpuSOKLyLBzrg
+    js/HqdxvwkZkkxwXft0Vi7GN0/LGHMdPrDE499pJelAHbxUpBoOk8sfXSNnfESI8
+    v20LybATGrKXFlOhMKshY1a3JLAoZTaW0xkG4PM4HT7lK/ovr/NLF6kdP7HxJFFQ
+    +4WjySiwbjusNobpTaQWVwCjD3Imf+2feAqJeX3z+P3EhxfrmvEojLOdUiiPcjMn
+    CFl9u2EpSlax53umSkcBQJGfFKM17fZzhtakjqhCTzAfXXlxZy4hsHDtfzRcfaoB
+    Zz/PiJ0TtJSC7TvxZZoPpGexQdoeT7kJEksTvGi5OSfJD7HaKe5RQSZGgpz7ATB+
+    YMBKbjhMXVu+TAdcO1qZRN0UCncrliN61ki9F0tE8kNgpS9biZ6kheoJMZUCAwEA
+    AaMjMCEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcN
+    AQELBQADggIBAFN1At11d/+Ue6cEjCklnRQpLuD5agXaE+pi5oHNfnggSlO8UjUj
+    lDzfnmxi3IAI6HruzDARM9J95qJboUxWOAEmK9SNnZNS/1b0GyikP3YcF1gIFqtW
+    6SgmVfC1cVUFKGt/+QQzZxhOLuoydv9Ozu12nxsTW/WY2t+cpeL+1wrsM22RA6W/
+    xV/bzYbwzCUT9v4tMZLZHk5CkXztVFXSJyW/8w9rVZdUGZM+mwwn81PLSE0v3s/2
+    wJH8eUqGl73i3Xgn7CVanN4cJYMGYVVlRD6Qr6muBN4Vh0ETY1PrjzsZ6XsV6vk9
+    glMbQnErYMZHZIDtnVdaBt60MecUD9O4eLeip0SE6bVRAucKBXA9xNMilMQEL8/X
+    FBMAxXl10rowBlbX2oaFSOM80eLTla4a3gSL7+0vua3C4Dtv2bLo8ZjCyqXwOJeb
+    N85cqcVZStz3qlgQN1nG2qGE3BS6scOs7+mUGe9CmS8acJV++PKm8CtmLH6/ibjV
+    tAc/6+Vg/keo0kty7H7L6oQd+gKpuaJ2I+wKYe4l9UwHi9zEb8YjzGB4n2AtsoWI
+    NDKi7/Gxha790QN3TD/U7p9GcEtfXKv8KISKfxnfD9o9Yx6pWOXcgvGefQkpXuRm
+    kGzDM+aY/VKs5Et5/RyKavBDihuOcumLsmoRg/bFoB/xpeURTyiehUdu
+    -----END CERTIFICATE-----

k8s/bootstrap/ingress/ingress.yaml

@@ -49,30 +49,6 @@ spec:
                 port:
                   number: 3100
 
----
-# ── Authentik ─────────────────────────────────────────────────────────────────
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: authentik
-  namespace: iam
-  annotations:
-    nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
-spec:
-  ingressClassName: nginx
-  rules:
-    - host: authentik.riotpiao.com
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: authentik-server
-                port:
-                  number: 80
-
 ---
 # ── Vault ─────────────────────────────────────────────────────────────────────
 apiVersion: networking.k8s.io/v1
@@ -299,28 +275,31 @@ spec:
                 name: temporal-web
                 port:
                   number: 8080
+
 ---
-# NOTE: api.riotpiao.com is deliberately NOT here. Its namespace `api` is
+# ── Authentik (identity provider) ─────────────────────────────────────────────
-# created in wave 7, and this Application syncs in wave 1 — an Ingress into a
+# The chart's ingress is disabled in k8s/infra/iam/authentik-values.yaml; the
-# namespace that doesn't exist yet would fail and mark this whole app
+# rule lives here so ingress-config owns every hostname. TLS terminates at
-# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced by the api-gw
+# nginx via the wildcard default-ssl-certificate, so the backend stays HTTP.
-# Application.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: homarr
+  name: authentik
-  namespace: dashboard
+  namespace: iam
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "16m"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
 spec:
   ingressClassName: nginx
   rules:
-    - host: homarr.riotpiao.com
+    - host: authentik.riotpiao.com
       http:
         paths:
           - path: /
             pathType: Prefix
             backend:
               service:
-                name: homarr
+                name: authentik-server
                 port:
-                  number: 7575
+                  number: 80
-

k8s/bootstrap/ingress/riotpiao-com-cert.yaml

@@ -1,13 +1,21 @@
 # k8s/bootstrap/ingress/riotpiao-com-cert.yaml
 # Public wildcard cert for *.riotpiao.com, signed by Let's Encrypt (DNS-01 via
-# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml). Distinct from
+# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml).
-# homelab-tls (self-signed homelab-ca, internal *.riotpiao.com) —
-# reference riotpiao-com-tls explicitly on ingresses that expose riotpiao.com
-# hosts to the internet; it is not nginx's default-ssl-certificate.
 #
-# issuerRef starts on letsencrypt-staging. Switch to letsencrypt-prod once the
+# nginx serves this secret as its default-ssl-certificate
-# staging cert issues cleanly, then delete this Certificate's Secret to force
+# (../ingress/nginx-values.yaml), so every Ingress that omits a tls: block uses
-# a re-issue against prod (staging certs aren't browser-trusted).
+# it. Do not add per-host tls: blocks for *.riotpiao.com hosts: they create a
+# second, single-host certificate and split the wildcard coverage.
+#
+# Renewal is automatic and does not drop traffic:
+#   - cert-manager starts a new order 30 days before expiry (renewBefore).
+#   - cert-manager writes the renewed pair into the same secret.
+#   - ingress-nginx watches the secret and hot-reloads. No pod restart.
+#   - The old pair stays valid until the new one is written, so no gap exists.
+#
+# The DNS-01 self-check must reach the public resolvers directly. Without the
+# --dns01-recursive-nameservers flags in ../../cert-manager/cert-manager-values.yaml
+# the self-check never sees the TXT record and renewal stalls until expiry.
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -45,7 +45,13 @@ gitea:
       SSH_DOMAIN: git.riotpiao.com
       SSH_PORT: 2222
       SSH_LISTEN_PORT: 2222
-      
+
+    # Self-signup was open (GET /user/sign_up returned 200), so anyone reaching
+    # forgejo.riotpiao.com could create an account. Accounts are provisioned by
+    # the admin only; admin create-user via UI/API is unaffected by this flag.
+    service:
+      DISABLE_REGISTRATION: true
+
     database:
       DB_TYPE: postgres
       HOST: forgejo-db-rw.cicd.svc.cluster.local:5432
@@ -101,7 +107,9 @@ ingress:
   enabled: true
   className: nginx
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation. forgejo.riotpiao.com is covered by the
+    # *.riotpiao.com wildcard, served as nginx's default-ssl-certificate.
+    #
     # This chart Ingress and the hand-written one in
     # k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com.
     # ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's
@@ -119,10 +127,9 @@ ingress:
       paths:
         - path: /
           pathType: Prefix
-  tls:
+  # tls: intentionally omitted. The chart would create a second, single-host
-    - secretName: forgejo-tls
+  # certificate (forgejo-tls) that expired on 2026-10-23 and competed with the
-      hosts:
+  # wildcard. The wildcard covers this host.
-        - forgejo.riotpiao.com
 
 # Resources
 resources:

k8s/bootstrap/phase4-argocd/argocd-values.yaml

@@ -138,12 +138,15 @@ configs:
     #   cicd  — automation-only; apiKey (token) for the CD pipeline, no UI login.
     accounts.rock: apiKey,login
     accounts.cicd: apiKey
-    # Authentik OIDC. clientSecret pulled from the argocd `oidc-secret` Secret
+    # Zitadel OIDC. clientSecret pulled from the argocd `oidc-secret` Secret.
-    # (created by authentik-provision). The groups claim drives RBAC below.
+    # The groups claim drives RBAC below.
     oidc.config: |
-      name: Authentik
+      name: Zitadel
-      issuer: https://authentik.riotpiao.com/application/o/argocd/
+      issuer: https://zitadel.riotpiao.com
-      clientID: argocd
+      # MUST be quoted: an unquoted 18-digit number is parsed as a float by the
+      # YAML parser and ArgoCD rejects it ("cannot unmarshal number into ...
+      # clientID of type string"). Same class of bug as Grafana's client_id.
+      clientID: "396757801210282367"
       clientSecret: $oidc-secret:client-secret
       requestedScopes:
         - openid
@@ -158,7 +161,7 @@ configs:
     server.insecure: false
 
   # RBAC. local `admin` + `rock` + the `cicd` pipeline account all get role:admin;
-  # the Authentik `homelab-admins` group (rock is a member) maps to admin so SSO
+  # the Zitadel `homelab-admins` role (rock is a member) maps to admin so SSO
   # logins are admin too.
   rbac:
     policy.default: role:readonly

k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml

@@ -0,0 +1,177 @@
+# etcd backend maintenance: compact, defrag, disarm NOSPACE.
+#
+# Why this exists
+# ---------------
+# etcd's backend file only grows. The apiserver compacts every 5m, but
+# compaction marks superseded revisions free *inside* the file; only `defrag`
+# returns that space to the filesystem. Nothing ran defrag, so the file grew
+# until it crossed the 2 GiB quota:
+#
+#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
+#               -> etcd raised NOSPACE on those two members
+#               -> etcd rejected ALL writes cluster-wide
+#               -> kubelets could not post node status: 3 nodes NotReady
+#               -> apiserver writes returned 500, the tunnel API path 502
+#
+# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
+# dead pages. Recovery was `compact <current-revision>` then `defrag` per
+# member then `alarm disarm`. Compaction had also stalled while the alarm was
+# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
+# worked. Order matters: compact first, then defrag.
+#
+# The earlier revision of this file never ran, for two independent reasons:
+#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
+#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
+#      cluster never had the CronJob or the Secret.
+#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
+#      `exec: "/bin/sh": no such file or directory`). Any container that runs
+#      a shell script needs an image that ships one.
+# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
+# 3.6.4, matching the 3.6.11 servers) pinned by digest.
+#
+# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
+# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
+# and the script exits unless the UTC epoch day is a multiple of 15. That gives
+# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
+# on the 1st, 16th and 31st, which puts two runs one day apart).
+#
+# Safety
+# ------
+# 1. Compaction uses the MINIMUM revision across members. Compacting to a
+#    revision a member has not reached wedges that member - measured on
+#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
+#    needed a reboot to rejoin.
+# 2. A member whose raft applied index lags the leader by more than 100000 is
+#    skipped and reported. Defragging a frozen member reclaims nothing.
+# 3. Members are defragged one at a time. A defrag pauses the member's apply
+#    loop for the duration of the file copy.
+#
+# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
+# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
+# Its client certificate is signed by the cluster's etcd CA and expires
+# 2036-10; rotate it before then by re-running the mint and re-encrypting.
+#
+# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
+# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-defrag
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: etcd-defrag
+    app.kubernetes.io/part-of: cluster-maintenance
+spec:
+  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 600
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: defrag
+              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
+              env:
+                - name: ETCDCTL_API
+                  value: "3"
+                - name: ETCDCTL_CACERT
+                  value: /etc/etcd/ca.crt
+                - name: ETCDCTL_CERT
+                  value: /etc/etcd/client.crt
+                - name: ETCDCTL_KEY
+                  value: /etc/etcd/client.key
+                - name: EPS
+                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  set -u
+
+                  DAY=$(( $(date -u +%s) / 86400 ))
+                  if [ $(( DAY % 15 )) -ne 0 ]; then
+                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
+                    exit 0
+                  fi
+
+                  echo "== before =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+
+                  MINREV=""
+                  MAXAPPLIED=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
+                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
+                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
+                      MINREV="$rev"
+                    fi
+                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
+                      MAXAPPLIED="$app"
+                    fi
+                  done
+
+                  if [ -z "$MINREV" ]; then
+                    echo "FATAL: no member returned a revision; nothing attempted"
+                    exit 1
+                  fi
+
+                  echo "== compact to revision $MINREV =="
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
+
+                  # Physical deletion of superseded keys runs in the background
+                  # after the compaction commit; give it a moment to drain.
+                  sleep 20
+
+                  echo "== defrag each member (sequential, one at a time) =="
+                  rc=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
+                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
+                      rc=1
+                      continue
+                    fi
+                    echo "-- $ep"
+                    if ! etcdctl --endpoints="$ep" defrag; then
+                      echo "DEFRAG-FAILED $ep"
+                      rc=1
+                    fi
+                  done
+
+                  echo "== disarm NOSPACE =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
+                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
+                  done
+
+                  echo "== after =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
+                  exit $rc
+              resources:
+                requests: { cpu: 10m, memory: 32Mi }
+                limits:   { cpu: 200m, memory: 128Mi }
+              volumeMounts:
+                - name: creds
+                  mountPath: /etc/etcd
+                  readOnly: true
+          volumes:
+            - name: creds
+              secret:
+                secretName: etcd-defrag-creds
+                defaultMode: 0444

k8s/infra/databases/kustomization.yaml

@@ -7,5 +7,6 @@ kind: Kustomization
 resources:
   - namespaces.yaml
   - authentik-db.yaml
+  - zitadel-db.yaml
   - temporal-db.yaml
   - paperless-db.yaml

k8s/infra/databases/zitadel-db.yaml

@@ -0,0 +1,42 @@
+# Dedicated CNPG Postgres for ZITADEL (GitOps, wave 2 — before the zitadel
+# chart at wave 3). Same shape as authentik-db.yaml: CNPG generates the secret
+# `zitadel-db-app` and the service `zitadel-db-rw` in ns iam, and
+# k8s/infra/iam/zitadel-values.yaml reads the DSN from that secret locally (no
+# cross-namespace secret copy).
+#
+# ZITADEL requires PostgreSQL 14+; the repo convention image is 16.2.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: zitadel-db
+  namespace: iam
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 3
+  imageName: ghcr.io/cloudnative-pg/postgresql:16.2
+  bootstrap:
+    initdb:
+      database: zitadel
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 10Gi
+    storageClass: longhorn-cnpg
+  monitoring:
+    enablePodMonitor: true
+  affinity:
+    # preferred + tolerations: HA across nodes without deadlocking a partly-
+    # schedulable 3-CP (same as authentik-db and forgejo-db).
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/infra/iam/vault-values.yaml

@@ -81,6 +81,15 @@ server:
     - envName: VAULT_UNSEAL_KEY_3
       secretName: vault-unseal-keys
       secretKey: key3
+    # Zitadel OIDC client credentials for the Vault UI login method. Consumed by
+    # the postStart hook below, which enables auth/oidc via the Vault API after
+    # unsealing. Vault auth methods cannot be declared in HCL, only via API/CLI.
+    - envName: OIDC_CLIENT_ID
+      secretName: vault-oidc-creds
+      secretKey: OIDC_CLIENT_ID
+    - envName: OIDC_CLIENT_SECRET
+      secretName: vault-oidc-creds
+      secretKey: OIDC_CLIENT_SECRET
 
   # ── Auto-unseal ─────────────────────────────────────────────────────────────
   # Vault starts sealed after every pod restart and can't serve requests until
@@ -97,6 +106,28 @@ server:
       vault operator unseal "$VAULT_UNSEAL_KEY_2" || true
       vault operator unseal "$VAULT_UNSEAL_KEY_3" || true
 
+      # ── Zitadel OIDC login method ────────────────────────────────────────
+      # Vault auth methods are API-driven: they cannot be declared in HCL.
+      # Enable auth/oidc and write its config on every pod start. Both calls
+      # are idempotent (auth enable returns 400 "already enabled", which we
+      # tolerate; the config write always succeeds and overwrites).
+      vault auth enable oidc 2>/dev/null || true
+      vault write auth/oidc/config \
+        oidc_discovery_url="https://zitadel.riotpiao.com" \
+        oidc_client_id="$OIDC_CLIENT_ID" \
+        oidc_client_secret="$OIDC_CLIENT_SECRET" \
+        default_role="default" 2>/dev/null || true
+      # The role binds the Zitadel client_id as audience and whitelists the UI
+      # callback URL. Policy `default` gives a logged-in operator standard
+      # read access; admin actions still require a token with root/sudo.
+      vault write auth/oidc/role/default \
+        bound_audiences="$OIDC_CLIENT_ID" \
+        allowed_redirect_uris="https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback" \
+        allowed_redirect_uris="http://localhost:8200/ui/vault/auth/oidc/oidc/callback" \
+        user_claim="preferred_username" \
+        oidc_scopes="openid,profile,email,groups" \
+        policies="default" 2>/dev/null || true
+
   # ── Vault config (HCL) ──────────────────────────────────────────────────────
   standalone:
     enabled: true

k8s/infra/iam/zitadel-values.yaml

@@ -0,0 +1,226 @@
+# k8s/infra/iam/zitadel-values.yaml
+# ZITADEL — Authentik replacement. Coexistence is deliberate: Authentik keeps
+# running, read-only, in the same `iam` namespace until the last consumer moves.
+# Migration plan: ~/workplace/homelab-doc/ZITADEL_MIGRATION.html (outside git).
+#
+# Chart: zitadel/zitadel 10.4.0, app v4.19.2. The plan cites v4.19.4; the chart
+# has not published it yet. Pin the chart, never `*`, until the v4.19.4 chart
+# exists — a silent app bump invalidates the verified masterkey/passwap facts.
+#
+# URL: https://zitadel.riotpiao.com. TLS terminates at ingress-nginx, which
+# serves the wildcard riotpiao-com-tls as its default-ssl-certificate, so
+# ExternalSecure=true and TLS.Enabled=false (no per-rule tls: block needed).
+#
+# Database: CNPG cluster zitadel-db (wave 2). The DSN comes from CNPG's
+# generated `zitadel-db-app` secret. When ZITADEL_DATABASE_POSTGRES_DSN is set,
+# ZITADEL ignores the discrete Database.Postgres.* fields, so neither a
+# password nor a host is written into this file.
+#
+# Masterkey: Secret `zitadel-masterkey` (k8s/argocd/secrets/
+# zitadel-masterkey.enc.yaml), key `masterkey`, exactly 32 bytes. It wraps the
+# rows in system.encryption_keys, which encrypt every client secret and private
+# key. Losing it makes those values unreadable: keep the offline sealed copy.
+#
+# Ingress uses the chart's own objects (controller: nginx adds the gRPC backend
+# annotation ZITADEL needs for its HTTP/2 API). Do not hand-write a second
+# Ingress for this host.
+
+zitadel:
+  configmapConfig:
+    ExternalSecure: true
+    ExternalDomain: zitadel.riotpiao.com
+    TLS:
+      Enabled: false
+
+    # First boot is COMPLETE (2026-10-26): the instance, the system org and the
+    # `iam-admin` machine key all exist. Skip stops the setup hook from running
+    # the first-instance step again. Without it, every sync re-runs the
+    # machinekeyWriter, which silently rotates the iam-admin key Terraform
+    # reads. ZITADEL objects are declared by scripts/iam/zitadel-provision.py.
+    # Never set this back to false.
+    FirstInstance:
+      Skip: true
+
+  # Rendered into a Secret by the chart; read by pods and init jobs alike.
+  masterkeySecretName: zitadel-masterkey
+
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+
+  # Best practice: run as non-root, drop all capabilities, and refuse
+  # privilege escalation. The chart's image already supports a non-root UID.
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+
+  # FirstInstance defaults stay for the FIRST sync: the chart creates org
+  # `zitadel` plus the `iam-admin` machine user and its machine-key Secret.
+  # Terraform consumes that key to declare projects, roles, OIDC apps and the
+  # groups action.
+  #
+  # SECOND-SYNC HAZARD — read before enabling automated sync.
+  # The chart marks eight objects as `pre-install,pre-upgrade` hooks (two
+  # ServiceAccounts, two ConfigMaps, Role, RoleBinding, zitadel-init,
+  # zitadel-setup), and one cleanup Job as `post-delete`. ArgoCD maps the first
+  # eight to PreSync with `before-hook-creation`: it deletes and re-creates them
+  # on EVERY sync, not only on a chart change.
+  #   - zitadel-init  (`init zitadel`) — schema step
+  #   - zitadel-setup (`setup`)        — first instance + iam-admin machine key
+  # On an initialized instance the setup step can fail, which aborts the sync
+  # before the Deployment is applied. If it succeeds instead, its
+  # machinekeyWriter container re-applies the iam-admin key Secret that
+  # Terraform reads, which silently rotates a credential in use.
+  # After the first sync reports healthy:
+  #   1. set configmapConfig.FirstInstance.Skip: true here — permanently: the
+  #      setup steps are schema-only once the instance exists, and Skip alone
+  #      stops the machinekeyWriter rotation
+  #   2. set initJob.enabled: false below
+  #   3. commit, sync once, then enable automated+selfHeal on the Application
+  # For a later schema upgrade, re-enable initJob.enabled only — never set
+  # FirstInstance.Skip back to false.
+  # ArgoCD never prunes a hook Job, so a Job that failed before step 2 stays and
+  # keeps ZitadelSetupJobFailed firing until a human deletes it.
+
+# DSN mode (chart renders this through the zitadel.dbEnv helper). CNPG's `uri`
+# key is a postgresql:// URL, which ZITADEL accepts as a DSN.
+env:
+  - name: ZITADEL_DATABASE_POSTGRES_DSN
+    valueFrom:
+      secretKeyRef:
+        name: zitadel-db-app
+        key: uri
+
+# Bundled Bitnami Postgres is disabled: the CNPG cluster above is the system of
+# record. Enabling it would create a second, unmanaged database.
+postgresql:
+  enabled: false
+
+# CNPG already created the database, the `app` user and its grants, so the init
+# job runs only the ZITADEL-internal step. Running the full default command
+# would try to create a database and a superuser it does not own.
+#
+# enabled: false after the first successful boot (2026-10-26). The schema is in
+# place; the init hook has nothing left to do and re-running it on every sync
+# only adds a PreSync failure surface. Re-enable it for a deliberate schema
+# upgrade, then set it back to false.
+initJob:
+  enabled: false
+  command: "zitadel"
+  activeDeadlineSeconds: 600
+
+# The setup job runs the full first-instance creation: schema migrations, the
+# system org, the iam-admin machine key, and 13 projection replays. 20 minutes
+# leaves room for a cold database without masking a genuine hang.
+#
+# DONE, 2026-10-26 — first boot completed. The annotations are moved from
+# pre-install/pre-upgrade to post-delete because a PreSync setup run is not
+# harmless after boot:
+#  - it re-runs every repeatable migration on every sync;
+#  - interrupting one (a sync aborted, or a human deleting the Job) leaves a
+#    `system.migration.started` event with no matching `done`, and ZITADEL then
+#    waits forever on `migration already started`. That is exactly how this
+#    instance deadlocked on projections.projects4 and later on
+#    repeatable_delete_stale_org_fields.
+# A post-delete hook is never executed by ArgoCD sync, so setup runs only on
+# first install. Re-enable it deliberately (and only with initJob too) for a
+# schema upgrade, then revert.
+setupJob:
+  activeDeadlineSeconds: 1200
+  annotations:
+    helm.sh/hook: post-delete
+    helm.sh/hook-delete-policy: hook-succeeded
+    helm.sh/hook-weight: "2"
+
+# Two API replicas: the read path survives one node drain, and both share the
+# same masterkey and database. The login UI stays at one replica — a session
+# cookie is not shared, so a second pod buys nothing.
+replicaCount: 2
+revisionHistoryLimit: 3
+
+# Best practice: a PDB guarantees at least one API pod stays available during
+# voluntary disruptions (node drain, rolling upgrade). Without it a drain can
+# take both replicas down at once. minAvailable 1 is correct for 2 replicas;
+# minAvailable 2 would block every drain forever.
+pdb:
+  enabled: true
+  minAvailable: 1
+
+# Best practice: spread replicas across nodes so a single node failure cannot
+# take both API pods. Anti-affinity alone only prefers separation; topology
+# spread enforces the failure-domain budget.
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: ScheduleAnyway
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: zitadel
+
+ingress:
+  enabled: true
+  className: nginx
+  controller: nginx
+
+login:
+  enabled: true
+  # ksops owns this Secret (k8s/argocd/secrets/zitadel-login-service-key.enc.yaml,
+  # rendered by the sops-secrets Application). Pointing the chart at it stops the
+  # chart from auto-generating an RSA keypair every render, which changed
+  # checksum/secret-login-service-key and kept both Deployments permanently
+  # OutOfSync. The keypair must stay RSA: the login container signs JWTs (RS256).
+  loginServiceKeySecretName: zitadel-login-service-key
+  replicaCount: 1
+  revisionHistoryLimit: 3
+  ingress:
+    enabled: true
+    className: nginx
+  # Best practice: the login UI is the public attack surface. Harden the pod
+  # even at one replica. The chart has no login PDB key; a single replica has
+  # nothing to protect with minAvailable.
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    runAsGroup: 1000
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL
+    seccompProfile:
+      type: RuntimeDefault
+
+# Chart-owned ServiceMonitor. The cluster's Prometheus runs with
+# serviceMonitorSelectorNilUsesHelmValues=false, so the release label is not
+# required for discovery. Endpoint is /debug/metrics on port http2-server.
+metrics:
+  enabled: true
+  serviceMonitor:
+    enabled: true
+    scrapeInterval: 30s
+    scrapeTimeout: 10s
+
+# The 3-CP / 2-worker cluster mixes control-plane taints with worker capacity.
+# Without the toleration a rolling upgrade can leave both replicas Pending.
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+affinity:
+  podAntiAffinity:
+    preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 100
+        podAffinityTerm:
+          topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: zitadel

k8s/infra/local-path/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No commonLabels on purpose: this is a vendored third-party manifest and
+# kustomize would also rewrite the provisioner Deployment's selector, which
+# is immutable on re-apply.
+resources:
+  - local-path-storage.yaml

k8s/infra/local-path/local-path-storage.yaml

@@ -0,0 +1,200 @@
+# Vendored from https://github.com/rancher/local-path-provisioner
+#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
+#
+# Local modifications (the only three; re-apply them on any re-vendor):
+#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
+#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
+#      baseline policy, which forbids that;
+#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
+#      filesystem read-only, so upstream's /opt path cannot be created;
+#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: local-path-provisioner-role
+  namespace: local-path-storage
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-path-provisioner-bind
+  namespace: local-path-storage
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+        - name: local-path-provisioner
+          image: docker.io/rancher/local-path-provisioner:v0.0.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - local-path-provisioner
+            - --debug
+            - start
+            - --config
+            - /etc/config/config.json
+          ports:
+            - name: health
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 12
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config/
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_MOUNT_PATH
+              value: /etc/config/
+            - name: HEALTH_PORT
+              value: "8080"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-path
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["/var/lib/local-path-provisioner"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node.kubernetes.io/disk-pressure
+          operator: Exists
+          effect: NoSchedule
+      containers:
+      - name: helper-pod
+        image: docker.io/library/busybox:1.36
+        imagePullPolicy: IfNotPresent

k8s/infra/logging/grafana-values.yaml

@@ -60,28 +60,28 @@ grafana.ini:
   database:
     wal: true
 
-  # ── Authentik OIDC (generic OAuth2) ───────────────────────────────────────
+  # ── ZITADEL OIDC (generic OAuth2) ────────────────────────────────────────
   # Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
-  # Authentik's discovery endpoint is internal-only (no external DNS for iam.svc).
+  # ZITADEL's discovery endpoint is internal-only (no external DNS for iam.svc).
   # All URLs use the external hostname so CoreDNS rewrites them to
-  # authentik-server.iam.svc — this keeps the Host header correct so Authentik
+  # zitadel.iam.svc — this keeps the Host header correct so ZITADEL
   # doesn't return localhost redirects in its token responses.
   #
   # role_attribute_path: JMESPath expression evaluated against the userinfo
   # response. akadmin gets GrafanaAdmin (server admin, can impersonate);
-  # homelab-admins members get Admin (org admin); everyone else Viewer.
+  # homelab-admins role members get Admin (org admin); everyone else Viewer.
   auth.generic_oauth:
     enabled: true
-    name: Authentik
+    name: Zitadel
     allow_sign_up: true  # Allow sync to create/update users from OAuth claims
-    client_id: grafana
+    client_id: "396757801965191169"
     scopes: openid email profile groups
-    auth_url: https://authentik.riotpiao.com/application/o/authorize/
+    auth_url: https://zitadel.riotpiao.com/oauth/v2/authorize
-    token_url: https://authentik.riotpiao.com/application/o/token/
+    token_url: https://zitadel.riotpiao.com/oauth/v2/token
-    api_url: https://authentik.riotpiao.com/application/o/userinfo/
+    api_url: https://zitadel.riotpiao.com/oidc/v1/userinfo
     # Read identity straight from the userinfo/id_token claims. Without these,
     # Grafana falls back to the GitHub-style "<api_url>/emails" call, which
-    # Authentik doesn't serve — request 404s with "Error getting email address"
+    # ZITADEL doesn't serve — request 404s with "Error getting email address"
     # and the whole OAuth login fails.
     email_attribute_path: email
     login_attribute_path: preferred_username  # Use preferred_username — OpenID standard claim, always present
@@ -91,10 +91,10 @@ grafana.ini:
     use_pkce: false
     use_refresh_token: false
     skip_org_role_sync: false  # Sync org roles from OAuth groups
-    tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
+    tls_skip_verify_insecure: true   # In-cluster cert; verify in prod
 
 # GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
-# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts
+# Secret (k8s/argocd/secrets/grafana-oidc-secrets.enc.yaml). envFromSecret mounts
 # every key in that secret as an environment variable — keeps secrets out of
 # values files and out of git.
 envFromSecret: grafana-oidc

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -41,13 +41,24 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          # - /var/lib/longhorn-disk4 (sde1, 2TB)
+          #
+          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
+          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
+          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
+          # process that touched it blocked in unkillable D state. Those unkillable
+          # processes held Longhorn's per-node rebuild slots forever, so no new
+          # replica could start on talos-cp-2 and every new volume there failed to
+          # attach. It is removed from the Terraform disk list too, so a re-render
+          # does not put it back.
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          if [ "$CURRENT_DISKS" -lt 5 ]; then
+          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
+          # existed; leaving the old threshold would re-apply the patch on every
+          # run once disk4 is gone.
+          if [ "$CURRENT_DISKS" -lt 4 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -75,14 +86,6 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
-                  },
-                  "disk4": {
-                    "allowScheduling": true,
-                    "diskType": "filesystem",
-                    "evictionRequested": false,
-                    "path": "/var/lib/longhorn-disk4",
-                    "storageReserved": 21474836480,
-                    "tags": []
                   }
                 }
               }

k8s/infra/minio/minio-provision-paperless-job.yaml

@@ -52,14 +52,15 @@ spec:
       restartPolicy: Never
       initContainers:
         - name: kubectl-copy
-          image: bitnami/kubectl:latest
+          image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
           command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
           volumeMounts:
             - name: shared
               mountPath: /shared
       containers:
         - name: provision
-          image: quay.io/minio/mc:latest
+          # bitnamilegacy/minio-client:2025.7.21-debian-12-r3 replaces the dead quay.io/minio/mc image
+          image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
           volumeMounts:
             - name: shared
               mountPath: /shared

k8s/infra/minio/minio-tenant.yaml

@@ -11,7 +11,15 @@ metadata:
     prometheus.io/port: "9000"
     prometheus.io/scrape: "true"
 spec:
-  image: minio/minio:RELEASE.2025-07-23T15-54-02Z
+  # Was minio/minio:RELEASE.2025-07-23T15-54-02Z — that Docker Hub repository no longer
+  # exists (404 for every tag), so the tenant sat in ImagePullBackOff and Vault's S3
+  # backend could not resolve minio-cluster-hl. Official images now need registry auth
+  # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
+  # release; swap it for the official image once MinIO registry credentials exist.
+  # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
+  # the pull again. The same digest already sits in the containerd cache on talos-cp-1,
+  # so the first start needs no registry access.
+  image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
 
   # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
   # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
@@ -95,17 +103,17 @@ spec:
         - https://minio-api.riotpiao.com
       console: https://minio.riotpiao.com
 
-  # ── OIDC via Authentik (server-side env, valid in v2 schema) ────────────────
+  # ── OIDC via ZITADEL (server-side env, valid in v2 schema) ──────────────────
-  # Use in-cluster URL for config fetch (pod→authentik); browser redirects use
+  # Use external URL for config fetch (hairpins through ingress-nginx); browser
-  # public URLs embedded in the OIDC metadata response (issuer stays public).
+  # redirects use public URLs embedded in the OIDC metadata response.
   env:
     - name: MINIO_IDENTITY_OPENID_CONFIG_URL
       # Must use external URL — well-known response contains external issuer/jwks_uri.
       # MinIO validates issuer in JWT matches well-known issuer. Internal URL = mismatch.
       # Hairpins through ingress-nginx but stays in-cluster.
-      value: "https://authentik.riotpiao.com/application/o/minio/.well-known/openid-configuration"
+      value: "https://zitadel.riotpiao.com/.well-known/openid-configuration"
     - name: MINIO_IDENTITY_OPENID_CLIENT_ID
-      value: "minio"
+      value: "396757803257036801"
     - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
       valueFrom:
         secretKeyRef:
@@ -118,7 +126,7 @@ spec:
     - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
       value: "https://minio.riotpiao.com/oauth_callback"
     - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME
-      value: "Authentik"
+      value: "Zitadel"
     - name: MINIO_IDENTITY_OPENID_SCOPES
       value: "openid,profile,email,groups"
     - name: MINIO_BROWSER_LOGIN_ANIMATION

k8s/infra/monitoring/alerts/api-gateway-alerts.yaml

@@ -158,3 +158,66 @@ spec:
             severity: warning
           annotations:
             summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
+
+    - name: api-gateway.admission
+      # The admission gate fails OPEN when its view of the engine gauges goes
+      # stale, so a broken observer degrades silently to the pre-#64 behaviour
+      # of admitting everything. These two series are the only detection —
+      # the gate has no automated tests.
+      rules:
+        # The gate fails OPEN when it cannot read the engine, so "no data" must
+        # alert: otherwise a missing emitter, a renamed metric, or a broken
+        # scrape is indistinguishable from a healthy observer. Both expressions
+        # therefore OR in an absent() guard on the metric itself.
+        - alert: APIGatewayObserverDegraded
+          expr: observer_degraded == 1 or absent(observer_degraded)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
+            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."
+
+        # Engine gauges went stale, or are not being reported at all
+        - alert: GatewayEngineStateStale
+          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
+            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."
+
+        # The acceptance metric for reasoning.yaml's prefill change. Preemption
+        # of a 65k prompt costs a full re-prefill (~250s at the measured
+        # 262 tok/s), so any sustained rate is a regression, not noise.
+        - alert: LLMPreemptions
+          # Guarded absent() is deliberately NOT used bare here: the engine
+          # restarts (strategy Recreate) and the series legitimately vanishes
+          # for the length of a boot, so a bare absent() would false-fire on
+          # every rollout. It only fires when the metric is missing WHILE the
+          # predictor is up, which is the real "renamed or unscraped" case.
+          expr: |
+            increase(vllm:num_preemptions_total[15m]) > 0
+            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
+            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."
+
+        # This change trades single-request latency for fairness: capping a long
+        # prompt at 2048 tokens/step means it takes more steps to finish, so a
+        # lone large prompt can now be slower even while preemptions stay 0.
+        # Without this alert the change has no way to show a regression.
+        - alert: LLMPrefillStarvation
+          expr: |
+            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
+            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
+            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."

k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

@@ -7,19 +7,24 @@ metadata:
     release: prometheus
 spec:
   groups:
-    # Recording rules: derive vLLM prefix cache hit rate from raw counters
+    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
+    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
+    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
+    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
+    # prefix_cache_queries_total — so this rule produced no data and
+    # VLLMPrefixCacheHitRateLow could never fire.
     - name: vllm.recording
       rules:
         - record: vllm:prefix_cache_hit_rate
           expr: |
-            rate(vllm:cache_query_hit_total[5m])
+            rate(vllm:prefix_cache_hits_total[5m])
             /
             clamp_min(
-              rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]),
+              rate(vllm:prefix_cache_queries_total[5m]),
               0.001
             )
           labels:
-            job: "vllm"
+            job: "reasoning-predictor"
 
     # Alert rules: CAG pipeline health
     - name: cag.alerts

k8s/infra/monitoring/alerts/cluster-alerts.yaml

@@ -27,6 +27,24 @@ spec:
           annotations:
             summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
 
+        # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
+        - alert: PodImagePullFailure
+          expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
+
+        # Pod not ready
+        - alert: PodNotReady
+          expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
+
         # CrashLoopBackOff
         - alert: PodCrashLooping
           expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0

k8s/infra/monitoring/alerts/svc-zitadel-rules.yaml

@@ -0,0 +1,68 @@
+# ZITADEL alert parity with the Authentik rules it will replace.
+#
+# Only two signals are ported at this point. The metrics endpoint
+# (/debug/metrics, scraped by the chart's own ServiceMonitor in ns iam) exposes
+# no request or error counters that this cluster has verified, so no error-rate
+# rule is written here. Writing one against a guessed series name would produce
+# a permanently-firing or permanently-silent alert — both worse than none.
+# AuthentikOutpostDown is deliberately not ported: it monitors an outpost that
+# carries zero providers and is deleted with the migration, not migrated.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: zitadel-rules
+  namespace: iam
+spec:
+  groups:
+    - name: zitadel.rules
+      interval: 30s
+      rules:
+        - alert: ZitadelDown
+          # absent() is required: equality alone cannot fire when the series is
+          # gone (ServiceMonitor deleted, Service selector changed, pods scaled
+          # to zero). Those are exactly the outages a target-down rule exists for.
+          expr: absent(up{job="zitadel"}) or up{job="zitadel"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "ZITADEL API target is down"
+            description: "Prometheus cannot scrape the ZITADEL API for 10 minutes. Token issuance and every OIDC login depend on it. Check the zitadel pods and the zitadel-db cluster in namespace iam. Value: {{ $value }}"
+
+        - alert: ZitadelLoginUIDown
+          expr: absent(kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"}) or kube_deployment_status_replicas_available{namespace="iam",deployment="zitadel-login"} == 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ZITADEL login UI has no available replica"
+            description: "The login UI serves /ui/v2/login. With zero available replicas a browser login completes at the IdP but cannot render. Value: {{ $value }}"
+
+        - alert: ZitadelRestarting
+          # pod!~"zitadel-db-.*" matters: the CNPG database pods share the
+          # `zitadel` name prefix, and a database restart must not page as an
+          # IdP crash loop.
+          expr: absent(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}) or increase(kube_pod_container_status_restarts_total{namespace="iam",pod=~"zitadel.*",pod!~"zitadel-db-.*"}[1h]) > 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "ZITADEL pod restarts more than 3 times per hour"
+            description: "A crash loop on the IdP is a cluster-wide login outage. Check the pod logs and the masterkey/database secret references. Value: {{ $value }}"
+
+        # The chart's zitadel-init and zitadel-setup Jobs are PreSync hooks with
+        # restartPolicy: Never, so their restart counter stays 0. Without this
+        # rule a failed schema init or first-instance setup reaches nobody but
+        # the person who runs `kubectl -n iam get job`.
+        # No absent() guard here on purpose: the chart ships only
+        # `before-hook-creation`, so a successful hook Job is replaced at the
+        # next PreSync. Once the jobs are retired, absent() would fire on a
+        # healthy cluster.
+        - alert: ZitadelSetupJobFailed
+          expr: kube_job_status_failed{namespace="iam",job_name=~"zitadel-(init|setup).*"} > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "ZITADEL init/setup job failed"
+            description: "A PreSync job failed, so ArgoCD aborted the sync before the Deployment was applied. Read the job pod logs, then delete the Job object to let the next sync retry. Value: {{ $value }}"

k8s/infra/monitoring/blackbox-exporter-values.yaml

@@ -43,8 +43,8 @@ serviceMonitor:
       url: https://grafana.riotpiao.com/
     - name: loki
       url: https://loki.riotpiao.com/ready
-    - name: authentik
+    - name: zitadel
-      url: https://authentik.riotpiao.com/-/health/ready/
+      url: https://zitadel.riotpiao.com/debug/ready
     - name: vault
       url: https://vault.riotpiao.com/v1/sys/health
     - name: minio-console

k8s/infra/monitoring/dashboards/generate-dashboards.py

@@ -26,20 +26,23 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr
     return p
 
 
-def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
     targets = []
     for e in exprs:
         if isinstance(e, tuple):
             targets.append({"expr": e[0], "legendFormat": e[1]})
         else:
             targets.append({"expr": e, "legendFormat": "{{pod}}"})
-    return {
+    p = {
         "id": id, "title": title, "type": "timeseries",
         "gridPos": {"h": h, "w": w, "x": x, "y": y},
         "datasource": DS_PROM,
         "fieldConfig": {"defaults": {"unit": unit}},
         "targets": targets,
     }
+    if description:
+        p["description"] = description
+    return p
 
 
 def table_panel(id, title, expr, x, y, w=12, h=8):
@@ -265,6 +268,237 @@ def build_api_gateway():
     }
 
 
+# ============================================================================
+# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
+# ============================================================================
+#
+# Two independent metric sources, deliberately both shown:
+#
+#   vLLM + TEI (llm-serving)  — engine-truth, but only for the three engines
+#                               that expose /metrics. Ollama (ornith:35b,
+#                               qwen2.5:3b-instruct) exposes nothing at all.
+#   api-gateway (/metrics)    — covers EVERY model including the two Ollama
+#                               ones, derived from the SSE byte stream: TTFT is
+#                               request-start to first byte, ITL is the gap
+#                               between SSE writes carrying "data: ", so it
+#                               counts SSE chunks rather than true tokens.
+#                               vLLM's own histograms remain authoritative for
+#                               `reasoning`.
+#
+# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
+# verified against the running gateway and Prometheus:
+#   llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
+#
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
+# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
+# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
+# homelab-frontend#63 fixed exactly that — the wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
+# The "dead code" note that used to live in this header is superseded.
+#
+# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
+# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
+#
+#
+# All metric names below were verified against a live scrape, not assumed:
+#   vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
+#   NOT cache_query_hit_total / cache_query_miss_total.
+
+def build_llm_observability():
+    def hq(metric, q, by="le"):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
+            f"p{int(q * 100)}",
+        )
+
+    def hq_gw(metric, q):
+        return (
+            f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
+            f"p{int(q * 100)} · {{{{model}}}}",
+        )
+
+    def avg_of(metric):
+        # mean = rate(_sum) / rate(_count) — valid for cumulative histograms
+        return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
+
+    panels = []
+
+    # ── TTFT ────────────────────────────────────────────────────────────────
+    panels.append(row(100, "Latency — Time To First Token", 0, [
+        stat_panel(101, "TTFT p95 (gateway)",
+                   "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
+                   0, 1, w=4, h=4, unit="s"),
+        stat_panel(102, "TTFT p95 (vLLM)",
+                   "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
+                   4, 1, w=4, h=4, unit="s"),
+        stat_panel(103, "KV cache in use",
+                   "max(vllm:kv_cache_usage_perc)",
+                   8, 1, w=4, h=4, unit="percentunit"),
+        stat_panel(104, "Requests running",
+                   "sum(vllm:num_requests_running)",
+                   12, 1, w=4, h=4, unit="short"),
+        stat_panel(105, "Requests waiting",
+                   "sum(vllm:num_requests_waiting)",
+                   16, 1, w=4, h=4, unit="short"),
+        stat_panel(106, "Preemptions (5m)",
+                   "increase(vllm:num_preemptions_total[5m])",
+                   20, 1, w=4, h=4, unit="short"),
+        ts_panel(110, "TTFT percentile by model (gateway)",
+                 [hq_gw("llm_ttft_seconds", 0.50),
+                  hq_gw("llm_ttft_seconds", 0.95),
+                  hq_gw("llm_ttft_seconds", 0.99)],
+                 0, 2, w=12, h=8, unit="s"),
+        ts_panel(111, "TTFT percentile — vLLM reasoning",
+                 [hq("vllm:time_to_first_token_seconds", 0.50),
+                  hq("vllm:time_to_first_token_seconds", 0.95),
+                  hq("vllm:time_to_first_token_seconds", 0.99)],
+                 12, 2, w=12, h=8, unit="s"),
+        ts_panel(112, "Prefill vs decode (mean)",
+                 [(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
+                  (avg_of("vllm:request_decode_time_seconds"), "decode")],
+                 0, 3, w=12, h=8, unit="s"),
+        ts_panel(113, "Queue wait before scheduling (mean)",
+                 [(avg_of("vllm:request_queue_time_seconds"), "queue")],
+                 12, 3, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Inter-token latency ─────────────────────────────────────────────────
+    panels.append(row(120, "Latency — Inter-Token", 4, [
+        ts_panel(121, "Inter-token latency by model (gateway)",
+                 [hq_gw("llm_itl_seconds", 0.50),
+                  hq_gw("llm_itl_seconds", 0.95),
+                  hq_gw("llm_itl_seconds", 0.99)],
+                 0, 5, w=12, h=8, unit="s"),
+        ts_panel(122, "Inter-token latency — vLLM reasoning",
+                 [hq("vllm:inter_token_latency_seconds", 0.50),
+                  hq("vllm:inter_token_latency_seconds", 0.95),
+                  hq("vllm:inter_token_latency_seconds", 0.99)],
+                 12, 5, w=12, h=8, unit="s"),
+        ts_panel(123, "Decode throughput (tokens/s per request)",
+                 [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
+                 0, 6, w=12, h=8, unit="short"),
+        ts_panel(124, "End-to-end request latency p50/p95/p99",
+                 [hq("vllm:e2e_request_latency_seconds", 0.50),
+                  hq("vllm:e2e_request_latency_seconds", 0.95),
+                  hq("vllm:e2e_request_latency_seconds", 0.99)],
+                 12, 6, w=12, h=8, unit="s"),
+    ]))
+
+    # ── Batching / scheduling ───────────────────────────────────────────────
+    panels.append(row(130, "Batching & Scheduling", 7, [
+        ts_panel(131, "vLLM running vs waiting requests (batch depth)",
+                 [("sum(vllm:num_requests_running)", "running (in batch)"),
+                  ("sum(vllm:num_requests_waiting)", "waiting (queued)")],
+                 0, 8, w=12, h=8, unit="short"),
+        ts_panel(132, "Preemption rate (KV cache eviction pressure)",
+                 [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
+                 12, 8, w=12, h=8, unit="short"),
+        ts_panel(133, "TEI batch size (embeddings / reranker)",
+                 [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
+                   "{{app}}")],
+                 0, 9, w=12, h=8, unit="short"),
+        ts_panel(134, "TEI queue depth",
+                 [("max(te_queue_size) by (app)", "{{app}}")],
+                 12, 9, w=12, h=8, unit="short"),
+    ]))
+
+    # ── KV cache ────────────────────────────────────────────────────────────
+    panels.append(row(140, "KV Cache", 10, [
+        ts_panel(141, "KV cache utilisation (GPU memory pressure)",
+                 [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
+                 0, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
+                 [("vllm:prefix_cache_hit_rate", "{{model_name}}")],
+                 12, 11, w=12, h=8, unit="percentunit"),
+        ts_panel(143, "Prefix cache hits vs queries",
+                 [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
+                  ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
+                 0, 12, w=12, h=8, unit="short"),
+        ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
+                 [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
+                 12, 12, w=12, h=8, unit="short"),
+    ]))
+
+    # ── Throughput & health ─────────────────────────────────────────────────
+    panels.append(row(150, "Throughput & Health", 13, [
+        ts_panel(151, "Token throughput",
+                 [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
+                  ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
+                  ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
+                 0, 14, w=12, h=8, unit="short"),
+        ts_panel(152, "Successful requests/s",
+                 [("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
+                  ("sum(rate(te_request_count[5m]))", "TEI")],
+                 12, 14, w=12, h=8, unit="short"),
+        ts_panel(153, "Backpressure — queue-full rejects",
+                 [("rate(queue_full_requests_total[5m])", "rejected/s")],
+                 0, 15, w=8, h=8, unit="short"),
+        ts_panel(154, "KV warmup requests delayed",
+                 [("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
+                 8, 15, w=8, h=8, unit="short"),
+        ts_panel(155, "Embedding / rerank latency p95",
+                 [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
+                   "{{app}} embed"),
+                  ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
+                   "{{app}} queue")],
+                 16, 15, w=8, h=8, unit="s"),
+    ]))
+
+    # ── Admission control ───────────────────────────────────────────────────
+    # These four panels read the gateway's admission gate, which is the
+    # companion change in homelab-frontend#64. That emitter is NOT deployed
+    # yet (verified against the live /metrics endpoint: 0 series matching
+    # ^(admission_|observer_|engine_state_)), so they render "No data" until
+    # #64 ships. That is expected, and each panel carries a description
+    # saying so — an unexplained blank panel would read as a broken
+    # dashboard, which is exactly the failure mode this row exists to catch.
+    panels.append(row(160, "Admission Control", 16, [
+        ts_panel(161, "Admission parked (5m)",
+                 [("increase(admission_parked_total[5m])", "parked")],
+                 0, 17, w=8, h=8, unit="short",
+                 description="Requests the gateway held back rather than admitting. "
+                             "Emitted by the admission gate (homelab-frontend#64); "
+                             "empty until that change is deployed."),
+        ts_panel(162, "Admission rejected by reason",
+                 [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
+                 8, 17, w=8, h=8, unit="short",
+                 description="Admission rejections split by the gate's own reason label. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(163, "Engine state stale",
+                 [("increase(engine_state_stale_total[5m])", "stale")],
+                 16, 17, w=8, h=8, unit="short",
+                 description="Times the gateway's view of the engine gauges went stale. "
+                             "The gate fails OPEN when this happens, so any non-zero rate "
+                             "means admission is silently degraded to admit-everything. "
+                             "Emitted by homelab-frontend#64; empty until deployed."),
+        ts_panel(164, "Gateway view vs engine truth",
+                 [("max(vllm_requests_running)", "gateway: running"),
+                  ("sum(vllm:num_requests_running)", "engine: running"),
+                  ("max(vllm_kv_usage_perc)", "gateway: kv (fraction)  [same as engine if healthy]"),
+                  ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
+                 0, 18, w=24, h=8, unit="short",
+                 description="The gateway's own reading of the engine against the engine's "
+                             "self-reported value. A disagreement between a pair is how a "
+                             "stale observer or a mistuned admission target shows itself, "
+                             "and nothing else in this dashboard reveals it. The two "
+                             "gateway series (vllm_requests_running, vllm_kv_usage_perc) "
+                             "ship with homelab-frontend#64 and are absent until it is "
+                             "deployed — until then this panel shows the engine's pair only."),
+    ]))
+
+    return {
+        "title": "LLM Observability",
+        "uid": "llm-observability",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["llm", "vllm", "tei", "gpu", "observability"],
+        "panels": panels,
+    }
+
+
 # ============================================================================
 # Generate
 # ============================================================================
@@ -277,4 +511,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
 # Dashboard 3
 write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
 
+# Dashboard 4
+write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
+
 print("Done.")

k8s/infra/monitoring/dashboards/llm-observability.yaml

@@ -0,0 +1,95 @@
+apiVersion: v1
+data:
+  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
+    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
+    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
+    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
+    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
+    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
+    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
+    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
+    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
+    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
+    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
+    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
+    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
+    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
+    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
+    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
+    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
+    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
+    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
+    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
+    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
+    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
+    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
+    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
+    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
+    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
+    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
+    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
+    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
+    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
+    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
+    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
+    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
+    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
+    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
+    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
+    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
+    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
+    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
+    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
+    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
+    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
+    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
+    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
+    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
+    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
+    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
+    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
+    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
+    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
+    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
+    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
+    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
+    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
+    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
+    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
+    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
+    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
+    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
+    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
+    the engine''s self-reported value. A disagreement between a pair is how a stale
+    observer or a mistuned admission target shows itself, and nothing else in this
+    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
+    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
+    then this panel shows the engine''s pair only."}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: LLM
+  labels:
+    grafana_dashboard: '1'
+  name: llm-observability-dashboard
+  namespace: logging

k8s/infra/monitoring/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - alerts/ingress-alerts.yaml
   - alerts/svc-argocd-rules.yaml
   - alerts/svc-authentik-rules.yaml
+  - alerts/svc-zitadel-rules.yaml
   - alerts/svc-forgejo-rules.yaml
   - alerts/svc-grafana-rules.yaml
   - alerts/svc-minio-rules.yaml
@@ -20,5 +21,8 @@ resources:
   - servicemonitors/forgejo.yaml
   - servicemonitors/minio.yaml
   - servicemonitors/ingress-nginx.yaml
+  - servicemonitors/vllm.yaml
+  - servicemonitors/api-gateway.yaml
   - dashboards/cluster-infrastructure.yaml
   - dashboards/api-gateway.yaml
+  - dashboards/llm-observability.yaml

k8s/infra/monitoring/prometheus-values.yaml

@@ -55,7 +55,10 @@ alertmanager:
 prometheus:
   prometheusSpec:
     retention: 30d
-    retentionSize: "95GB"
+    # Must stay under the PVC size below (25Gi). At 95GB the limit never
+    # triggered, the volume filled, and Prometheus crashlooped with
+    # "no space left on device" writing /prometheus/queries.active.
+    retentionSize: "18GB"
 
     # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
     # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
@@ -78,7 +81,7 @@ prometheus:
           accessModes: ["ReadWriteOnce"]
           resources:
             requests:
-              storage: 100Gi
+              storage: 25Gi
 
     resources:
       requests:

k8s/infra/monitoring/servicemonitors/api-gateway.yaml

@@ -0,0 +1,62 @@
+# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the
+# three latency series below recorded nothing. internal/proxy/llm_metrics.go
+# declared a capture wrapper (LLMMetricsCapture) that was never constructed
+# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels
+# stayed empty.
+#
+# homelab-frontend#63 fixed exactly that. The wrapper is now
+# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
+# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy
+# request path. Verified live on 2026-09-29:
+#   llm_ttft_seconds  count=7  sum=598.212s
+#   llm_itl_seconds   count=16116
+#   llm_tokens_total  3 series
+#
+# These derived-from-SSE series cover EVERY model including the two Ollama ones
+# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all.
+# They are proxy-side: TTFT is request-start to first byte and ITL is the gap
+# between SSE writes, so both are granular to SSE write boundaries rather than
+# to the engine's own token cadence. llm_tokens_total counts `data:` events,
+# and the documented assumption in internal/proxy/llm_metrics.go is that these
+# upstreams emit one such event per token — so it is a token count only as far
+# as that assumption holds. vLLM's own histograms remain authoritative.
+#
+# Also recorded and working (3 series, one per replica):
+#   queue_full_requests_total           counter    backpressure rejects
+#   warmup_requests_delayed_total       counter    vLLM slot contention
+#   vllm_prefix_cache_hit_tokens_total  counter    prefix-cache hits
+#
+# NOT working, do not build on it: request_duration_seconds is declared and
+# exported but has 0 samples, and its only writer Metrics.RecordRequest has
+# zero call sites outside _test.go — verified by ast-grep, not assumed. It is
+# the same dead-code shape this branch already corrected for the latency
+# series, so it is listed here rather than left to be rediscovered.
+#
+# Also note the Deployment already carried prometheus.io/scrape annotations,
+# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,
+# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being
+# collected before this ServiceMonitor existed.
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api-gateway
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: api-gateway
+    app.kubernetes.io/part-of: api
+spec:
+  namespaceSelector:
+    matchNames:
+      - api
+  selector:
+    matchLabels:
+      app: api-gateway
+  endpoints:
+    # api-gateway Service port IS named `http` (unlike the KServe ones), so
+    # the name form works here.
+    - port: http
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -11,20 +11,39 @@ spec:
     matchNames:
       - llm-serving
   selector:
-    matchLabels:
+    # Only the three engines that actually serve /metrics.
-      app.kubernetes.io/part-of: llm-serving
+    # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus
+    # endpoint (/metrics returns 404) — selecting them would add permanently
+    # DOWN targets. Ollama-model visibility comes from the api-gateway
+    # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are
+    # measured at the proxy, so they cover every model including Ollama).
+    matchExpressions:
+      - key: app.kubernetes.io/name
+        operator: In
+        values:
+          - llm-reasoning   # vLLM 0.11.0 — full engine metrics
+          - llm-embeddings  # TEI 1.8.2 — CPU
+          - llm-reranker    # TEI 1.8.2 — CPU
   endpoints:
-    - port: http
+    # targetPort, not port: the KServe-generated Services on llm-serving have
+    # UNNAMED ports, so `port: <name>` can never match. Naming them would mean
+    # editing the InferenceServices (predictor restart = full weights reload,
+    # tens of seconds on the vLLM slot), so scrape the pod port directly.
+    # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor.
+    - targetPort: 8080
       interval: 30s
       scrapeTimeout: 10s
       path: /metrics
       scheme: http
       relabelings:
-        - sourceLabels: [__meta_kubernetes_namespace]
+        # Prometheus's Kubernetes service discovery already supplies the
-          targetLabel: namespace
+        # namespace, pod and service labels; the only one that needs deriving
-        - sourceLabels: [__meta_kubernetes_pod_name]
+        # is `app`, since the TEI panels aggregate by it (by (app)) and SD does
-          targetLabel: pod
+        # not promote arbitrary pod labels.
-        - sourceLabels: [__meta_kubernetes_service_name]
+        #
-          targetLabel: service
+        # `action: replace` is written explicitly. It is the API server's
-        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+        # default for relabeling rules, so omitting it leaves the live object
+        # diverged from this manifest and ArgoCD reports permanent OutOfSync.
+        - action: replace
+          sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
           targetLabel: app

k8s/infra/oidc-bootstrap/bootstrap.yaml

@@ -0,0 +1,33 @@
+# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
+# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
+# tokens were never accepted -> every login got 401). Delete this whole directory after.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: oidc-bootstrap
+    namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: oidc-bootstrap

k8s/infra/poimen/kustomization.yaml

@@ -6,3 +6,7 @@ kind: Kustomization
 # deployments (wave 0) are applied.
 resources:
   - memory-db.yaml
+  - poimen-sandbox-namespace.yaml
+  - poimen-sandbox-quota.yaml
+  - pi-session-netpol.yaml
+  - pi-agent-workspace-pv.yaml

k8s/infra/poimen/pi-agent-workspace-pv.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  finalizers:
+  - kubernetes.io/pv-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace-pv
+  annotations:
+    argocd.argoproj.io/sync-wave: '-1'
+spec:
+  accessModes:
+  - ReadWriteOnce
+  capacity:
+    storage: 10Gi
+  claimRef:
+    apiVersion: v1
+    kind: PersistentVolumeClaim
+    name: pi-agent-workspace
+    namespace: poimen-sandbox
+  hostPath:
+    path: /var/lib/pi-agent-sandbox
+    type: DirectoryOrCreate
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+      - matchExpressions:
+        - key: kubernetes.io/hostname
+          operator: In
+          values:
+          - talos-cp-2
+  persistentVolumeReclaimPolicy: Retain
+  volumeMode: Filesystem
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    pv.kubernetes.io/bind-completed: 'yes'
+    argocd.argoproj.io/sync-wave: '-1'
+  finalizers:
+  - kubernetes.io/pvc-protection
+  labels:
+    app: pi-coding-agent-sandbox
+    part-of: poimen-toolkit
+  name: pi-agent-workspace
+  namespace: poimen-sandbox
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  storageClassName: ''
+  volumeMode: Filesystem
+  volumeName: pi-agent-workspace-pv

k8s/infra/poimen/pi-session-netpol.yaml

@@ -0,0 +1,12 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: pi-session-deny-kube-api
+  namespace: poimen-sandbox
+spec:
+  endpointSelector:
+    matchLabels:
+      app: pi-session
+  egressDeny:
+    - toEntities:
+        - kube-apiserver

k8s/infra/poimen/poimen-sandbox-namespace.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: poimen-sandbox
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/warn: restricted

k8s/infra/poimen/poimen-sandbox-quota.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: sandbox-quota
+  namespace: poimen-sandbox
+spec:
+  hard:
+    pods: "20"
+    requests.cpu: "10"
+    requests.memory: 20Gi
+    limits.cpu: "20"
+    limits.memory: 40Gi

k8s/infra/rbac/homarr-operator-role.yaml

@@ -1,32 +0,0 @@
-# Scoped operator access for homarr-admins: the homarr Deployment + app
-# secrets only. homarr-oidc, auth-oidc-secret, db-encryption stay excluded
-# (security-managed, not app config). Inert until kube-apiserver's OIDC
-# wiring lands.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: homarr-operator
-  namespace: dashboard
-rules:
-  - apiGroups: ["apps"]
-    resources: ["deployments"]
-    resourceNames: ["homarr"]
-    verbs: ["get", "list", "watch", "update", "patch"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    resourceNames: ["homarr-secrets"]
-    verbs: ["get", "list", "watch", "update", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: homarr-admins-binding
-  namespace: dashboard
-subjects:
-  - kind: Group
-    name: "oidc:homarr-admins"
-    apiGroup: rbac.authorization.k8s.io
-roleRef:
-  kind: Role
-  name: homarr-operator
-  apiGroup: rbac.authorization.k8s.io

k8s/infra/rbac/kustomization.yaml

@@ -10,12 +10,12 @@ resources:
   - grafana-operator-role.yaml
   - minio-operator-role.yaml
   - forgejo-operator-role.yaml
-  - homarr-operator-role.yaml
   - portainer-operator-role.yaml
   - kmsvc-operator-role.yaml
   - temporal-operator-role.yaml
   - llm-serving-operator-role.yaml
   - poimen-memory-operator-role.yaml
+  - sandbox-observer-role.yaml
 # paperless's Role/RoleBinding lives in k8s/apps/paperless/rbac.yaml instead -
 # that app already has its own kustomization + namespace, no need to
 # duplicate it here. All of these stay inert (grant nothing) until

k8s/infra/rbac/sandbox-observer-role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sandbox-observer
+  namespace: poimen-sandbox
+rules:
+  - apiGroups: [""]
+    resources: [pods, pods/log, services]
+    verbs: [get, list, watch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: poimen-devs-sandbox
+  namespace: poimen-sandbox
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: "oidc:poimen-devs"
+roleRef:
+  kind: Role
+  name: sandbox-observer
+  apiGroup: rbac.authorization.k8s.io

k8s/infra/talos-repair/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Stable secret name: the job mounts talos-repair-config by name, so the generated
+# Secret must not get a hash suffix (same convention as k8s/argocd/secrets).
+generatorOptions:
+  disableNameSuffixHash: true
+# ksops decrypts talosconfig-secret.enc.yaml into the Secret the job mounts.
+# Kept local to this app on purpose: no edits to the shared secrets generator.
+generators:
+  - secret-generator.yaml
+resources:
+  - restart-kubelet.yaml

k8s/infra/talos-repair/restart-kubelet.yaml

@@ -0,0 +1,57 @@
+# One-shot repair: the kubelets on cp-1, cp-2 and worker-1 stopped posting node status at
+# 04:11-04:14 UTC and never reconnected, so those nodes are NotReady/unreachable and every
+# pod on them (Cilium included) shows NotReady. This job runs on the one healthy node (cp-3),
+# talks to the Talos API on the LAN, and uses Talos' endpoint->node proxying to reach the
+# other three: it reports the kubelet service state, then restarts it.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: talos-restart-stalled-kubelets
+  namespace: cicd
+spec:
+  backoffLimit: 1
+  ttlSecondsAfterFinished: 86400
+  template:
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: talosctl
+          image: docker.io/curlimages/curl:8.22.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -u
+              echo "fetching talosctl v1.13.3 (matches the node version)"
+              curl -sSL -o /tmp/talosctl \
+                https://github.com/siderolabs/talos/releases/download/v1.13.3/talosctl-linux-amd64 \
+                || { echo "DOWNLOAD FAILED"; exit 1; }
+              chmod +x /tmp/talosctl
+              T="/tmp/talosctl --talosconfig /talos/config -e 192.168.1.162"
+              NODES="192.168.1.166 192.168.1.214 192.168.1.223"
+              echo "=== BEFORE: kubelet service state ==="
+              for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
+              echo "=== kernel log tail (last 15 lines per node) ==="
+              for n in $NODES; do echo "--- $n"; $T -n $n dmesg --tail 15 2>&1 | tail -15 || true; done
+              echo "=== ACTION: restart kubelet ==="
+              for n in $NODES; do echo "--- restarting $n"; $T -n $n restart kubelet 2>&1 || true; sleep 20; done
+              sleep 60
+              echo "=== AFTER ==="
+              for n in $NODES; do echo "--- $n"; $T -n $n service kubelet 2>&1 || true; done
+          volumeMounts:
+            - name: talos
+              mountPath: /talos
+              readOnly: true
+      volumes:
+        - name: talos
+          secret:
+            secretName: talos-repair-config
+            defaultMode: 0444
+            items:
+              - key: config
+                path: config

k8s/infra/talos-repair/secret-generator.yaml

@@ -0,0 +1,10 @@
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: talos-repair-secret-generator
+  annotations:
+    config.kubernetes.io/function: |
+      exec:
+        path: ksops
+files:
+  - talosconfig-secret.enc.yaml

k8s/infra/talos-repair/talosconfig-secret.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: talos-repair-config
+    namespace: cicd
+type: Opaque
+stringData:
+    config: ENC[AES256_GCM,data:WO+jinh/2Rq5VeGLoDDJOQL4zfYJINSTYZCwGWndbzfIHw4XRwxDB7BixjCdpPuLOkNIBAXHQpKPBhlQpvRTso70L26njgVdVENUGeNhxvb6/cwl3VYRsfQQxKoQW9Qu6DyCD7ItuyazXMdIbENKBoih7GA2IjKdGeEtpB+a6PO+6rpI0AQRIjM9h65yJ6gvWeLZmb6U0qCFW61SelHmVSsfehxH/R9EKQROKwD0Z/TgYnoS28OuDtixAQVfnrI3gtgUAeXgnPQTJf1Vb9iwbRBvaSWkPSIqa7hOqWEIM2qyrvhTsEb8XDRUHTmp4BCxYktIxoiSWZEbeh0y0HKRzekZIj4rHA+c/JYSPqS0dvn1Ga+8PTc1LNdXDWIcKI58YQ2jzA0oE2vm0YzmOjcDTzTGLDBk49gIql+AMfwoxCop7t1PB7BrJw6M4bj9M5aHIq8ZlGXDvtWZcSEBj0ChZ0q20cB4v4bvMkbuJR/PS25uNkeG+UBCPidBQqCUw37aNf8SDwOaAC+6PeHdLmcJ/dkKntGFlU5MQzWQXfqs/fp95ze3mr7f3et7SqVl7fAlYSdEt6dA2xxnFnqgNwAm3UacV/We6SRnjrFlBAHZ3Q5hIuKkxmszsS4Bh/VuNtZI5YJ4ldnoIZC3aWKDt+rVB6P+Qxp1T1XqZj8GY715DZRP1q+Z7iQ9RcXiJBTD5qWVpnkESOvbTEYv/W3oXTufVRrJ8Ps0on/xWRF5NU04+tqvToC8+MIQDz5rb+eXYCEWjZG9ZhInY+M2HRTLg+c49dRlyqVnnqfNWBejSS/hoQbf5HmkUbtzsCftvu2NwS2nOMhm55xcCsAunB+0uY+Bj4jqyqRvWUjA99yHIdQH5JTJXE4FQPxdBTmBFp8xVdYWvlnCojK4qfv5EurLUunBugdSdYidi2TKYEoW2mX19RdVnkQgBjW9caaZUX9BIi+zNhObKenoWxRORCb0dHw72mwXMaSQM4IDygcuwi/79OtkYik3juECVmI58bu/GFRo2NtmgkGdMD29SMV6rerDtYjGjetgFvWkDy6hrB/Vz4dI37eGt/EH/7G0z/IT9Q2sM8nzJDtCYdicepLQ1WaM0P4BEn2rrqhv81VvDH7M9e7wtz4xly5RYQUr5wkzvoSEIglFekPvPmhIxEsNdaBj6MlVmLTbEbv85oBLwm0ZLRJcgUsrBbZghGKQaI5udAk1dgwOng9v0Ai+ilJ2RSt2VI6WG1rwRnDG+Hv+H1cKT/NwA8ocmM/kxQlUE3BxESrxhTnKq6W0rzPInIj+xeS8v1X4sloGJIiweCCI8YrqUnmYKKQ6Rtd5B4jS5fAa8jp274/R4xvLaR2r70wKp890Iwfb948PwqjNpH/XlOUcfIxPu3jHO65OEyX3OG/+qm+g91AmxlrwZRDYeSioHztz55Du+JKX2Twnb5ttEwVm6I7eQWGbzw50Qgg6I0cHM1YRrXv/h35C8pkXxV7vMlyR0g5ELAoRWYkuveniruKNIIPBw6SY81RtjIIoPvKzNAFssNdiNxfYEsHiDlfK9kUvTLs8qREmLt1rzgD/vHtanmKh7WMtHN42tSUp+NhoHPlEyaaJLUMkFS7suh2S9xPmhisw61npROXrc2PBxLR52i8t0cJYFa1XuZ40KMCZ7kw0UmKiUxeFDTlZkaOTocVqw3qLY906argYnVdkpaoJMUXsnE3JmXch9HG1qpP+hY8LJSOl5ejAiQADHYDm8P/+9VVlNL6eo2O6qFUs9lC+I8hVeSOvJd8+aLeWisFeboKjarhw/52eXwZdTEpPHa7yMDsIQgciqN82yagJYrEL6BevvZ/kGHP5TszYYfcKcN28k9y/R2JYND2TvYeIKb2Nmj9rhwuXBOFnlep4Exw1QS0If80gJG1BCub+u6GKk9FaFHUSH/x3MIRHwVzCHRnZxqbwc1uGZ2I3rRxVq5MHo5p7htS/KYi7+cQjSdRvfdlCofo6YKP38qQwh9y9yAUqMEOABtjUaEJSeWlGq2yh0onm+Uqm54fcCSr7S5MvBTZ1oblLqVUrZBDTIkY5cdbuiyyoj7GPLHrxUo25yiGI+FXHKy136E0Rv7p5N8o3U5O8ycPdk5vOVYgvM3pdLS1zHHsVYIYY/Y/svtDUnMhJONl5AwQ2sDZ2GiWx1nWJ4VumLuwb/Yow/BHtnW6SOeWFSwRkgJdNL5mzrNoexytbrDaSWR0IpuLUbvK/Gi4MJaEkm1acQB1CMOjIymVu1pcih0H1Zl4O+fhQM+bVke6NKrBSUKErHWca9iNH16Du7iXBbaK0Dz73nIhALJKCivr1rWviXXxeQuzTFMRXdK/SalJySHM7ov6qOikQTG/Q97zQ+UYiUnLo3UxlWngXrnzGy8fAb58v9bHhEtnvCrAhiJTer0x4GYQJGwYjQDbN9ucpu00PDnQU0GqYJHqCf4Vm9Y3SYTs92T6Lftvu3nw/ZH6PblcznC6sueEPON5QzzQppepezdolRNyx4g8XYsI+XJlCNrExWxDxErLBTOm3f7aB19RYCS1tG0G/aKEJTXh9RE5VyWSyBzPNMuLZF9HIIHp3yGpqGM7xsYKsMtngBiyXC+i/C/geIgUFJym/sqWn4iPDWtd0YeELg46B5krcolTg87KEeiqDAebieqL5ZiftnVi10kV2L2XPs4ofJjpVI1Q53RsgT89zWXYv97DTZeDwSE8FY2vbADnUpGk/UZ2WMQlX8iwFOTo1woE6AHyqrq6kzGamUGsyoAarqmJk77o2ktzO2M4OqsVqJErSWTcCLZOlxsDWUn/OsQX5+9vaAhHwDHMmrqWo6cJuAtvaLYOHVB/BUOiI/MJLSYeDIrKUML3Ew0GbX89jkR45SPjRdsXQ/oVCJDxm7WjF9jd6bly2wPTmWCzoqeTJU48jPk47tDHWA7QjLEah6PajxbAs/xSN4c2uK9ltR3K/urNpOTYwsFMLgRfzontPe8RMB9A6aZcwQnIhLTpgX8fw/j+aqwqJwVcIk57/KvcOksBjN4rwPQ78kCmpLJ/KEjYh1z8EwIdVX3UDvq8c8YSZnzwsTAVfhWxigvLGzQDTXrr8aNMDXwB2iUVwhNdGw/E1UCSvsKbvw+p9YRaCAzPnnKa8AVAJySAc1Ys/DAYskdKYaTPUwdCXyZfLzfOsFvEzBxqrRgxxK77ij5UQXKINjQpAIP6pkDze+JTwWQsPfcMZ5xtb2tpiUF9FvCVqR1dfblppq3/f3ZHSl7f4s/sOM2/ZRkiI4m8yxhWDMtauxyrnkgiLsOBUcK4MclD5E3H6KtaJODFiCyH+NPtAMmaZ1iduRM51qxmrHlcI5B4ErJum/iZg+zIokXqIi/U6B+tbjj0EKugN+5V03JaoTvdaGQKNvJHnyP0odFN05ZgZnYCKGGh41EGyCZ8/lmrNbAfZRPjRtHw5ypBUiW8Yz+SkumgWA637ASf1AklCNs9MBBuOPmGckDWy8zTvUCgWCwYehIjKSQ+1hYoWpraLNOZy7tHlTQZ7G5zIFR3cJljO4bfmpAGDsSeDQtBsAI90gD4P3ipKFXVQTfl7c3CD1kB8TiOpG0xVGlYQ4dP15z1wTUfHRz82eFxfwn2fc/R/AKmINq9P5PEL31jhLM+gTxctb2BiGn8cCRuorLCg9FtkwC4jy2AluVj6uZZOGvDCYyGqFsdObgwb+FdIoCYDdeZXo+1dxlxhL1258L6h5+yJBgSy6QDN26N6MMNnK2iD7QTQwtB9CfaX3Bk6T0nZ0TT6zSxu7ix7UG+E5AKwoMVcD0EuLlmJ0IhLI+4ZZ8z/QUIzeTZ9ve+r8uyHKmWr2oR8ec5m95PGG5JzWmw30sDelFFl6S3KxsHwzlJsXeXbE5I/Z0RWZQBa8xPbmY54qYXPzp3O+9WYys6A697UzL6fcCORYokB/TR5exBS2saW4OhkwX5ss9aUJGZiMFSgfF1paxqXkGIx3IuDH5xP3QxhGxNMB+T+Yn+meApf2A00dEUng1HRtgBU5Mk/TM19g8l3nVtwTrmAZDm50jIn+G51GpeoBeYPONhGfb1Ngao383TPkvShaNujE3zsJLfJCrNda76rIechIhAC25wWauUMGUU8daL8cdtJnAEeidnolR+lVf5WFKUxvO36X84FCSaR4PqzvlNk7hOvd607h7IGOEVx7/XRLpWDDrrT6hpY653lNf3RCsxdQ7kS9e5hi/RpU1mf9mdTOsf+AtHkkog336jKdyl1/plDV/AlRS8/QjlPfeyvGmA6gEC8cZmBkvyZPbs81tYmif8q2Amd9La3wldh3evYOOkC/b1QQ3fLryoeUl5/9oVprW/aEfmcTxG9CytaFfsx6cAejAb7g6fz/iaWE57fXfth7K7ATi827ZwAs0XX9lLkyGn60ePtqg771isK5/v1jCsbDQDwHwNLehH+BX9zZWWjaQxK85/o09Oaiy0oGY1nivv5kHelmGwj8qqd,iv:a4NHhErZolhOuzNCUU51GQiCfCT8GzZ3GQ24s7fkuyU=,tag:w+GrXmaYWADTrBxMko3xtg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmk5aE1GN2dpVlREZU1s
+            N3V5ZUQweWxMaVo3Y1VjdzZTYW96eFBYZGxvCnNJcjVXKzI4VnliTWFXQy9iSy9k
+            eEZmWlVPQzhZK09NZGowZUVXc1BoZTQKLS0tIHIrMEpTM0lMU0w0eG02aUxJY2VZ
+            aUQ0cmkvK0UwdzhqcDJwM1RDcnpEU0UKQRiTa9/yNrqlkIdlNeaIDzIKa/xvx5nW
+            7wTNfjq2M3go0DMnaVjJA3e15JZBOpOs0QP9z9ydKF/FS4PeCIEFyQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T04:27:33Z"
+    mac: ENC[AES256_GCM,data:CxZdlGJdIFFTwaeeS9LW2XDDgvUmSsBeBKtFYyDM1JliS8BHtgEv48yuCWxP5sR3SaYtcgNSIQ55zUa89jM6dQHqvjK+1jSYu1xzFWPOlxUu8gVSlU0QLS4GU1RChXuPpTUOjxzn4ymZukGgclZk9iuMvI0/U+nGD8nvs8k0zpM=,iv:MGZ7YsOsjelJPjLYWtALOrM6r8Vd1HgbU/PKL4c3r3k=,tag:dIE38TCobESY3j56NVEvIw==,type:str]
+    version: 3.13.2

scripts/iam/provision-rbac.py

@@ -351,11 +351,6 @@ OAuth2_PROVIDERS = {
         "client_secret": "QxyWfESXqTD55aUyh6miYnny1QTCEuEwyC4escw9",
         "redirect_uris": ["app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login", "https://img.riotpiao.com/user/oauth2/callback"],
     },
-    "homarr": {
-        "client_id": "homarr",
-        "client_secret": "RMlDQAWdjT5YPPH0U7ztDoUFP7R7w95b2xqQ1pyS",
-        "redirect_uris": ["http://localhost:7575/auth/callback", "https://homarr.riotpiao.com/auth/callback"],
-    },
     "argocd": {
         "client_id": "argocd",
         "client_secret_env": "AUTHENTIK_PROVIDER_ARGOCD_SECRET",

scripts/iam/rotate-secrets.sh

@@ -53,7 +53,7 @@ generate_secret() {
 
 declare -A NEW_SECRETS
 
-for svc in api-gw minio poimen paperless grafana argocd forgejo homarr immich vault portfolio-agent memory-agent local-llm; do
+for svc in api-gw minio poimen paperless grafana argocd forgejo immich vault portfolio-agent memory-agent local-llm; do
   NEW_SECRETS[$svc]=$(generate_secret)
   log "  $svc: ${NEW_SECRETS[$svc]:0:15}..."
 done
@@ -70,7 +70,6 @@ declare -A PROVIDER_PKS=(
   [grafana]=6
   [argocd]=7
   [forgejo]=8
-  [homarr]=9
   [immich]=10
   [vault]=11
 )
@@ -203,7 +202,7 @@ fi
 log ""
 log "=== Summary ==="
 log "Rotated 13 credentials:"
-log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, homarr, immich, vault"
+log "  OAuth2 Providers: api-gw, minio, poimen, paperless, grafana, argocd, forgejo, immich, vault"
 log "  Service Accounts: portfolio-agent, memory-agent, local-llm"
 
 log ""

scripts/iam/zitadel-provision.py

@@ -0,0 +1,627 @@
+#!/usr/bin/env python3
+"""Provision ZITADEL end to end: machine key, Terraform, SOPS secrets, users.
+
+One command replaces the manual sequence the Authentik script
+(scripts/iam/provision-rbac.py) used to do in a single pass. Phases run in a
+fixed order and every phase is safe to re-run:
+
+  keys         extract the iam-admin machine key and PAT from namespace iam
+  apply        create project, roles and OIDC apps through the ZITADEL API
+  secrets      write generated client ids/secrets back into the SOPS files
+  users        import one human user, converting the Django password hash
+  verify       check discovery, client count and the groups-claim gate
+
+`all` runs keys, apply, secrets, verify. It never runs users: the password hash
+must be supplied by a human, one user at a time.
+
+Nothing here talks to Authentik. Authentik keeps serving until a consumer is
+moved deliberately.
+
+Usage:
+  scripts/iam/zitadel-provision.py keys
+  scripts/iam/zitadel-provision.py apply --org-id <id>
+  scripts/iam/zitadel-provision.py secrets --dry-run
+  scripts/iam/zitadel-provision.py secrets --apply-config
+  scripts/iam/zitadel-provision.py users --django-hash '<pbkdf2_sha256$...>'
+  scripts/iam/zitadel-provision.py verify
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import json
+import os
+import re
+import signal
+import stat
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import yaml
+
+REPO = Path(__file__).resolve().parents[2]
+SOPS_DIR = REPO / "k8s" / "argocd" / "secrets"
+SOPS_CONFIG = REPO / ".sops.yaml"
+CONF_DIR = Path(os.environ.get("ZITADEL_CONF_DIR", Path.home() / ".config" / "zitadel"))
+STATE_FILE = CONF_DIR / "state.json"
+NAMESPACE = os.environ.get("ZITADEL_NAMESPACE", "iam")
+BASE_URL = os.environ.get("ZITADEL_BASE_URL", "https://zitadel.riotpiao.com")
+
+# ── desired ZITADEL state ──────────────────────────────────────────────────
+# Declared here, applied through the API by the `apply` phase. This replaces the
+# retired terraform/zitadel root module: same objects, no state file to drift and
+# no second writer.
+PROJECT_NAME = os.environ.get("ZITADEL_PROJECT_NAME", "homelab")
+PROJECT_GROUP = "homelab"
+
+# Role keys EQUAL the Authentik group names, so the `groups` claim keeps working
+# for every consumer that maps on it (kube-apiserver RBAC, MinIO policies,
+# Grafana roles).
+ROLES = {
+    "homelab-admins": "Cluster administrators with full access",
+    "minio-admins": "MinIO administrators",
+    "minio-photos": "Photos bucket (Immich) access",
+    "minio-documents": "Documents bucket (Paperless) access",
+    "minio-backups": "Backups bucket read-only access",
+    "poimen-admins": "Poimen memory administrators",
+    "poimen-devs": "Dev and staging projects access",
+    "poimen-prod-readonly": "Production projects read-only access",
+    "paperless-admins": "Paperless administrators",
+    "paperless-finance": "Finance documents",
+    "paperless-legal": "Legal documents",
+    "paperless-hr": "HR documents",
+    "grafana-admins": "Grafana administrators",
+    "grafana-editors": "Grafana dashboard editors",
+    "grafana-viewers": "Grafana dashboard viewers",
+    "sqs-users": "SQS/Temporal queue read access",
+    "sqs-writers": "SQS/Temporal queue read/write access",
+    "s3-users": "S3 read access",
+    "s3-writers": "S3 read/write access",
+}
+
+_WEB = {"app_type_id": "OIDC_APP_TYPE_WEB", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_BASIC"}
+_NATIVE = {"app_type_id": "OIDC_APP_TYPE_NATIVE", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+_PKCE = {"app_type_id": "OIDC_APP_TYPE_USER_AGENT", "auth_method_id": "OIDC_AUTH_METHOD_TYPE_NONE"}
+
+OIDC_APPS = {
+    "api-gw": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://api.riotpiao.com/callback"]},
+    "argocd": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/auth/callback",
+        "https://argocd.riotpiao.com/auth/callback"]},
+    "forgejo": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/user/oauth2/authentik/callback",
+        "https://forgejo.riotpiao.com/user/oauth2/authentik/callback"]},
+    "grafana": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/login/generic_oauth",
+        "https://grafana.riotpiao.com/login/generic_oauth"]},
+    "immich": {**_WEB, "app_type_id": "OIDC_APP_TYPE_NATIVE", "redirect_uris": [
+        "app.immich:///oauth-callback", "https://img.riotpiao.com/auth/login",
+        "https://img.riotpiao.com/user/oauth2/callback"]},
+    "kubernetes": {**_PKCE, "dev_mode": True, "redirect_uris": [
+        "http://localhost:8000", "http://localhost:18000"]},
+    "minio": {**_WEB, "redirect_uris": [
+        "http://localhost:9000/auth/sso/oauth2/code",
+        "https://minio.riotpiao.com/auth/sso/oauth2/code"]},
+    "paperless": {**_WEB, "redirect_uris": [
+        "http://localhost:8000/accounts/oidc/authentik/login/callback/",
+        "https://paperless.riotpiao.com/accounts/oidc/authentik/login/callback/"]},
+    "poimen": {**_WEB, "redirect_uris": [
+        "http://localhost:3000/callback", "https://poimen.riotpiao.com/callback"]},
+    "queue-sqs": {**_WEB, "redirect_uris": [
+        "http://localhost:8080/callback", "https://queue.riotpiao.com/callback"]},
+    "vault": {**_WEB, "redirect_uris": [
+        "http://localhost:8200/ui/vault/auth/oidc/oidc/callback",
+        "https://vault.riotpiao.com/ui/vault/auth/oidc/oidc/callback"]},
+}
+
+
+_TEMP_FILES: set[str] = set()
+
+
+def _cleanup_temps(*_a) -> None:
+    for path in list(_TEMP_FILES):
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+        _TEMP_FILES.discard(path)
+
+
+for _sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP):
+    signal.signal(_sig, lambda s, f: (_cleanup_temps(), sys.exit(130)))
+
+
+def die(msg: str) -> None:
+    _cleanup_temps()
+    print(f"FATAL: {msg}", file=sys.stderr)
+    sys.exit(1)
+
+
+def run(cmd: list[str], **kw) -> subprocess.CompletedProcess:
+    return subprocess.run(cmd, check=False, text=True, capture_output=True, **kw)
+
+
+def need(binary: str) -> None:
+    if subprocess.run(["sh", "-c", f"command -v {binary}"], capture_output=True).returncode != 0:
+        die(f"{binary} is not on PATH")
+
+
+def write_private(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    try:
+        os.fchmod(fd, 0o600)  # open() mode applies at creation only
+        with os.fdopen(fd, "wb") as fh:
+            fh.write(data)
+    finally:
+        pass
+
+
+def secret_value(name: str, key: str) -> str:
+    cp = run(["kubectl", "-n", NAMESPACE, "get", "secret", name, "-o", f"jsonpath={{{key}}}"])
+    if cp.returncode != 0:
+        die(f"cannot read secret {NAMESPACE}/{name}: {cp.stderr.strip()}")
+    raw = cp.stdout.strip()
+    if not raw:
+        die(f"secret {NAMESPACE}/{name} has no key {key}")
+    return base64.b64decode(raw).decode()
+
+
+def load_env_file(path: Path = REPO / ".env") -> None:
+    """Load KEY=VALUE lines from .env without overriding the real environment.
+
+    The file is git-ignored and holds the bootstrap token a human pastes in.
+    Sourcing it in the shell would work too, but then every phase depends on the
+    caller remembering to do it.
+    """
+    if not path.exists():
+        return
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        key = key.strip()
+        value = value.strip().strip('"').strip("'")
+        if key and key not in os.environ:
+            os.environ[key] = value
+
+
+def bootstrap_token() -> str | None:
+    """Resolve the iam-admin PAT: environment, then cached file, then cluster.
+
+    Order matters. ZITADEL_BOOTSTRAP_TOKEN in .env is filled by a human after the
+    first sync and wins. The cached file under ~/.config/zitadel keeps a run
+    working when the API server is unreachable. The cluster Secret is the last
+    resort and the only source that always reflects a rotation.
+    """
+    token = os.environ.get("ZITADEL_BOOTSTRAP_TOKEN", "").strip()
+    if token:
+        return token
+    cached = CONF_DIR / "iam-admin.pat"
+    if cached.exists():
+        return cached.read_text().strip()
+    try:
+        return secret_value("iam-admin-pat", ".data.pat").strip()
+    except SystemExit:
+        return None
+
+
+def curl_json(method: str, url: str, token: str | None = None, body: dict | None = None,
+              timeout: int = 20) -> tuple[int, str]:
+    """Call the API without putting a secret in the process argv.
+
+    `curl --config -` reads the options from stdin, so neither the bearer token
+    nor the password hash of a user import shows up in `ps`.
+    """
+    def esc(value: str) -> str:
+        return value.replace("\\", "\\\\").replace('"', '\\"')
+
+    cfg = [f'url = "{esc(url)}"', f'request = "{method}"', "silent", "show-error",
+           f"max-time = {timeout}", 'write-out = "\\n%{http_code}"',
+           'header = "Content-Type: application/json"']
+    # Reaching the instance by name needs either public DNS through the tunnel
+    # or a pinned address. ZITADEL picks the instance from the Host header, so a
+    # plain port-forward fails with "Instance not found". Set this to bypass DNS:
+    #   ZITADEL_CURL_RESOLVE=zitadel.riotpiao.com:443:192.168.1.160
+    pinned = os.environ.get("ZITADEL_CURL_RESOLVE", "").strip()
+    if pinned:
+        cfg.append(f'resolve = "{esc(pinned)}"')
+    if token:
+        cfg.append(f'header = "Authorization: Bearer {esc(token)}"')
+    if body is not None:
+        cfg.append(f'data = "{esc(json.dumps(body))}"')
+    cp = run(["curl", "--config", "-"], input="\n".join(cfg) + "\n")
+    if cp.returncode != 0:
+        die(f"{method} {url} failed: {cp.stderr.strip()}")
+    payload, _, code = cp.stdout.rpartition("\n")
+    return int(code.strip() or 0), payload
+
+
+# ── phase: keys ────────────────────────────────────────────────────────────
+
+def phase_keys(_args: argparse.Namespace) -> int:
+    write_private(CONF_DIR / "iam-admin.json", secret_value("iam-admin", ".data.iam-admin\\.json").encode())
+    write_private(CONF_DIR / "iam-admin.pat", secret_value("iam-admin-pat", ".data.pat").strip().encode())
+    print(f"machine key  -> {CONF_DIR / 'iam-admin.json'} (0600)")
+    print(f"PAT          -> {CONF_DIR / 'iam-admin.pat'} (0600)")
+    print("both files stay outside the repo; never commit them")
+    return 0
+
+
+# ── phase: apply ───────────────────────────────────────────────────────────
+
+def _api(method: str, path: str, token: str, body: dict | None = None,
+         timeout: int = 30) -> tuple[int, dict]:
+    code, payload = curl_json(method, f"{BASE_URL}{path}", token=token, body=body, timeout=timeout)
+    try:
+        return code, (json.loads(payload) if payload.strip() else {})
+    except json.JSONDecodeError:
+        return code, {"_raw": payload}
+
+
+def _ensure_project(token: str, name: str) -> str:
+    code, d = _api("POST", "/management/v1/projects/_search", token, {})
+    for row in d.get("result") or []:
+        if row.get("name") == name:
+            return row["id"]
+    code, d = _api("POST", "/management/v1/projects", token, {"name": name})
+    if code not in (200, 201) or not d.get("id"):
+        die(f"create project {name} failed: {code} {d}")
+    return d["id"]
+
+
+def _ensure_roles(token: str, project_id: str, roles: dict[str, str]) -> None:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/roles/_search", token, {})
+    have = {r.get("roleKey") for r in (d.get("result") or [])}
+    for key, display in roles.items():
+        if key in have:
+            continue
+        code, d = _api("POST", f"/management/v1/projects/{project_id}/roles", token,
+                       {"roleKey": key, "displayName": display, "group": PROJECT_GROUP})
+        if code not in (200, 201):
+            die(f"create role {key} failed: {code} {d}")
+
+
+def _ensure_oidc_apps(token: str, project_id: str, apps: dict[str, dict]) -> tuple[dict, dict]:
+    code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/_search", token, {})
+    existing = {a.get("name"): a for a in (d.get("result") or [])}
+    ids: dict[str, str] = {}
+    secrets: dict[str, str] = {}
+    for name, spec in apps.items():
+        row = existing.get(name)
+        if row:
+            ids[name] = (row.get("oidcConfig") or {}).get("clientId", "")
+            app_id = row.get("id")
+        else:
+            body = {
+                "name": name,
+                "redirectUris": spec["redirect_uris"],
+                "responseTypes": ["OIDC_RESPONSE_TYPE_CODE"],
+                "grantTypes": ["OIDC_GRANT_TYPE_AUTHORIZATION_CODE",
+                               "OIDC_GRANT_TYPE_REFRESH_TOKEN"],
+                "appType": spec.get("app_type_id", "OIDC_APP_TYPE_WEB"),
+                "authMethodType": spec.get("auth_method_id", "OIDC_AUTH_METHOD_TYPE_BASIC"),
+                "devMode": bool(spec.get("dev_mode")),
+                "accessTokenType": "OIDC_TOKEN_TYPE_BEARER",
+                "version": "OIDC_VERSION_1_0",
+                "idTokenRoleAssertion": True,
+            }
+            code, d = _api("POST", f"/management/v1/projects/{project_id}/apps/oidc", token, body)
+            if code not in (200, 201) or not d.get("appId"):
+                die(f"create app {name} failed: {code} {d}")
+            app_id = d["appId"]
+            ids[name] = d.get("clientId", "")
+        if spec.get("auth_method_id") == "OIDC_AUTH_METHOD_TYPE_NONE":
+            continue
+        code, d = _api("POST",
+                       f"/management/v1/projects/{project_id}/apps/{app_id}"
+                       "/oidc_config/_generate_client_secret", token, {})
+        if code in (200, 201) and d.get("clientSecret"):
+            secrets[name] = d["clientSecret"]
+    return ids, secrets
+
+
+def phase_apply(_args: argparse.Namespace) -> int:
+    """Declare the ZITADEL objects through the API. No Terraform, no state drift."""
+    token = bootstrap_token()
+    if not token:
+        die("no iam-admin PAT — run the keys phase first")
+
+    project_id = _ensure_project(token, PROJECT_NAME)
+    print(f"project   : {PROJECT_NAME} ({project_id})")
+    _ensure_roles(token, project_id, ROLES)
+    print(f"roles     : {len(ROLES)} ensured")
+    ids, secrets = _ensure_oidc_apps(token, project_id, OIDC_APPS)
+    print(f"oidc apps : {len(ids)} ensured, {len(secrets)} secrets issued")
+
+    state = {
+        "project_id": {"value": project_id},
+        "client_ids": {"value": ids},
+        "client_secrets": {"value": secrets},
+        "machine_keys": {"value": {}},
+        "groups_claim_enabled": {"value": False},
+    }
+    write_private(STATE_FILE, json.dumps(state, indent=2, sort_keys=True).encode())
+    print(f"state     -> {STATE_FILE} (0600)")
+    return 0
+
+
+# ── phase: secrets ─────────────────────────────────────────────────────────
+
+# client -> [(sops file, Secret name inside the file, key, kind)]
+# Only secrets with a git-tracked SOPS file appear here. Every other client is
+# printed as manual: guessing a path writes a credential into the wrong Secret.
+SECRET_MAP = {
+    "grafana": [("grafana-oidc-secrets.enc.yaml", "grafana-oidc",
+                 "GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET", "secret")],
+    "minio": [("minio-secrets.enc.yaml", "minio-oidc",
+               "MINIO_IDENTITY_OPENID_CLIENT_SECRET", "secret")],
+    "paperless": [("paperless-ai-secrets.enc.yaml", "paperless-ai-config",
+                   "LLM_AUTH_CLIENT_SECRET", "secret")],
+}
+
+# Clients whose client_id lives in a plain (unencrypted) manifest. They need the
+# id update as much as the secret: ZITADEL generates a new client_id for every
+# application, so a secret-only re-point yields invalid_client.
+CLIENT_ID_MAP = {
+    "grafana": ("k8s/infra/logging/grafana-values.yaml",
+                r"(?m)^(\s*client_id:\s*)\S+\s*$", r"\g<1>{value}"),
+    "minio": ("k8s/infra/minio/minio-tenant.yaml",
+              r'(?m)^(\s*value:\s*)"(?:minio|\d+)"\s*$', r'\g<1>"{value}"'),
+    "paperless": ("k8s/apps/paperless/paperless-ai.yaml",
+                  r"client_id=(?:paperless-ai-agent|\d+)", "client_id={value}"),
+}
+
+# Secrets that a live workload mounts but no SOPS file in git defines. The phase
+# cannot re-point them safely, so it names them.
+MANUAL_ONLY: dict[str, str] = {}
+
+
+def state_outputs() -> dict:
+    """Read the object map written by the apply phase."""
+    if not STATE_FILE.exists():
+        die(f"{STATE_FILE} missing — run the apply phase first")
+    return json.loads(STATE_FILE.read_text())
+
+
+def sops_env() -> dict:
+    env = dict(os.environ)
+    env.setdefault("SOPS_AGE_KEY_FILE", str(Path.home() / ".sops" / "key.txt"))
+    return env
+
+
+def write_secret(rel_path: str, secret_name: str, key: str, value: str,
+                 dry_run: bool) -> None:
+    path = SOPS_DIR / rel_path
+    if not path.exists():
+        die(f"{path} does not exist — the mapping and the repo disagree")
+    env = sops_env()
+    dec = run(["sops", "--config", str(SOPS_CONFIG), "decrypt", str(path)], env=env)
+    if dec.returncode != 0:
+        die(f"cannot decrypt {rel_path}: {dec.stderr.strip()}")
+    docs = [d for d in yaml.safe_load_all(dec.stdout) if d]
+    matching = [d for d in docs if (d.get("metadata") or {}).get("name") == secret_name]
+    if len(matching) != 1:
+        die(f"{rel_path}: expected exactly one document named {secret_name}, found {len(matching)}")
+
+    current = (matching[0].get("stringData") or {}).get(key)
+    if current == value:
+        print(f"  unchanged {rel_path}:{secret_name}:{key}")
+        return
+    if dry_run:
+        print(f"  would set {rel_path}:{secret_name}:{key} (len {len(value)})")
+        return
+
+    if len(docs) == 1:
+        cp = run(["sops", "--config", str(SOPS_CONFIG), "set", "--value-stdin", "--idempotent",
+                  str(path), f'["stringData"]["{key}"]'], env=env, input=json.dumps(value))
+        if cp.returncode != 0:
+            die(f"sops set failed on {rel_path}: {cp.stderr.strip()}")
+        print(f"  wrote {rel_path}:{secret_name}:{key}")
+        return
+
+    # Multi-document file. `sops set` cannot address one document, so the whole
+    # file is rewritten. sops encrypt reads the creation rule from the FILENAME,
+    # so the temp path is overridden with the repo-relative path; without that
+    # override sops finds no matching rule and refuses.
+    matching[0].setdefault("stringData", {})[key] = value
+    plain = "---\n".join(yaml.safe_dump(d, sort_keys=False) for d in docs)
+    fd, tmp = tempfile.mkstemp(suffix=".yaml", dir=tempfile.gettempdir())
+    _TEMP_FILES.add(tmp)
+    try:
+        with os.fdopen(fd, "w") as fh:
+            fh.write(plain)
+        enc = run(["sops", "--config", str(SOPS_CONFIG), "encrypt",
+                   "--filename-override", str(path), tmp], env=env)
+        if enc.returncode != 0:
+            die(f"sops encrypt failed for {rel_path}: {enc.stderr.strip()}")
+        mode = stat.S_IMODE(path.stat().st_mode)
+        fd2, staged = tempfile.mkstemp(suffix=".enc.yaml", dir=str(path.parent))
+        try:
+            with os.fdopen(fd2, "w") as fh:
+                fh.write(enc.stdout)
+            os.chmod(staged, mode)
+            os.replace(staged, path)  # same directory: atomic, never EXDEV
+        finally:
+            if os.path.exists(staged):
+                os.unlink(staged)
+        print(f"  wrote {rel_path}:{secret_name}:{key} (multi-document rewrite)")
+    finally:
+        if os.path.exists(tmp):
+            os.unlink(tmp)
+        _TEMP_FILES.discard(tmp)
+
+
+def update_client_id(client: str, value: str, dry_run: bool, apply_config: bool) -> None:
+    spec = CLIENT_ID_MAP.get(client)
+    if not spec:
+        return
+    rel_path, pattern, repl = spec
+    path = REPO / rel_path
+    if not path.exists():
+        die(f"{path} does not exist — the client-id mapping is stale")
+    text = path.read_text()
+    matches = list(re.finditer(pattern, text))
+    if len(matches) != 1:
+        die(f"{rel_path}: expected 1 match for the {client} client id, found {len(matches)}")
+    new_line = matches[0].expand(repl.format(value=value))
+    if matches[0].group(0) == new_line:
+        print(f"  unchanged {rel_path}: {client} client id")
+        return
+    if dry_run or not apply_config:
+        print(f"  would change {rel_path}: {matches[0].group(0).strip()} -> {new_line.strip()}")
+        if not apply_config:
+            print("    re-run with --apply-config to write this plaintext manifest edit")
+        return
+    path.write_text(text[:matches[0].start()] + new_line + text[matches[0].end():])
+    print(f"  wrote {rel_path}: {client} client id")
+
+
+def phase_secrets(args: argparse.Namespace) -> int:
+    need("sops")
+    out = state_outputs()
+    ids = out.get("client_ids", {}).get("value", {})
+    secrets = out.get("client_secrets", {}).get("value", {})
+    machine_keys = out.get("machine_keys", {}).get("value", {})
+
+    if not ids and not machine_keys:
+        die("no ZITADEL objects recorded yet — run the apply phase first")
+
+    print("[1/3] client secrets into SOPS")
+    for client, entries in SECRET_MAP.items():
+        for rel_path, secret_name, key, kind in entries:
+            value = ids.get(client) if kind == "id" else secrets.get(client)
+            if value:
+                write_secret(rel_path, secret_name, key, value, args.dry_run)
+
+    print("[2/3] client ids in plain manifests")
+    for client in list(CLIENT_ID_MAP) + list(SECRET_MAP):
+        if ids.get(client):
+            update_client_id(client, ids[client], args.dry_run, args.apply_config)
+
+    print("[3/3] machine keys")
+    key_dir = CONF_DIR / "machine-keys"
+    for name, key_json in machine_keys.items():
+        if args.dry_run:
+            print(f"  would write {key_dir / (name + '.json')}")
+        else:
+            write_private(key_dir / f"{name}.json", key_json.encode())
+            print(f"  wrote {key_dir / (name + '.json')} (0600)")
+
+    unhandled = sorted(set(ids) - set(SECRET_MAP) - set(CLIENT_ID_MAP) - set(MANUAL_ONLY))
+    if MANUAL_ONLY or unhandled:
+        print("\n  MANUAL — no safe write path exists for these clients:")
+        for client, why in MANUAL_ONLY.items():
+            print(f"    {client}: {why}")
+        for client in unhandled:
+            print(f"    {client}: set its client_id and secret in its own manifest or Secret file")
+    return 0
+
+
+# ── phase: users ───────────────────────────────────────────────────────────
+
+HASH_RE = re.compile(r"^pbkdf2_sha256\$(?P<rounds>\d+)\$(?P<salt>[^$]+)\$(?P<hash>[^$]+)$")
+
+
+def convert_hash(django_hash: str) -> str:
+    """Django pbkdf2_sha256 -> passwap MCF $pbkdf2-sha256$...
+
+    ZITADEL imports one MCF string via hashedPassword.hash (max 200 chars) and
+    pins passwap, whose pbkdf2 format is $pbkdf2-sha256$<rounds>$<salt>$<hash>
+    with alt-base64 (standard base64, + -> ., no padding). Rounds, salt and hash
+    bytes are unchanged, so the password still verifies.
+    """
+    m = HASH_RE.match(django_hash.strip())
+    if not m:
+        die("unsupported hash format — only pbkdf2_sha256 is convertible")
+    alt = m.group("hash").rstrip("=").replace("+", ".")
+    mcf = f"$pbkdf2-sha256${m.group('rounds')}${m.group('salt')}${alt}"
+    if len(mcf) > 200:
+        die(f"converted hash is {len(mcf)} chars, ZITADEL accepts 200")
+    return mcf
+
+
+def phase_users(args: argparse.Namespace) -> int:
+    token = bootstrap_token()
+    if not token:
+        die("no bootstrap token — set ZITADEL_BOOTSTRAP_TOKEN in .env, or run the keys phase")
+    if not args.django_hash:
+        die("pass --django-hash with the pbkdf2_sha256 value exported from Authentik")
+    body = {
+        "username": args.username,
+        "profile": {"givenName": args.given_name, "familyName": args.family_name},
+        "email": {"email": args.email, "isVerified": True},
+        "hashedPassword": {"hash": convert_hash(args.django_hash)},
+        "passwordChangeRequired": False,
+    }
+    code, payload = curl_json("POST", f"{BASE_URL}/v2/users/human", token=token, body=body)
+    if code in (200, 201):
+        print(f"  {args.username}: created")
+        return 0
+    if code == 409:
+        print(f"  {args.username}: already exists — the password was NOT updated")
+        return 1
+    die(f"create {args.username} -> {code} {payload[:300]}")
+
+
+# ── phase: verify ──────────────────────────────────────────────────────────
+
+def phase_verify(_args: argparse.Namespace) -> int:
+    code, _ = curl_json("GET", f"{BASE_URL}/.well-known/openid-configuration", timeout=10)
+    print(f"discovery      : {code}")
+    if code != 200:
+        die("discovery endpoint is not 200 — the instance or the ingress is not ready")
+
+    out = state_outputs()
+    ids = out.get("client_ids", {}).get("value", {})
+    print(f"client apps    : {len(ids)}")
+    enabled = bool(out.get("groups_claim_enabled", {}).get("value"))
+    print(f"groups action  : {enabled}")
+    if not enabled:
+        print("BLOCKED for cutover: the groups claim action is disabled. Every consumer")
+        print("that reads `groups` (kube-apiserver, MinIO, Grafana) sees no groups.")
+        print("Enable it only after the live gate passes, with the script path set.")
+    return 0
+
+
+PHASES = {"keys": phase_keys, "apply": phase_apply, "secrets": phase_secrets,
+          "users": phase_users, "verify": phase_verify}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Provision ZITADEL end to end.")
+    parser.add_argument("phase", choices=[*PHASES, "all"])
+    parser.add_argument("--org-id", help="ZITADEL org id (required for apply)")
+    parser.add_argument("--dry-run", action="store_true", help="print actions, write nothing")
+    parser.add_argument("--apply-config", action="store_true",
+                        help="also rewrite client ids inside plain manifests")
+    parser.add_argument("--username", default="rock")
+    parser.add_argument("--email", default="[email protected]")
+    parser.add_argument("--given-name", default="Rock")
+    parser.add_argument("--family-name", default="Liang")
+    parser.add_argument("--django-hash", help="pbkdf2_sha256 value exported from Authentik")
+    args = parser.parse_args()
+
+    load_env_file()
+    need("kubectl")
+    if args.phase == "all":
+        if not args.org_id:
+            die("all requires --org-id")
+        for name in ("keys", "apply"):
+            rc = PHASES[name](args)
+            if rc:
+                return rc
+        rc = phase_secrets(args)
+        if rc:
+            return rc
+        return phase_verify(args)
+    return PHASES[args.phase](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

terraform/edge/cloudflare.tf

@@ -0,0 +1,51 @@
+# Cloudflare side of the edge.
+#
+# The tunnel itself is remotely managed today: its token lives in
+# k8s/apps/cloudflared and the public hostnames are configured in the Zero
+# Trust dashboard. Terraform can own the DNS records without owning the tunnel,
+# which is the safe half: a record change is reversible, a tunnel config rewrite
+# removes every hostname the plan does not list.
+
+locals {
+  # CNAME target of every remotely-managed tunnel.
+  tunnel_target = "${var.tunnel_id}.cfargotunnel.com"
+}
+
+resource "cloudflare_dns_record" "tunnel" {
+  for_each = toset(var.cloudflare_tunnel_hostnames)
+
+  zone_id = var.cloudflare_zone_id
+  name    = each.value
+  type    = "CNAME"
+  content = local.tunnel_target
+  proxied = var.cloudflare_dns_proxied
+  ttl     = var.cloudflare_dns_ttl
+  comment = "cloudflared tunnel; managed by terraform/edge"
+
+  # A proxied CNAME must point at the tunnel target. A CNAME to any other
+  # hostname breaks the tunnel mapping.
+  lifecycle {
+    precondition {
+      condition     = var.cloudflare_dns_proxied
+      error_message = "Tunnel CNAMEs must be proxied; an unproxied record bypasses the tunnel."
+    }
+  }
+}
+
+# Optional: take the tunnel configuration into Terraform. Off by default.
+# Applying it REPLACES the whole ingress list. Import first:
+#   terraform import cloudflare_zero_trust_tunnel_cloudflared_config.this \
+#     <account_id>/<tunnel_id>
+resource "cloudflare_zero_trust_tunnel_cloudflared_config" "this" {
+  count = var.manage_tunnel_config ? 1 : 0
+
+  account_id = var.cloudflare_account_id
+  tunnel_id  = var.tunnel_id
+
+  config = {
+    ingress = concat(
+      [for host, service in var.tunnel_ingress : { hostname = host, service = service }],
+      [{ service = "http_status:404" }],
+    )
+  }
+}

terraform/edge/outputs.tf

@@ -0,0 +1,40 @@
+output "cloudflare_records" {
+  description = "Hostnames now managed in the Cloudflare zone."
+  value       = sort(keys(cloudflare_dns_record.tunnel))
+}
+
+output "tunnel_config_managed" {
+  description = "Whether Terraform owns the tunnel ingress list. False means the dashboard still does."
+  value       = var.manage_tunnel_config
+}
+
+output "relay_public_ip" {
+  description = "Public address of the Tencent relay. Point the .cn DNS at it, or let the module do it."
+  value       = tencentcloud_instance.relay.public_ip
+}
+
+output "china_hostnames" {
+  description = "Fully qualified China hostnames served by the relay."
+  value       = [for host in var.china_hostnames : "${host}.${var.china_domain}"]
+}
+
+output "wireguard_peer" {
+  description = "Peer stanza for the homelab side. Add it to the cluster's wg0.conf; the relay already trusts this public key."
+  value       = <<-EOT
+    [Peer]
+    PublicKey = <relay public key; see /etc/wireguard/wg0.conf on the relay>
+    Endpoint = ${tencentcloud_instance.relay.public_ip}:${var.wireguard_port}
+    AllowedIPs = ${var.wireguard_relay_address}
+    PersistentKeepalive = 25
+  EOT
+}
+
+output "next_manual_steps" {
+  description = "What this module cannot do for you."
+  value = [
+    "Confirm the ICP filing for ${var.china_domain} if the relay is in a mainland region.",
+    "Issue a certificate for the China hostnames on the relay (Caddy ACME needs the DNS-01 challenge for a .cn domain).",
+    "Run a WireGuard peer in the cluster that dials ${tencentcloud_instance.relay.public_ip}:${var.wireguard_port}.",
+    "Add the .cn origins to the OIDC client redirect URIs before any China user logs in.",
+  ]
+}

terraform/edge/providers.tf

@@ -0,0 +1,47 @@
+# terraform/edge — the China-facing edge, plus DNS ownership for the existing zone
+#
+# Purpose: give users in mainland China a short, stable path to the homelab
+# without touching the Cloudflare path that serves everyone else.
+#
+#   China user -> <service>.<china_domain> -> Tencent CVM (this module)
+#              -> WireGuard tunnel -> ingress-nginx (192.168.1.160) -> service
+#
+#   Everyone else -> Cloudflare -> cloudflared tunnel -> same ingress (unchanged)
+#
+# The homelab still opens no inbound port: the tunnel is established outbound
+# from the cluster to the CVM.
+#
+# Credentials come from the environment, never from a file in git:
+#   export CLOUDFLARE_API_TOKEN=...
+#   export TENCENTCLOUD_SECRET_ID=...
+#   export TENCENTCLOUD_SECRET_KEY=...
+#   terraform -chdir=terraform/edge init
+#   terraform -chdir=terraform/edge plan -var china_domain=example.cn ...
+#
+# State is local and git-ignored. It carries no
+# credential today, but it will once InstanceData or certificates are added, so
+# treat it as a secret and move it to MinIO before a second operator applies.
+terraform {
+  required_version = ">= 1.6.0"
+
+  required_providers {
+    cloudflare = {
+      source  = "cloudflare/cloudflare"
+      version = "5.28.1"
+    }
+    tencentcloud = {
+      source  = "tencentcloudstack/tencentcloud"
+      version = "1.83.38"
+    }
+  }
+}
+
+provider "cloudflare" {
+  api_token = var.cloudflare_api_token
+}
+
+provider "tencentcloud" {
+  secret_id  = var.tencent_secret_id
+  secret_key = var.tencent_secret_key
+  region     = var.tencent_region
+}

terraform/edge/templates/Caddyfile.tpl

@@ -0,0 +1,22 @@
+# Rendered by terraform/edge. Do not edit on the instance.
+#
+# Caddy terminates TLS for the .cn hostnames and forwards over the WireGuard
+# tunnel to ingress-nginx. flush_interval -1 disables response buffering so
+# Server-Sent Events reach the client chunk-by-chunk.
+{
+	email ${acme_email}
+	admin off
+}
+
+${hosts} {
+	reverse_proxy ${ingress_upstream} {
+		flush_interval -1
+	}
+
+	header {
+		Strict-Transport-Security "max-age=31536000; includeSubDomains"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "strict-origin-when-cross-origin"
+	}
+}

terraform/edge/tencent.tf

@@ -0,0 +1,169 @@
+# The relay: one small CVM in Tencent Cloud that terminates TLS for the .cn
+# domain and forwards over WireGuard to the homelab ingress.
+#
+# Why a relay and not Cloudflare: mainland clients reach a Tencent edge far
+# faster than a Cloudflare edge, and Cloudflare's China network is an
+# Enterprise product. The homelab keeps zero inbound exposure because the
+# tunnel is dialled outbound from the cluster.
+#
+# ICP FILING: a mainland resource serving HTTP/HTTPS for a domain requires an
+# ICP filing naming that domain. Without it this instance accepts the
+# connection and the mainland network resets the HTTP layer. Put the relay in a
+# non-mainland region (ap-hongkong, ap-singapore) when no filing exists.
+
+resource "tencentcloud_vpc" "edge" {
+  name       = "${replace(var.china_domain, ".", "_")}_edge"
+  cidr_block = var.edge_vpc_cidr
+}
+
+resource "tencentcloud_subnet" "edge" {
+  name              = "${replace(var.china_domain, ".", "_")}_edge"
+  vpc_id            = tencentcloud_vpc.edge.id
+  availability_zone = var.tencent_availability_zone
+  cidr_block        = var.edge_subnet_cidr
+}
+
+resource "tencentcloud_key_pair" "edge" {
+  key_name   = "${replace(var.china_domain, ".", "_")}_edge"
+  public_key = var.relay_ssh_public_key
+}
+
+resource "tencentcloud_security_group" "edge" {
+  name        = "${replace(var.china_domain, ".", "_")}_edge"
+  description = "TLS + WireGuard for the ${var.china_domain} relay; SSH from the operator only"
+}
+
+resource "tencentcloud_security_group_rule" "ssh" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  port_range        = "22"
+  cidr_ip           = var.relay_admin_cidr
+  policy            = "accept"
+}
+
+resource "tencentcloud_security_group_rule" "tls" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "tcp"
+  port_range        = "80,443"
+  cidr_ip           = "0.0.0.0/0"
+  policy            = "accept"
+}
+
+resource "tencentcloud_security_group_rule" "wireguard" {
+  security_group_id = tencentcloud_security_group.edge.id
+  type              = "ingress"
+  ip_protocol       = "udp"
+  port_range        = tostring(var.wireguard_port)
+  # The homelab dials outbound and has no static public address, so this is
+  # 0.0.0.0/0 by default. WireGuard authenticates every packet by public key,
+  # and an unauthenticated packet is dropped before it reaches the tunnel.
+  # Narrow this to the homelab egress CIDR when that address is stable.
+  cidr_ip = var.wireguard_allowed_cidr
+  policy  = "accept"
+}
+
+# cloud-init: WireGuard listener, forwarding sysctls, and Caddy for TLS.
+# The relay never holds application credentials; it forwards bytes.
+#
+# NOTE ON THE PRIVATE KEY: user_data is visible in the instance metadata and in
+# Terraform state. That is a known limitation of bootstrapping WireGuard this
+# way. The argument is wrapped in sensitive() to keep it out of plan/apply
+# output, but anyone with console or state access can read it. Rotate the key
+# if the state or the instance metadata is ever exposed.
+locals {
+  china_fqdns = [for host in var.china_hostnames : "${host}.${var.china_domain}"]
+
+  caddyfile = templatefile("${path.module}/templates/Caddyfile.tpl", {
+    acme_email       = var.acme_email
+    hosts            = join(", ", local.china_fqdns)
+    ingress_upstream = var.ingress_upstream
+  })
+
+  relay_user_data = <<-EOT
+    #cloud-config
+    package_update: true
+    package_upgrade: false
+    manage_etc_hosts: true
+    timezone: Asia/Shanghai
+    packages: [wireguard, wireguard-tools, caddy]
+    write_files:
+      - path: /etc/wireguard/wg0.conf
+        permissions: "0600"
+        content: |
+          [Interface]
+          Address = ${var.wireguard_relay_address}
+          ListenPort = ${var.wireguard_port}
+          PrivateKey = ${var.wireguard_relay_private_key}
+          # Masquerade forwarded traffic so the homelab peer only has to accept
+          # packets sourced from the relay's tunnel address. Without this the
+          # homelab peer's AllowedIPs would drop the original client source IP.
+          PostUp = iptables -t nat -A POSTROUTING -o wg0 -j MASQUERADE
+          PostDown = iptables -t nat -D POSTROUTING -o wg0 -j MASQUERADE
+          [Peer]
+          PublicKey = ${var.wireguard_homelab_public_key}
+          AllowedIPs = ${var.wireguard_homelab_allowed_ips}
+          PersistentKeepalive = 25
+      - path: /etc/sysctl.d/99-wireguard.conf
+        content: |
+          net.ipv4.ip_forward = 1
+      - path: /etc/caddy/Caddyfile
+        permissions: "0644"
+        content: ${jsonencode(local.caddyfile)}
+    runcmd:
+      - sysctl --system
+      - systemctl enable --now wg-quick@wg0
+      - caddy validate --config /etc/caddy/Caddyfile
+      - systemctl enable --now caddy
+      - systemctl restart caddy
+  EOT
+}
+
+resource "tencentcloud_instance" "relay" {
+  instance_name     = "${replace(var.china_domain, ".", "_")}_edge"
+  availability_zone = var.tencent_availability_zone
+  image_id          = var.relay_image_id
+  instance_type     = var.relay_instance_type
+
+  vpc_id             = tencentcloud_vpc.edge.id
+  subnet_id          = tencentcloud_subnet.edge.id
+  security_groups    = [tencentcloud_security_group.edge.id]
+  key_ids            = [tencentcloud_key_pair.edge.id]
+  allocate_public_ip = true
+
+  # Outbound only needs a few Mbps; inbound TLS is the same order.
+  internet_max_bandwidth_out = var.relay_internet_max_bandwidth_out
+  system_disk_type           = var.relay_system_disk_type
+  system_disk_size           = var.relay_system_disk_size
+
+  # The private key is embedded in this bootstrap script. sensitive() keeps it
+  # out of CLI output; it does not remove it from state or instance metadata.
+  user_data = sensitive(local.relay_user_data)
+
+  tags = {
+    role   = "china-edge"
+    domain = var.china_domain
+  }
+
+  # Accidental `terraform destroy` would take the only China-facing edge down.
+  # Remove this block deliberately when a replacement path exists.
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+# DNS for the .cn domain lives at DNSPod (Tencent). Each subdomain resolves to
+# the relay's public address, and the relay proxies to the homelab over the
+# tunnel. The "中国境内" line is the default line; no separate line records.
+resource "tencentcloud_dnspod_record" "china" {
+  for_each = toset(var.china_hostnames)
+
+  domain      = var.china_domain
+  sub_domain  = each.value
+  record_type = "A"
+  record_line = var.china_dns_record_line
+  value       = tencentcloud_instance.relay.public_ip
+  ttl         = var.china_dns_ttl
+  remark      = "China edge relay; origin is the homelab ingress over WireGuard"
+}