feat: add temporal:admin role to portfolio-agent

- portfolio-agent can now call Temporal API in addition to LLM, memory, S3, SQS

.env.example

@@ -4,8 +4,8 @@
 
 # ── Cluster Configuration ──────────────────────────────────────────────────────
 # Base domain for external services (Authentik, MinIO, Forgejo, etc.)
-#   Example: riotpiao.homelab.com
-CLUSTER_DOMAIN=riotpiao.homelab.com
+#   Example: riotpiao.com
+CLUSTER_DOMAIN=riotpiao.com
 
 # Internal Kubernetes DNS names (svc.cluster.local)
 # Only change these if your cluster domain differs
@@ -68,3 +68,14 @@ AUTHENTIK_TEMPORAL_CLIENT_ID=
 # ── CI/CD ──────────────────────────────────────────────────────────────────────
 # Forgejo Personal Access Token (from rock user) for pushing images to registry
 FORGEJO_RIOTPIAO_PAT=
+
+# ── Cloudflare Tunnel (remote off-LAN access to kubectl/talosctl) ──────────────
+# From Cloudflare Zero Trust dashboard → Networks → Tunnels
+#   CLOUDFLARE_CONNECTOR_TOKEN: full tunnel token (JWT-like base64 string)
+#   CLOUDFLARE_ACCOUNT_ID: your account ID (hex string)
+#   CLOUDFLARE_TUNNEL_ID: tunnel UUID
+#   CLOUDFLARE_API_TOKEN: API token for programmatic tunnel config (optional)
+CLOUDFLARE_CONNECTOR_TOKEN=
+CLOUDFLARE_ACCOUNT_ID=
+CLOUDFLARE_TUNNEL_ID=
+CLOUDFLARE_API_TOKEN=

.forgejo/workflows/cluster-ci.yaml

@@ -0,0 +1,269 @@
+name: Cluster CI Pipeline
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - 'k8s/**'
+      - '.forgejo/workflows/cluster-ci.yaml'
+  pull_request:
+    paths:
+      - 'k8s/**'
+
+jobs:
+  ci:
+    runs-on: docker
+    steps:
+      # === Checkout ===
+      - name: Checkout
+        run: |
+          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
+          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
+          git clone --depth 1 "$CLONE_URL" .
+          git fetch origin main
+          git checkout main
+
+      # === Install Tools ===
+      - name: Install Tools
+        run: |
+          unset GITHUB_TOKEN
+          apt-get update && apt-get install -y \
+            yamllint \
+            python3-pip \
+            curl \
+            jq
+
+          # kubeval
+          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
+          mv -f kubeval /usr/local/bin/
+
+          # kustomize
+          rm -f kustomize
+          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
+          mv -f kustomize /usr/local/bin/
+
+          # argocd
+          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
+          chmod +x /usr/local/bin/argocd
+
+          # trivy
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
+
+          # polaris
+          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
+          chmod +x /usr/local/bin/polaris
+
+      # === YAML Lint ===
+      - name: YAML Lint
+        run: |
+          echo "=== Linting YAML files ==="
+          yamllint k8s/ -c .yamllint.yaml || true
+
+      # === Kubeval - Validate K8s Syntax ===
+      - name: Kubeval - Validate K8s Syntax
+        run: |
+          echo "=== Validating Kubernetes manifests ==="
+          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
+            echo "Validating $file..."
+            kubeval "$file" -d 2>/dev/null || true
+          done
+
+      # === Kustomize Build - All overlays ===
+      - name: Kustomize Build - Infrastructure
+        run: |
+          echo "=== Building k8s/infrastructure/ ==="
+          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
+          echo "✓ Infrastructure built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
+
+      - name: Kustomize Build - Bootstrap
+        run: |
+          echo "=== Building k8s/bootstrap/ ==="
+          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
+          echo "✓ Bootstrap built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
+
+      - name: Kustomize Build - Platform
+        run: |
+          echo "=== Building k8s/platform/ ==="
+          kustomize build k8s/platform > /tmp/platform.yaml
+          echo "✓ Platform built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
+
+      - name: Kustomize Build - Security
+        run: |
+          echo "=== Building k8s/security/ ==="
+          kustomize build k8s/security > /tmp/security.yaml
+          echo "✓ Security built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
+
+      - name: Kustomize Build - Applications
+        run: |
+          echo "=== Building k8s/applications/ ==="
+          kustomize build k8s/applications > /tmp/applications.yaml
+          echo "✓ Applications built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
+
+      - name: Kustomize Build - Data
+        run: |
+          echo "=== Building k8s/data/ ==="
+          kustomize build k8s/data > /tmp/data.yaml
+          echo "✓ Data built successfully"
+          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
+
+      - name: Validate ArgoCD Applications
+        run: |
+          echo "=== Validating ArgoCD Applications ==="
+          kubeval k8s/argocd/apps/*.yaml
+
+      # === Trivy - Scan Dockerfile ===
+      - name: Trivy - Scan Dockerfile
+        run: |
+          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
+            echo "=== Scanning Dockerfiles with Trivy ==="
+            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
+          else
+            echo "No Dockerfiles found"
+          fi
+
+      # === Trivy - Scan Helm Charts ===
+      - name: Trivy - Scan Helm Charts
+        run: |
+          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
+            echo "=== Scanning Helm charts with Trivy ==="
+            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
+              echo "Scanning $chart..."
+              trivy config "$chart" || true
+            done
+          else
+            echo "No Helm charts found"
+          fi
+
+      # === Polaris - K8s Security Audit ===
+      - name: Polaris - K8s Security Audit
+        run: |
+          echo "=== Running Polaris K8s security audit ==="
+          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
+
+          if [ -f /tmp/polaris-audit.json ]; then
+            echo "Security issues found:"
+            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
+          fi
+
+      # === Check for Secrets in Code ===
+      - name: Check for Secrets in Code
+        run: |
+          echo "=== Scanning for hardcoded secrets ==="
+          # BLOCKING. This step used to only count findings and then exit 0, so a
+          # plaintext deploy key rode through it into a public remote. Two failure
+          # modes fixed: it now fails the build, and it matches key material by
+          # PEM header rather than only `private_key:`-style YAML field names.
+          # Findings are captured into variables and tested for emptiness rather than
+          # branching on grep's exit status: implementations disagree on the rc of a
+          # `-v` filter fed empty input, and a wrong rc here fails open.
+          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
+          # and silently scans nothing.
+          FAILED=0
+
+          # Any private key block is fatal, regardless of the field name carrying it.
+          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
+                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
+                 | grep -v "\.enc\.yaml" || true)
+          if [ -n "$KEYS" ]; then
+            echo "❌ Unencrypted private key material found:"
+            echo "$KEYS"
+            FAILED=1
+          fi
+
+          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
+          # so encrypted files never trip this.
+          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
+                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
+                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
+          if [ -n "$VALS" ]; then
+            echo "❌ Plaintext secret value found:"
+            echo "$VALS"
+            FAILED=1
+          fi
+
+          if [ "$FAILED" -ne 0 ]; then
+            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
+            exit 1
+          fi
+          echo "✓ No hardcoded secrets found"
+
+      # === Check K8s Security Best Practices ===
+      - name: Check K8s Security Best Practices
+        run: |
+          echo "=== Checking K8s security best practices ==="
+
+          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
+            echo "⚠️  Found privileged containers"
+          fi
+
+          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
+            echo "⚠️  Found hostNetwork usage"
+          fi
+
+          echo "Checking for missing resource limits..."
+          MISSING=0
+          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
+            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
+              if ! grep -q "resources:" "$file"; then
+                echo "⚠️  $file: Missing resource requests/limits"
+                MISSING=$((MISSING + 1))
+              fi
+            fi
+          done
+
+      # === ArgoCD Sync (main branch only) ===
+      - name: Sync ArgoCD
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        env:
+          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
+          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
+        run: |
+          echo "=== Syncing homelab-root ==="
+          argocd app sync homelab-root --force
+          argocd app wait homelab-root --timeout 5m
+
+      - name: Check Sync Status
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        env:
+          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
+          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
+        run: |
+          echo "=== ArgoCD Applications Status ==="
+          argocd app list -o table
+
+          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
+          if [ "$STATUS" != "Synced" ]; then
+            echo "❌ Root app sync failed: $STATUS"
+            exit 1
+          fi
+          echo "✓ Root app synced successfully"
+
+      - name: Health Check
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        env:
+          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
+          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
+        run: |
+          echo "=== Checking Application Health ==="
+          argocd app get homelab-root -o wide
+
+      # === Summary ===
+      - name: Summary
+        if: always()
+        run: |
+          echo "=== CI Pipeline Summary ==="
+          echo "✓ YAML linted"
+          echo "✓ Manifests validated"
+          echo "✓ Kustomizations built"
+          echo "✓ Security scans completed"
+          echo "✓ Secrets check passed"
+          echo "✓ Best practices verified"
+          echo ""
+          echo "✓ All checks passed"

.gitignore

@@ -1,5 +1,6 @@
 # Environment files — real values must never be committed
 .env
+.env.terraform.sh
 
 # Private CA key and generated TLS certs — ca.key must never enter the cluster or git.
 # Only ca.crt is safe to share, but we exclude the whole dir to avoid accidents.
@@ -8,6 +9,9 @@ forge/pki/
 # Talos machine configs — contain WireGuard private keys, bootstrap tokens, PKI
 cluster-config/controlplane.yaml
 cluster-config/worker*.yaml
+cluster-config/talos-worker*.yaml
+cluster-config/cp-*.yaml
+cluster-config/talos-cp-*.yaml
 cluster-config/secrets.yaml
 cluster-config/talosconfig
 talos-forge-trust.yaml
@@ -34,7 +38,31 @@ k8s/storage/test/test
 *.key
 *.conf
 
-# Allowed markdown: CLAUDE.example.md, README.md, TROUBLESHOOTING.md
-CLAUDE.md
+# CLAUDE.md is now version-controlled (was previously excluded as a
+# private-notes file; contains no secrets - just architecture, IPs
+# [private RFC1918 space], and operational lessons, same bar as README.md).
 
+# Terraform state and cache (local files, remote state in MinIO)
+.terraform/
+terraform/.terraform/
+terraform/*.tfstate
+terraform/*.tfstate.*
+terraform.tfvars.local
 skills-lock.json
+secrets-plaintext.yaml
+
+# Saved plan files — binary, environment-specific, may embed resource attributes
+terraform/tfplan
+terraform/tfplan-*
+
+.DS_Store
+CLAUDE.md
+docs/
+
+bootstrap-argocd.log
+
+# Any plaintext (non-SOPS) secret manifest. Encrypted ones are *.enc.yaml and
+# ARE committed — see .sops.yaml. A missing newline once merged two patterns on
+# one line here, which is how a plaintext deploy key reached a public remote.
+k8s/**/*-secret.yaml
+!k8s/**/*.enc.yaml

.sops.yaml

@@ -0,0 +1,5 @@
+creation_rules:
+  # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
+  # and was committed in plaintext to a public remote.
+  - path_regex: k8s/.*secrets?.*\.ya?ml
+    age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla

.yamllint.yaml

@@ -0,0 +1,33 @@
+---
+extends: default
+
+rules:
+  line-length:
+    max: 120
+    level: warning
+  indentation:
+    spaces: 2
+  brackets:
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+  braces:
+    min-spaces-inside: 0
+    max-spaces-inside: 0
+  comments:
+    min-spaces-from-content: 2
+  comments-indentation: {}
+  document-end: disable
+  document-start: disable
+  empty-lines:
+    max: 3
+  empty-values:
+    forbid-in-block-mappings: true
+    forbid-in-flow-mappings: true
+  key-duplicates: enable
+  key-ordering: disable
+  new-line-at-end-of-file: enable
+  new-lines:
+    type: unix
+  trailing-spaces: enable
+  truthy:
+    level: warning

AUTH_INTEGRATION.md

@@ -0,0 +1,206 @@
+# Authentik Auth Integration for NextJS
+
+## Current State
+
+### Gateway Auth Status
+
+| Endpoint | Auth Status | Notes |
+|----------|-------------|-------|
+| `/v1/chat/completions` | ❌ **OFF** | LLM routes have no auth middleware |
+| `/v1/embeddings` | ❌ **OFF** | Same - no auth |
+| `/v1/rerank` | ❌ **OFF** | Same - no auth |
+| `X-Service: sqs` | ✅ **ON** | JWT validated via `internal/auth/jwt.go` |
+| `/workflow` | ❌ **OFF** | Pass-through to Temporal |
+
+**Auth module exists** at `homelab-frontend/internal/auth/jwt.go` but only wired for SQS.
+LLM routes in `internal/proxy/proxy.go` have no auth middleware.
+
+### Authentik App
+
+Authentik app `local-llm` exists for LLM API auth:
+- **Client ID**: `local-llm`
+- **Client Secret**: `kubectl -n llm-serving get secret local-llm-jwt -o jsonpath='{.data.client-secret}' | base64 -d`
+- **Token endpoint**: `https://authentik.riotpiao.com/application/o/token/`
+- **Userinfo endpoint**: `https://authentik.riotpiao.com/application/o/userinfo/`
+- **OIDC discovery**: `https://authentik.riotpiao.com/application/o/local-llm/.well-known/openid-configuration`
+
+## Sign-in Methods
+
+### 1. Resource Owner Password Credentials (ROPC)
+
+Direct username/password login. Server-side only (needs client_secret).
+
+```typescript
+// API Route: app/api/auth/login/route.ts
+const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+  method: 'POST',
+  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+  body: new URLSearchParams({
+    grant_type: 'password',
+    client_id: 'local-llm',
+    client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+    username: '[email protected]',
+    password: 'userpassword',
+    scope: 'openid email profile groups',
+  }),
+});
+
+const tokens = await response.json();
+// { access_token, refresh_token, expires_in, token_type }
+```
+
+### 2. Authorization Code Flow (Browser Redirect)
+
+Requires adding redirect URIs to `local-llm` Authentik app:
+
+```python
+# In k8s/infra/iam/scripts/authentik-provision.py, update:
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",  # dev
+        "https://your-nextjs-app.com/api/auth/callback",  # prod
+    ],
+}
+```
+
+Then standard OIDC flow:
+1. Redirect to `https://authentik.riotpiao.com/application/o/authorize/?client_id=local-llm&redirect_uri=...&response_type=code&scope=openid email profile groups`
+2. User logs in via Authentik UI
+3. Callback receives `code`, exchange for tokens
+
+## JWT Token Persistence
+
+### Browser (localStorage)
+
+```typescript
+const TOKEN_KEY = 'llm_auth_token';
+
+// Save
+localStorage.setItem(TOKEN_KEY, JSON.stringify({
+  access_token: tokens.access_token,
+  refresh_token: tokens.refresh_token,
+  expires_at: Date.now() + tokens.expires_in * 1000,
+}));
+
+// Load
+const stored = JSON.parse(localStorage.getItem(TOKEN_KEY) || 'null');
+if (stored && stored.expires_at > Date.now()) {
+  // Token valid
+}
+
+// Clear (logout)
+localStorage.removeItem(TOKEN_KEY);
+```
+
+### Server-side (HTTP-only cookies)
+
+```typescript
+// app/api/auth/login/route.ts
+import { cookies } from 'next/headers';
+
+// After successful login
+cookies().set('llm_auth_token', JSON.stringify(tokens), {
+  httpOnly: true,
+  secure: process.env.NODE_ENV === 'production',
+  sameSite: 'lax',
+  maxAge: tokens.expires_in,
+  path: '/',
+});
+
+// Read in middleware or API routes
+const tokenCookie = cookies().get('llm_auth_token');
+const tokens = JSON.parse(tokenCookie?.value || 'null');
+```
+
+## Token Refresh
+
+```typescript
+async function refreshAccessToken(refresh_token: string) {
+  const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'refresh_token',
+      client_id: 'local-llm',
+      client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
+      refresh_token,
+    }),
+  });
+  return response.json();
+}
+```
+
+## Environment Variables
+
+```bash
+# .env.local
+AUTHENTIK_URL=https://authentik.riotpiao.com
+AUTHENTIK_CLIENT_ID=local-llm
+AUTHENTIK_CLIENT_SECRET=<from-secret>
+
+# For client-side (public)
+NEXT_PUBLIC_AUTHENTIK_URL=https://authentik.riotpiao.com
+NEXT_PUBLIC_AUTHENTIK_CLIENT_ID=local-llm
+```
+
+## Using Token with LLM API
+
+```typescript
+const token = await getValidToken(); // from localStorage or cookie
+
+const response = await fetch('https://api.riotpiao.com/v1/chat/completions', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${token}`,  // JWT from Authentik
+  },
+  body: JSON.stringify({
+    model: 'reasoning',
+    messages: [{ role: 'user', content: 'Hello' }],
+  }),
+});
+```
+
+## TODO
+
+### Gateway-side (homelab-frontend)
+
+- [ ] Wire `internal/auth/jwt.go` into LLM proxy handler (`internal/proxy/proxy.go`)
+- [ ] Add `authRequired: true` to model config or create LLM-specific middleware
+- [ ] Example pattern from SQS (in `internal/serviceadapter/router.go`):
+
+```go
+// In proxy.go ServeHTTP, before dispatching to LLM upstream:
+if strings.HasPrefix(r.URL.Path, "/v1/") {
+    authHeader := r.Header.Get("Authorization")
+    claims, err := llmJWTAuth.ValidateBearerToken(authHeader)
+    if err != nil {
+        // Return 401/403
+    }
+    if !llmJWTAuth.CheckPermissions(claims, "llm:inference", "*") {
+        // Return 403 insufficient permissions
+    }
+}
+```
+
+### Authentik-side
+
+- [ ] Enable ROPC grant in Authentik provider settings (if not already)
+- [ ] Add redirect URIs to `local-llm` app if browser OAuth flow needed:
+
+```python
+# k8s/infra/iam/scripts/authentik-provision.py
+"local-llm": {
+    ...
+    "redirect_uris": [
+        "http://localhost:3000/api/auth/callback",
+        "https://your-app.com/api/auth/callback",
+    ],
+}
+```
+
+### NextJS-side
+
+- [ ] Until gateway auth is wired, LLM API works without token
+- [ ] Once wired, add `Authorization: Bearer <token>` to all LLM requests

CLAUDE.example.md

@@ -1,72 +1,210 @@
-# CLAUDE.md — Homelab Integration Guide
+# CLAUDE.md — Homelab Integration Guide (Example / Reusable Template)
 
-**Homelab:** A bare-metal three-node Kubernetes cluster running Talos Linux with a full observability stack, SSO via Authentik, secret management via Vault, and CI/CD infrastructure (Forgejo + Argo CD, deployed).
+> **This is a sanitized template.** Copy to `CLAUDE.md`, fill in your own
+> node IPs/hostnames/secrets, and delete this notice. Nothing in this file
+> should contain real credentials, real IPs beyond illustrative examples, or
+> anything that would matter if this file became public. It's meant to be
+> shared across homelabs running similar hardware/topology (3-node bare-metal
+> Talos Kubernetes + ArgoCD GitOps), not just this one.
+
+**Homelab:** A bare-metal N-node Kubernetes cluster running Talos Linux with a
+full observability stack, SSO via Authentik, secret management via Vault, and
+CI/CD infrastructure (self-hosted git forge + Argo CD).
+
+## Cluster Topology (adjust to your hardware)
+
+| Node | IP | Zone | Scheduling | Storage |
+|------|----|----|-----------|---------|
+| `<node-1>` | `<ip>` | az-a | schedulable (all workloads) | sole storage node (if single-node storage) |
+| `<node-2>` | `<ip>` | az-b | dedicated (`NoSchedule`) | none |
+| `<node-3>` | `<ip>` | az-c | dedicated (`NoSchedule`) | none |
+
+If your storage layer (Longhorn, local-path, etc.) only runs on one node,
+every stateful workload pinned to that storage class is effectively
+single-instance regardless of your control-plane HA count — document that
+explicitly here, it changes your failure-mode assumptions everywhere else.
+
+## Deployment Model: ArgoCD GitOps (app-of-apps)
+
+```
+git commit → git push (your git forge) → ArgoCD auto-sync → cluster
+```
+
+Structure:
+- One root `Application` (`k8s/argocd/root/`) pointing at a directory of
+  child `Application` manifests (`k8s/argocd/apps/*.yaml`)
+- Each child Application is either:
+  - A remote Helm chart + a **second** git source (`ref: values`) supplying
+    just the values file — lets you pin an upstream chart version while
+    keeping your values under normal git history/review
+  - A plain git directory of raw manifests (optionally with a
+    `kustomization.yaml`)
+- `argocd.argoproj.io/sync-wave` annotations control ordering across
+  Applications (lower number syncs first)
+
+**Never `kubectl apply`/`patch`/`delete` a resource ArgoCD manages**, except:
+- Pure cleanup of stuck/dead state (e.g. deleting a permanently-failed hook
+  Job so the next real sync can create a fresh one) — this is not a config
+  change, just clearing wreckage that GitOps itself won't clean up
+  automatically (see Gotchas below)
+- Genuine one-time bootstrap circular dependencies (e.g. Vault
+  `operator init`/unseal — nothing can configure Vault's own unseal keys
+  before Vault has generated them)
+
+## Hard Rules (adapt freely, but keep something like these)
+
+🔴 **Identify and document your storage/stateful-singleton node explicitly.**
+Whatever node holds your CSI driver's data (Longhorn, local-path, etc.),
+renaming or wiping it orphans every PVC pinned there. Name it here, in
+caps, so nobody "cleans up" it by accident.
+
+🔴 **If you run multi-member etcd across a LAN + VPN/WireGuard overlay,
+pin the advertised subnet explicitly** (e.g. Talos's
+`cluster.etcd.advertisedSubnets`). Without it, etcd may advertise on the
+wrong interface and new members hang as non-promoting learners.
+
+🔴 **Run your IaC formatter (terraform fmt, etc.) before every commit
+that touches infra code.** Wire this into CI as a hard gate, not a
+suggestion.
+
+🔴 **Whatever your source of truth is (Terraform, ArgoCD, both) — never
+manually mutate resources it manages.** State drift is the single most
+common cause of "why did my last apply undo my manual fix" confusion.
+Fix the source, re-apply/re-sync, never bypass.
+
+🔴 **Never delete a PVC without confirming replica count / backup
+freshness first.** This is always a one-way door.
+
+🔴 **Decide your commit message convention up front and enforce it.**
+(This template's origin project uses: no co-authored-by footers, single-line
+commit summarizing what/why, solo-authorship assumption — adjust to your
+team's norms.)
+
+🔴 **Decide your git workflow (rebase vs merge) up front and stick to it**
+cluster-wide, across every contributor/agent working in the repo.
+
+🔴 **Long-running commands should not block a synchronous session** — run
+them in the background and poll, especially anything that waits on a
+Kubernetes rollout, an image pull, or a Terraform apply.
+
+## GitOps / ArgoCD Gotchas (transferable to any ArgoCD-based homelab)
+
+🟠 **A `kustomization.yaml` with an explicit `resources:` allowlist
+silently drops anything you forget to list.** No error, no drift shown in
+ArgoCD's UI — it just reports `Synced/Healthy` against a manifest set that
+never included your new file. Always run `kubectl kustomize <dir>/`
+locally before pushing to confirm exactly what ArgoCD will build.
+
+🟠 **A top-level `namespace:` transformer in `kustomization.yaml` rewrites
+`metadata.namespace` on every resource it builds** — including RBAC
+bindings deliberately targeting a *different* namespace (e.g. granting a
+ServiceAccount in namespace A read access to Secrets in namespace B). If
+any manifest needs cross-namespace RBAC, either drop the transformer
+(safe if every resource already sets its own explicit namespace) or give
+that manifest its own Application/directory.
+
+🟠 **PreSync hooks run before an Application's own normal resources are
+synced.** A PreSync Job that depends on RBAC/ServiceAccounts defined as
+plain (non-hook) resources in the *same* Application will deadlock — it
+tries to start before its own permissions exist. Use PostSync instead if
+the hook needs resources from its own Application, or move the
+prerequisite RBAC into an earlier sync-wave Application.
+
+🟠 **ArgoCD hooks are not continuously reconciled by `selfHeal`.** Once a
+hook Job finishes (success, or exhausts `backoffLimit`), it's only
+deleted+recreated during an *actual new Sync operation* — not by passive
+drift detection, even with `automated.selfHeal: true` on. If you fix a
+broken hook's spec and push, the Application's `status.sync.revision` can
+show "caught up" while the live hook resource is still the old, broken
+one, because no new operation actually re-ran it. To force it: delete the
+stuck hook (clear `argocd.argoproj.io/hook-finalizer` manually if it's
+stuck `Terminating`), and if that alone doesn't trigger a fresh full sync,
+delete + re-`kubectl apply -f` the Application object itself.
+
+🟠 **If you route ArgoCD's own `repoURL` through an ingress/reverse-proxy
+hostname that only listens on 80/443, don't use a non-standard port in the
+URL** — it'll silently time out trying to reach a port the proxy never
+opened, and depending on your setup this can block *every* Application's
+sync simultaneously (repo-server can't fetch git refs for anything).
+
+🟠 **Don't pin exact version tags for images from registries that don't
+guarantee tag retention** (Bitnami stopped publishing versioned tags for
+free-tier images in 2025 — only `latest` + sha256 digests remain). Verify
+a tag actually exists before pinning it, or prefer minimal base images +
+a stdlib-only runtime download (e.g. Python's `urllib.request` to fetch a
+static binary) to avoid depending on any third party's tagging policy.
+
+🟠 **Non-root containers can't `apk add`/`apt install` in most default
+base images** — package manager directories are root-owned. Use a
+world-writable scratch dir (`/tmp`) for anything you need to
+download/install at runtime instead.
+
+🟠 **Helm does not validate unknown `values.yaml` keys.** A typo, or a
+values schema copied from the wrong chart *version's* docs/examples, is
+silently a no-op — not an error. Before concluding "this chart doesn't
+support X," clone the chart at your exact pinned version/tag and run
+`helm template` against your real values file, then diff the rendered
+output. Don't trust a chart's current `main`-branch example values file
+if you're pinned to an older release — schemas do change between major
+versions without warning in your own values file.
 
 ## Service Integration Routes
 
-**New service? Pick your stack below:**
+**New service? Pick your stack below** (adjust doc paths to match your repo):
 
 | Need | Doc | Example |
 |------|-----|---------|
 | **Authentication** | `project-usage/authentik-oidc.md` | OAuth2 login, RBAC groups, JWT tokens |
-| **Async messaging** | `project-usage/sqs-messaging.md` | Kafka topic consumers, fire-and-forget, DLQ |
+| **Async messaging** | `project-usage/sqs-messaging.md` | Queue consumers, fire-and-forget, DLQ |
 | **Object storage** | `project-usage/minio-s3.md` | File uploads, backups, log backend |
-| **CI/CD pipeline** | `project-usage/cicd-workflow.md` | GitHub Actions syntax, image push, Argo CD sync |
+| **CI/CD pipeline** | `project-usage/cicd-workflow.md` | Pipeline syntax, image push, ArgoCD sync |
 | **Workflows** | `project-usage/temporal-workflows.md` | Long-running jobs, retries, state machines |
 | **Database** | `project-usage/database-postgres.md` | CloudNativePG setup, schema migrations, replicas |
 | **Monitoring** | `project-usage/monitoring-metrics.md` | Prometheus scrape, Grafana dashboard, alerts |
 | **Secrets** | `project-usage/vault-secrets.md` | Store credentials, rotate tokens, seal/unseal |
 | **Networking** | `project-usage/networking-ingress.md` | Public HTTPS, hostname routing, TLS |
 
-## Cluster Essentials
+## Cluster Essentials (fill in your own inventory)
 
-**22 namespaces, 18 releases:**
-```
-Core: cert-manager, ingress-nginx, kube-system, cilium
-Storage: longhorn-system, storage (MinIO)
-Data: ddb (PostgreSQL), iam (Authentik + Vault)
-Observability: logging (Loki + Grafana), monitoring (Prometheus)
-Apps: cicd (Forgejo + Argo CD), sqs (Kafka + kmsvc), temporal, story-crater-backend
-```
+**Architecture principles (adjust to taste, but these travel well):**
+- Immutable OS (Talos, or similar — no SSH, fully declarative config)
+- Secrets in a proper secrets backend (Vault) + SOPS-encrypted manifests in
+  git (`*.enc.yaml`, age-encrypted); never commit plaintext secrets or `.env`
+- ArgoCD app-of-apps as the single CD source of truth; two-phase bootstrap
+  documented separately (chicken-and-egg: ArgoCD needs to exist before it
+  can deploy itself declaratively — document your exact bootstrap steps)
+- Pull-based GitOps — no kubeconfig/cluster credentials ever touch your CI
+  runner; the runner only needs push access to git, ArgoCD does the rest
+- Federated OIDC (one identity provider fronting every service that
+  supports it)
 
-**Architecture principles:**
-- Immutable OS (Talos — no SSH, declarative config)
-- Secrets in Vault (never commit `.env`, credentials in Secret volumes)
-- Helmfile = single source of truth (`helmfile.yaml.gotmpl`)
-- Pull-based GitOps (Argo CD, no kubeconfig in CI)
-- Federated OIDC (Authentik provider for all services)
+## Deployment Checklist (per new service)
 
-
-## Deployment Checklist
-
-- [ ] Service has Prometheus `/metrics` endpoint or ServiceMonitor
-- [ ] All credentials in Vault (never in pod env, ConfigMap, or code)
-- [ ] Ingress rule in `k8s/ingress/` with TLS cert
-- [ ] Grafana dashboard in `k8s/monitoring/dashboards/svc-<name>.yaml`
-- [ ] Alert rules in `k8s/monitoring/alerts/svc-<name>-rules.yaml` (if needed)
-- [ ] Helm release in `helmfile.yaml.gotmpl` with correct `needs:` dependencies
-
-## Hard Rules
-
-1. **No kubeconfig in CI** — Argo CD bridges gap (pull-based, never push secrets to runner)
-2. **Field name = variable name** — In Vault: `talos put cluster/KAFKA_BOOTSTRAP KAFKA_BOOTSTRAP="..."`
-3. **Secrets via volumes** — Never `--env` flag in pod specs (exposes in `kubectl describe`)
-4. **External services via Ingress** — All public endpoints via TLS (homelab-ca)
-5. **Never commit `.env`** — Only `.env.example` in git; real secrets in Vault
+- [ ] Prometheus `/metrics` endpoint or ServiceMonitor, if it exposes metrics
+- [ ] All credentials in your secrets backend (never in plain values.yaml,
+      pod env directly, or committed anywhere in cleartext)
+- [ ] Ingress rule with TLS, if externally reachable
+- [ ] Dashboard + alert rules, if metrics are exposed
+- [ ] ArgoCD `Application` manifest added to the appropriate sync-wave file,
+      **not** a standalone `helm install`/`kubectl apply` run by hand
+- [ ] Validated locally before push: `kubectl apply --dry-run=client -f`,
+      `kubectl kustomize <dir>/` (if applicable), or `helm template` against
+      the exact pinned chart version (if Helm-sourced)
+- [ ] After push: confirmed ArgoCD's `status.sync.revision` actually matches
+      your new commit — not just that `status.sync.status` says `Synced`
+      (see Gotchas — a stale hook can hide behind an otherwise-current app)
 
 ## Git & Release
 
-**Multi-remote push:**
-```bash
-git push origin main
-```
-
-**Incremental commits (service-layer grouped):**
+**Incremental commits (service-layer grouped) tend to age well:**
 - Foundation & Docs
-- Helmfile & Core Infra
-- Storage Layer
+- Core Infra (CNI, ingress, cert management, storage)
 - Observability Stack
 - IAM & Secrets
 - CI/CD & GitOps
-- Messaging Infrastructure
+- Messaging / Data Infrastructure
 - Applications & Utilities
+
+Grouping by layer (rather than by day or by "misc fixes") makes it much
+easier to `git log --oneline -- <path>` your way back to *why* a given
+piece of config looks the way it does, months later.

Makefile

@@ -1,21 +1,21 @@
-# ── Node IPs ──────────────────────────────────────────────────────────────────
-# CP_IP has a default. All W{N}_IP variables are expected to be exported from
-# ~/.zshrc (e.g. export W1_IP=192.168.1.162). No guards — assumed always set.
-CP_IP ?= 192.168.1.213
-
-export CP_IP
+# ── Node IPs (3-CP HA topology) ───────────────────────────────────────────────
+CP1_IP  := 192.168.1.166  # talos-cp-1
+CP2_IP  := 192.168.1.214  # talos-cp-2 (storage: 3 disks)
+CP3_IP  := 192.168.1.162  # talos-cp-3
+CP_VIP  := 192.168.1.166  # controlplane VIP (currently .166)
 
 # ── Paths ─────────────────────────────────────────────────────────────────────
-TALOSCONFIG := cluster-config/coreconfig
-CP_CONFIG   := cluster-config/controlplane.yaml
-SECRETS     := cluster-config/secrets.yaml
+TALOSCONFIG := cluster-config/talosconfig
+CP1_CONFIG  := cluster-config/talos-cp-1.yaml
+CP2_CONFIG  := cluster-config/talos-cp-2.yaml
+CP3_CONFIG  := cluster-config/talos-cp-3.yaml
 KUBECONFIG  := cluster-config/kubeconfig
 
-CLUSTER_NAME := homelab-cluster
-CP_ENDPOINT  := https://$(CP_IP):6443
-TALOS_IMAGE  := factory.core.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.13.3
+CLUSTER_NAME := homelab
+CP_ENDPOINT  := https://$(CP_VIP):6443
 
-TALOSCTL := corectl --coreconfig $(TALOSCONFIG)
+# Use talosctl (not corectl). Needs TALOSCONFIG env var pointing to talosconfig file.
+TALOSCTL := talosctl
 KUBECTL  := kubectl --kubeconfig $(KUBECONFIG)
 
 # Derive IP and config from worker number N (used by generic targets).
@@ -28,236 +28,157 @@ W_CONFIG = cluster-config/worker-$(N).yaml
 # ── Help ──────────────────────────────────────────────────────────────────────
 .PHONY: help
 help:
-	@echo "Homelab cluster — available targets"
+	@echo "Homelab cluster (3-CP HA: .166/.214/.162) — available targets"
 	@echo ""
-	@echo "  Status"
+	@echo "  Status & Services"
 	@echo "    nodes             kubectl get nodes"
-	@echo "    status-cp         core node overview  (control plane)"
-	@echo "    status-w1         core node overview  (worker-1)"
-	@echo "    services-cp       list core services  (control plane)"
-	@echo "    services-w1       list core services  (worker-1)"
+	@echo "    status-all        etcd members on all 3 CPs"
+	@echo "    status-cp1/2/3    etcd members on specific CP"
+	@echo "    services-cp1/2/3  list Talos services on specific CP"
 	@echo ""
 	@echo "  Logs"
-	@echo "    logs-cp           stream kubelet logs  (control plane)"
-	@echo "    logs-w1           stream kubelet logs  (worker-1)"
-	@echo "    dmesg-cp          kernel dmesg         (control plane)"
-	@echo "    dmesg-w1          kernel dmesg         (worker-1)"
-	@echo "    log-svc-cp        stream a service log (control plane)  SVC=<name>"
-	@echo "    log-svc-w1        stream a service log (worker-1)       SVC=<name>"
+	@echo "    logs-cp1/2/3      stream kubelet logs from CP{1,2,3}"
+	@echo "    dmesg-cp1/2/3     stream kernel dmesg from CP{1,2,3}"
+	@echo "    log-svc-cp1/2/3   stream service logs (SVC=<name>)"
 	@echo ""
-	@echo "  Config"
-	@echo "    gen-config        regenerate controlplane.yaml + worker-N.yaml from secrets"
-	@echo "    apply-cp          apply controlplane.yaml to CP node (live cluster)"
-	@echo "    apply-w1          apply cluster-config/worker-1.yaml to worker-1"
-	@echo "    apply-w1-insecure first-time apply to worker-1 (no certs yet)"
-	@echo "    apply-worker      apply cluster-config/worker-N.yaml   N=<num> W<N>_IP=<ip>"
-	@echo "    apply-worker-new  first-time apply (--insecure)        N=<num> W<N>_IP=<ip>"
+	@echo "  Config Apply"
+	@echo "    apply-all         apply configs to all 3 CPs (talos-cp-{1,2,3}.yaml)"
+	@echo "    apply-cp1/2/3     apply config to specific CP"
 	@echo ""
-	@echo "  Upgrade"
-	@echo "    upgrade-cp        upgrade Talos on control plane"
-	@echo "    upgrade-w1        upgrade Talos on worker-1"
-	@echo "    upgrade-worker    upgrade any worker                    N=<num> W<N>_IP=<ip>"
-	@echo ""
-	@echo "  Shutdown / Reboot"
-	@echo "    shutdown-cluster  graceful full shutdown (drain w1 → off w1 → off cp)"
-	@echo "    shutdown-cp       shut down control plane only"
-	@echo "    shutdown-w1       shut down worker-1 only"
-	@echo "    shutdown-worker   shut down any worker                  N=<num> W<N>_IP=<ip>"
-	@echo "    reboot-cp         reboot control plane"
-	@echo "    reboot-w1         reboot worker-1"
-	@echo "    reboot-worker     reboot any worker                     N=<num> W<N>_IP=<ip>"
-	@echo ""
-	@echo "  Inspect (node filesystem)"
-	@echo "    node-ls   <ip> <path>   list files on a node"
-	@echo "    node-read <ip> <path>   read a file on a node"
-	@echo ""
-	@echo "  Maintenance"
-	@echo "    clean-pods        delete Evicted/Failed/Terminating pods cluster-wide"
+	@echo "  Reboot"
+	@echo "    reboot-all        reboot all 3 CPs"
+	@echo "    reboot-cp1/2/3    reboot specific CP"
 	@echo ""
 	@echo "  Port-forwards"
 	@echo "    pf-grafana        localhost:3000 → Grafana"
-	@echo "    pf-minio          localhost:9001 → MinIO console  /  localhost:9000 → S3 API"
-	@echo "    pf-loki           localhost:3100 → Loki HTTP API"
-	@echo "    pf-portainer      localhost:9000 → Portainer UI (dashboard ns)"
-	@echo "    pf-prometheus     localhost:9090 → Prometheus UI (monitoring ns)"
 	@echo "    pf-longhorn       localhost:8080 → Longhorn UI"
-	@echo "    pf-iam            localhost:7000 → Authentik IAM (when deployed)"
+	@echo "    pf-prometheus     localhost:9090 → Prometheus UI"
 	@echo ""
-	@echo "  CLI"
-	@echo "    cli               build core-cli and install to ~/.local/bin/core"
-	@echo ""
-	@echo "  Variables"
-	@echo "    CP_IP   (default: 192.168.1.160)"
-	@echo "    W1_IP   (export from ~/.zshrc — e.g. export W1_IP=192.168.1.162)"
-	@echo "    N       (required for generic targets — worker number, e.g. N=2)"
-	@echo "    W<N>_IP (export from ~/.zshrc — e.g. export W2_IP=192.168.1.163)"
-	@echo "    SVC     (required for log-svc-* targets, e.g. SVC=kubelet)"
+	@echo "  IPs"
+	@echo "    CP1 (talos-cp-1): $(CP1_IP)  — NVMe, wg0/wg1, VIP"
+	@echo "    CP2 (talos-cp-2): $(CP2_IP)  — 3 Longhorn disks"
+	@echo "    CP3 (talos-cp-3): $(CP3_IP)  — NVMe"
 
 # ── Status ────────────────────────────────────────────────────────────────────
 .PHONY: nodes
 nodes:
 	$(KUBECTL) get nodes -o wide
 
+.PHONY: status-all
+status-all: status-cp1 status-cp2 status-cp3
+
+.PHONY: status-cp1
+status-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) etcd members
+
+.PHONY: status-cp2
+status-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) etcd members
+
+.PHONY: status-cp3
+status-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) etcd members
+
 .PHONY: status-cp
-status-cp:
-	$(TALOSCTL) --nodes $(CP_IP) get members
+status-cp: status-all
 
-.PHONY: status-w1
-status-w1:
-	$(TALOSCTL) --nodes $(W1_IP) get members
+.PHONY: services-cp1
+services-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) service
 
-.PHONY: services-cp
-services-cp:
-	$(TALOSCTL) --nodes $(CP_IP) service
+.PHONY: services-cp2
+services-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) service
 
-.PHONY: services-w1
-services-w1:
-	$(TALOSCTL) --nodes $(W1_IP) service
+.PHONY: services-cp3
+services-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) service
 
-# ── Logs ──────────────────────────────────────────────────────────────────────
-.PHONY: logs-cp
-logs-cp:
-	$(TALOSCTL) --nodes $(CP_IP) logs kubelet -f
+# ── Logs (3-CP) ───────────────────────────────────────────────────────────────
+.PHONY: logs-cp1
+logs-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs kubelet -f
 
-.PHONY: logs-w1
-logs-w1:
-	$(TALOSCTL) --nodes $(W1_IP) logs kubelet -f
+.PHONY: logs-cp2
+logs-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs kubelet -f
 
-.PHONY: dmesg-cp
-dmesg-cp:
-	$(TALOSCTL) --nodes $(CP_IP) dmesg --follow
+.PHONY: logs-cp3
+logs-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs kubelet -f
 
-.PHONY: dmesg-w1
-dmesg-w1:
-	$(TALOSCTL) --nodes $(W1_IP) dmesg --follow
+.PHONY: dmesg-cp1
+dmesg-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) dmesg --follow
 
-# Usage: make log-svc-cp SVC=etcd
-.PHONY: log-svc-cp
-log-svc-cp:
+.PHONY: dmesg-cp2
+dmesg-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) dmesg --follow
+
+.PHONY: dmesg-cp3
+dmesg-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) dmesg --follow
+
+# Usage: make log-svc-cp1 SVC=etcd
+.PHONY: log-svc-cp1
+log-svc-cp1:
 ifndef SVC
-	$(error SVC is not set — run: make log-svc-cp SVC=<service-name>)
+	$(error SVC is not set — run: make log-svc-cp1 SVC=<service-name>)
 endif
-	$(TALOSCTL) --nodes $(CP_IP) logs $(SVC) -f
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs $(SVC) -f
 
-.PHONY: log-svc-w1
-log-svc-w1:
+.PHONY: log-svc-cp2
+log-svc-cp2:
 ifndef SVC
-	$(error SVC is not set — run: make log-svc-w1 SVC=<service-name>)
+	$(error SVC is not set — run: make log-svc-cp2 SVC=<service-name>)
 endif
-	$(TALOSCTL) --nodes $(W1_IP) logs $(SVC) -f
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs $(SVC) -f
 
-# ── Config generation ─────────────────────────────────────────────────────────
-.PHONY: gen-config
-gen-config:
-	corectl gen config $(CLUSTER_NAME) $(CP_ENDPOINT) \
-	  --with-secrets $(SECRETS) \
-	  --output-dir cluster-config/ \
-	  --force
+.PHONY: log-svc-cp3
+log-svc-cp3:
+ifndef SVC
+	$(error SVC is not set — run: make log-svc-cp3 SVC=<service-name>)
+endif
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs $(SVC) -f
+
+# ── Config Apply (3-CP) ───────────────────────────────────────────────────────
+.PHONY: apply-all
+apply-all: apply-cp1 apply-cp2 apply-cp3
+	@echo "✓ All 3 control planes configured"
+
+.PHONY: apply-cp1
+apply-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) apply-config -f $(CP1_CONFIG)
+
+.PHONY: apply-cp2
+apply-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) apply-config -f $(CP2_CONFIG)
+
+.PHONY: apply-cp3
+apply-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) apply-config -f $(CP3_CONFIG)
 
-# ── Config apply ──────────────────────────────────────────────────────────────
 .PHONY: apply-cp
-apply-cp:
-	$(TALOSCTL) apply-config \
-	  --nodes $(CP_IP) \
-	  --file $(CP_CONFIG)
+apply-cp: apply-all
 
-.PHONY: apply-w1
-apply-w1:
-	$(TALOSCTL) apply-config \
-	  --nodes $(W1_IP) \
-	  --file cluster-config/worker-1.yaml
+.PHONY: reboot-all
+reboot-all: reboot-cp1 reboot-cp2 reboot-cp3
+	@echo "✓ All 3 control planes rebooting"
 
-# First-time apply to worker-1 (no certs yet)
-.PHONY: apply-w1-insecure
-apply-w1-insecure:
-	$(TALOSCTL) apply-config \
-	  --nodes $(W1_IP) \
-	  --file cluster-config/worker-1.yaml \
-	  --insecure
+.PHONY: reboot-cp1
+reboot-cp1:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) reboot
 
-# Generic targets — derive both IP and config from N.
-# Usage: make apply-worker N=2 W2_IP=192.168.1.162
-#        make apply-worker N=3 W3_IP=192.168.1.163
-.PHONY: apply-worker
-apply-worker:
-ifndef N
-	$(error N is not set — run: make apply-worker N=<num> W<N>_IP=<ip>)
-endif
-	$(TALOSCTL) apply-config \
-	  --nodes $(W_IP) \
-	  --file $(W_CONFIG)
+.PHONY: reboot-cp2
+reboot-cp2:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) reboot
 
-.PHONY: apply-worker-new
-apply-worker-new:
-ifndef N
-	$(error N is not set — run: make apply-worker-new N=<num> W<N>_IP=<ip>)
-endif
-	$(TALOSCTL) apply-config \
-	  --nodes $(W_IP) \
-	  --file $(W_CONFIG) \
-	  --insecure
-
-# ── Upgrade ───────────────────────────────────────────────────────────────────
-.PHONY: upgrade-cp
-upgrade-cp:
-	$(TALOSCTL) upgrade \
-	  --nodes $(CP_IP) \
-	  --image $(TALOS_IMAGE) \
-	  --preserve
-
-.PHONY: upgrade-w1
-upgrade-w1:
-	$(TALOSCTL) upgrade \
-	  --nodes $(W1_IP) \
-	  --image $(TALOS_IMAGE) \
-	  --preserve
-
-# Usage: make upgrade-worker N=2 W2_IP=192.168.1.162
-.PHONY: upgrade-worker
-upgrade-worker:
-ifndef N
-	$(error N is not set — run: make upgrade-worker N=<num> W<N>_IP=<ip>)
-endif
-	$(TALOSCTL) upgrade \
-	  --nodes $(W_IP) \
-	  --image $(TALOS_IMAGE) \
-	  --preserve
-
-# ── Shutdown / Reboot ─────────────────────────────────────────────────────────
-# Full cluster: drain workers first so pods stop cleanly, then workers off,
-# then CP last (etcd must be the final process to stop).
-.PHONY: shutdown-cluster
-shutdown-cluster:
-	@echo "--- draining core-worker-1 ---"
-	$(KUBECTL) drain core-worker-1 --ignore-daemonsets --delete-emptydir-data
-	@echo "--- shutting down worker-1 ---"
-	$(TALOSCTL) --nodes $(W1_IP) shutdown
-	@echo "--- shutting down control plane (last) ---"
-	$(TALOSCTL) --nodes $(CP_IP) shutdown
-
-.PHONY: shutdown-cp
-shutdown-cp:
-	$(TALOSCTL) --nodes $(CP_IP) shutdown
-
-.PHONY: shutdown-w1
-shutdown-w1:
-	$(TALOSCTL) --nodes $(W1_IP) shutdown
-
-# Usage: make shutdown-worker N=2 W2_IP=192.168.1.162
-.PHONY: shutdown-worker
-shutdown-worker:
-ifndef N
-	$(error N is not set — run: make shutdown-worker N=<num> W<N>_IP=<ip>)
-endif
-	$(TALOSCTL) --nodes $(W_IP) shutdown
+.PHONY: reboot-cp3
+reboot-cp3:
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) reboot
 
 .PHONY: reboot-cp
-reboot-cp:
-	$(TALOSCTL) --nodes $(CP_IP) reboot
-
-.PHONY: reboot-w1
-reboot-w1:
-	$(TALOSCTL) --nodes $(W1_IP) reboot
+reboot-cp: reboot-all
 
 # Usage: make reboot-worker N=2 W2_IP=192.168.1.162
 .PHONY: reboot-worker
@@ -268,18 +189,16 @@ endif
 	$(TALOSCTL) --nodes $(W_IP) reboot
 
 # ── Inspect ───────────────────────────────────────────────────────────────────
-# Positional args: make node-ls 192.168.1.160 /etc/kubernetes/manifests
-# $(word 2/3, $(MAKECMDGOALS)) captures the extra words; the % rule absorbs
-# them so Make doesn't error with "No rule to make target".
+# Positional args: make node-ls 192.168.1.166 /etc/kubernetes/manifests
 .PHONY: node-ls
 node-ls:
-	$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
+	$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
 
 .PHONY: node-read
 node-read:
-	$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
+	$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
 
-# Absorb positional arguments passed to node-ls / node-read
+# Absorb positional arguments
 %:
 	@:
 

README.md

@@ -42,6 +42,86 @@ All logs + metrics centralized in Grafana for debugging
 - **Secrets at rest** — Vault + encrypted etcd; credentials never in logs or ConfigMaps
 - **Infrastructure-as-code** — Every service deployed via Helmfile; one `helmfile apply` recovers from total failure
 
+## ArgoCD — GitOps Deployment Flow
+
+**ArgoCD** pulls infrastructure changes from git and syncs the cluster automatically. 
+No manual `kubectl apply` — push to git, ArgoCD detects the change, and deploys within ~3 minutes.
+
+```
+Developer pushes to git
+       ↓
+ArgoCD detects change (every 3 min or webhook)
+       ↓
+Syncs manifests to cluster
+       ↓
+Workloads reconcile automatically
+```
+
+Applications are deployed in waves (numbered 00, 10, 20, 30, ...) to respect dependencies — 
+storage deploys before databases, databases before applications.
+
+### Tracked Git Repositories
+
+ArgoCD monitors these repos for changes:
+
+| Repository | Purpose |
+|------------|----------|
+| `https://github.com/Riotpiaole/riotpiao.homelab.com` | Main infrastructure repo (all manifests in `k8s/argocd/apps/`) |
+| `https://forgejo.riotpiao.com/rock/*` | Any `rock/*` repo in in-cluster Forgejo (apps + configs) |
+| `https://github.com/Riotpiaole/Poimen-*` | External Poimen services (memory, workflows) |
+
+To deploy a new application: create a git repo, add an Application manifest to the homelab repo's 
+`k8s/argocd/apps/`, commit + push, and ArgoCD syncs within 3 minutes.
+
+## Management Planes — Talos vs Kubernetes
+
+This cluster has **two separate management planes**, each with different workflows:
+
+| Plane | What it manages | Workflow | Tool |
+|-------|-----------------|----------|------|
+| **Talos (OS)** | Node configuration, kernel params, networking, CoreDNS, machine state | Edit `terraform/` → `terraform apply` → `make apply-cp` | `terraform` + `talosctl` |
+| **Kubernetes (workloads)** | All pods, services, deployments, ingresses, databases | Edit `k8s/argocd/apps/` → `git push` → ArgoCD syncs | `git` + ArgoCD |
+
+**Critical distinction:**
+- **Kubernetes resources** (`k8s/**`) flow through **git → ArgoCD** — never use `kubectl apply`
+- **Talos machine config** (`terraform/**`) uses **local `terraform apply`** (sanctioned exception — CI can't hold node credentials)
+
+Example: To add a CoreDNS hostname rewrite, you edit `terraform/files/coredns/Corefile`, then:
+```bash
+cd terraform && terraform apply -var-file=terraform.tfvars.local
+cd .. && make apply-cp     # talosctl apply-config to all 3 control planes
+```
+
+But to add a new Kubernetes Deployment or update an Ingress, you only `git push` — **never `kubectl apply`**.
+
+### CoreDNS ConfigMap Ownership — Critical
+
+⚠️ **Warning:** The `coredns` ConfigMap in `kube-system` namespace is **owned by Talos**, not ArgoCD or kubectl. 
+It is rendered from `terraform/files/coredns/Corefile` into Talos's machine config at bootstrap time.
+
+**Do not `kubectl apply` or `kubectl edit` this ConfigMap directly.** Doing so transfers field ownership to kubectl's 
+client-side-apply mechanism, and Talos's inline-manifest controller will silently no-op on every future reconcile 
+(server-side-apply conflict, no error surfaced).
+
+**To update CoreDNS (e.g., add a hostname rewrite):**
+1. Edit `terraform/files/coredns/Corefile`
+2. Commit + push
+3. Run `cd terraform && terraform apply -var-file=terraform.tfvars.local`
+4. Run `make apply-cp` to push config to all control planes
+5. CoreDNS picks up changes via its `reload` plugin — no pod restart needed
+
+**If you accidentally edited the ConfigMap directly and broke Talos's ownership:**
+```bash
+kubectl delete configmap coredns -n kube-system
+# Wait ~30s for Talos's k8s.ManifestApplyController to recreate it
+kubectl get configmap coredns -n kube-system -w
+```
+
+Or as a stopgap, apply the correct content yourself:
+```bash
+kubectl apply --server-side -f <(terraform output coredns_config)
+```
+
 ## Quick Start — Deploying the Cluster
 
 ### 1. Bootstrap Talos Nodes
@@ -60,7 +140,7 @@ Edit `.env` and fill in cluster configuration. See `.env.example` for all option
 
 ```bash
 # Cluster configuration
-CLUSTER_DOMAIN=riotpiao.homelab.com           # Your cluster domain
+CLUSTER_DOMAIN=riotpiao.com           # Your cluster domain
 POSTGRES_HOST=ddb-cluster-rw.ddb.svc.cluster.local
 MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000
 KAFKA_BOOTSTRAP=kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092
@@ -285,26 +365,39 @@ Add to `/etc/hosts` on every client machine (Mac/Linux):
 
 ```
 # WireGuard access (remote — via talos-cp-1)
-10.6.0.1  grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com
+10.6.0.1  grafana.riotpiao.com authentik.riotpiao.com vault.riotpiao.com minio.riotpiao.com prometheus.riotpiao.com portainer.riotpiao.com longhorn.riotpiao.com loki.riotpiao.com forgejo.riotpiao.com temporal.riotpiao.com temporal-grpc.riotpiao.com kmsvc.riotpiao.com
 
 # LAN access (on the home network — use actual LoadBalancer IP from above)
-192.168.1.160  grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com
+192.168.1.160  grafana.riotpiao.com authentik.riotpiao.com vault.riotpiao.com minio.riotpiao.com prometheus.riotpiao.com portainer.riotpiao.com longhorn.riotpiao.com loki.riotpiao.com forgejo.riotpiao.com temporal.riotpiao.com temporal-grpc.riotpiao.com kmsvc.riotpiao.com
 ```
 
 **Note:** `192.168.1.160` is an example Cilium LB-IPAM assignment. Verify with `kubectl get svc -n ingress-nginx ingress-nginx`.
 
+**There is no real DNS wildcard for `*.riotpiao.com`** — every hostname must be added to `/etc/hosts` explicitly (as above) before it resolves. Adding a new Ingress host doesn't make it reachable by itself; add the line too.
+
+### kubectl Context
+
+Two contexts exist in `cluster-config/kubeconfig`, pointed at the same cluster over different paths:
+
+| Context | Server | Use when |
+|---|---|---|
+| `admin@homelab-cluster` | `192.168.1.213:6443` (LAN) | On the home network |
+| `admin@homelab-cluster-1` | `10.6.0.1:6443` (WireGuard) | Remote / off-LAN |
+
+If `kubectl` commands hang or refuse the connection, switch: `kubectl config use-context admin@homelab-cluster-1`.
+
 Then access services at:
 
 | Service | URL | Credentials |
 |---------|-----|-------------|
-| Grafana | http://grafana.riotpiao.homelab.com | admin / `GRAFANA_ADMIN_PASSWORD` or Authentik SSO |
-| Authentik | http://authentik.riotpiao.homelab.com | akadmin / see `.env` |
-| Vault | http://vault.riotpiao.homelab.com | root token / see `setup_vault.sh` output |
-| MinIO console | http://minio.riotpiao.homelab.com | `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD` |
-| Prometheus | http://prometheus.riotpiao.homelab.com | no auth |
-| Portainer | http://portainer.riotpiao.homelab.com | set on first visit |
-| Longhorn | http://longhorn.riotpiao.homelab.com | no auth |
-| Forgejo *(planned)* | https://forgejo.forge.riotpiao.homelab.com | `rock` / `FORGEJO_ADMIN_PASSWORD`, or Authentik SSO |
+| Grafana | http://grafana.riotpiao.com | admin / `GRAFANA_ADMIN_PASSWORD` or Authentik SSO |
+| Authentik | http://authentik.riotpiao.com | akadmin / see `.env` |
+| Vault | http://vault.riotpiao.com | root token / see `setup_vault.sh` output |
+| MinIO console | http://minio.riotpiao.com | `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD` |
+| Prometheus | http://prometheus.riotpiao.com | no auth |
+| Portainer | http://portainer.riotpiao.com | set on first visit |
+| Longhorn | http://longhorn.riotpiao.com | no auth |
+| Forgejo *(planned)* | https://forgejo.forge.riotpiao.com | `rock` / `FORGEJO_ADMIN_PASSWORD`, or Authentik SSO |
 | Argo CD *(planned)* | `kubectl port-forward -n argocd svc/argocd-server 8080:443` | Authentik SSO (admins only) |
 
 Grafana → "Homelab" folder has the operator dashboards (sidecar-loaded from `k8s/monitoring/dashboards/`, no restart needed on change):
@@ -366,7 +459,7 @@ Authentik is the central OIDC identity provider. Vault stores secrets and delega
        │
        │  OAuth2 / OIDC
        ▼
-  Authentik  (authentik.riotpiao.homelab.com)
+  Authentik  (authentik.riotpiao.com)
   ├── grafana app      → Grafana OIDC login (group → Admin/Viewer role)
   ├── minio app        → MinIO OIDC login (group → readwrite/readonly policy)
   ├── vault-browser    → Vault UI OIDC login / `vault login -method=oidc`
@@ -374,13 +467,13 @@ Authentik is the central OIDC identity provider. Vault stores secrets and delega
        │
        │  JWKS endpoint for JWT validation
        ▼
-  HashiCorp Vault  (vault.riotpiao.homelab.com)
+  HashiCorp Vault  (vault.riotpiao.com)
   ├── auth/jwt   — core-cli authenticates with device code JWT
   ├── auth/oidc  — browser/UI login via Authentik
   └── secret/    — KV v2: mcp/*, cluster/*, cloud/*
 ```
 
-**talos-cli device code login:**
+**core CLI device code login:**
 ```bash
 core secrets login        # prints URL + code → approve in browser → Vault token cached
 core put cluster/DUCKDNS_TOKEN DUCKDNS_TOKEN="abc"            # field name = variable name, never `value`
@@ -389,8 +482,8 @@ core put cluster/DUCKDNS_TOKEN DUCKDNS_TOKEN="abc"            # field name = var
 **One-time IAM setup (after `helmfile apply`):**
 ```bash
 # 1. Provision OIDC apps and groups in Authentik
-GRAFANA_URL=http://grafana.riotpiao.homelab.com \
-MINIO_URL=http://minio.riotpiao.homelab.com \
+GRAFANA_URL=http://grafana.riotpiao.com \
+MINIO_URL=http://minio.riotpiao.com \
 python3 k8s/talos-iam/provision_oidc.py
 
 # 2. Init Vault, wire JWT + OIDC auth, seed secrets
@@ -474,7 +567,7 @@ Pods / Talos journal (both nodes)
        │
   Grafana (logging ns)              queries Loki + Prometheus via dashboards
        │
-  Nginx Ingress → grafana.riotpiao.homelab.com   browser access
+  Nginx Ingress → grafana.riotpiao.com   browser access
 ```
 
 ## Example Applications & Workloads

USAGE.md

@@ -1,6 +1,18 @@
 ## Cluster Architecture at a Glance
 
-**Homelab** is a 2-node bare-metal Kubernetes cluster deployed with Talos Linux, designed for self-hosted services, observability, and GitOps-ready CI/CD.
+**Homelab** is a 3-node bare-metal Kubernetes cluster deployed with Talos Linux, designed for self-hosted services, observability, and GitOps-ready CI/CD.
+
+### Node Topology (3 control-plane HA, since 2026-07-20)
+
+| Node | IP | Zone | Role | Scheduling | Storage |
+|------|----|----|------|-----------|---------|
+| `talos-cp-1` | 192.168.1.213 | az-a | control-plane | **schedulable** (runs all workloads) | sole Longhorn node (sdb/sdc/sdd) |
+| `talos-cp-2` | 192.168.1.163 | az-b | control-plane | dedicated (`NoSchedule`) | none |
+| `talos-cp-3` | 192.168.1.166 | az-c | control-plane | dedicated (`NoSchedule`) | none |
+
+- **etcd** has 3 voting members peering over the LAN (`cluster.etcd.advertisedSubnets: 192.168.1.0/24` — without it Talos may advertise on the WireGuard IP and new members hang as learners). Tolerates 1 node loss.
+- Only `talos-cp-1` runs workloads and holds storage, so stateful services are **single-instance** (e.g. CNPG `ddb-cluster` = 1 instance). The kube-apiserver endpoint is single-homed to `.213` (no VIP yet).
+- Scheduling is declarative: `allowSchedulingOnControlPlanes: true` + per-node `machine.nodeTaints` re-adds the control-plane taint on the dedicated nodes only.
 
 ### Deployment Stack (18 Helm releases)
 
@@ -12,7 +24,7 @@
 | **Certificates** | cert-manager + homelab-ca | cert-manager | Self-signed CA, auto-renewal |
 | **Storage (Block)** | Longhorn v1.7.0 | longhorn-system | Persistent volumes, default StorageClass |
 | **Storage (Object)** | MinIO (3-node, site-repl) | storage | S3-compatible, multi-AZ replication |
-| **Database** | CloudNativePG (3 replicas) | ddb | PostgreSQL 16 + pgvector |
+| **Database** | CloudNativePG (1 instance) | ddb | PostgreSQL 16 + pgvector (single-node; see topology) |
 | **IAM / OIDC** | Authentik | iam | Federated OIDC provider for all services |
 | **Secrets** | HashiCorp Vault | iam | KV secrets backend, JWT auth |
 | **Logs** | Loki (SingleBinary) | logging | 10-day retention, MinIO backend |
@@ -73,101 +85,252 @@ sqs (Kafka + Message Queue)
 
 ---
 
-## Custom CLI — `talos`
+## Custom CLI — `core`
 
-Homelab cluster control CLI (`core/`). Manages cluster nodes and Vault secrets.
+Homelab cluster control CLI (source: `~/workplace/core/`). Manages cluster nodes, Authentik IAM, and Vault secrets.
 
-### Secret path convention
-
-All secrets live under `cluster/<VARIABLE_NAME>`. The field name is always the variable name itself (SCREAMING_SNAKE_CASE), matching the `.env` key. Example paths:
-
-```
-cluster/ANTHROPIC_API_KEY
-cluster/AUTHENTIK_FORGEJO_CLIENT_ID
-cluster/AUTHENTIK_ARGOCD_CLIENT_SECRET
+**Setup:**
+```bash
+make cli                     # builds + installs to ~/.local/bin/core
+core auth login-oob         # authenticate with Authentik (OOB flow)
+core nodes                  # verify cluster access
 ```
 
-### `talos put` — write a secret to Vault
+**Full documentation:** See [~/workplace/core/USAGE.md](../core/USAGE.md)
+
+---
+
+### Authentication
+
+#### OAuth2 Out-of-Band (OOB) Login
+
+Browser-based login with manual code entry (recommended).
 
 ```bash
-talos put cluster/VARIABLE_NAME VARIABLE_NAME="secret-value"
-talos put cluster/FORGEJO_ADMIN_PASSWORD FORGEJO_ADMIN_PASSWORD="$FORGEJO_ADMIN_PASSWORD"
+export CORE_CLI_SECRET="cyoVr96FB9oeq3o64DUG0BmoVzMPsOTIWxVEd8ZdTezKrEZYwrKpIRkOwrDQNEtF6QJyNUPH4mjr9jWokQY7KVBWX1KVUXFyyhgAHTqWRRWAYUJ8r3H35wLFiTfn5KsV"
+core auth login-oob
 ```
 
-Field name = variable name — never `value`.
+Token expires in 8 hours. Check status:
+```bash
+core auth status   # show expiry
+core auth clear    # force re-auth on next command
+```
 
-### `talos get` — fetch a secret from Vault
+---
+
+### Cluster Management
 
 ```bash
-talos get cluster/VARIABLE_NAME --key VARIABLE_NAME        # always specify --key
-talos get cluster/FORGEJO_ADMIN_PASSWORD --key FORGEJO_ADMIN_PASSWORD
-talos get cluster/VARIABLE_NAME --json                     # full secret as JSON
+core nodes                    # list cluster nodes
+core status <ip>              # Talos node overview
+core services <ip>            # list Talos services
+core logs <ip>                # stream kubelet logs
+core log-svc <ip> <svc>       # logs for specific service (etcd, kubelet, etc.)
+core pods clean               # delete Failed/Evicted/Terminating pods
 ```
 
-Note: `talos get` uses `--key` (long flag), not a positional arg — unlike `talos secrets get`.
+#### kubectl Context (LAN vs. WireGuard)
 
-### `vsource` — load a `.env` into the shell
-
-zsh function (lives in `~/.zshrc`, not in the repo — can reference but cannot run directly).
-Empty `.env` values are fetched from Vault at `cluster/<KEY>`; hardcoded values pass through.
-
-```zsh
-vsource            # loads .env in current directory
-vsource .env.local # loads a specific file
-```
-
-`.env` format — leave secrets empty, vsource resolves them from Vault:
+`cluster-config/kubeconfig` has two contexts pointed at the same cluster:
+`admin@homelab-cluster` (LAN, `192.168.1.213:6443`) and `admin@homelab-cluster-1`
+(WireGuard, `10.6.0.1:6443`). If `kubectl`/`core nodes` hangs or refuses the
+connection, you're likely off-LAN — switch contexts:
 
 ```bash
-ANTHROPIC_API_KEY=          # fetched from cluster/ANTHROPIC_API_KEY
-AUTHENTIK_ARGOCD_CLIENT_ID= # fetched from cluster/AUTHENTIK_ARGOCD_CLIENT_ID
-DEBUG=true                  # hardcoded, passed through as-is
+core config kube-list             # list contexts
+core config kube-use admin@homelab-cluster-1   # switch to WireGuard path
 ```
 
-### Typical workflow for a generated secret
+**Known gap:** `core config use <talos-context>` (the combined talosctl+kubectl
+switch) only maps to `admin@homelab-cluster` today — its WireGuard mapping
+(`home-cluster-wire-guard`) is stale, that kubectl context doesn't exist. Use
+`core config kube-use admin@homelab-cluster-1` directly until that's fixed.
+
+---
+
+### Secret Management (Vault)
+
+All secrets live under `cluster/<VARIABLE_NAME>`. Field name = variable name (SCREAMING_SNAKE_CASE).
+
+#### Write Secret to Vault
 
 ```bash
-# 1. Store immediately after generation (keeps secrets out of shell history)
-talos put cluster/AUTHENTIK_FORGEJO_CLIENT_SECRET AUTHENTIK_FORGEJO_CLIENT_SECRET="<paste>"
-
-# 2. Use via subshell when creating K8s secrets
-kubectl create secret generic my-secret \
-  --from-literal=client-secret="$(talos get cluster/AUTHENTIK_FORGEJO_CLIENT_SECRET --key AUTHENTIK_FORGEJO_CLIENT_SECRET)"
-
-# 3. Or load into shell via vsource for helmfile/env-driven tools
-vsource .env && helmfile apply
+core put cluster/ANTHROPIC_API_KEY ANTHROPIC_API_KEY="sk-ant-..."
+core put cluster/FORGEJO_ADMIN_PASSWORD FORGEJO_ADMIN_PASSWORD="secret123"
 ```
 
-### IAM Management (Federated OIDC, Phases 1–6 Complete)
+**Key rule:** Field name must match variable name — never `value=`.
 
-**Status:** ✅ Fully deployed (2026-07-02). Single federated OIDC provider (`talos-federation`) handles all service auth.
-
-**Quick reference:**
+#### Fetch Secret from Vault
 
 ```bash
-# View roles and capabilities
-talos iam roles list && talos iam roles describe admin
-
-# Service registry (Grafana, MinIO, Forgejo, etc.)
-talos iam services list && talos iam services describe grafana
-
-# Agents (admin-bot, ci-bot with auto-rotation)
-talos iam agents list && talos iam agents rotate ci-bot
-
-# Role bindings (user → role with TTL)
-talos iam bindings grant [email protected] devops --expires 2026-12-31
-talos iam bindings list
-
-# Audit trail (90-day retention, 12 event types)
-talos iam audit list && talos iam audit export --format json
-
-# OIDC provider sync with Authentik
-talos iam providers sync-authentik
+core get cluster/ANTHROPIC_API_KEY              # default field
+core get cluster/AUTHENTIK_ARGOCD_CLIENT_SECRET # full value
 ```
 
-**See `homelab/CLAUDE.md` § IAM Management for full reference** (roles, services, agents, bindings, audit, providers).
+#### List All Secrets
 
-**Vault paths:** All IAM state stored under `cluster/iam/{federation,roles,services,agents,bindings}`.
+```bash
+core secrets list
+```
+
+#### Load into Shell (vsource)
+
+zsh function in `~/.zshrc` — fetches empty `.env` values from Vault:
+
+```bash
+# .env format
+ANTHROPIC_API_KEY=              # fetched from Vault
+AUTHENTIK_ARGOCD_CLIENT_ID=     # fetched from Vault
+DEBUG=true                      # hardcoded, passed through
+
+# Usage
+vsource                         # loads .env in current dir
+vsource .env.prod               # loads specific file
+eval "$(vsource .env)" && helmfile apply   # inject + deploy
+```
+
+---
+
+### IAM Management (Authentik)
+
+Manage OAuth2 applications, groups, and user access via Authentik.
+
+#### List Groups
+
+```bash
+core iam list-groups
+```
+
+Output:
+```
+authentik Admins (id: 9d72cbf2-9d52-4d3c-bba9-0068525d7a91)
+grafana-admins (id: 5f2e1e79-7d7e-4ef0-9b99-3c6df19c0b88)
+minio-admins (id: e640d887-eb85-431b-b5b2-5b6a8a7e0a44)
+argocd-admins (id: 22a3c296-0d98-433f-8456-ebc2f1c8d489)
+forgejo-admins (id: 4084c8d6-0c12-46af-acf8-7372172b9016)
+```
+
+#### List OAuth2 Applications
+
+```bash
+core iam list-apps
+```
+
+#### Create New Application
+
+```bash
+core iam create-app "my-service" --slug my-service --redirect-uri "https://my-service.riotpiao.com/callback"
+```
+
+Returns client ID and secret (save immediately).
+
+#### Describe Application
+
+```bash
+core iam describe-app grafana
+```
+
+Shows:
+- Client ID
+- Client Secret
+- Redirect URIs
+- Scope claims
+
+#### Bind Group to Application
+
+```bash
+core iam bind-app grafana grafana-admins
+```
+
+Members of `grafana-admins` can log in to Grafana via OIDC.
+
+#### Create Group
+
+```bash
+core iam create-group "developers"
+```
+
+#### Add User to Group
+
+```bash
+core iam add-member grafana-admins newuser
+```
+
+#### Rotate Application Secret
+
+```bash
+core iam rotate-secret grafana
+```
+
+⚠️ Must update deployment after rotating.
+
+---
+
+### Port Forwarding
+
+```bash
+core pf grafana         # localhost:3000 → Grafana
+core pf prometheus      # localhost:9090 → Prometheus
+core pf minio           # localhost:9001 → MinIO console
+core pf iam             # localhost:7000 → Authentik
+```
+
+---
+
+### Workflow: Rotate OAuth2 Secret
+
+```bash
+# 1. Rotate in Authentik
+SECRET=$(core iam rotate-secret grafana | jq -r '.client_secret')
+
+# 2. Update deployment
+vi k8s/logging/grafana-values.yaml
+# Set: GRAFANA_OIDC_CLIENT_SECRET="$SECRET"
+
+# 3. Redeploy
+helmfile apply -l app=grafana
+
+# 4. Verify
+core iam describe-app grafana
+```
+
+---
+
+### Workflow: Add User to Service
+
+```bash
+# 1. Create or verify group exists
+core iam list-groups | grep minio-admins
+
+# 2. Add user to group
+core iam add-member minio-admins alice
+
+# 3. Verify
+# (User will have access next login via OIDC)
+```
+
+---
+
+### Vault Integration (Advanced)
+
+Vault paths for IAM state (if using federated OIDC):
+
+```
+cluster/iam/federation/
+cluster/iam/roles/
+cluster/iam/services/
+cluster/iam/agents/
+cluster/iam/bindings/
+```
+
+Query via:
+```bash
+core get cluster/iam/roles/admin --key roles
+```
+
+**See `CLAUDE.md` § IAM for full architecture** (roles, services, agents, audit).
 
 ### CI/CD Image Registry Authentication (Forgejo + Runner)
 
@@ -177,16 +340,16 @@ talos iam providers sync-authentik
 
 ```bash
 # 1. Get ci-bot JWT token (runner has this injected via ServiceAccount)
-export REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token)
+export REGISTRY_TOKEN=$(core get cluster/iam/agents/ci-bot --key token)
 
 # 2. Authenticate docker/podman to Forgejo registry
-docker login forgejo.riotpiao.homelab.com \
+docker login forgejo.riotpiao.com \
   --username ci-bot \
   --password "$REGISTRY_TOKEN"
 
 # 3. Tag and push image
-docker tag myapp:latest forgejo.riotpiao.homelab.com/rock/myapp:latest
-docker push forgejo.riotpiao.homelab.com/rock/myapp:latest
+docker tag myapp:latest forgejo.riotpiao.com/rock/myapp:latest
+docker push forgejo.riotpiao.com/rock/myapp:latest
 ```
 
 **Pull images in runner (automatic):**
@@ -194,7 +357,7 @@ docker push forgejo.riotpiao.homelab.com/rock/myapp:latest
 ```bash
 # Inside .forgejo/workflows/*.yml, runner pulls via K8s ServiceAccount
 # No explicit login needed — imagePullSecrets injected by runner pod
-image: forgejo.riotpiao.homelab.com/rock/myapp:latest
+image: forgejo.riotpiao.com/rock/myapp:latest
 ```
 
 **Runner pod setup:**

_FLUX_START_HERE.md

@@ -1,324 +0,0 @@
-# Flux CD Integration Planning — START HERE
-
-## What Just Happened?
-
-Your subagent completed **comprehensive planning documentation** for integrating Flux CD v2 with your homelab's helmfile-based infrastructure.
-
-**Three complete documents created:**
-
-1. **FLUX_INTEGRATION_PLAN.md** (1,810 lines)
-   - Full technical specification with code examples
-   - Phase-by-phase implementation roadmap
-   - Conflict resolution & safety procedures
-   - Testing strategy & risk assessment
-
-2. **FLUX_PLANNING_SUMMARY.md** (351 lines)
-   - Executive overview for stakeholders
-   - Decision matrices & quick reference
-   - Timeline & effort estimates
-   - Success metrics
-
-3. **FLUX_PLANNING_INDEX.md** (356 lines)
-   - Navigation guide across all documents
-   - Quick start for different audiences
-   - FAQ & next steps
-
-**Total:** 2,517 lines of planning documentation
-
----
-
-## The Plan in 60 Seconds
-
-### What Problem Are We Solving?
-
-Current helmfile workflow:
-- Manual `helmfile apply` required
-- No automatic drift detection
-- No Git audit trail for changes
-- No approval gates
-- Hard to scale to multi-cluster
-
-### What's the Solution?
-
-Deploy **Flux CD v2** (GitOps) to:
-- Continuously reconcile cluster state from Git
-- Auto-detect & correct drift
-- Maintain full audit trail
-- Support staged rollouts with approval gates
-- Keep helmfile.yaml.gotmpl as fallback during transition
-
-### How Do We Do It?
-
-**3 phases, 6–8 weeks, ~99 hours:**
-
-| Phase | Timeline | Work | Goal |
-|-------|----------|------|------|
-| **1** | Weeks 1–2 | Bootstrap Flux + helmfile bridge | Zero breaking changes |
-| **2** | Weeks 3–6 | Migrate 23 releases to HelmRelease CRDs | Parallel migration (4 streams) |
-| **3** | Weeks 7–8 | Enable auto-sync, metrics, runbooks | Full GitOps readiness |
-
-**Key:** No downtime. Helmfile stays functional as fallback throughout.
-
----
-
-## Architecture Simplified
-
-```
-Git (Forgejo) ← Source of Truth
-  └─→ Flux Reconciliation Loop (every 5 min)
-      └─→ Kubernetes Cluster
-          └─→ 23 Helm Releases (reconciled state)
-```
-
-That's it. Flux watches Git. When you push changes, Flux applies them. If someone manually changes the cluster (kubectl), Flux auto-corrects on next reconciliation.
-
----
-
-## Key Decisions (No Surprises)
-
-| Decision | Choice | Reasoning |
-|----------|--------|-----------|
-| **Controller** | Flux v2 | Stable, battle-tested; v3 still beta |
-| **Helm** | HelmRelease CRDs | Preserves values-based workflow |
-| **Secrets** | SOPS + age | Git-stored, audited, simple |
-| **Rollout** | Phased (3×8 weeks) | Lower risk, easier debugging |
-
-All decisions explained in detail in FLUX_INTEGRATION_PLAN.md §3 (Architecture Decision Matrix).
-
----
-
-## What You Get
-
-### By End of Phase 1 (Week 2)
-- ✅ Flux running in cluster
-- ✅ Git syncing every 60 seconds
-- ✅ Helmfile still works as fallback
-- ✅ Zero disruption to running workloads
-
-### By End of Phase 2 (Week 6)
-- ✅ All 23 releases migrated to Git-based HelmRelease CRDs
-- ✅ Helmfile no longer used for deployments
-- ✅ Every release tested & verified
-- ✅ Full test suite in place
-
-### By End of Phase 3 (Week 8)
-- ✅ Automatic reconciliation enabled
-- ✅ Drift detection + alerting working
-- ✅ Metrics flowing to Prometheus
-- ✅ Team trained on GitOps workflows
-- ✅ RTO < 2 hours (restore from Git if needed)
-
----
-
-## How to Read the Documentation
-
-### Quick Overview (10 min)
-→ **Read:** FLUX_PLANNING_SUMMARY.md
-
-Start here to understand what we're doing and why. Tables, diagrams, high-level summary. Perfect for stakeholder presentations.
-
-### Getting Ready to Build (1 hour)
-→ **Read:** FLUX_PLANNING_INDEX.md + FLUX_INTEGRATION_PLAN.md (Executive Summary)
-
-Learn the full architecture, decision rationale, and how phases fit together.
-
-### Phase 1 Implementation (Week 1–2)
-→ **Reference:** FLUX_INTEGRATION_PLAN.md §5.1 (Phase 1: Flux Bootstrap)
-
-Detailed tasks:
-- 1.1: Bootstrap Flux into cluster
-- 1.2: Create Git repo structure
-- 1.3: HelmRepository CRDs (13 repos)
-- 1.4: SOPS + age setup
-- 1.5: Helmfile-bridge CronJob
-
-### Phase 2 Migration (Weeks 3–6)
-→ **Reference:** FLUX_INTEGRATION_PLAN.md §5.2 (Phase 2: HelmRelease Migration)
-
-Four parallel streams:
-- Stream A: Low-risk (reloader, prometheus)
-- Stream B: Medium-risk (cert-manager, ingress)
-- Stream C: High-risk secrets (authentik, vault)
-- Stream D: Complex stateful (minio, forgejo)
-
-Per-release process: generate CRD → validate → deploy → test → commit
-
-### Phase 3 Production Readiness (Weeks 7–8)
-→ **Reference:** FLUX_INTEGRATION_PLAN.md §5.3 (Phase 3: Continuous Reconciliation)
-
-Auto-sync, metrics, runbooks, team training.
-
-### Troubleshooting & Rollback
-→ **Reference:** FLUX_INTEGRATION_PLAN.md §7 (Rollback & Safety Guardrails)
-
-How to recover if something breaks:
-- Suspend Flux + manual rollback
-- Git revert + auto-reconciliation
-- Disaster recovery from Git
-
-### Testing Strategy
-→ **Reference:** FLUX_INTEGRATION_PLAN.md §8 (Testing Strategy)
-
-Unit tests, integration tests, chaos tests, production deployment strategy.
-
----
-
-## Risk Summary
-
-### Main Risks & How We Handle Them
-
-| Risk | Mitigation |
-|------|-----------|
-| **Flux + helmfile conflict** | Stagger reconciliation (helmfile 30min, Flux 5min) |
-| **Secret injection breaks** | Three-tier approach (SOPS + ConfigMaps + .env fallback) |
-| **Secrets leak in Git** | SOPS encryption from start + pre-commit hooks |
-| **Cluster recovery fails** | Keep helmfile as fallback; test quarterly |
-
-All risks detailed with specific mitigations in FLUX_INTEGRATION_PLAN.md §9 (Risk Assessment).
-
----
-
-## Timeline Reality Check
-
-```
-Week 1–2:  Phase 1 bootstrap (20 hrs)
-           ├─ 1 DevOps engineer + 1 Security engineer
-           └─ 0 downtime to running workloads
-
-Week 3–6:  Phase 2 migration (40 hrs)
-           ├─ 4 parallel streams (DevOps + Ops + Security)
-           └─ Release-by-release (low risk)
-
-Week 7–8:  Phase 3 hardening (16 hrs)
-           ├─ DevOps + QA
-           └─ Runbooks + training
-
-Total:     ~99 hours (~2.5 FTE-weeks)
-           6–8 calendar weeks (with parallelization)
-```
-
-Actual timeline depends on:
-- Team size (4 engineers = 8 weeks; 2 engineers = 12 weeks)
-- Experience with Flux (learning curve ~40 hours)
-- Testing rigor (each phase adds 1–2 weeks)
-
----
-
-## Next Actions
-
-### Immediately (Today)
-
-1. **Review FLUX_PLANNING_SUMMARY.md** (15 min)
-   - Understand the approach
-   - Check decision matrix
-   - Confirm timeline is acceptable
-
-2. **Share with stakeholders**
-   - Security team: review SOPS approach
-   - Ops team: review rollback procedures
-   - Management: confirm timeline & resources
-
-3. **Get approval** for:
-   - Phased approach (6–8 weeks)
-   - Flux v2 + HelmRelease CRDs
-   - SOPS encryption for secrets
-   - ~99 hours effort
-
-### Week 1 (Phase 1 Kickoff)
-
-1. **Assign team members**
-   - DevOps lead
-   - Security engineer (SOPS)
-   - Ops engineer (testing)
-
-2. **Bootstrap Flux**
-   - `flux bootstrap git` command
-   - Set up Git repo structure
-   - Deploy HelmRepository CRDs
-
-3. **Start helmfile-bridge development**
-   - CronJob to run `helmfile apply` every 30 min
-   - Test alongside Flux (staggered intervals)
-
-### Weeks 3–8 (Phases 2 & 3)
-
-Follow the phase roadmap in FLUX_INTEGRATION_PLAN.md with weekly syncs.
-
----
-
-## Files Created
-
-All in `/Users/rockliang/workplace/homelab/`:
-
-1. **FLUX_INTEGRATION_PLAN.md** (55 KB)
-   - Complete technical specification
-   - Phase-by-phase breakdown
-   - Code examples & detailed procedures
-
-2. **FLUX_PLANNING_SUMMARY.md** (13 KB)
-   - Executive overview
-   - Decision matrices
-   - Quick reference tables
-
-3. **FLUX_PLANNING_INDEX.md** (13 KB)
-   - Navigation guide
-   - Quick start by audience
-   - FAQ & related docs
-
-4. **_FLUX_START_HERE.md** (this file)
-   - Quick orientation
-   - Next actions
-
----
-
-## Questions to Ask
-
-Before Phase 1 starts, clarify:
-
-1. **Team capacity?** How many FTE can we dedicate?
-   - 4 FTE → 8 weeks
-   - 2 FTE → 12 weeks
-
-2. **Timeline flexibility?** Hard deadline or can we adjust?
-   - If hard: compress with more parallel streams
-   - If flexible: add more testing/validation
-
-3. **Flux experience on team?** Anyone used Flux before?
-   - If no: add 1–2 weeks for learning curve
-   - If yes: can reduce onboarding time
-
-4. **Multi-cluster plans?** Will you add more clusters after homelab?
-   - If yes: design for portability from start
-   - If no: homelab-specific is fine
-
-5. **SOPS comfort?** Any concerns about secret encryption in Git?
-   - If yes: alternative is store in Vault (referenced from HelmRelease)
-   - If no: SOPS is recommended
-
----
-
-## Document Quality Checklist
-
-The planning documentation includes:
-
-- ✅ **Executive summary** — problem & solution in 1 page
-- ✅ **Current state analysis** — what we're migrating from
-- ✅ **Architecture decisions** — Flux v2, HelmRelease, SOPS (with reasoning)
-- ✅ **Detailed design** — GitRepository, Kustomization, HelmRelease CRDs
-- ✅ **3-phase roadmap** — specific tasks, timelines, deliverables, success criteria
-- ✅ **Conflict resolution** — helmfile + Flux, .env → SOPS, kubectl drift
-- ✅ **Rollback procedures** — what to do if something breaks
-- ✅ **Safety guardrails** — RBAC, audit logging, validation webhooks, approval gates
-- ✅ **Testing strategy** — unit, integration, chaos, production deployment
-- ✅ **Risk assessment** — probability, impact, mitigation for each risk
-- ✅ **Timeline & effort** — 99 hours, 6-8 weeks, team composition
-- ✅ **Useful commands** — Flux CLI cheatsheet
-- ✅ **FAQ** — downtime, rollback, recovery, cost
-
-Ready for review and implementation kickoff.
-
----
-
-**Status:** Planning phase complete. Ready for team discussion & approval.
-
-**Next:** Review FLUX_PLANNING_SUMMARY.md, approve approach, assign Phase 1 team.

bootstrap.sh

@@ -0,0 +1,269 @@
+#!/usr/bin/env bash
+#
+# Phase-0 bootstrap — bring a bare Talos cluster to a self-hosted GitOps control
+# plane, breaking the ArgoCD <-> Forgejo circle via a GitHub seed + cutover.
+# See docs/adr/0001-gitops-bootstrap-and-cd.md (Part A) and docs/plans/0001-EXECUTION.md.
+#
+# Order (all manual, once): Cilium -> Longhorn -> CNPG operator -> forgejo-db
+# (wait Ready) -> Forgejo -> ArgoCD (seeded from GitHub) -> cutover to Forgejo.
+# Everything ELSE is deployed by ArgoCD from the seed repo, in sync-wave order.
+#
+# Prereqs:
+#   - Talos cluster up; kubectl context points at it
+#   - helm 3, kubectl
+#   - SOPS age key at $SOPS_KEY (for the ArgoCD SOPS CMP plugin)
+#
+# The GitHub seed repo is public, so it is cloned anonymously over HTTPS — no
+# deploy key, no repository Secret, one less thing to bootstrap before ArgoCD.
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+BOOT="$SCRIPT_DIR/k8s/bootstrap"
+SOPS_KEY="${SOPS_KEY:-$HOME/.sops/key.txt}"
+
+log()  { echo "[$(date +%H:%M:%S)] $*"; }
+die()  { echo "ERROR: $*" >&2; exit 1; }
+phase(){ echo; echo "━━━ $* ━━━"; echo; }
+
+# Idempotent helm repo setup
+ensure_helm_repo() {
+  local name=$1 url=$2
+  helm repo list 2>/dev/null | grep -q "^$name" || helm repo add "$name" "$url" >/dev/null
+  helm repo update "$name" >/dev/null 2>&1 || true
+}
+
+preflight() {
+  log "preflight…"
+  kubectl cluster-info >/dev/null || die "kubectl not configured / cluster unreachable"
+  command -v helm >/dev/null || die "helm 3 not found"
+  [[ -f "$SOPS_KEY" ]]   || die "SOPS age key missing at $SOPS_KEY"
+  log "✅ preflight ok"
+}
+
+p1_cilium() {
+  phase "PHASE 1a: CNI (Cilium)"
+  if kubectl -n kube-system get ds cilium >/dev/null 2>&1; then log "cilium present, skip"; return; fi
+  ensure_helm_repo cilium https://helm.cilium.io
+  helm install cilium cilium/cilium -n kube-system \
+    --set ipam.mode=kubernetes --set kubeProxyReplacement=true --wait --timeout 10m
+  log "✅ cilium installed"
+}
+
+p1_longhorn() {
+  phase "PHASE 1b: STORAGE (Longhorn)"
+  
+  # Always ensure namespace + StorageClasses (idempotent, resumable)
+  kubectl apply -f "$BOOT/phase1-storage/namespace.yaml"
+  
+  # Install Longhorn if not present
+  if ! helm -n longhorn-system list 2>/dev/null | grep -q longhorn; then
+    ensure_helm_repo longhorn https://charts.longhorn.io
+    log "Installing Longhorn storage (this may take 5-10 minutes)..."
+    if helm install longhorn longhorn/longhorn -n longhorn-system \
+      --values "$BOOT/phase1-storage/longhorn-values.yaml" --wait --timeout 10m; then
+      log "✅ Longhorn installed"
+    else
+      log "⚠️  Helm install failed, but continuing to ensure resources..."
+    fi
+  fi
+  
+  # Always apply StorageClasses (even if helm install partially failed)
+  kubectl apply -f "$BOOT/phase1-storage/storageclasses.yaml"
+  
+  # Verify critical components (resumable check)
+  if kubectl -n longhorn-system wait --for=condition=available --timeout=300s deploy/longhorn-manager 2>/dev/null; then
+    log "✅ longhorn installed"
+  else
+    log "⚠️  longhorn-manager not ready yet, but StorageClasses applied. Re-run to verify."
+  fi
+}
+
+p1_ingress() {
+  phase "PHASE 1c: INGRESS (Nginx Ingress Controller)"
+  
+  # Install nginx-ingress if not present
+  if kubectl get ingressclass nginx >/dev/null 2>&1; then
+    log "nginx IngressClass present, skip install"
+    return
+  fi
+  
+  # Always ensure namespace with PodSecurity labels (idempotent)
+  kubectl apply -f "$BOOT/ingress/namespace.yaml"
+  
+  ensure_helm_repo ingress-nginx https://kubernetes.github.io/ingress-nginx
+  log "Installing nginx-ingress controller (this may take 2-3 minutes)..."
+  if helm install ingress-nginx ingress-nginx/ingress-nginx -n ingress-nginx \
+    --values "$BOOT/ingress/nginx-values.yaml" --timeout 5m; then
+    log "✅ nginx-ingress installed"
+  else
+    log "❌ nginx-ingress install failed"
+    return 1
+  fi
+  
+  # Apply additional ingress resources (cert, ingress rules)
+  log "Applying ingress manifests (ignoring cert-manager CRD errors)..."
+  kubectl apply -k "$BOOT/ingress/" 2>&1 | grep -v "no matches for kind" || true
+  log "✅ Ingress resources applied (cert-manager resources will be created by ArgoCD)"
+}
+
+p2_cnpg() {
+  phase "PHASE 2: CNPG OPERATOR"
+  if kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then log "cnpg CRD present, skip install"; return; fi
+  ensure_helm_repo cnpg https://cloudnative-pg.github.io/charts
+  log "Installing CloudNativePG operator (this may take 2-3 minutes)..."
+  if helm install cnpg cnpg/cloudnative-pg -n cnpg-system --create-namespace \
+    --values "$BOOT/phase2-cnpg/cnpg-values.yaml" --wait --timeout 5m; then
+    log "✅ CNPG operator installed"
+  else
+    log "❌ CNPG operator install failed"
+    return 1
+  fi
+  kubectl get crd clusters.postgresql.cnpg.io >/dev/null || die "CNPG CRD not registered"
+  log "✅ cnpg operator installed"
+}
+
+p3_forgejo() {
+  phase "PHASE 3: forgejo-db + Forgejo (ns cicd)"
+  
+  # Always ensure namespace + NetworkPolicy + Secrets (idempotent)
+  kubectl apply -f "$BOOT/phase3-forgejo/namespace.yaml"
+  
+  # Clean up old Valkey NetworkPolicy if it exists (from bundled chart)
+  kubectl delete networkpolicy forgejo-valkey-cluster -n cicd 2>/dev/null || true
+  
+  # Apply CNPG-specific NetworkPolicy
+  kubectl apply -f "$BOOT/phase3-forgejo/cnpg-networkpolicy.yaml"
+  
+  # Create Forgejo admin secret (bootstrap-time only, before ArgoCD exists)
+  # In GitOps mode, ArgoCD will sync the SOPS-encrypted version from git
+  if ! kubectl get secret forgejo-admin -n cicd >/dev/null 2>&1; then
+    log "Creating forgejo-admin secret from .env (bootstrap mode)"
+    [ -f "$HOME/workplace/homelab/.env" ] && source "$HOME/workplace/homelab/.env"
+    kubectl -n cicd create secret generic forgejo-admin \
+      --from-literal=username=rock \
+      --from-literal=password="${FORGEJO_ADMIN_PASSWORD}" \
+      --from-literal=email=[email protected]
+  else
+    log "forgejo-admin secret exists, skip (managed by ArgoCD in GitOps mode)"
+  fi
+  
+  # Check if forgejo-db cluster exists and is Ready
+  if kubectl get cluster forgejo-db -n cicd >/dev/null 2>&1; then
+    if kubectl get cluster forgejo-db -n cicd -o jsonpath='{.status.phase}' 2>/dev/null | grep -q "Cluster in healthy state"; then
+      log "forgejo-db already Ready, skip wait"
+    else
+      log "forgejo-db exists but not Ready, waiting for all 3 instances (up to 30 min)…"
+      if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
+        log "✅ forgejo-db cluster is Ready"
+      else
+        log "❌ forgejo-db cluster failed to become Ready"
+        return 1
+      fi
+    fi
+  else
+    log "creating forgejo-db cluster (3 instances)"
+    kubectl apply -f "$BOOT/phase3-forgejo/forgejo-db.yaml"
+    log "Waiting for all 3 CNPG instances to be Ready (up to 30 min)…"
+    if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
+      log "✅ forgejo-db cluster is Ready"
+    else
+      log "❌ forgejo-db cluster failed to become Ready"
+      return 1
+    fi
+  fi
+  
+  kubectl -n cicd get secret forgejo-db-app >/dev/null || die "CNPG did not create forgejo-db-app secret"
+  
+  # Install Forgejo if not present
+  if helm -n cicd list 2>/dev/null | grep -q forgejo; then log "forgejo helm release present, skip"; return; fi
+  ensure_helm_repo forgejo https://dl.gitea.io/charts/
+  log "Installing Forgejo (this may take 10-15 minutes on slow nodes)..."
+  if helm install forgejo forgejo/gitea -n cicd \
+    --values "$BOOT/phase3-forgejo/forgejo-values.yaml" --wait --timeout 10m; then
+    log "✅ Forgejo installed"
+  else
+    log "❌ Forgejo install failed"
+    return 1
+  fi
+  log "Forgejo is up — now push this repo to Forgejo and configure the GitHub pull-mirror"
+}
+
+p4_argocd() {
+  phase "PHASE 4: ArgoCD (seeded from GitHub)"
+  
+  # Always ensure namespace (idempotent). The seed repo is public — ArgoCD clones
+  # it anonymously over HTTPS, so there is no repository Secret to create.
+  kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
+
+  # Decrypt and apply any encrypted secrets from bootstrap dir (local SOPS)
+  if command -v sops &> /dev/null; then
+    export SOPS_AGE_KEY_FILE="$SOPS_KEY"
+    log "Decrypting encrypted secrets with local SOPS..."
+    local decrypted_count=0
+    for enc_file in "$BOOT"/phase*/**.enc.yaml; do
+      [ -f "$enc_file" ] || continue
+      log "  → Decrypting $(basename "$enc_file")..."
+      if sops -d "$enc_file" | kubectl apply -f -; then
+        decrypted_count=$((decrypted_count + 1))
+        log "    ✅ Applied"
+      else
+        log "    ⚠️  Failed (may already exist)"
+      fi
+    done
+    log "Decrypted and applied $decrypted_count secret(s)"
+  else
+    log "⚠️  SOPS not installed, skipping encrypted secret decryption"
+  fi
+  
+  # Install ArgoCD if not present
+  if ! helm -n argocd list 2>/dev/null | grep -q argocd; then
+    ensure_helm_repo argo https://argoproj.github.io/argo-helm
+    log "Installing ArgoCD via Helm (installing chart, pods will start afterward)..."
+    if helm install argocd argo/argo-cd -n argocd \
+      --values "$BOOT/phase4-argocd/argocd-values.yaml" --timeout 10m; then
+      log "✅ ArgoCD Helm release installed (pods starting...)"
+    else
+      log "❌ ArgoCD Helm install failed"
+      return 1
+    fi
+  else
+    log "ArgoCD Helm release already exists, skipping install"
+  fi
+  
+  # Wait for server ready (resumable - slow on talos-cp-2)
+  log "Waiting for argocd-server deployment to be available (max 10 minutes)..."
+  if kubectl -n argocd wait --for=condition=available --timeout=600s deploy/argocd-server; then
+    log "✅ argocd-server is available"
+  else
+    log "❌ argocd-server failed to become available within 10 minutes"
+    log "Check pods: kubectl get pods -n argocd"
+    return 1
+  fi
+  
+  # Always apply root app (idempotent)
+  kubectl apply -f "$BOOT/phase4-argocd/root-app-github.yaml"
+  log "✅ ArgoCD syncing from GitHub seed. Watch: kubectl get applications -n argocd"
+  log "NOTE: SOPS CMP plugin not installed yet (bootstrap uses local SOPS decryption)."
+  log "      To add SOPS plugin for GitOps, see k8s/bootstrap/phase4-argocd/argocd-cmp-cm.yaml"
+}
+
+p5_cutover() {
+  phase "PHASE 5: CUTOVER GitHub -> Forgejo"
+  read -rp "Forgejo healthy AND mirroring GitHub? (y/N) " r; [[ $r =~ ^[Yy]$ ]] || die "push+mirror to Forgejo first"
+  kubectl apply -f "$BOOT/phase5-cutover/root-app-forgejo.yaml"
+  log "✅ root app now sourced from Forgejo. GitHub mirror = DR seed. Circle dead."
+}
+
+case "${1:-all}" in
+  all)      preflight; p1_cilium; p1_longhorn; p1_ingress; p2_cnpg; p3_forgejo; p4_argocd
+            log "Phases 1-4 done. Push repo to Forgejo + set up pull-mirror, then: $0 cutover" ;;
+  cilium)   preflight; p1_cilium ;;
+  storage)  preflight; p1_longhorn ;;
+  ingress)  preflight; p1_ingress ;;
+  cnpg)     preflight; p2_cnpg ;;
+  forgejo)  preflight; p3_forgejo ;;
+  argocd)   preflight; p4_argocd ;;
+  cutover)  preflight; p5_cutover ;;
+  *) echo "usage: $0 {all|cilium|storage|ingress|cnpg|forgejo|argocd|cutover}"; exit 1 ;;
+esac

cluster-config/cilium-values.yaml

@@ -0,0 +1,86 @@
+# cluster-config/cilium-values.yaml
+# Cilium CNI — installed via talosctl (not helmfile) during cluster bootstrap.
+# Applied once: `helm install cilium cilium/cilium -n kube-system -f cilium-values.yaml`
+#
+# Why Cilium: Talos Linux does not ship kube-proxy. Cilium's eBPF dataplane
+# replaces it entirely (kubeProxyReplacement: true) and also handles LB-IPAM
+# so LoadBalancer services get real IPs without MetalLB.
+
+# ── cgroup ────────────────────────────────────────────────────────────────────
+# Talos mounts cgroups at boot before any container runtime starts.
+# autoMount: false tells Cilium to use the existing mount rather than trying
+# to mount its own — double-mounting on Talos causes init failures.
+cgroup:
+  autoMount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup   # where Talos exposes the cgroup v2 hierarchy
+
+# ── IPAM ──────────────────────────────────────────────────────────────────────
+# kubernetes mode: Cilium allocates pod IPs from the pod CIDR that Talos
+# configured for each node (--pod-cidr in the kubelet). Alternative is
+# Cilium's own cluster-pool IPAM, but that requires extra config and
+# conflicts with the Talos node CIDR assignment.
+ipam:
+  mode: kubernetes
+
+# ── Operator ──────────────────────────────────────────────────────────────────
+# Single replica is fine for a 3-node homelab. The operator manages CiliumNode
+# objects and LB-IPAM pools — it does not sit in the data path.
+operator:
+  replicas: 1
+
+# ── kube-proxy replacement ────────────────────────────────────────────────────
+# Talos is deliberately installed without kube-proxy (machineConfig
+# install.extensions excludes it). Cilium must replace it completely —
+# partial replacement would leave Service ClusterIPs unreachable.
+kubeProxyReplacement: true
+
+# ── L2 announcements ──────────────────────────────────────────────────────────
+# Without this, LB-IPAM (k8s/cilium/lb-ipam-pool.yaml) assigns real IPs to
+# LoadBalancer Services, but nothing ARPs for them on the LAN — the IP shows
+# up in `kubectl get svc` but is 100% unreachable from outside the cluster
+# (confirmed: forgejo's .165 and shadowsocks' .166 both had incomplete ARP
+# entries and 100% ping loss before this). This flag is what actually makes
+# k8s/cilium/l2-announcement-policy.yaml take effect instead of being inert.
+l2announcements:
+  enabled: true
+
+# ── API server endpoint ───────────────────────────────────────────────────────
+# Cilium needs to talk to the Kubernetes API to watch Nodes/Services/Endpoints.
+# On Talos the API server listens on 127.0.0.1:7445 locally (the external
+# port 6443 requires the node's external cert, which may not be available
+# during early bootstrap). This is the standard Talos Cilium bootstrap config.
+k8sServiceHost: 127.0.0.1
+k8sServicePort: 7445
+
+# ── Security context / capabilities ──────────────────────────────────────────
+# Cilium's eBPF programs run in the kernel and require elevated capabilities.
+# These are the minimum set needed — removing any of them breaks networking.
+#
+#   NET_ADMIN / NET_RAW  — manipulate iptables/nftables and raw sockets
+#   IPC_LOCK             — lock eBPF maps in memory (prevents paging out BPF state)
+#   SYS_ADMIN            — call bpf() syscall and mount BPF filesystem
+#   SYS_RESOURCE         — raise RLIMIT_MEMLOCK for BPF map memory
+#   DAC_OVERRIDE / FOWNER / SETGID / SETUID — file permission ops during init
+#   CHOWN / KILL         — container lifecycle management
+#
+# cleanCiliumState runs as a one-shot init container to wipe stale eBPF state
+# on upgrades — it needs NET_ADMIN, SYS_ADMIN, SYS_RESOURCE only.
+securityContext:
+  capabilities:
+    ciliumAgent:
+    - CHOWN
+    - KILL
+    - NET_ADMIN
+    - NET_RAW
+    - IPC_LOCK
+    - SYS_ADMIN
+    - SYS_RESOURCE
+    - DAC_OVERRIDE
+    - FOWNER
+    - SETGID
+    - SETUID
+    cleanCiliumState:
+    - NET_ADMIN
+    - SYS_ADMIN
+    - SYS_RESOURCE

cluster-config/longhorn_bootstrap.sh

@@ -0,0 +1,19 @@
+# bootstrap.sh
+#!/bin/bash
+
+# Wait for cluster to be ready
+kubectl wait --for=condition=Ready nodes --all --timeout=300s
+
+# Longhorn requires privileged pods and hostPath volumes
+kubectl create namespace longhorn-system --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace longhorn-system \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# Install Longhorn
+kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.7.0/deploy/longhorn.yaml
+
+# Set as default StorageClass
+kubectl patch storageclass longhorn \
+  -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'

cluster-config/patches/forgejo-registry-ca.yaml

@@ -0,0 +1,10 @@
+# Trust the homelab-ca CA for pulling from the Forgejo container registry.
+# Without this, containerd fails: x509 certificate signed by unknown authority
+# (nginx terminates forgejo.riotpiao.com TLS with a homelab-ca cert).
+# Apply: talosctl -n <node> patch mc --patch @cluster-config/patches/forgejo-registry-ca.yaml
+machine:
+  registries:
+    config:
+      forgejo.riotpiao.com:
+        tls:
+          ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJiVENDQVJTZ0F3SUJBZ0lVYTBVaGs3Rm81d3BiWjZsRzVEWWJUVkFic1k4d0NnWUlLb1pJemowRUF3SXcKRlRFVE1CRUdBMVVFQXhNS2FHOXRaV3hoWWkxallUQWVGdzB5TmpBM01UQXdORFF3TXpaYUZ3MHpOakEzTURjdwpORFF3TXpaYU1CVXhFekFSQmdOVkJBTVRDbWh2YldWc1lXSXRZMkV3V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPClBRTUJCd05DQUFTc1pNU2piUWI0YzNiUk00MjMxVEVrRXVLTnFLUUhaaW5uYnUzbWZGbStRc0wweTF3cjg1Uk0KUWJ6ZEZ2N01JZmN4REpMbHFqQTY1bEJ6TE9pdHRZZHRvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBcVF3RHdZRApWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVUVmcGJQL3FnYWsxaXUvQzdaQi9uZk5zc0hpd3dDZ1lJCktvWkl6ajBFQXdJRFJ3QXdSQUlnR2ltdnJiWU1xZjhGYThCeTBBM0M1ak1VL0d3dGU0NHgzOU4rRDRyaTJ1a0MKSUNkOEtIQXhhV0s2ZkVJcEFYZGdUQ1FxQmFiZjVZUDdhQzNDVzkzYkNsTjIKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

helmfile.yaml.gotmpl

@@ -0,0 +1,33 @@
+# helmfile.yaml — DEPRECATED
+#
+# This file is kept for historical reference only.
+# All Helm releases have been migrated to:
+#   1. Terraform (bootstrap releases: cert-manager, reloader, ingress-nginx)
+#   2. ArgoCD Applications (all workload releases)
+#
+# Deployment now uses:
+#   - terraform apply (for core infrastructure)
+#   - ArgoCD auto-sync (for all workloads)
+#
+# To view or manage releases:
+#   kubectl get applications -n argocd
+#
+# To modify releases, update k8s/argocd/apps/*.yaml files and commit to git.
+#
+# This file remains as a marker to prevent accidental `helmfile apply` usage.
+# DELETE if no longer needed after full migration verification.
+
+# Historical note:
+# - Phases 0-3 migrated: cert-manager, reloader, ingress-nginx, strimzi-operator,
+#   kmsvc-redis, kafka-cluster, queue-crd, management-service, promtail, portainer,
+#   cloudnative-pg, loki, grafana, prometheus, forgejo, forgejo-runner, authentik
+# - Phase 4 cutover: 2026-07-15 (helmfile stubbed, ArgoCD becomes sole convergence engine)
+
+# ── DO NOT USE ────────────────────────────────────────────────────────────────
+# helmfile apply                  # FORBIDDEN (use ArgoCD)
+# helmfile diff                   # FORBIDDEN (use argocd app diff)
+# helmfile destroy                # FORBIDDEN (use kubectl delete)
+
+# For drift detection (CI only):
+#   argocd app diff <app-name>    # Check what ArgoCD would change
+#   terraform plan                # Check what Terraform would change

k8s/apps/agent-pod/configmap.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+data:
+  settings.json: |
+    {
+      "defaultProvider": "homelab-ornith",
+      "defaultModel": "ornith:35b",
+      "defaultThinkingLevel": "medium",
+      "theme": "light",
+      "compaction": {
+        "enabled": true,
+        "reserveTokens": 16000,
+        "keepRecentTokens": 6000
+      }
+    }
+kind: ConfigMap
+metadata:
+  name: pi-config
+  namespace: agent-pod

k8s/apps/agent-pod/console-ingress.yaml

@@ -0,0 +1,49 @@
+# Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
+# (trigger a new session) -- both are routes on the same hub.js service.
+#
+# Was ingressClassName: kong until Kong was retired on 2026-08-19. Pointed
+# straight at nginx rather than through the replacement Go gateway because that
+# gateway has no WebSocket upgrade support yet -- routing /console through it
+# would break the console outright. nginx handles the upgrade natively.
+#
+# Path precedence: the nginx Ingress api/api catch-alls `/` on this same host
+# to the gateway. nginx matches longest prefix first, so these three paths win
+# over `/` and the rest of the host still reaches the gateway.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: console
+  namespace: agent-pod
+  annotations:
+    # A console WebSocket stays open across a whole agent session; nginx's 60s
+    # default read timeout would drop it mid-run.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /console
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /run
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /sessions
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090

k8s/apps/agent-pod/coordinator-configmap.yaml

@@ -0,0 +1,982 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coordinator-src
+  namespace: agent-pod
+data:
+  coordinator.js: |
+    #!/usr/bin/env node
+    // coordinator: local CLI that drives a multi-phase, multi-task pipeline of
+    // planner/investigator/implementer/judge stages, same state machine as
+    // hub.js's old /pipeline handler. Every interactive stage runs through the
+    // patched agent-manager fork's `spawn` subcommand, so it is a real tracked
+    // session (tmux pane on agent-manager's private socket + a state.db row)
+    // from the moment it exists -- attachable and visible in agent-manager's
+    // own TUI the whole time it runs.
+    //
+    // Per repo, each role (planner/investigator/implementer/judge) is ONE
+    // persistent agent-manager session, not a fresh spawn per task: the first
+    // task to need a role spawns it, every later task for that role reuses the
+    // same tmux pane via `tmux send-keys` (see runOnPool) -- the same nudge
+    // mechanism that used to only fire on a stall now doubles as "give this
+    // agent its next task." Every role reads everything it needs fresh off disk
+    // each call, so every pane gets a `/new` before every reuse instead of
+    // accumulating history that degrades and eventually errors out task after
+    // task -- same pane, same agent-manager session, zero memory of the last
+    // task it handled.
+    // Because only one implementer/judge/etc. exists per repo, tasks within a
+    // phase run strictly sequentially against the pool -- no per-task worktree,
+    // no per-task branch, no merge-back step; every task commits straight onto
+    // the phase branch in the repo's one shared clone.
+    //
+    // The unit of concurrency is now the REPO, not the task: runCoordinator
+    // takes a list of repos and runs up to REPO_CONCURRENCY of them at once,
+    // each with its own clone (under WORK_DIR/<repoId>) and its own 4-agent
+    // pool. The coordinator never kills a role's session; it rests at an idle
+    // prompt between tasks, and agent-manager's session list becomes the audit
+    // trail of everything every repo's pipeline ran. Completion is signaled by
+    // sentinel files under the repo's clone (unchanged convention), waited on
+    // with fs.watch instead of polling.
+    const fs = require("node:fs");
+    const path = require("node:path");
+    const { spawn } = require("node:child_process");
+    
+    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
+    
+    // Never rely on a bare `pi`/`agent-manager` on $PATH -- see PI_BIN's own
+    // comment below; the same collision risk applies to any CLI name. Always
+    // invoke explicit pinned paths.
+    const PI_BIN =
+      process.env.PI_BIN ||
+      path.join(__dirname, "..", ".pi-cli", "node_modules", "@earendil-works", "pi-coding-agent", "dist", "cli.js");
+    
+    const AGENT_MANAGER_BIN = process.env.AGENT_MANAGER_BIN || path.join(__dirname, "..", ".bin", "agent-manager-fork");
+    
+    // agent-manager's own session-state DB -- used to detect a session that has
+    // actually died (process crashed/exited, status flips to "errored"/"dead")
+    // instead of one that's merely slow. Read-only introspection plus the one
+    // UPDATE in killDeadSession below, same class of operation as the tmux
+    // nudges already done directly against agent-manager's internals.
+    const AGENT_MANAGER_DB =
+      process.env.AGENT_MANAGER_DB || path.join(require("node:os").homedir(), ".config", "agent-manager", "state.db");
+    
+    // Empty means "let pi fall back to ~/.pi/agent/settings.json's default"
+    // (currently anthropic/claude-sonnet-4-5, real paid usage). Set both to
+    // route every stage -- headless (spawnPi) and interactive (runOnPool) --
+    // at the homelab model instead, e.g. AGENT_PROVIDER=homelab-ornith
+    // AGENT_MODEL=ornith:35b.
+    const AGENT_PROVIDER = process.env.AGENT_PROVIDER || "";
+    const AGENT_MODEL = process.env.AGENT_MODEL || "";
+    
+    // judge can run a different model than the rest of the chain, e.g.
+    // homelab-reasoning instead of homelab-ornith now that verifier/PRM is
+    // retired. Falls back to AGENT_PROVIDER/AGENT_MODEL when unset, so a run
+    // that doesn't care keeps one uniform model everywhere.
+    const JUDGE_PROVIDER = process.env.JUDGE_PROVIDER || AGENT_PROVIDER;
+    const JUDGE_MODEL = process.env.JUDGE_MODEL || AGENT_MODEL;
+    
+    function providerModelFor(role) {
+      return role === "judge" ? { provider: JUDGE_PROVIDER, model: JUDGE_MODEL } : { provider: AGENT_PROVIDER, model: AGENT_MODEL };
+    }
+    
+    // agent-manager's private tmux server and session-naming scheme
+    // (internal/tmux/tmux.go: defaultSocket = "agentmgr", sessionName(id) =
+    // "am_"+id) -- stable, documented internals of the fork, used here only
+    // for read-only introspection (pane capture) and role nudges, exactly the
+    // class of operation hub.js already ran directly against its own sessions
+    // rather than asking a model to do it.
+    const AM_SOCKET = "agentmgr";
+    function amSessionName(id) {
+      return `am_${id}`;
+    }
+    function runAmTmux(args) {
+      return runCmd("tmux", ["-L", AM_SOCKET, ...args]);
+    }
+    
+    const ROLE_SKILLS = new Set(["planner", "investigator", "info-collector", "implementer", "judge", "resolver"]);
+    
+    // Every role reads everything it needs fresh off disk each call -- PLAN.md,
+    // the task spec, judge's verdict file, `git diff` against baseBranch --
+    // nothing depends on remembering earlier tasks. Left to accumulate, a
+    // pooled session's conversation grows without bound across every task in a
+    // repo and both correctness and reliability degrade hard once it does
+    // (observed: a planner session at ~1.5M cumulative tokens started erroring
+    // out every call, an investigator session that far gone started narrating a
+    // different codebase entirely). So every role gets reset to a clean
+    // conversation before every reuse instead of just being nudged with the
+    // next prompt -- same pane, same agent-manager session (still
+    // visible/attachable), zero history carried between tasks.
+    
+    function sleep(ms) {
+      return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    
+    const HARD_RULES =
+      "Read and follow ~/.pi/agent/skills/karpathy-guidelines/SKILL.md and " +
+      "~/.pi/agent/skills/caveman/SKILL.md as hard rules for this entire task, before anything else. ";
+    
+    // judge (routed to homelab-reasoning) has been observed narrating an
+    // entire review in prose -- "I should run git diff, then check X..." --
+    // and then writing a verdict based on that narration without ever calling
+    // a real tool. Live example: a phase-judge call produced a page of
+    // "I would check..." reasoning, declared VERDICT: PASS, and showed the
+    // touch command as a fenced code block IN ITS OWN TEXT rather than
+    // executing it. Coordinator just timed out waiting on a sentinel that was
+    // never going to appear, since nothing was ever actually run. Spelled out
+    // explicitly since "use the judge skill" alone apparently isn't enough to
+    // rule this out.
+    const REQUIRE_REAL_TOOL_CALLS =
+      "Do not narrate what you would check -- actually run the commands via a real tool call and read their real " +
+      "output before writing anything. A verdict based on describing checks instead of executing them is invalid. " +
+      "Writing the verdict file and touching the sentinel are themselves tool calls you must execute, not text to " +
+      "display in your response. ";
+    
+    function parseVerdictLine(text, label) {
+      if (!text) return null;
+      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
+      const m = text.match(re);
+      return m ? m[1].toUpperCase() : null;
+    }
+    
+    function runCmd(bin, args, cwd) {
+      return new Promise((resolve) => {
+        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+        let out = "";
+        child.stdout.on("data", (c) => (out += c));
+        child.stderr.on("data", (c) => (out += c));
+        child.on("close", (code) => resolve({ code, out: out.trim() }));
+      });
+    }
+    
+    function runGit(cwd, args) {
+      return runCmd("git", args, cwd);
+    }
+    
+    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
+    // in practice: a stage writes a real file and simply never runs `git add`/
+    // `git commit`, leaving it untracked and invisible to every later `git diff`.
+    // Sweep and commit anything left dirty after every stage, deterministically.
+    async function commitPending(cwd, message) {
+      await runGit(cwd, ["add", "-A"]);
+      const status = await runGit(cwd, ["status", "--porcelain"]);
+      if (!status.out) return { committed: false };
+      const commit = await runGit(cwd, ["commit", "-m", message]);
+      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
+    }
+    
+    // Headless one-shot pi call (`pi -p --mode json <prompt>`), used only for
+    // quick diagnostic/mechanical calls that don't need to be a watchable
+    // session: resolver's crash/stall diagnosis, and the initial clone. Kept
+    // exactly as before -- only the interactive per-role stages (runOnPool,
+    // below) go through agent-manager.
+    //
+    // Bounded by SPAWN_PI_TIMEOUT_MS -- unlike runOnPool's pooled sessions
+    // (which now have status polling to catch a dead session fast, see
+    // waitForSentinel/killDeadSession), this is a raw child_process with no
+    // equivalent escape hatch. Observed live: a resolver call shared the
+    // default backend with a concurrently-busy repo's implementer and sat for
+    // 6+ minutes producing nothing -- with no timeout here, that blocks the
+    // entire calling repo's pipeline forever, since askResolver is always
+    // awaited before the next stage can run.
+    const SPAWN_PI_TIMEOUT_MS = 5 * 60 * 1000;
+    
+    function spawnPi({ agent, prompt, cwd }) {
+      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${HARD_RULES}${prompt}` : prompt;
+      const args = ["-p", "--mode", "json"];
+      if (AGENT_PROVIDER) args.push("--provider", AGENT_PROVIDER);
+      if (AGENT_MODEL) args.push("--model", AGENT_MODEL);
+      args.push(finalPrompt);
+      const child = spawn(PI_BIN, args, { stdio: ["ignore", "pipe", "pipe"], cwd });
+      let lastText = "";
+      let stderrTail = "";
+      let buf = "";
+      child.stdout.on("data", (chunk) => {
+        buf += chunk;
+        let idx;
+        while ((idx = buf.indexOf("\n")) !== -1) {
+          const line = buf.slice(0, idx);
+          buf = buf.slice(idx + 1);
+          if (!line.trim()) continue;
+          try {
+            const event = JSON.parse(line);
+            if (event.type === "message_end" && event.message && Array.isArray(event.message.content)) {
+              const text = event.message.content
+                .filter((c) => c.type === "text")
+                .map((c) => c.text)
+                .join("\n");
+              if (text) lastText = text;
+            }
+          } catch {
+            // non-JSON stdout noise, ignore
+          }
+        }
+      });
+      child.stderr.on("data", (chunk) => {
+        process.stderr.write(chunk);
+        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
+      });
+      return new Promise((resolve) => {
+        let settled = false;
+        const timer = setTimeout(() => {
+          if (settled) return;
+          settled = true;
+          child.kill("SIGKILL");
+          resolve({ code: null, lastText, stderrTail, timedOut: true });
+        }, SPAWN_PI_TIMEOUT_MS);
+        child.on("close", (code) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(timer);
+          resolve({ code, lastText, stderrTail });
+        });
+      });
+    }
+    
+    async function askResolver(cwd, repoId, diagnosticPrompt) {
+      const result = await spawnPi({ agent: "resolver", prompt: diagnosticPrompt, cwd });
+      return parseVerdictLine(result.lastText, "RESOLUTION");
+    }
+    
+    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
+    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
+    
+    // Resolves as soon as filePath appears (fs.watch on its directory, same as
+    // before), as soon as target's agent-manager status flips to "errored" or
+    // "dead" (polled -- state.db has no watch mechanism), or after limitMs with
+    // neither. A session that has actually crashed will never touch the
+    // sentinel, so without the status poll this just burns the full STAGE_
+    // TIMEOUT_MS waiting on a file that was never coming, same as a genuine
+    // stall -- polling status catches that in ~pollMs instead.
+    function waitForSentinel(filePath, target, limitMs, pollMs = 5000) {
+      return new Promise((resolve) => {
+        if (fs.existsSync(filePath)) return resolve({ ok: true });
+        const dir = path.dirname(filePath);
+        const id = target.replace(/^am_/, "");
+        let settled = false;
+        let watcher;
+        let poller;
+        let timer;
+        const finish = (result) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(timer);
+          clearInterval(poller);
+          if (watcher) {
+            try {
+              watcher.close();
+            } catch {
+              // already closed
+            }
+          }
+          resolve(result);
+        };
+        try {
+          watcher = fs.watch(dir, () => {
+            if (fs.existsSync(filePath)) finish({ ok: true });
+          });
+        } catch {
+          // dir missing at watch time is a real bug elsewhere (cwd should
+          // already exist); surface it as a timeout rather than hang forever.
+          return finish({ timedOut: true });
+        }
+        // Closes the race between the existsSync check above and the watcher
+        // actually being attached.
+        if (fs.existsSync(filePath)) return finish({ ok: true });
+        poller = setInterval(async () => {
+          const { out } = await runCmd("sqlite3", [AGENT_MANAGER_DB, `SELECT status FROM sessions WHERE id='${id}'`]);
+          const status = out.trim();
+          if (status === "errored" || status === "dead") finish({ dead: true, status });
+        }, pollMs);
+        timer = setTimeout(() => finish({ timedOut: true }), limitMs);
+      });
+    }
+    
+    // Kills a session that's actually crashed (not just slow) and archives it
+    // in agent-manager's own DB so it stops showing up as a live, unattended
+    // pane -- otherwise every crash leaves an orphaned tmux session + state.db
+    // row behind permanently, identical to the manually-cleaned-up poiman-
+    // planner ghost session found earlier this same run.
+    async function killDeadSession(target) {
+      await runAmTmux(["kill-session", "-t", target]);
+      const id = target.replace(/^am_/, "");
+      await runCmd("sqlite3", [AGENT_MANAGER_DB, `UPDATE sessions SET archived=1 WHERE id='${id}'`]);
+    }
+    
+    // Runs one task's worth of work on a persistent per-role agent: spawns the
+    // role's session the first time it's ever needed for this repo, sends every
+    // later prompt into that same tmux pane via send-keys -- prefixed with a
+    // `/new` first, so the pane and agent-manager session stay the same but the
+    // model starts that prompt with a clean conversation, no history carried
+    // over from whatever task this role last handled. pool is a plain object
+    // keyed by role name ("planner"/"investigator"/"implementer"/"judge"),
+    // shared across every task in a repo's pipeline (see runRepoPipeline) -- it
+    // IS the 4-agent pool, one entry per role, filled in lazily as each role
+    // gets its first task.
+    // A dead/errored session gets one respawn-and-retry (same prompt, fresh
+    // session) before this stage is abandoned -- matches resolver-SKILL.md's
+    // own documented contract of retrying a failed stage at most once.
+    const DEAD_SESSION_RETRIES = 1;
+    
+    async function runOnPool(pool, cwd, repoId, role, prompt, sentinelFile) {
+      fs.rmSync(sentinelFile, { force: true });
+      const label = `${repoId}-${role}`;
+    
+      // A pooled session's shell cwd drifts as it explores the repo (e.g. cd
+      // into a Rust workspace subdirectory to read source) and nothing resets
+      // it back between turns. Seen in practice: a repo whose own internal
+      // workspace folder is one letter off from the repo's own directory name
+      // ("poiman" the repo vs. "poimen" the crate workspace inside it) was
+      // enough for the agent to touch its sentinel one level off from where
+      // this function is watching for it -- coordinator waits out the full
+      // STAGE_TIMEOUT_MS for a file that already exists, just in the wrong
+      // place. State the absolute target directory and use absolute paths for
+      // every filesystem instruction, so there's nothing for the agent to get
+      // wrong by reasoning about a relative "current directory."
+      const cwdReminder = `Your working directory for this task is ${cwd} -- if your shell isn't already there, run: cd ${cwd}\n\n`;
+    
+      const spawnFresh = async () => {
+        const spawnArgs = ["spawn", "--tool", "pi", "--cwd", cwd, "--name", label, "--group", repoId, "--prompt", cwdReminder + HARD_RULES + prompt];
+        const { provider, model } = providerModelFor(role);
+        if (provider) spawnArgs.push("--provider", provider);
+        if (model) spawnArgs.push("--model", model);
+        const spawned = await runCmd(AGENT_MANAGER_BIN, spawnArgs);
+        return spawned.code === 0 ? amSessionName(spawned.out) : null;
+      };
+    
+      let target = pool[role];
+      if (!target) {
+        target = await spawnFresh();
+        if (!target) return { ok: false, crashed: true, error: "spawn failed", sessionName: label };
+        pool[role] = target;
+      } else {
+        await runAmTmux(["send-keys", "-t", target, "/new", "Enter"]);
+        await sleep(1000);
+        await runAmTmux(["send-keys", "-t", target, cwdReminder + HARD_RULES + prompt, "Enter"]);
+      }
+    
+      for (let deadRetries = 0; ; deadRetries++) {
+        const outcome = await waitForSentinel(sentinelFile, target, STAGE_TIMEOUT_MS);
+        if (outcome.ok) return { ok: true, sessionName: label };
+    
+        if (outcome.dead) {
+          await killDeadSession(target);
+          if (pool[role] === target) delete pool[role];
+          if (deadRetries >= DEAD_SESSION_RETRIES) {
+            return { ok: false, crashed: true, error: `session died (status: ${outcome.status})`, sessionName: label };
+          }
+          target = await spawnFresh();
+          if (!target) return { ok: false, crashed: true, error: "respawn after death failed", sessionName: label };
+          pool[role] = target;
+          continue;
+        }
+    
+        // Plain stall -- session still alive, just slow. Ask resolver once,
+        // nudge if it says worth it, and stop here either way (this is not
+        // the death path, so no respawn/retry loop).
+        const pane = await runAmTmux(["capture-pane", "-t", target, "-p", "-S", "-200"]);
+        const resolution = await askResolver(
+          cwd,
+          repoId,
+          `Repo ${repoId}'s "${role}" agent hasn't finished its current task after 10 minutes. Its pane tail:\n${pane.out.slice(-3000)}\n\n` +
+            `Decide: is it still making real progress and worth nudging to wrap up, or stuck and worth abandoning?`
+        );
+        let ok = false;
+        if (resolution === "RETRY") {
+          await runAmTmux(["send-keys", "-t", target, `Please wrap up now and run: touch ${sentinelFile}`, "Enter"]);
+          const nudged = await waitForSentinel(sentinelFile, target, NUDGE_TIMEOUT_MS);
+          ok = nudged.ok === true;
+          if (nudged.dead) {
+            await killDeadSession(target);
+            if (pool[role] === target) delete pool[role];
+          }
+        }
+        return { ok, sessionName: label };
+      }
+    }
+    
+    function plannerPrompt(task, specHint, judgeOnly, cwd) {
+      // judgeOnly (auto-discovered tasks only, see parseTaskBoard): planner
+      // itself decides whether the task is already done before planning it,
+      // reading tasks/INDEX.md's own status notes plus git log/current code --
+      // replaces what used to be a separate judge pre-check call. One LLM round
+      // trip instead of two, and the same agent that's about to plan the task
+      // is the one deciding whether planning it is even necessary.
+      const resultFile = path.join(cwd, `.task-result-${task}`);
+      const decideStep = judgeOnly
+        ? `First, decide whether task ${task} is already fully implemented on this branch: check ` +
+          `\`git log --oneline --grep '${task}'\`, tasks/INDEX.md's own status notes for this task, and the current ` +
+          `code directly against its spec (${specHint})'s acceptance criteria. Write your decision to ` +
+          `${resultFile} as a single "VERDICT: PASS" (already done, no further work needed) or ` +
+          `"VERDICT: FAIL" (needs work) line plus one line of rationale. If VERDICT is FAIL, continue below and ` +
+          `draft the plan in this same turn; if VERDICT is PASS, skip the rest and go straight to the touch step.\n\n`
+        : "";
+      return (
+        `${decideStep}Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). ` +
+        `PLAN.md is scratch state for this harness, not a deliverable -- do NOT commit it or add it to git. ` +
+        `Then run: touch ${path.join(cwd, `.stage-done-${task}-planner`)}`
+      );
+    }
+    
+    function investigatorPrompt(task, cwd) {
+      return (
+        `Use the investigator skill to confirm PLAN.md against real sources for task ${task}, append findings. ` +
+        `PLAN.md is scratch state for this harness, not a deliverable -- do NOT commit it or add it to git. ` +
+        `Then run: touch ${path.join(cwd, `.stage-done-${task}-investigator`)}`
+      );
+    }
+    
+    function implementerPrompt(task, attempt, feedbackHint, cwd) {
+      return (
+        `Use the implementer skill to implement what the current PLAN.md specifies for task ${task} (commit as you go). ` +
+        `${feedbackHint} Then run: touch ${path.join(cwd, `.stage-done-${task}-implementer-${attempt}`)}`
+      );
+    }
+    
+    function judgePrompt(task, baseBranch, attempt, cwd) {
+      return (
+        `${REQUIRE_REAL_TOOL_CALLS}Use the judge skill to review the diff against ${baseBranch}...HEAD for task ${task}. ` +
+        `Write your verdict to ${path.join(cwd, `.task-result-${task}`)} as a single "VERDICT: PASS" or "VERDICT: FAIL" ` +
+        `line plus one line of rationale, then run: touch ${path.join(cwd, `.stage-done-${task}-judge-${attempt}`)}`
+      );
+    }
+    
+    const MAX_IMPLEMENT_ATTEMPTS = 5;
+    const MAX_PLAN_REVISIONS = 3;
+    
+    // Runs one task against the repo's shared role pool: planner drafts
+    // PLAN.md (for auto-discovered tasks, first deciding off tasks/INDEX.md and
+    // the repo's own state whether the task is already done -- see
+    // plannerPrompt's judgeOnly branch; judge never does this pre-check),
+    // investigator confirms it, then implementer and judge go back and
+    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
+    // next implementer attempt is told to read and address. After
+    // MAX_IMPLEMENT_ATTEMPTS straight fails, the planner role is asked to judge
+    // whether the plan itself is wrong -- fresh conversation, same as any other
+    // planner call, reading PLAN.md/the judge feedback/the
+    // diff off disk rather than remembering having drafted the original plan.
+    // If it decides the approach is wrong it revises PLAN.md and the implementer
+    // gets a fresh attempt budget.
+    // MAX_PLAN_REVISIONS caps this from looping forever on a task that's
+    // genuinely stuck. All work happens directly in cwd (the repo's one shared
+    // clone, currently checked out to the phase branch) -- no worktree, since
+    // only one implementer/judge exist per repo and tasks run strictly one at a
+    // time (see runPhase).
+    async function runTaskOnPool(cwd, baseBranch, task, pool, repoId, pipelineSession, judgeOnly) {
+      const resultFile = path.join(cwd, `.task-result-${task}`);
+      fs.rmSync(resultFile, { force: true });
+    
+      const specHint = `the file under tasks/ starting with "${task}-"`;
+    
+      const stage = async (role, prompt, sentinel, displayLabel) => {
+        const label = displayLabel || role;
+        pipelineSession.activeTasks[task] = { stage: label, startedAt: new Date().toISOString() };
+        logProgress(pipelineSession);
+        const result = await runOnPool(pool, cwd, repoId, role, prompt, sentinel);
+        await commitPending(cwd, `task: ${task} (${label})`);
+        return result;
+      };
+    
+      const abandon = (stageLabel, result, attempt) => {
+        delete pipelineSession.activeTasks[task];
+        logProgress(pipelineSession);
+        return {
+          task,
+          status: result.crashed ? "spawn-crashed" : "timed-out",
+          error: result.error,
+          stoppedAt: stageLabel,
+          ...(attempt !== undefined ? { attempt } : {}),
+        };
+      };
+    
+      // PLAN.md is scratch state for this one task, not a deliverable (see
+      // plannerPrompt/investigatorPrompt -- it's gitignored too, as a backstop
+      // in case an agent commits it anyway). Discard it once the task is done,
+      // whatever the outcome, so it never bleeds into the next task's planner
+      // call or sits around as stale harness clutter in the shared clone.
+      try {
+        let result = await stage("planner", plannerPrompt(task, specHint, judgeOnly, cwd), path.join(cwd, `.stage-done-${task}-planner`));
+        if (!result.ok) return abandon("planner", result);
+    
+        if (judgeOnly) {
+          const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+          if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
+            delete pipelineSession.activeTasks[task];
+            logProgress(pipelineSession);
+            return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
+          }
+        }
+    
+        result = await stage("investigator", investigatorPrompt(task, cwd), path.join(cwd, `.stage-done-${task}-investigator`));
+        if (!result.ok) return abandon("investigator", result);
+    
+        let planRevisions = 0;
+        let implementAttempt = 0;
+        let verdict = null;
+        let resultText = "";
+        let justRevisedPlan = false;
+    
+        while (true) {
+          implementAttempt++;
+          const feedbackHint = fs.existsSync(resultFile)
+            ? justRevisedPlan
+              ? `${resultFile} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
+                `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
+              : `A previous judge review exists at ${resultFile} -- read it and address every issue it raises.`
+            : "";
+          justRevisedPlan = false;
+    
+          result = await stage(
+            "implementer",
+            implementerPrompt(task, implementAttempt, feedbackHint, cwd),
+            path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
+          );
+          if (!result.ok) return abandon("implementer", result, implementAttempt);
+    
+          result = await stage("judge", judgePrompt(task, baseBranch, implementAttempt, cwd), path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`));
+          if (!result.ok) return abandon("judge", result, implementAttempt);
+    
+          resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+          verdict = parseVerdictLine(resultText, "VERDICT");
+          if (verdict === "PASS") break;
+    
+          if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
+            if (planRevisions >= MAX_PLAN_REVISIONS) break;
+            planRevisions++;
+            result = await stage(
+              "planner",
+              `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
+                `the judge's feedback in ${resultFile}, and the current diff against ${baseBranch}...HEAD. Decide ` +
+                `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md. If ` +
+                `you change the approach, also use the investigator skill to confirm the new approach against real ` +
+                `sources. If the plan is sound, note why in PLAN.md and leave it as-is. PLAN.md is scratch state for ` +
+                `this harness, not a deliverable -- do NOT commit it or add it to git. Then run: ` +
+                `touch ${path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)}`,
+              path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`),
+              "planner-revise"
+            );
+            if (!result.ok) return abandon("planner-revise", result, planRevisions);
+            implementAttempt = 0;
+            justRevisedPlan = true;
+          }
+        }
+    
+        delete pipelineSession.activeTasks[task];
+        logProgress(pipelineSession);
+    
+        if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
+          return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
+        }
+        return { task, status: verdict === "PASS" ? "done" : "done-with-concerns", judgeRationale: resultText };
+      } finally {
+        fs.rmSync(path.join(cwd, "PLAN.md"), { force: true });
+      }
+    }
+    
+    // Committed (never gitignored) so it survives a resumed phase branch --
+    // one task id per line, appended as each task resolves. This is what lets
+    // a resumed run skip straight past already-resolved tasks instead of
+    // re-running planner's judgeOnly decision on every one of them again:
+    // resuming the git branch alone only recovers the CODE, not "which tasks
+    // are already settled," and re-deciding that from scratch for every task
+    // burns a full LLM call per already-done task before ever reaching the
+    // first one that actually needs work.
+    function progressLedgerPath(cwd) {
+      return path.join(cwd, ".agent-progress");
+    }
+    
+    function readCompletedTasks(cwd) {
+      const file = progressLedgerPath(cwd);
+      if (!fs.existsSync(file)) return new Set();
+      return new Set(
+        fs
+          .readFileSync(file, "utf8")
+          .split("\n")
+          .map((line) => line.trim())
+          .filter(Boolean)
+      );
+    }
+    
+    async function recordTaskComplete(cwd, task) {
+      fs.appendFileSync(progressLedgerPath(cwd), `${task}\n`);
+      await runGit(cwd, ["add", path.basename(progressLedgerPath(cwd))]);
+      await runGit(cwd, ["commit", "-m", `chore: mark ${task} complete in progress ledger`]);
+    }
+    
+    // Runs every task in a phase (no declared dependency between them) strictly
+    // one at a time against the repo's shared role pool -- only one implementer/
+    // judge/etc. exists per repo, so there is no per-task concurrency to have
+    // here anymore (see REPO_CONCURRENCY below for where concurrency now
+    // lives). No worktrees: every task commits directly onto phaseBranch in the
+    // one shared cwd. baseBranch here is the TRUE base (e.g. "main") -- judge
+    // reviews `git diff baseBranch...HEAD`, not phaseBranch...HEAD, which would
+    // always be empty since HEAD *is* phaseBranch while it's checked out.
+    //
+    // Pushes phaseBranch after every task, not just once at full-phase-end: the
+    // pod is ephemeral and every restart re-clones baseBranch fresh (see
+    // runRepoPipeline) -- without this, a redeploy mid-phase silently discards
+    // every task committed so far, and the next run re-decides "is this done?"
+    // from a clone that never saw any of that work.
+    async function runPhase(cwd, baseBranch, phaseBranch, phaseTasks, pool, repoId, pipelineSession) {
+      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
+      const completed = readCompletedTasks(cwd);
+    
+      for (const entry of entries) {
+        if (completed.has(entry.id)) {
+          const result = { task: entry.id, status: "done", resumed: true };
+          pipelineSession.taskResults.push(result);
+          logProgress(pipelineSession);
+          continue;
+        }
+    
+        const result = await runTaskOnPool(cwd, baseBranch, entry.id, pool, repoId, pipelineSession, entry.judgeOnly);
+        pipelineSession.taskResults.push(result);
+        if (result.status === "done" || result.status === "done-with-concerns") {
+          await recordTaskComplete(cwd, entry.id);
+        }
+        await runGit(cwd, ["push", "-u", "origin", phaseBranch]);
+        logProgress(pipelineSession);
+      }
+    }
+    
+    // Discovers phases/tasks from the repo's own tasks/INDEX.md instead of
+    // requiring the caller to pass --tasks. Matches this convention's board
+    // shape (see e.g. Poimen/agent-rust's tasks/INDEX.md): a numbered phase
+    // heading ("## 1 — Foundations · T0.x"), followed by a markdown table
+    // whose rows link to each task's own spec file ("| [T0.1](T0.1-....md) |
+    // ... |"). Headings that aren't a numbered phase (prose sections like
+    // "## Ordering — declared, never derived", "## Progress") are skipped --
+    // only "## <digits> — ..." starts a new phase. Returns null if
+    // tasks/INDEX.md doesn't exist; an empty array if it exists but no phase
+    // yielded any task rows.
+    function parseTaskBoard(cwd) {
+      const indexPath = path.join(cwd, "tasks", "INDEX.md");
+      if (!fs.existsSync(indexPath)) return null;
+    
+      const phaseHeaderRe = /^##\s+\d+\s+—/;
+      const taskRowRe = /^\|\s*\[([A-Za-z0-9.]+)\]\(/;
+    
+      const phases = [];
+      let current = null;
+      for (const line of fs.readFileSync(indexPath, "utf8").split("\n")) {
+        if (phaseHeaderRe.test(line)) {
+          current = [];
+          phases.push(current);
+          continue;
+        }
+        const m = line.match(taskRowRe);
+        if (m && current) current.push(m[1]);
+      }
+      return phases.filter((phase) => phase.length > 0);
+    }
+    
+    function phaseLabelFor(phaseTasks, index) {
+      const first = phaseTasks[0];
+      const id = typeof first === "string" ? first : first.id;
+      const dot = id.indexOf(".");
+      return dot === -1 ? `phase-${index}` : id.slice(0, dot);
+    }
+    
+    function logProgress(pipelineSession) {
+      console.log(`[repo ${pipelineSession.id}] ${JSON.stringify(pipelineSession)}`);
+    }
+    
+    // Runs one repo's full pipeline: clone, then phases strictly sequentially.
+    // tasks: array of phases, each phase an array of task ids with no declared
+    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]).
+    // A flat array of ids is also accepted and treated as one single phase. If
+    // omitted, phases are discovered from the repo's own tasks/INDEX.md and run
+    // judgeOnly first (a cheap "is this already done" check against the
+    // board's possibly-stale checkmarks). Each phase gets its own branch
+    // (agent-run/<repoId>/<phaseLabel>, e.g. .../T1); once every task in that
+    // phase lands "done" or "done-with-concerns" AND the phase judge (the same
+    // pooled judge agent that reviewed each task) passes the integration
+    // review, the phase branch is squash-merged into baseBranch and pushed,
+    // then the next phase branches off that updated base. Any failure halts
+    // this repo's pipeline before merging -- it does not affect other repos
+    // running concurrently (see runCoordinator).
+    async function runRepoPipeline({ repoId, repo, baseBranch, tasks, branchName }, pipelineSession) {
+      const cwd = path.join(WORK_DIR, repoId);
+      // repoId is a slug derived from the repo URL now (see slugFor), not a
+      // fresh UUID -- reusable across separate `runCoordinator` invocations
+      // against the same repo, so a stale clone from a prior run has to be
+      // wiped before this one starts, not merged into.
+      fs.rmSync(cwd, { recursive: true, force: true });
+      fs.mkdirSync(cwd, { recursive: true });
+      const pool = {};
+    
+      const finish = (status) => {
+        pipelineSession.status = status;
+        pipelineSession.endedAt = new Date().toISOString();
+        logProgress(pipelineSession);
+        return pipelineSession;
+      };
+    
+      // Deterministic, not routed through an LLM -- clone is 100% mechanical
+      // (same reasoning as commitPending/the squash-merge sequence below), and
+      // was the one place left that broke that pattern: a headless spawnPi
+      // call here meant a crash gave zero diagnostic output, just a silent
+      // exit code with nothing to debug from.
+      const clone = await runGit(cwd, ["clone", "--branch", baseBranch, repo, "."]);
+      if (clone.code !== 0) {
+        pipelineSession.gitError = clone.out;
+        return finish("clone-crashed");
+      }
+      if (!fs.existsSync(path.join(cwd, ".git"))) return finish("clone-missing");
+    
+      let phases = tasks ? (Array.isArray(tasks[0]) ? tasks : [tasks]) : parseTaskBoard(cwd);
+      if (!phases || phases.length === 0) {
+        pipelineSession.gitError = "no tasks given and tasks/INDEX.md not found or empty";
+        return finish("no-tasks-found");
+      }
+      if (!tasks) {
+        phases = phases.map((phase) => phase.map((id) => ({ id, judgeOnly: true })));
+      }
+      pipelineSession.totalTasks = phases.flat().length;
+    
+      for (let i = 0; i < phases.length; i++) {
+        const phaseTasks = phases[i];
+        const phaseLabel = phaseLabelFor(phaseTasks, i);
+        const phaseBranch = branchName ? `${branchName}/${phaseLabel}` : `agent-run/${repoId}/${phaseLabel}`;
+    
+        // Resume a phase branch a prior (since-restarted) run already pushed,
+        // instead of always branching fresh off baseBranch -- otherwise every
+        // redeploy silently discards whatever tasks that prior run already
+        // committed and pushed (see runPhase's per-task push below).
+        const fetchExisting = await runGit(cwd, ["fetch", "origin", phaseBranch]);
+        const resuming = fetchExisting.code === 0;
+        const branchResult = resuming
+          ? await runGit(cwd, ["checkout", "-b", phaseBranch, "FETCH_HEAD"])
+          : await runGit(cwd, ["checkout", "-b", phaseBranch]);
+        if (branchResult.code !== 0) {
+          pipelineSession.gitError = branchResult.out;
+          return finish("branch-crashed");
+        }
+    
+        // Idempotent and run every phase, NOT gated on a fresh (non-resumed)
+        // start -- every run this session was a resume, so the old i===0 &&
+        // !resuming gate meant this setup permanently never ran on poiman's
+        // branch, and portfolio's PLAN.md stayed tracked from before this rule
+        // ever existed (gitignore has no effect on an already-tracked file --
+        // observed live: it kept getting swept back in by every `git add -A`
+        // regardless of the ignore rule). Check-and-fix on every phase instead
+        // of once-at-genesis so a repo that's missing either self-heals on its
+        // very next run rather than carrying the gap forever.
+        const gitignorePath = path.join(cwd, ".gitignore");
+        const currentGitignore = fs.existsSync(gitignorePath) ? fs.readFileSync(gitignorePath, "utf8").split("\n") : [];
+        const requiredGitignoreLines = [
+          "*.tar.gz",
+          "*.tgz",
+          "*.crate",
+          "*.zip",
+          "*.bin",
+          "*.whl",
+          "vendor/",
+          "node_modules/",
+          ".task-result-*",
+          ".phase-result-*",
+          ".stage-done-*",
+          "PLAN.md",
+        ];
+        const missingGitignoreLines = requiredGitignoreLines.filter((line) => !currentGitignore.includes(line));
+        if (missingGitignoreLines.length > 0) {
+          fs.appendFileSync(
+            gitignorePath,
+            "\n# agent-harness: build artifacts, vendored archives, and harness bookkeeping never belong in source control\n" +
+              missingGitignoreLines.join("\n") +
+              "\n"
+          );
+          await runGit(cwd, ["add", ".gitignore"]);
+          await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
+        }
+        const trackedFiles = await runGit(cwd, ["ls-tree", "-r", "HEAD", "--name-only"]);
+        if (trackedFiles.out.split("\n").includes("PLAN.md")) {
+          await runGit(cwd, ["rm", "--cached", "PLAN.md"]);
+          await runGit(cwd, ["commit", "-m", "chore: untrack PLAN.md (already gitignored, was committed pre-rule)"]);
+        }
+    
+        await runPhase(cwd, baseBranch, phaseBranch, phaseTasks, pool, repoId, pipelineSession);
+    
+        const phaseTaskIds = new Set(phaseTasks.map((t) => (typeof t === "string" ? t : t.id)));
+        const phaseResults = pipelineSession.taskResults.filter((r) => phaseTaskIds.has(r.task));
+        const phaseClean =
+          phaseResults.length === phaseTaskIds.size && phaseResults.every((r) => r.status === "done" || r.status === "done-with-concerns");
+    
+        if (!phaseClean) {
+          pipelineSession.haltedAt = phaseLabel;
+          return finish("halted-phase-failed");
+        }
+    
+        const phaseResultFile = path.join(cwd, `.phase-result-${phaseLabel}`);
+        fs.rmSync(phaseResultFile, { force: true });
+        const phaseJudge = await runOnPool(
+          pool,
+          cwd,
+          repoId,
+          "judge",
+          `${REQUIRE_REAL_TOOL_CALLS}Use the judge skill to review the full phase diff for phase ${phaseLabel} against ` +
+            `${baseBranch}...HEAD (covers every task in this phase: ${[...phaseTaskIds].join(", ")}). Every ` +
+            `individual task already passed its own judge review -- your job here is different: confirm the ` +
+            `tasks integrate correctly as one coherent narrative, and that real integration tests (not just ` +
+            `each task's isolated unit checks) exist and actually exercise the phase's intended use case end ` +
+            `to end. Write your verdict to ${phaseResultFile} as a single "VERDICT: PASS" or ` +
+            `"VERDICT: FAIL" line plus rationale, then run: touch ${path.join(cwd, `.stage-done-phase-${phaseLabel}-judge`)}`,
+          path.join(cwd, `.stage-done-phase-${phaseLabel}-judge`)
+        );
+        await commitPending(cwd, `phase: ${phaseLabel} integration review`);
+    
+        if (!phaseJudge.ok) {
+          pipelineSession.haltedAt = phaseLabel;
+          pipelineSession.gitError = phaseJudge.error;
+          return finish("phase-judge-crashed");
+        }
+    
+        const phaseJudgeText = fs.existsSync(phaseResultFile) ? fs.readFileSync(phaseResultFile, "utf8") : "";
+        if (parseVerdictLine(phaseJudgeText, "VERDICT") !== "PASS") {
+          pipelineSession.haltedAt = phaseLabel;
+          pipelineSession.phaseJudgeRationale = phaseJudgeText;
+          return finish("halted-phase-judge-failed");
+        }
+    
+        const checkoutBase = await runGit(cwd, ["checkout", baseBranch]);
+        if (checkoutBase.code !== 0) {
+          pipelineSession.gitError = checkoutBase.out;
+          return finish("squash-crashed");
+        }
+        const squash = await runGit(cwd, ["merge", "--squash", phaseBranch]);
+        if (squash.code !== 0) {
+          await runGit(cwd, ["merge", "--abort"]);
+          pipelineSession.gitError = squash.out;
+          return finish("squash-crashed");
+        }
+        const commit = await runGit(cwd, ["commit", "-m", `feat: ${phaseLabel} (${[...phaseTaskIds].join(", ")})`]);
+        if (commit.code !== 0) {
+          pipelineSession.gitError = commit.out;
+          return finish("squash-crashed");
+        }
+        const push = await runGit(cwd, ["push", "origin", baseBranch]);
+        if (push.code !== 0) {
+          pipelineSession.gitError = push.out;
+          return finish("squash-push-crashed");
+        }
+    
+        // Milestone's content now lives in baseBranch as one squashed commit --
+        // the phase branch (and whatever a prior restart already pushed of it)
+        // has no further reason to exist. Delete it both places so a future run
+        // never tries to resume a phase that's already done, and so origin
+        // doesn't accumulate one dangling branch per completed phase forever.
+        await runGit(cwd, ["branch", "-D", phaseBranch]);
+        await runGit(cwd, ["push", "origin", "--delete", phaseBranch]);
+    
+        logProgress(pipelineSession);
+      }
+    
+      return finish("completed");
+    }
+    
+    async function runConcurrent(items, limit, worker) {
+      const results = new Array(items.length);
+      let i = 0;
+      async function next() {
+        while (i < items.length) {
+          const idx = i++;
+          results[idx] = await worker(items[idx], idx);
+        }
+      }
+      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
+      return results;
+    }
+    
+    // How many repos can be mid-flight at once. Each repo gets its own clone
+    // and its own 4-agent pool (planner/investigator/implementer/judge), so
+    // this is now the real concurrency knob -- tasks within one repo are
+    // already serialized against that repo's pool (see runPhase). The backend
+    // (homelab-ornith) actually runs 2 GPU replicas behind one Kubernetes
+    // Service, each with its own copy of the model loaded (see homelab's
+    // k8s/apps/llm-serving/ornith.yaml) -- so up to 2 concurrent LLM calls get
+    // real independent instances; a 3rd+ concurrent call queues inside
+    // whichever replica the Service's own load-balancing lands it on (each
+    // replica runs OLLAMA_NUM_PARALLEL=1). REPO_CONCURRENCY above 2 is still
+    // useful (more repos in flight overlaps git/file work, not just LLM calls)
+    // but past 2 simultaneous LLM calls, extra concurrency mostly means queueing
+    // rather than added throughput -- bump the backend's replica count to
+    // change that, not this constant.
+    const REPO_CONCURRENCY = Number(process.env.REPO_CONCURRENCY) || 3;
+    
+    // repoId is the repo's own name, not a random id -- it's what every role
+    // session's --name is built from (see runOnPool: `${repoId}-${role}`), so
+    // agent-manager's own session list groups naturally by repo ("portfolio-
+    // planner", "portfolio-judge", "poiman-planner", ...) instead of by opaque
+    // UUID. Takes the last path segment of the URL, strips a trailing `.git`,
+    // and sanitizes anything that isn't safe in a tmux session name / directory
+    // name / git branch name. Two different repos that happen to share a
+    // basename (e.g. two orgs' "portfolio") would collide -- not handled, since
+    // nothing about this harness's usage has needed more than one org per run.
+    function slugFor(repoUrl) {
+      const last = repoUrl.replace(/\/+$/, "").split("/").pop() || repoUrl;
+      return last.replace(/\.git$/, "").replace(/[^a-zA-Z0-9._-]/g, "-");
+    }
+    
+    // Top-level entry point: runs every repo in `repos` to completion, up to
+    // REPO_CONCURRENCY at a time. Returns a map of repoId -> final
+    // pipelineSession, one per repo, independent of how the others fared.
+    async function runCoordinator({ repos, base, tasks, branchName }) {
+      const sessions = {};
+      await runConcurrent(repos, REPO_CONCURRENCY, async (repoUrl) => {
+        const repoId = slugFor(repoUrl);
+        const pipelineSession = {
+          id: repoId,
+          repo: repoUrl,
+          status: "running",
+          taskResults: [],
+          activeTasks: {},
+          totalTasks: 0,
+          startedAt: new Date().toISOString(),
+        };
+        sessions[repoId] = pipelineSession;
+        await runRepoPipeline({ repoId, repo: repoUrl, baseBranch: base, tasks, branchName }, pipelineSession);
+      });
+      return sessions;
+    }
+    
+    function parseArgs(argv) {
+      const opts = { base: "main" };
+      for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === "--repo") opts.repo = argv[++i];
+        else if (a === "--repos") opts.repos = argv[++i];
+        else if (a === "--base") opts.base = argv[++i];
+        else if (a === "--tasks") opts.tasks = argv[++i];
+        else if (a === "--branch") opts.branch = argv[++i];
+      }
+      return opts;
+    }
+    
+    async function main() {
+      const opts = parseArgs(process.argv.slice(2));
+      const repos = opts.repos ? opts.repos.split(",") : opts.repo ? [opts.repo] : null;
+      if (!repos || repos.length === 0) {
+        console.error(
+          "usage: coordinator.js --repos <url1,url2,...> [--tasks T0.1,T0.2;T1.1,T1.2,...] [--base main] [--branch <name>]\n" +
+            "  --repo <url> also accepted for a single repo\n" +
+            "  --tasks applies to every repo listed; omitted: each repo discovers its own phases from tasks/INDEX.md\n" +
+            "  REPO_CONCURRENCY env var (default 3): how many repos run at once"
+        );
+        // process.exitCode + natural exit, not process.exit() -- stdout piped
+        // through kubectl exec (not a TTY) can drop buffered console.log/
+        // console.error output if the process exits before it flushes. Setting
+        // exitCode and letting the event loop drain naturally is the
+        // documented-safe way to exit with a specific code without racing it.
+        process.exitCode = 1;
+        return;
+      }
+      const phases = opts.tasks ? opts.tasks.split(";").map((phase) => phase.split(",")) : null;
+      const sessions = await runCoordinator({ repos, base: opts.base, tasks: phases, branchName: opts.branch });
+      process.exitCode = Object.values(sessions).every((s) => s.status === "completed") ? 0 : 1;
+    }
+    
+    if (require.main === module) {
+      main();
+    }
+    
+    module.exports = { runCoordinator, runRepoPipeline, runOnPool, parseTaskBoard };
+    

k8s/apps/agent-pod/deployment.yaml

@@ -0,0 +1,168 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent-pod
+  namespace: agent-pod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: agent-pod
+  template:
+    metadata:
+      labels:
+        app: agent-pod
+    spec:
+      # api.riotpiao.com has no in-cluster DNS record (only resolves from the
+      # home network's own resolver) -- pin it to ingress-nginx-controller's
+      # ClusterIP so pi's models.json baseUrl works unchanged. TLS still
+      # terminates correctly since SNI/Host still say api.riotpiao.com.
+      hostAliases:
+        - ip: "10.101.128.185"
+          hostnames:
+            - "api.riotpiao.com"
+      containers:
+        # hub.js runs in the same container as pi (not a sidecar) so it can
+        # spawn `pi -p --mode json` directly via child_process -- a separate
+        # container can't exec into another container's filesystem/PATH.
+        # It IS the container's long-running process now; no more `sleep
+        # infinity` placeholder.
+        #
+        # Also builds the agent-manager fork (github.com/Riotpiaole/
+        # agent-manager, add-headless-spawn branch) from source and drops
+        # coordinator.js in beside hub.js -- neither is the container's
+        # foreground process. hub.js keeps that role unchanged; coordinator.js
+        # itself now owns multi-repo concurrency (REPO_CONCURRENCY env,
+        # default 3), so one invocation handles every repo:
+        # `kubectl exec <pod> -- node /root/coordinator.js --repos
+        # repoA,repoB,... --tasks ...`. Each repo gets its own clone and its
+        # own persistent 4-agent pool (planner/investigator/implementer/
+        # judge, one agent-manager session per role, reused across every
+        # task in that repo) on the container's local tmux server --
+        # `kubectl exec -it <pod> -- agent-manager` attaches its TUI live
+        # against those same sessions, no cross-machine visibility problem
+        # since spawner, tmux server, and viewer are all colocated here.
+        #
+        # No prebuilt Linux binary is shipped for agent-manager: the local
+        # .bin/ build is macOS arm64 (wrong OS/arch for this container
+        # anyway) and it's 27MB, well over a ConfigMap's ~1MiB cap. Debian's
+        # `apt-get golang-go` is far too old for this fork's go 1.26.5
+        # requirement, so the real Go toolchain is fetched directly from
+        # go.dev instead.
+        - name: pi
+          image: node:22-slim
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              apt-get update && apt-get install -y git curl jq openssh-client tmux python3 sqlite3 gcc build-essential
+              ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub
+              eval "$(ssh-agent -s)"
+              ssh-add /root/.ssh/id_forgejo
+              npm install -g @earendil-works/[email protected]
+              npm install --prefix /root ws
+
+              curl -fsSL "https://go.dev/dl/go1.26.5.linux-$(dpkg --print-architecture).tar.gz" | tar -C /usr/local -xz
+              export PATH="$PATH:/usr/local/go/bin"
+              git clone --branch add-headless-spawn --depth 1 \
+                https://github.com/Riotpiaole/agent-manager.git /root/agent-manager-src
+              (cd /root/agent-manager-src && go build -o /usr/local/bin/agent-manager .)
+
+              # Language toolchains for whatever repos the implementer/investigator/
+              # judge roles actually build and test -- go was already fetched above
+              # only for building agent-manager itself, and its PATH export above is
+              # local to this script, invisible to `kubectl exec` sessions into the
+              # already-running container. Symlinking both into /usr/local/bin (on
+              # PATH for every exec session, interactive or not) instead of relying
+              # on shell rc sourcing, which pi's non-interactive tool calls don't do.
+              ln -sf /usr/local/go/bin/go /usr/local/bin/go
+              curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+              ln -sf /root/.cargo/bin/cargo /usr/local/bin/cargo
+              ln -sf /root/.cargo/bin/rustc /usr/local/bin/rustc
+              ln -sf /root/.cargo/bin/rustup /usr/local/bin/rustup
+
+              node /root/hub.js
+          env:
+            - name: PI_BIN
+              value: pi
+            - name: AGENT_MANAGER_BIN
+              value: /usr/local/bin/agent-manager
+            - name: HUB_WORK_DIR
+              value: /root/agent-harness-work
+            # planner/investigator/implementer stay on the default
+            # (homelab-ornith/ornith:35b, pi's settings.json default). Judge
+            # moves to the separate homelab-reasoning backend (DeepSeek-R1,
+            # its own 2 GPU replicas) so judge calls stop contending with the
+            # other 3 roles for the 2 ornith pods -- an entire role's worth
+            # of traffic moves onto otherwise-idle capacity instead.
+            - name: JUDGE_PROVIDER
+              value: homelab-reasoning
+            - name: JUDGE_MODEL
+              value: reasoning
+          ports:
+            - containerPort: 9090
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+            limits:
+              cpu: "8"
+              memory: 16Gi
+          volumeMounts:
+            - name: pi-config
+              mountPath: /root/.pi/agent/settings.json
+              subPath: settings.json
+            - name: pi-models
+              mountPath: /root/.pi/agent/models.json
+              subPath: models.json
+            - name: pi-skills
+              mountPath: /root/.pi/agent/skills
+            - name: hub-src
+              mountPath: /root/hub.js
+              subPath: hub.js
+            - name: coordinator-src
+              mountPath: /root/coordinator.js
+              subPath: coordinator.js
+            - name: ssh-key
+              mountPath: /root/.ssh/id_forgejo
+              subPath: id_forgejo
+            - name: ssh-config
+              mountPath: /root/.ssh/config
+              subPath: config
+      volumes:
+        - name: pi-config
+          configMap:
+            name: pi-config
+        - name: pi-models
+          secret:
+            secretName: pi-models
+        - name: pi-skills
+          configMap:
+            name: pi-skills
+            items:
+              - key: planner-SKILL.md
+                path: planner/SKILL.md
+              - key: investigator-SKILL.md
+                path: investigator/SKILL.md
+              - key: info-collector-SKILL.md
+                path: info-collector/SKILL.md
+              - key: implementer-SKILL.md
+                path: implementer/SKILL.md
+              - key: judge-SKILL.md
+                path: judge/SKILL.md
+              - key: resolver-SKILL.md
+                path: resolver/SKILL.md
+        - name: hub-src
+          configMap:
+            name: hub-src
+        - name: coordinator-src
+          configMap:
+            name: coordinator-src
+        - name: ssh-key
+          secret:
+            secretName: agent-pod-ssh-key
+            defaultMode: 0600
+        - name: ssh-config
+          configMap:
+            name: agent-pod-ssh-config

k8s/apps/agent-pod/hub-configmap.yaml

@@ -0,0 +1,700 @@
+apiVersion: v1
+data:
+  hub.js: |
+    #!/usr/bin/env node
+    // agent-hub: lives inside the pi container (not a sidecar) so it can spawn
+    // `pi` directly, and control the pod's own tmux server. One persistent
+    // in-cluster service -- POST /run to trigger a single ad-hoc headless agent
+    // run, POST /pipeline to run an ordered list of task phases against a
+    // repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within
+    // a phase run concurrently, each in its own git worktree). Within one task,
+    // planner/investigator/implementer/judge are separate agents in separate
+    // named tmux sessions (task-<id>-<role>, attachable via `kubectl exec -it --
+    // tmux attach -t <name>` while running), coordinating only through what's on
+    // disk in that task's worktree -- not one shared conversation. GET /console
+    // (WebSocket) watches every concurrent headless run live, relaying pi's own
+    // session protocol verbatim (same event shape Claude Code sessions use).
+    const http = require("node:http");
+    const crypto = require("node:crypto");
+    const fs = require("node:fs");
+    const path = require("node:path");
+    const { spawn } = require("node:child_process");
+    const readline = require("node:readline");
+    const { WebSocketServer } = require("ws");
+
+    const PORT = process.env.HUB_PORT || 9090;
+    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
+
+    // Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide
+    // with unrelated tools on this machine (a Rust CLI and a Datadog TUI,
+    // respectively, discovered the hard way this session). Always invoke the
+    // exact pinned @earendil-works/[email protected] installed locally under
+    // .pi-cli/, by explicit path.
+    const PI_BIN =
+      process.env.PI_BIN ||
+      path.join(
+        __dirname,
+        "..",
+        ".pi-cli",
+        "node_modules",
+        "@earendil-works",
+        "pi-coding-agent",
+        "dist",
+        "cli.js"
+      );
+
+    // Job-type skills under pi/skills/<name>/SKILL.md (mounted at
+    // ~/.pi/agent/skills/<name>/ in agent-pod). When `agent` matches one of
+    // these, the prompt is forced through pi's `/skill:<name> <args>` mechanism
+    // instead of being sent bare -- see pi's skills.md docs on single-shot skill
+    // loading. `resolver` is never dispatched directly by a caller; only the
+    // pipeline driver invokes it, on stage crashes.
+    const ROLE_SKILLS = new Set([
+      "planner",
+      "investigator",
+      "info-collector",
+      "implementer",
+      "judge",
+      "resolver",
+    ]);
+
+    const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?}
+    const viewers = new Set(); // WebSocket connections watching /console
+
+    function broadcast(type, session) {
+      const msg = JSON.stringify({ type, session });
+      for (const ws of viewers) {
+        if (ws.readyState === ws.OPEN) ws.send(msg);
+      }
+    }
+
+    function startSession(agent, extra = {}) {
+      const id = extra.id || crypto.randomUUID();
+      const session = {
+        ...extra,
+        id,
+        agent,
+        status: "running",
+        events: [],
+        startedAt: new Date().toISOString(),
+      };
+      sessions.set(id, session);
+      broadcast("start", session);
+      return session;
+    }
+
+    function addEvent(session, rawLine) {
+      const event = JSON.parse(rawLine);
+      session.events.push(event);
+      broadcast("event", session);
+      return event;
+    }
+
+    function endSession(session, status) {
+      session.status = status;
+      session.endedAt = new Date().toISOString();
+      broadcast("end", session);
+    }
+
+    // Extracts the plain-text content of a message_end event, if any -- used to
+    // find the VERDICT:/RESOLUTION: line judge/resolver skills are required to
+    // end their final message with.
+    function textOf(event) {
+      if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) {
+        return null;
+      }
+      return event.message.content
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join("\n");
+    }
+
+    // Core spawn primitive. Spawns `pi -p --mode json <extraArgs> <prompt>`,
+    // relays every line as a session event exactly like before. Returns
+    // { session, done } -- `session` is available synchronously (so an HTTP
+    // handler can respond with its id right away, same as the old runAgent),
+    // `done` is a Promise resolving once the process exits, for callers that
+    // need to wait on a stage (the pipeline driver) rather than fire-and-forget.
+    function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) {
+      const session = startSession(agent, sessionExtra);
+      const args = ["-p", "--mode", "json"];
+      if (provider) args.push("--provider", provider);
+      if (model) args.push("--model", model);
+      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt;
+      args.push(finalPrompt);
+
+      const child = spawn(PI_BIN, args, {
+        stdio: ["ignore", "pipe", "pipe"],
+        cwd,
+      });
+      const rl = readline.createInterface({ input: child.stdout });
+      let lastText = "";
+      let stderrTail = "";
+
+      rl.on("line", (line) => {
+        if (!line.trim()) return;
+        try {
+          const event = addEvent(session, line);
+          const text = textOf(event);
+          if (text) lastText = text;
+        } catch {
+          // non-JSON stdout noise, ignore
+        }
+      });
+      child.stderr.on("data", (chunk) => {
+        process.stderr.write(chunk);
+        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
+      });
+
+      const done = new Promise((resolve) => {
+        child.on("close", (code) => {
+          endSession(session, code === 0 ? "done" : "error");
+          resolve({ code, session, lastText, stderrTail });
+        });
+      });
+
+      return { session, done };
+    }
+
+    function runAgent(agent, prompt, extraArgs = {}) {
+      // Fire-and-forget: caller (the /run handler) doesn't await `done`.
+      return spawnPi({ agent, prompt, ...extraArgs }).session;
+    }
+
+    function parseVerdictLine(text, label) {
+      if (!text) return null;
+      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
+      const m = text.match(re);
+      return m ? m[1].toUpperCase() : null;
+    }
+
+    // Deterministic git operations, run directly by hub.js rather than left to
+    // the model -- branch creation and pushing after each task are mechanical,
+    // not judgment calls, and need to happen reliably every time regardless of
+    // what a task's stages did or didn't remember to do.
+    function runCmd(bin, args, cwd) {
+      return new Promise((resolve) => {
+        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+        let out = "";
+        child.stdout.on("data", (c) => (out += c));
+        child.stderr.on("data", (c) => (out += c));
+        child.on("close", (code) => resolve({ code, out: out.trim() }));
+      });
+    }
+
+    function runGit(cwd, args) {
+      return runCmd("git", args, cwd);
+    }
+
+    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
+    // in practice: a stage writes a real file and simply never runs `git add`/
+    // `git commit`, leaving it untracked and invisible to every later `git diff`.
+    // Sweep and commit anything left dirty after every stage, deterministically.
+    async function commitPending(cwd, message) {
+      await runGit(cwd, ["add", "-A"]);
+      const status = await runGit(cwd, ["status", "--porcelain"]);
+      if (!status.out) return { committed: false };
+      const commit = await runGit(cwd, ["commit", "-m", message]);
+      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
+    }
+
+    // Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide
+    // RETRY vs ABORT. Shared by both crash-recovery paths below (headless
+    // exit-code failures and interactive sentinel-file timeouts) -- the
+    // diagnostic prompt differs per caller, but "ask resolver, parse the
+    // RESOLUTION: line" is identical either way.
+    async function askResolver(pipelineId, cwd, task, diagnosticPrompt) {
+      const resolverResult = await spawnPi({
+        agent: "resolver",
+        prompt: diagnosticPrompt,
+        cwd,
+        sessionExtra: { pipelineId, stage: "resolver", task },
+      }).done;
+      return parseVerdictLine(resolverResult.lastText, "RESOLUTION");
+    }
+
+    // Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic
+    // judge FAIL, which is handled separately), asks the resolver to diagnose
+    // and decide RETRY vs ABORT. Retries the failed stage at most once,
+    // regardless of what resolver recommends a second time -- a hard cap, not
+    // indefinite trust in the model's judgment.
+    async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) {
+      let result = await spawnPi({
+        agent: stage,
+        prompt,
+        cwd,
+        sessionExtra: { pipelineId, stage, task },
+      }).done;
+      if (result.code === 0) return result;
+
+      const resolution = await askResolver(
+        pipelineId,
+        cwd,
+        task,
+        `Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}`
+      );
+
+      if (resolution === "RETRY") {
+        result = await spawnPi({
+          agent: stage,
+          prompt,
+          cwd,
+          sessionExtra: { pipelineId, stage, task },
+        }).done;
+      }
+      return result;
+    }
+
+    // Deterministic tmux operations -- same rationale as runGit: mechanical,
+    // not a judgment call, run directly rather than trusted to a prompt.
+    function runTmux(args) {
+      return runCmd("tmux", args);
+    }
+
+    function tmuxSessionName(task) {
+      return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`;
+    }
+
+    // Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in
+    // flight at once. No external dep; a plain in-order index cursor shared by
+    // `limit` runner loops.
+    async function runConcurrent(items, limit, worker) {
+      const results = new Array(items.length);
+      let i = 0;
+      async function next() {
+        while (i < items.length) {
+          const idx = i++;
+          results[idx] = await worker(items[idx], idx);
+        }
+      }
+      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
+      return results;
+    }
+
+    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
+    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
+    const POLL_MS = 10 * 1000;
+
+    async function waitForFile(filePath, limitMs) {
+      const start = Date.now();
+      while (!fs.existsSync(filePath)) {
+        if (Date.now() - start > limitMs) return false;
+        await new Promise((r) => setTimeout(r, POLL_MS));
+      }
+      return true;
+    }
+
+    const MAX_IMPLEMENT_ATTEMPTS = 5;
+    const MAX_PLAN_REVISIONS = 3;
+
+    // Runs one role as its own fresh interactive pi session in its own named
+    // tmux session -- planner, investigator, implementer, and judge are
+    // separate agents with separate context, not turns in one shared
+    // conversation. They coordinate only through what's on disk in the task's
+    // worktree: PLAN.md, committed code, judge's result file. Each session is
+    // attachable while it runs (kubectl exec -it -- tmux attach -t <name>) and
+    // killed once its sentinel file lands or it's abandoned after resolver
+    // escalation.
+    async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) {
+      const sessionName = `${tmuxSessionName(task)}-${stageLabel}`;
+      fs.rmSync(sentinelFile, { force: true });
+
+      const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]);
+      if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName };
+
+      let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS);
+      if (!ok) {
+        const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]);
+        const resolution = await askResolver(
+          pipelineId,
+          cwd,
+          task,
+          `Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` +
+            `Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` +
+            `real progress and worth nudging to wrap up, or stuck and worth abandoning?`
+        );
+        if (resolution === "RETRY") {
+          await runTmux([
+            "send-keys",
+            "-t",
+            sessionName,
+            `Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`,
+            "Enter",
+          ]);
+          ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS);
+        }
+      }
+
+      await runTmux(["kill-session", "-t", sessionName]);
+      return { ok, sessionName };
+    }
+
+    // Runs one task in its own git worktree (see runPhase): planner drafts
+    // PLAN.md, investigator confirms it, then implementer and judge go back and
+    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
+    // next implementer attempt is told to read and address. After
+    // MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to
+    // judge whether the *plan* itself is wrong, not just the implementation; if
+    // so it revises PLAN.md and the implementer gets a fresh attempt budget
+    // against the new plan. MAX_PLAN_REVISIONS caps this from looping forever
+    // on a task that's genuinely stuck.
+    async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) {
+      const resultFile = path.join(cwd, `.task-result-${task}`);
+      fs.rmSync(resultFile, { force: true });
+
+      const specHint = `the file under tasks/ starting with "${task}-"`;
+
+      const runRole = async (stageLabel, prompt, sentinel) => {
+        pipelineSession.activeTasks[task] = {
+          stage: stageLabel,
+          sessionName: `${tmuxSessionName(task)}-${stageLabel}`,
+          startedAt: new Date().toISOString(),
+        };
+        broadcast("event", pipelineSession);
+        const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel);
+        await commitPending(cwd, `task: ${task} (${stageLabel})`);
+        return result;
+      };
+
+      const abandon = (stageLabel, result, attempt) => {
+        delete pipelineSession.activeTasks[task];
+        broadcast("event", pipelineSession);
+        return {
+          task,
+          status: result.crashed ? "spawn-crashed" : "timed-out",
+          error: result.error,
+          stoppedAt: stageLabel,
+          ...(attempt !== undefined ? { attempt } : {}),
+        };
+      };
+
+      // Task's implementation is inherited already-committed (e.g. from a base
+      // branch of prior work) -- try one judge pass against the spec directly
+      // (no PLAN.md exists yet) before paying for a full planner/investigator
+      // redo. PASS ends the task here; FAIL falls through into the normal flow
+      // below, so planner/implementer pick up with the judge's real feedback.
+      if (judgeOnly) {
+        const quick = await runRole(
+          "judge",
+          `Task ${task} may already be implemented on this branch -- check ` +
+            `\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` +
+            `(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` +
+            `verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` +
+            `one line of rationale, then run: touch .stage-done-${task}-judge-0`,
+          path.join(cwd, `.stage-done-${task}-judge-0`)
+        );
+        if (!quick.ok) return abandon("judge", quick, 0);
+
+        const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+        if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
+          delete pipelineSession.activeTasks[task];
+          broadcast("event", pipelineSession);
+          return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
+        }
+      }
+
+      let result = await runRole(
+        "planner",
+        `Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`,
+        path.join(cwd, `.stage-done-${task}-planner`)
+      );
+      if (!result.ok) return abandon("planner", result);
+
+      result = await runRole(
+        "investigator",
+        `Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`,
+        path.join(cwd, `.stage-done-${task}-investigator`)
+      );
+      if (!result.ok) return abandon("investigator", result);
+
+      let planRevisions = 0;
+      let implementAttempt = 0;
+      let verdict = null;
+      let resultText = "";
+      let justRevisedPlan = false;
+
+      while (true) {
+        implementAttempt++;
+        const feedbackHint = fs.existsSync(resultFile)
+          ? justRevisedPlan
+            ? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
+              `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
+            : `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.`
+          : "";
+        justRevisedPlan = false;
+
+        result = await runRole(
+          "implementer",
+          `Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`,
+          path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
+        );
+        if (!result.ok) return abandon("implementer", result, implementAttempt);
+
+        result = await runRole(
+          "judge",
+          `Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` +
+            `.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` +
+            `rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`,
+          path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`)
+        );
+        if (!result.ok) return abandon("judge", result, implementAttempt);
+
+        resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+        verdict = parseVerdictLine(resultText, "VERDICT");
+        if (verdict === "PASS") break;
+
+        if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
+          if (planRevisions >= MAX_PLAN_REVISIONS) break;
+          planRevisions++;
+          result = await runRole(
+            "planner-revise",
+            `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
+              `the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` +
+              `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` +
+              `commit. If you change the approach, also use the investigator skill to confirm the new approach against ` +
+              `real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` +
+              `touch .stage-done-${task}-planner-revise-${planRevisions}`,
+            path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)
+          );
+          if (!result.ok) return abandon("planner-revise", result, planRevisions);
+          implementAttempt = 0;
+          justRevisedPlan = true;
+        }
+      }
+
+      delete pipelineSession.activeTasks[task];
+      broadcast("event", pipelineSession);
+
+      if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
+        return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
+      }
+      return {
+        task,
+        status: verdict === "PASS" ? "done" : "done-with-concerns",
+        judgeRationale: resultText,
+      };
+    }
+
+    const PHASE_CONCURRENCY = 3;
+
+    // Runs one phase (a batch of tasks with no declared dependency on each
+    // other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task
+    // gets its own git worktree off workBranch -- concurrent pi sessions writing
+    // into one shared working tree would corrupt the index; worktrees share the
+    // same object database but give each task an isolated checkout. After a
+    // task's session ends, its branch is merged back into workBranch and pushed,
+    // one merge at a time (git ref updates aren't safe to run concurrently even
+    // though the worktrees themselves are isolated).
+    async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) {
+      // Each entry is either a plain task id, or { id, judgeOnly: true } when
+      // the task's implementation already exists (e.g. inherited from a base
+      // branch) and just needs a real judge pass rather than a full
+      // planner/investigator/implementer redo.
+      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
+
+      const worktrees = {};
+      for (const entry of entries) {
+        const task = entry.id;
+        const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`);
+        const taskBranch = `task/${task}`;
+        const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]);
+        if (add.code !== 0) {
+          pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out });
+          continue;
+        }
+        worktrees[task] = { wtDir, taskBranch };
+      }
+
+      const runnable = entries.filter((e) => worktrees[e.id]);
+      await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => {
+        const { wtDir } = worktrees[entry.id];
+        const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly);
+        pipelineSession.taskResults.push(result);
+        return result;
+      });
+
+      // Merge + push sequentially -- ref updates on the shared repo, one at a
+      // time, in the declared task order for this phase.
+      for (const entry of runnable) {
+        const task = entry.id;
+        const { wtDir, taskBranch } = worktrees[task];
+        const result = pipelineSession.taskResults.find((r) => r.task === task);
+
+        const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]);
+        if (merge.code !== 0) {
+          await runGit(cwd, ["merge", "--abort"]);
+          if (result) {
+            result.status = "merge-conflict";
+            result.mergeError = merge.out;
+          }
+        } else {
+          const push = await runGit(cwd, ["push", "-u", "origin", workBranch]);
+          if (result) {
+            result.pushed = push.code === 0;
+            if (!result.pushed) result.pushError = push.out;
+          }
+        }
+
+        await runGit(cwd, ["worktree", "remove", wtDir, "--force"]);
+        await runGit(cwd, ["branch", "-D", taskBranch]);
+        broadcast("event", pipelineSession);
+      }
+    }
+
+    // tasks: array of phases, each phase an array of task ids with no declared
+    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) --
+    // caller's responsibility to supply real phase grouping (see tasks/INDEX.md;
+    // filename/numeric sort does NOT match execution order on boards like this).
+    // A flat array of ids is also accepted and treated as one single phase.
+    // Phases run strictly sequentially (a phase boundary is a real dependency
+    // gate); tasks within a phase run concurrently, each in its own worktree --
+    // see runPhase.
+    function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) {
+      const cwd = path.join(WORK_DIR, pipelineId);
+      fs.mkdirSync(cwd, { recursive: true });
+
+      const phases = Array.isArray(tasks[0]) ? tasks : [tasks];
+
+      const pipelineSession = startSession("pipeline", {
+        id: pipelineId,
+        pipelineId,
+        stage: "pipeline",
+        taskResults: [],
+        activeTasks: {},
+        totalTasks: phases.flat().length,
+      });
+
+      (async () => {
+        const clone = await runStageWithResolver(
+          pipelineId,
+          cwd,
+          "planner",
+          `Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`,
+          "clone"
+        );
+        if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed");
+        if (!fs.existsSync(path.join(cwd, ".git"))) {
+          // The model deciding to `cd` elsewhere before cloning (instead of
+          // cloning into the assigned cwd) is a real failure mode seen in
+          // practice, not a hypothetical -- exit code 0 doesn't mean the clone
+          // landed where every later stage's cwd assumes it did.
+          return endSession(pipelineSession, "clone-missing");
+        }
+
+        // Dedicated branch, never main -- and pushed after every single task
+        // (not just at the end) so a pod restart mid-run loses at most the
+        // in-progress task's work, not everything since the start.
+        const workBranch = branchName || `agent-run/${pipelineId}`;
+        const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]);
+        if (branchResult.code !== 0) {
+          pipelineSession.gitError = branchResult.out;
+          return endSession(pipelineSession, "branch-crashed");
+        }
+
+        // Seen in practice: a task manually downloads a dependency tarball
+        // (crates.io registry access isn't guaranteed from every sandboxed
+        // checkout) and it lands at repo root, outside whatever .gitignore
+        // already covers -- then git add -A (ours or the model's own) commits
+        // it. Append broad build-artifact/archive patterns before any task
+        // runs, so it's excluded regardless of who stages files later.
+        const gitignoreAdditions = [
+          "",
+          "# agent-harness: build artifacts and vendored archives never belong in source control",
+          "*.tar.gz",
+          "*.tgz",
+          "*.crate",
+          "*.zip",
+          "*.bin",
+          "*.whl",
+          "vendor/",
+          "node_modules/",
+          "",
+          "# agent-harness: task completion sentinel files, harness bookkeeping only",
+          ".task-result-*",
+          ".stage-done-*",
+        ].join("\n");
+        fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
+        await runGit(cwd, ["add", ".gitignore"]);
+        await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
+
+        for (const phaseTasks of phases) {
+          await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession);
+        }
+
+        const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed"));
+        endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed");
+      })();
+
+      return pipelineSession;
+    }
+
+    const server = http.createServer((req, res) => {
+      const url = new URL(req.url, "http://localhost");
+
+      if (url.pathname === "/healthz") {
+        res.writeHead(200).end();
+        return;
+      }
+
+      if (url.pathname === "/sessions" && req.method === "GET") {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify([...sessions.values()]));
+        return;
+      }
+
+      if (url.pathname === "/run" && req.method === "POST") {
+        let body = "";
+        req.on("data", (chunk) => (body += chunk));
+        req.on("end", () => {
+          try {
+            const { agent, prompt, provider, model } = JSON.parse(body);
+            if (!agent || !prompt) throw new Error("agent and prompt are required");
+            const session = runAgent(agent, prompt, { provider, model });
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ id: session.id }));
+          } catch (err) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      if (url.pathname === "/pipeline" && req.method === "POST") {
+        let body = "";
+        req.on("data", (chunk) => (body += chunk));
+        req.on("end", () => {
+          try {
+            const { repo, baseBranch, tasks, branchName } = JSON.parse(body);
+            if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) {
+              throw new Error("repo, baseBranch, and a non-empty tasks array are required");
+            }
+            const pipelineId = crypto.randomUUID();
+            runPipeline({ pipelineId, repo, baseBranch, tasks, branchName });
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ id: pipelineId }));
+          } catch (err) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      res.writeHead(404).end();
+    });
+
+    const wss = new WebSocketServer({ server, path: "/console" });
+    wss.on("connection", (ws) => {
+      for (const session of sessions.values()) {
+        ws.send(JSON.stringify({ type: "snapshot", session }));
+      }
+      viewers.add(ws);
+      ws.on("close", () => viewers.delete(ws));
+    });
+
+    server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
+kind: ConfigMap
+metadata:
+  name: hub-src
+  namespace: agent-pod

k8s/apps/agent-pod/hub-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-hub
+  namespace: agent-pod
+spec:
+  selector:
+    app: agent-pod
+  ports:
+    - port: 9090
+      targetPort: 9090

k8s/apps/agent-pod/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-pod
+resources:
+  - deployment.yaml
+  - configmap.yaml
+  - hub-configmap.yaml
+  - coordinator-configmap.yaml
+  - pi-skills-configmap.yaml
+  - ssh-configmap.yaml
+  - hub-service.yaml
+  - console-ingress.yaml

k8s/apps/agent-pod/pi-skills-configmap.yaml

@@ -0,0 +1,122 @@
+apiVersion: v1
+data:
+  implementer-SKILL.md: |
+    ---
+    name: implementer
+    description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run.
+    allowed-tools: Read Grep Find Ls Write Edit Bash
+    ---
+    
+    Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions.
+    
+    - Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history.
+    - Push only if the task explicitly asks for it.
+    
+    Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch.
+    
+    **UI/frontend changes:** don't trust that the code compiles as proof it works. Start the app (or its dev server) and use `npx playwright` via Bash to actually load the page and look — screenshot the affected view before and after your change, and click through the golden path the plan describes. `npx playwright screenshot <url> out.png` for a quick visual check; for interaction (clicks, form fills, navigation), write a small throwaway script under a scratch path (e.g. `/tmp/`, never committed) using `playwright` the library, run it with `node`, then delete it. This doesn't apply to non-UI work (a Rust library, a CLI, a backend-only change) — use judgment.
+    
+    **Hard rules:**
+    - Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing.
+    - Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it.
+    
+    **Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it.
+    
+  info-collector-SKILL.md: |
+    ---
+    name: info-collector
+    description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill).
+    allowed-tools: Read Bash
+    ---
+
+    **Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there.
+
+    **Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job).
+
+    **Modes:**
+
+    - **Collect mode** (default) — search the web (`curl` against `https://api.search.brave.com/res/v1/web/search`, header `X-Subscription-Token: $BRAVE_API_KEY`, `--data-urlencode "q=<query>"`) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill.
+    - If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response.
+  investigator-SKILL.md: |
+    ---
+    name: investigator
+    description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct.
+    allowed-tools: Read Bash
+    ---
+
+    Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one.
+
+    - Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), search the web (`curl` against `https://api.search.brave.com/res/v1/web/search`, header `X-Subscription-Token: $BRAVE_API_KEY`) to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`.
+    - No `PLAN.md`? Just answer the question asked, citing sources.
+
+    Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next.
+
+    **Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not.
+  judge-SKILL.md: |
+    ---
+    name: judge
+    description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria.
+    allowed-tools: Read Bash
+    ---
+    
+    Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp.
+    
+    - Run `git diff <base-branch>...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection?
+    - **UI/frontend changes:** a diff that reads correctly can still render broken. Start the app and use `npx playwright` via Bash to actually look — screenshot the affected view, click through the golden path the plan/spec describes. FAIL on a visual defect the diff alone wouldn't show (broken layout, a control that doesn't do what its code claims, a state the plan promised that never renders). Doesn't apply to non-UI work — use judgment.
+    - FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale.
+    - No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly.
+    
+    You MUST end your final message with a literal verdict line, exactly one of:
+    
+    ```
+    VERDICT: PASS
+    ```
+    ```
+    VERDICT: FAIL
+    ```
+    
+    followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline.
+    
+  planner-SKILL.md: |
+    ---
+    name: planner
+    description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it.
+    allowed-tools: Read Grep Find Ls Write Bash
+    ---
+
+    Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms.
+
+    - If cwd is empty: `git clone` the given repo/branch first (only at pipeline start).
+    - Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity.
+    - Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/<name>.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: <summary>"`.
+
+    Don't touch any other file.
+  resolver-SKILL.md: |
+    ---
+    name: resolver
+    description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion.
+    allowed-tools: Read Bash
+    ---
+
+    **Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again.
+
+    **Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work.
+
+    **Modes:**
+
+    - **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix).
+
+    You MUST end your final message with a literal resolution line, exactly one of:
+
+    ```
+    RESOLUTION: RETRY
+    ```
+    ```
+    RESOLUTION: ABORT
+    ```
+
+    followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`.
+kind: ConfigMap
+metadata:
+  name: pi-skills
+  namespace: agent-pod

k8s/apps/agent-pod/ssh-configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-pod-ssh-config
+  namespace: agent-pod
+data:
+  config: |
+    Host git.riotpiao.com
+        IdentityFile /root/.ssh/id_forgejo
+        Port 2222
+        User git
+        StrictHostKeyChecking accept-new

k8s/apps/api/ingress.yaml

@@ -0,0 +1,46 @@
+# Edge route for the API gateway.
+#
+# Lives here rather than in the central k8s/bootstrap/ingress/ingress.yaml
+# because that Application syncs in wave 1, before namespace `api` exists.
+#
+# nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its
+# default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards
+# plain HTTP to the gateway.
+#
+# Backend was kong-proxy:80 until Kong was retired on 2026-08-19; it is now the
+# Go gateway's Service, api-gateway:8080, deployed from rock/homelab-frontend.
+# Reverting the cutover is a change to these two lines and nothing else.
+#
+# Catch-all `/` on purpose: everything under this host belongs to the gateway.
+# Listing per-API paths here would duplicate the gateway's routing table inside
+# nginx, and the two copies would drift.
+#
+# In-cluster callers should prefer http://api-gateway.api.svc.cluster.local:8080
+# directly. Resolving api.riotpiao.com sends them out to nginx and back in,
+# which is a pointless hairpin unless they need TLS or the public hostname.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: api
+  namespace: api
+  annotations:
+    # An API gateway carries streaming responses (SSE, gRPC-web, LLM token
+    # streams). nginx's 60s default read timeout and its response buffering
+    # would truncate or stall those.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: api-gateway
+                port:
+                  number: 8080

k8s/apps/api/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist. Anything added to this directory and not listed here is
+# silently dropped — no error, no drift shown.
+#
+# Down to a single Ingress since Kong was retired (2026-08-19). The Kong Helm
+# values, the KongClusterPlugin for Prometheus, the six KongPlugin CRs behind
+# the path-per-model LLM surface, the KongConsumer and the key-auth plan all
+# went with it.
+resources:
+  - ingress.yaml
+# No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
+# namespace, and the transformer rewrites metadata.namespace on every resource
+# it builds, which is a trap for anything cross-namespace added later.

k8s/apps/cloudflared/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cloudflared
+  namespace: cloudflared
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: cloudflared
+  template:
+    metadata:
+      labels:
+        app: cloudflared
+    spec:
+      containers:
+        - name: cloudflared
+          image: cloudflare/cloudflared:latest
+          args:
+            - tunnel
+            - --no-autoupdate
+            - run
+            - --token
+            - $(TUNNEL_TOKEN)
+          env:
+            - name: TUNNEL_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: cloudflared-token
+                  key: token
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi

k8s/apps/cloudflared/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: cloudflared
+resources:
+  - deployment.yaml

k8s/apps/homarr/homarr-values.yaml

@@ -0,0 +1,78 @@
+# Homarr landing page with Authentik SSO
+
+# Probe tuning (chart DOES expose these — the old PostSync patch-job was
+# unnecessary and fragile: it only patched one Deployment revision, so any later
+# rollout reverted to the chart's aggressive defaults). Homarr's first-boot icon
+# updater blocks the event loop for ~50s ("icons updater took 49553ms"), during
+# which /api/health/live can't answer within the default 10s×3 window → kubelet
+# SIGTERMs the pod → CrashLoopBackOff (247 restarts, 503 at the ingress). Give
+# liveness a wide window so the icon import can finish without a kill.
+livenessProbe:
+  initialDelaySeconds: 60
+  periodSeconds: 30
+  timeoutSeconds: 5
+  failureThreshold: 10
+readinessProbe:
+  initialDelaySeconds: 30
+  periodSeconds: 15
+  timeoutSeconds: 5
+  failureThreshold: 6
+
+image:
+  repository: ghcr.io/homarr-labs/homarr
+  tag: "latest"
+  pullPolicy: Always
+
+replicaCount: 1
+
+# Configure SSO via environment variables
+# Chart supports these via top-level env dict (not array)
+env:
+  AUTH_PROVIDERS: "oidc,credentials"
+  AUTH_OIDC_ISSUER: "https://authentik.riotpiao.com/application/o/homarr/"
+  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
+  # hides the "Sign in with Authentik" button entirely when it's absent (per the
+  # authentik Homarr integration + homarr SSO docs). This was the missing var.
+  AUTH_OIDC_URI: "https://authentik.riotpiao.com/application/o/authorize/"
+  AUTH_OIDC_CLIENT_NAME: "Authentik"
+  AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
+  AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
+  AUTH_OIDC_AUTO_LOGIN: "false"
+  # Link the OIDC identity to an existing homarr account with the same email.
+  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
+  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
+  # per run ("callback took longer than expected"), compounding CPU pressure.
+  DISABLE_ANALYTICS: "true"
+  BASE_URL: "https://homarr.riotpiao.com"
+  NEXTAUTH_URL: "https://homarr.riotpiao.com"
+
+# Client credentials from homarr-oidc secret
+# Chart doesn't support envFrom, so we add via extraEnv
+extraEnv:
+  - name: AUTH_OIDC_CLIENT_ID
+    valueFrom:
+      secretKeyRef:
+        name: homarr-oidc
+        key: client-id
+  - name: AUTH_OIDC_CLIENT_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: homarr-oidc
+        key: client-secret
+
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+resources:
+  requests:
+    cpu: 250m
+    memory: 384Mi
+  limits:
+    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
+    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
+    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
+    # real CPU headroom.
+    cpu: "2"
+    memory: 1Gi

k8s/apps/homarr/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: dashboard
+# Probes are now tuned via homarr-values.yaml (chart-native); the old
+# fix-probes-job PostSync hook is removed. homarr-secrets/auth-oidc/db-encryption
+# Secrets are delivered by the sops-secrets (ksops) Application.
+resources: []

k8s/apps/immich/configmap.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: immich-config
+data:
+  DB_HOSTNAME: "immich-db-rw"
+  DB_DATABASE_NAME: "immich"
+  # Only pgvector is installed (see db.yaml) - no vectorchord extension image
+  # exists for pg18 in CNPG's catalog yet. Explicit instead of relying on
+  # auto-detect's vectorchord-first preference order.
+  DB_VECTOR_EXTENSION: "pgvector"
+  REDIS_HOSTNAME: "immich-redis"
+  IMMICH_MACHINE_LEARNING_URL: "http://immich-machine-learning:3003"
+  TZ: "America/Los_Angeles"

k8s/apps/immich/db.yaml

@@ -0,0 +1,58 @@
+# Dedicated CNPG Postgres for Immich. Same recipe as paperless-db/authentik-db
+# (2 instances, default longhorn storage class) except the operand is
+# PostgreSQL 18, not 16.2 - the official CNPG pgvector extension image
+# (ghcr.io/cloudnative-pg/pgvector) is only published for pg18, no pg16 tags
+# exist in that registry. Immich itself supports pg18 fine (immich-app's own
+# postgres image already ships 18-vectorchord builds).
+#
+# pgvector loaded via CNPG's ImageVolume extension mechanism (CNPG 1.27+,
+# k8s ImageVolume feature - both present here: operator is 1.30.0, cluster is
+# v1.36.1). No shared_preload_libraries needed - pgvector doesn't require
+# preload, just CREATE EXTENSION, which immich-server issues itself at
+# startup. Distro/pg-major must match between the operand image and the
+# extension image (both "18"+"trixie" here) - CNPG's own compatibility rule.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: immich-db
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+spec:
+  instances: 2
+  imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie
+  postgresql:
+    extensions:
+      - name: pgvector
+        image:
+          reference: ghcr.io/cloudnative-pg/pgvector:0.8.1-18-trixie
+  bootstrap:
+    initdb:
+      database: immich
+      owner: app
+      encoding: UTF8
+      localeCollate: C
+      localeCType: C
+      # CREATE EXTENSION vector requires superuser (pgvector's control file
+      # isn't marked trusted) and the "app" owner role isn't one
+      # (enableSuperuserAccess: false, repo convention) - postInitApplicationSQL
+      # runs as superuser during initdb, before the app ever connects. Only
+      # fires on a fresh bootstrap; the live cluster already had this run
+      # manually once (kubectl exec ... psql -U postgres -c 'CREATE EXTENSION').
+      postInitApplicationSQL:
+        - "CREATE EXTENSION IF NOT EXISTS vector;"
+        - "CREATE EXTENSION IF NOT EXISTS cube;"
+        - "CREATE EXTENSION IF NOT EXISTS earthdistance;"
+  enableSuperuserAccess: false
+  resources:
+    requests: { memory: "512Mi", cpu: "250m" }
+    limits: { memory: "2Gi", cpu: "1" }
+  storage:
+    size: 20Gi
+    storageClass: longhorn
+  affinity:
+    podAntiAffinityType: preferred
+    topologyKey: kubernetes.io/hostname
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule

k8s/apps/immich/deployment.yaml

@@ -0,0 +1,100 @@
+# immich-server: pinned to talos-cp-3, same reasoning as paperless
+# (deployment.yaml comment there) - immich-media is a ReadWriteOnce Longhorn
+# volume with a single replica physically on that node's disk (shared with
+# paperless-media on the same 4TB HDD). Recreate strategy for the same
+# reason: two pods can't both attach an RWO volume.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-server
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: immich-server
+  template:
+    metadata:
+      labels:
+        app: immich-server
+    spec:
+      serviceAccountName: immich
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      containers:
+        - name: immich-server
+          image: ghcr.io/immich-app/immich-server:release
+          ports:
+            - containerPort: 2283
+          envFrom:
+            - configMapRef:
+                name: immich-config
+          env:
+            - name: DB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: immich-db-app
+                  key: username
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: immich-db-app
+                  key: password
+            # Composed by k8s/infra/iam's provisioning script (system-config
+            # JSON, oauth section) - see immich-oidc Secret.
+            - name: IMMICH_CONFIG_FILE
+              value: /config/immich.json
+          resources:
+            requests: { cpu: "500m", memory: "1Gi" }
+            limits: { cpu: "2", memory: "4Gi" }
+          volumeMounts:
+            - name: media
+              mountPath: /usr/src/app/upload
+            - name: oidc-config
+              mountPath: /config
+              readOnly: true
+      volumes:
+        - name: media
+          persistentVolumeClaim:
+            claimName: immich-media
+        - name: oidc-config
+          secret:
+            secretName: immich-oidc
+            items:
+              - key: config.json
+                path: immich.json
+---
+# CPU-only for now - the cluster's one GPU node (worker-1) is already
+# dedicated to llm-serving predictors. Not node-pinned: its cache PVC is on
+# the default 3-replica pool, not the single-disk cp-3 HDD.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-machine-learning
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich-machine-learning
+  template:
+    metadata:
+      labels:
+        app: immich-machine-learning
+    spec:
+      serviceAccountName: immich
+      containers:
+        - name: immich-machine-learning
+          image: ghcr.io/immich-app/immich-machine-learning:release
+          ports:
+            - containerPort: 3003
+          resources:
+            requests: { cpu: "500m", memory: "1Gi" }
+            limits: { cpu: "2", memory: "4Gi" }
+          volumeMounts:
+            - name: ml-cache
+              mountPath: /cache
+      volumes:
+        - name: ml-cache
+          persistentVolumeClaim:
+            claimName: immich-ml-cache

k8s/apps/immich/ingress.yaml

@@ -0,0 +1,24 @@
+# Direct nginx ingress, same reasoning as paperless: large uploads (photos/
+# videos) and long-lived operations (video transcode, big batch uploads) need
+# proxy-body-size/timeouts raised past nginx's defaults.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: immich
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: img.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: immich-server
+                port:
+                  number: 2283

k8s/apps/immich/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: immich
+resources:
+  - db.yaml
+  - pvc.yaml
+  - configmap.yaml
+  - redis.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+  - rbac.yaml
+# immich-oidc Secret written by the PostSync provisioning Job in
+# k8s/infra/iam (same as paperless-oidc) - not duplicated here.

k8s/apps/immich/pvc.yaml

@@ -0,0 +1,38 @@
+# Two volumes:
+#
+# - media: original photos/videos + generated thumbnails/encoded videos.
+#   Shares the cp-3 USB HDD with paperless-media, same StorageClass/disk tag,
+#   single replica (single disk, no redundancy possible - same tradeoff
+#   paperless already accepts). Sized 1400Gi, not 2000Gi: the disk's real
+#   usable capacity (~3724GiB, formatting overhead) minus paperless-media's
+#   2000Gi and ~231GiB of other apps' default-class replicas that Longhorn
+#   placed here anyway (disk tags only pull matching volumes in, they don't
+#   exclude non-matching ones when the untagged pool elsewhere is full) only
+#   leaves ~1493Gi of real scheduling headroom right now.
+# - ml-cache: downloaded ML model weights for immich-machine-learning
+#   (face detection / CLIP embeddings). Small, disposable (re-downloads on
+#   loss), but persisted so a pod restart doesn't re-pull multi-GB models -
+#   default 3-replica pool, not node-pinned.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-media
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-paperless-media
+  resources:
+    requests:
+      storage: 1400Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-ml-cache
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 5Gi

k8s/apps/immich/rbac.yaml

@@ -0,0 +1,43 @@
+# Scoped operator access for immich-admins: restart/config-edit rights on
+# just this service's own resources, nothing CNPG-managed (immich-db-*) or
+# provisioning-managed (immich-oidc). Same pattern as
+# k8s/apps/paperless/rbac.yaml. Inert until kube-apiserver's OIDC wiring
+# lands (--oidc-groups-claim=groups, --oidc-groups-prefix=oidc:).
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: immich
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: immich-operator
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    resourceNames: ["immich-server", "immich-machine-learning"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["immich-config"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["immich-oidc"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: immich-admins-binding
+subjects:
+  - kind: Group
+    name: "oidc:immich-admins"
+    apiGroup: rbac.authorization.k8s.io
+  - kind: ServiceAccount
+    name: immich
+    namespace: immich
+roleRef:
+  kind: Role
+  name: immich-operator
+  apiGroup: rbac.authorization.k8s.io

k8s/apps/immich/redis.yaml

@@ -0,0 +1,37 @@
+# Job queue broker for immich-server. No PVC: queue state is disposable - a
+# lost queue on restart just re-triggers the affected background jobs
+# (thumbnail generation, ML jobs, etc.), no photo data loss since originals
+# live on immich-media.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich-redis
+  template:
+    metadata:
+      labels:
+        app: immich-redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          ports:
+            - containerPort: 6379
+          resources:
+            requests: { cpu: "50m", memory: "64Mi" }
+            limits: { cpu: "250m", memory: "256Mi" }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-redis
+spec:
+  selector:
+    app: immich-redis
+  ports:
+    - port: 6379
+      targetPort: 6379

k8s/apps/immich/service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-server
+spec:
+  selector:
+    app: immich-server
+  ports:
+    - port: 2283
+      targetPort: 2283
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-machine-learning
+spec:
+  selector:
+    app: immich-machine-learning
+  ports:
+    - port: 3003
+      targetPort: 3003

k8s/apps/llm-serving/embeddings.yaml

@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-embeddings
+    app.kubernetes.io/part-of: llm-serving
+  name: embeddings
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=nomic-ai/nomic-embed-text-v2-moe
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/llm-serving/kustomization.yaml

@@ -0,0 +1,18 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist, matching k8s/apps/api. Anything added to this directory
+# and not listed here is silently dropped — no error, no drift shown.
+#
+# These five were adopted from live state on 2026-08-15; they had been applied
+# by hand and carried no ArgoCD ownership. Each was exported and verified with
+# `kubectl diff -f <file>` returning empty before the Application below was
+# created, so the first sync was a no-op rather than a redeploy. Re-verify that
+# way after any edit here: a GPU predictor restart is a weights reload measured
+# in tens of seconds, not a rolling update.
+resources:
+  - embeddings.yaml
+  - ornith.yaml
+  - reasoning.yaml
+  - reranker.yaml
+# No namespace transformer: every file sets its own, and the transformer would
+# rewrite metadata.namespace on anything cross-namespace added later.

k8s/apps/llm-serving/ornith.yaml

@@ -0,0 +1,108 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    # The konghq.com/{connect,read,write}-timeout annotations that used to live
+    # here went with Kong (retired 2026-08-19). They existed because Kong read
+    # its upstream timeouts off the Kubernetes Service, and its 60s default cut
+    # off the first request after any pod restart — a restart flushes VRAM and
+    # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
+    # problem in steady state.
+    #
+    # The equivalent budget now belongs to the Go gateway's per-route timeout
+    # config in rock/homelab-frontend, not to an annotation on this object.
+  labels:
+    app.kubernetes.io/name: llm-ornith
+    app.kubernetes.io/part-of: llm-serving
+  name: ornith
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - command:
+      - /bin/sh
+      - -c
+      - 'set -e
+
+        ollama serve &
+
+        SERVE_PID=$!
+
+        until ollama list >/dev/null 2>&1; do sleep 2; done
+
+        ollama pull ornith:35b
+
+        ollama pull qwen2.5:3b-instruct
+
+        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
+
+        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+
+        wait $SERVE_PID
+
+        '
+      env:
+      - name: OLLAMA_HOST
+        value: 0.0.0.0:8080
+      - name: OLLAMA_MODELS
+        value: /mnt/models/ollama
+      - name: OLLAMA_CONTEXT_LENGTH
+        value: '32768'
+      - name: OLLAMA_KEEP_ALIVE
+        value: '-1'
+      - name: OLLAMA_NUM_PARALLEL
+        value: '1'
+      - name: OLLAMA_MAX_LOADED_MODELS
+        value: '2'
+      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
+        failureThreshold: 120
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    deploymentStrategy:
+      type: Recreate
+    # 2 replicas -- each its own GPU, each loading both ornith:35b and
+    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
+    # get an independent instance instead of contending on one, at the
+    # cost of judge/qwen traffic still sharing whichever replica an
+    # implementer call also lands on.
+    maxReplicas: 2
+    minReplicas: 2
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/llm-serving/reasoning.yaml

@@ -0,0 +1,122 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reasoning
+    app.kubernetes.io/part-of: llm-serving
+  name: reasoning
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then-
+      # matmul is two slow kernel launches instead of one fused int4 GEMM,
+      # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to
+      # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same
+      # hermes/qwen3 parser stack (no narration-bug risk, same as before),
+      # only the quant format changes. Plain (non-Marlin) GPTQ kernel is
+      # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try
+      # to auto-upgrade to it, so --quantization is pinned explicitly to
+      # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB
+      # (summed from the real safetensors index, not bits-per-param math).
+      # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN:
+      # factor=4.0, original_max_position_embeddings=32768) -- 200k was
+      # asked for but exceeds this architecturally regardless of VRAM.
+      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
+      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
+      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
+      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
+      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
+      # above the 131072 target with room to spare -- the old
+      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
+      # model's much smaller 16384 context) is no longer needed and is
+      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
+      # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no
+      # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth
+      # trying later, but not risking a parallelism-strategy change in the
+      # same rollout as the quant+context-length change.
+      # This GPTQ requant's own config.json ships max_position_embeddings=
+      # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw
+      # config.json), the base Qwen3-32B repo's YaRN block did NOT carry
+      # over during quantization. Re-applying it explicitly here restores
+      # the same math the base model documents (32768 * 4.0 = 131072);
+      # without this, --max-model-len=131072 fails ModelConfig validation
+      # against the checkpoint's own (unscaled) 40960 ceiling.
+      - --model=JunHowie/Qwen3-32B-GPTQ-Int4
+      - --served-model-name=reasoning
+      - --quantization=gptq
+      - --dtype=float16
+      - --kv-cache-dtype=auto
+      - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
+      - --tensor-parallel-size=1
+      - --pipeline-parallel-size=2
+      - --max-model-len=131072
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
+      - --enable-chunked-prefill
+      - --enable-prefix-caching
+      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
+      # blocks.
+      - --reasoning-parser=qwen3
+      # hermes is the documented tool-call parser for general (non-Coder)
+      # Qwen3 models -- native chat template support, not narrated text.
+      - --enable-auto-tool-choice
+      - --tool-call-parser=hermes
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: '0'
+      - name: VLLM_ATTENTION_BACKEND
+        value: TRITON_ATTN
+      - name: HF_HOME
+        value: /mnt/models
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 36Gi
+          nvidia.com/gpu: '2'
+        requests:
+          cpu: '8'
+          memory: 12Gi
+          nvidia.com/gpu: '2'
+      startupProbe:
+        failureThreshold: 80
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        sizeLimit: 2Gi
+      name: shm
+

k8s/apps/llm-serving/reranker.yaml

@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reranker
+    app.kubernetes.io/part-of: llm-serving
+  name: reranker
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=BAAI/bge-reranker-base
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/messaging/kafka-cluster/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: kafka-cluster
+description: Strimzi Kafka/KafkaNodePool CRs for the kmsvc Kafka cluster (design.md §7)
+type: application
+version: 0.1.0

k8s/apps/messaging/kafka-cluster/templates/kafka-cluster.yaml

@@ -0,0 +1,30 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: Kafka
+metadata:
+  name: {{ .Values.clusterName }}
+  namespace: {{ .Values.namespace }}
+  annotations:
+    strimzi.io/node-pools: enabled
+    strimzi.io/kraft: enabled
+spec:
+  kafka:
+    version: 4.0.0
+    metadataVersion: 4.0-IV3
+    listeners:
+      - name: plain
+        port: 9092
+        type: internal
+        tls: false
+      - name: tls
+        port: 9093
+        type: internal
+        tls: true
+    config:
+      default.replication.factor: {{ .Values.kafka.replicationFactor }}
+      min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }}
+      offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }}
+      transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }}
+      transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }}
+  entityOperator:
+    topicOperator: {}
+    userOperator: {}

k8s/apps/messaging/kafka-cluster/templates/kafka-nodepool.yaml

@@ -0,0 +1,39 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaNodePool
+metadata:
+  name: {{ .Values.clusterName }}-pool
+  namespace: {{ .Values.namespace }}
+  labels:
+    strimzi.io/cluster: {{ .Values.clusterName }}
+spec:
+  replicas: {{ .Values.nodePool.replicas }}
+  roles:
+    - controller
+    - broker
+  storage:
+    type: persistent-claim
+    size: {{ .Values.nodePool.storage.sizeGi }}Gi
+    class: {{ .Values.nodePool.storage.class }}
+    deleteClaim: false
+  resources:
+    limits:
+      memory: {{ .Values.nodePool.resources.memory }}
+      cpu: {{ .Values.nodePool.resources.cpu | quote }}
+    requests:
+      memory: {{ .Values.nodePool.resources.memory }}
+      cpu: {{ .Values.nodePool.resources.cpu | quote }}
+  template:
+    pod:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }}
+                labelSelector:
+                  matchLabels:
+                    strimzi.io/cluster: {{ .Values.clusterName }}
+    kafkaContainer:
+      env:
+        - name: KAFKA_HEAP_OPTS
+          value: {{ .Values.nodePool.heapOpts | quote }}

k8s/apps/messaging/kafka-cluster/values.yaml

@@ -0,0 +1,26 @@
+clusterName: kmsvc
+namespace: sqs
+
+nodePool:
+  replicas: 3
+  storage:
+    class: longhorn
+    # Longhorn's per-node scheduling budget on the current 2-node cluster has
+    # only ~36Gi of headroom left (other PVCs already reserve the rest), and
+    # each node hosts one replica of all 3 broker volumes -- so 3 * sizeGi
+    # must fit in that headroom. Revisit once the 3rd node joins.
+    sizeGi: 10
+  resources:
+    memory: 5Gi
+    cpu: "2"
+  heapOpts: "-Xms2g -Xmx2g"
+  # design.md §7: 3 real zones now exist (talos-cp-1=az-a, talos-worker-1=az-b,
+  # talos-worker-2=az-c), so anti-affinity keys off zone instead of hostname —
+  # spreads the 3 broker pods one-per-zone/one-per-node (equivalent today,
+  # but zone is the correct long-term key if a node ever gets replaced within
+  # the same zone).
+  antiAffinityTopologyKey: topology.kubernetes.io/zone
+
+kafka:
+  replicationFactor: 3
+  minInsyncReplicas: 2

k8s/apps/messaging/management-service/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: management-service
+description: kmsvc message-plane gRPC+REST server (design.md §1, §7a, §9)
+type: application
+version: 0.1.0

k8s/apps/messaging/management-service/templates/configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: management-service-config
+  namespace: {{ .Values.namespace }}
+data:
+  KMSVC_KAFKA_BROKERS: {{ .Values.env.kafkaBrokers | quote }}
+  KMSVC_REDIS_ADDR: {{ .Values.env.redisAddr | quote }}
+  KMSVC_AUTHENTIK_ISSUER_URL: {{ .Values.env.authentikIssuerURL | quote }}
+  KMSVC_AUTHENTIK_AUDIENCE: {{ .Values.env.authentikAudience | quote }}
+  KMSVC_GRPC_LISTEN_ADDR: ":{{ .Values.grpcPort }}"
+  KMSVC_HTTP_LISTEN_ADDR: ":{{ .Values.httpPort }}"

k8s/apps/messaging/management-service/templates/deployment.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: management-service
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: management-service
+  template:
+    metadata:
+      labels:
+        app: management-service
+    spec:
+      serviceAccountName: kmsvc
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: management-service
+      containers:
+        - name: management-service
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: grpc
+              containerPort: {{ .Values.grpcPort }}
+            - name: http
+              containerPort: {{ .Values.httpPort }}
+          env:
+            - name: GOMEMLIMIT
+              value: {{ .Values.goMemLimit | quote }}
+          envFrom:
+            - configMapRef:
+                name: management-service-config
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ .Values.httpPort }}
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.httpPort }}
+            initialDelaySeconds: 10
+            periodSeconds: 20

k8s/apps/messaging/management-service/templates/hpa.yaml

@@ -0,0 +1,27 @@
+{{- if .Values.hpa.enabled }}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: management-service
+  namespace: {{ .Values.namespace }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: management-service
+  minReplicas: {{ .Values.hpa.minReplicas }}
+  maxReplicas: {{ .Values.hpa.maxReplicas }}
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.hpa.targetMemoryUtilizationPercentage }}
+{{- end }}

k8s/apps/messaging/management-service/templates/ingress-grpc.yaml

@@ -0,0 +1,27 @@
+{{- if and .Values.ingress.enabled .Values.ingress.grpcEnabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: management-service-grpc
+  namespace: {{ .Values.namespace }}
+  annotations:
+    cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
+    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.host }}
+      secretName: {{ .Values.ingress.tlsSecretName }}
+  rules:
+    - host: {{ .Values.ingress.host }}
+      http:
+        paths:
+          - path: {{ .Values.ingress.grpcPathPrefix }}
+            pathType: Prefix
+            backend:
+              service:
+                name: management-service
+                port:
+                  number: {{ .Values.grpcPort }}
+{{- end }}

k8s/apps/messaging/management-service/templates/ingress.yaml

@@ -0,0 +1,26 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: management-service
+  namespace: {{ .Values.namespace }}
+  annotations:
+    cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.host }}
+      secretName: {{ .Values.ingress.tlsSecretName }}
+  rules:
+    - host: {{ .Values.ingress.host }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: management-service
+                port:
+                  number: {{ .Values.httpPort }}
+{{- end }}

k8s/apps/messaging/management-service/templates/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: management-service
+  namespace: {{ .Values.namespace }}
+spec:
+  selector:
+    app: management-service
+  ports:
+    - name: grpc
+      port: {{ .Values.grpcPort }}
+      targetPort: {{ .Values.grpcPort }}
+    - name: http
+      port: {{ .Values.httpPort }}
+      targetPort: {{ .Values.httpPort }}
+  type: ClusterIP

k8s/apps/messaging/management-service/values.yaml

@@ -0,0 +1,50 @@
+namespace: sqs
+replicaCount: 3
+
+image:
+  repository: forgejo.riotpiao.com/rock/kmsvc-manage
+  tag: latest
+  pullPolicy: Always
+
+grpcPort: 9090
+httpPort: 8080
+
+env:
+  kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
+  redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
+  authentikIssuerURL: ""
+  authentikAudience: ""
+
+resources:
+  requests:
+    cpu: 200m
+    memory: 256Mi
+  limits:
+    cpu: "1"
+    memory: 512Mi
+
+# Go's GC only reacts to GOGC by default and has no idea about the cgroup
+# memory limit above -- it'll happily grow heap until the kernel OOMKills it.
+# Setting GOMEMLIMIT to ~90% of the container limit makes the GC self-throttle
+# before that happens. Keep this in sync with resources.limits.memory.
+goMemLimit: "460MiB"
+
+hpa:
+  enabled: true
+  minReplicas: 3
+  maxReplicas: 9
+  targetCPUUtilizationPercentage: 70
+  targetMemoryUtilizationPercentage: 80
+
+ingress:
+  enabled: true
+  className: nginx
+  clusterIssuer: homelab-ca
+  host: kmsvc.riotpiao.com
+  tlsSecretName: kmsvc-tls
+  # kmsvc-cli connects via gRPC directly to --server/KMSVC_SERVER (default
+  # kmsvc.riotpiao.com:443, see kmsvc-cli's README), so raw gRPC needs an
+  # external path too — scoped to the gRPC service's own path prefix on the
+  # same host/port, rather than opening the whole host to gRPC passthrough.
+  grpcEnabled: true
+  grpcPathPrefix: /kafkamgmt.v1.QueueService/

k8s/apps/messaging/memory-queues/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: memory-queues
+description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
+type: application
+version: 0.1.0
+appVersion: "1.0"

k8s/apps/messaging/memory-queues/templates/queues.yaml

@@ -0,0 +1,20 @@
+{{- range .Values.queues }}
+---
+apiVersion: kmsvc.io/v1alpha1
+kind: Queue
+metadata:
+  name: {{ .name }}
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: memory-service
+    queue: dlq
+spec:
+  name: {{ .name }}
+  description: {{ .description }}
+  partitions: {{ .partitions }}
+  replicationFactor: {{ .replicationFactor }}
+  config:
+    retention.ms: "{{ .config.retention.ms }}"
+    message.retention.seconds: "{{ .config.message.retention.seconds }}"
+    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
+{{- end }}

k8s/apps/messaging/memory-queues/values.yaml

@@ -0,0 +1,25 @@
+# Poimen Memory Service Kafka Queues (kmsvc)
+# Phase 6.6: DLQ topics for webhook + metrics failures
+
+queues:
+  # DLQ for extraction, webhook, and agent failures
+  - name: poimen-memory-dlq
+    description: "DLQ for extraction, webhook, and agent failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+  # DLQ for metrics persistence failures  
+  - name: poimen-memory-metric-dlq
+    description: "DLQ for metrics persistence failures"
+    partitions: 3
+    replicationFactor: 1
+    config:
+      retention.ms: "1209600000"  # 14 days
+      message.retention.seconds: "1209600"
+      visibility.timeout.seconds: "300"
+
+namespace: sqs

k8s/apps/messaging/queue-crd/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: queue-crd
+description: Queue CRD definition + queue-operator Deployment/RBAC (design.md §2a)
+type: application
+version: 0.1.0

k8s/apps/messaging/queue-crd/templates/crd.yaml

@@ -0,0 +1,274 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.21.0
+  name: queues.kmsvc.io
+spec:
+  group: kmsvc.io
+  names:
+    kind: Queue
+    listKind: QueueList
+    plural: queues
+    shortNames:
+    - queue
+    - queues
+    singular: queue
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .spec.fifoQueue
+      name: FIFO
+      type: boolean
+    - jsonPath: .status.phase
+      name: Phase
+      type: string
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: Queue is the Schema for the queues API — see design.md §2a.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: QueueSpec defines the desired state of a Queue (design.md
+              §2a).
+            properties:
+              deadLetterTargetQueue:
+                description: |-
+                  DeadLetterTargetQueue is the name of another Queue to route exhausted
+                  messages to. Must not point at itself or at another DLQ (design.md §5).
+                type: string
+              delaySeconds:
+                description: DelaySeconds is the default delivery delay applied to
+                  sent messages.
+                format: int32
+                maximum: 900
+                minimum: 0
+                type: integer
+              fifoQueue:
+                default: false
+                description: FIFOQueue enables per-MessageGroupId ordering and deduplication
+                  semantics.
+                type: boolean
+              isDLQ:
+                description: |-
+                  IsDLQ marks this queue as itself a dead-letter queue, used to enforce
+                  the no-DLQ-chaining validation rule in design.md §5.
+                type: boolean
+              maxReceiveCount:
+                default: 5
+                description: |-
+                  MaxReceiveCount is how many times a message may be redelivered before
+                  being routed to DeadLetterTargetQueue.
+                format: int32
+                minimum: 1
+                type: integer
+              maxShards:
+                default: 8
+                description: MaxShards is the ceiling on shard count the operator
+                  may split up to (design.md §2c).
+                format: int32
+                minimum: 1
+                type: integer
+              messageRetentionPeriodSeconds:
+                default: 345600
+                description: MessageRetentionPeriodSeconds maps to the underlying
+                  Kafka topic's retention.ms.
+                format: int32
+                maximum: 1209600
+                minimum: 60
+                type: integer
+              minShards:
+                default: 1
+                description: MinShards is the floor on shard count; the operator never
+                  merges below this.
+                format: int32
+                minimum: 1
+                type: integer
+              partitionsPerShard:
+                default: 6
+                description: PartitionsPerShard is the Kafka partition count on each
+                  shard's topic.
+                format: int32
+                minimum: 1
+                type: integer
+              shardSplitCooldownSeconds:
+                default: 300
+                description: |-
+                  ShardSplitCooldownSeconds is the minimum age a shard must reach before it
+                  is eligible to be split again, preventing rapid re-splitting of a child
+                  that hasn't yet absorbed its share of traffic.
+                format: int32
+                minimum: 0
+                type: integer
+              shardSplitThresholdBytesPerSec:
+                default: 5242880
+                description: |-
+                  ShardSplitThresholdBytesPerSec is the sustained per-shard throughput that
+                  triggers a split into two child shards (design.md §2c).
+                format: int64
+                minimum: 1
+                type: integer
+              visibilityTimeoutSeconds:
+                default: 30
+                description: |-
+                  VisibilityTimeoutSeconds is how long a received-but-unacked message stays
+                  invisible to other consumers before being redelivered.
+                format: int32
+                maximum: 43200
+                minimum: 0
+                type: integer
+            type: object
+          status:
+            description: QueueStatus defines the observed state of a Queue.
+            properties:
+              conditions:
+                description: Conditions hold detailed status information.
+                items:
+                  description: Condition contains details for one aspect of the current
+                    state of this API Resource.
+                  properties:
+                    lastTransitionTime:
+                      description: |-
+                        lastTransitionTime is the last time the condition transitioned from one status to another.
+                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                      format: date-time
+                      type: string
+                    message:
+                      description: |-
+                        message is a human readable message indicating details about the transition.
+                        This may be an empty string.
+                      maxLength: 32768
+                      type: string
+                    observedGeneration:
+                      description: |-
+                        observedGeneration represents the .metadata.generation that the condition was set based upon.
+                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                        with respect to the current state of the instance.
+                      format: int64
+                      minimum: 0
+                      type: integer
+                    reason:
+                      description: |-
+                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                        Producers of specific condition types may define expected values and meanings for this field,
+                        and whether the values are considered a guaranteed API.
+                        The value should be a CamelCase string.
+                        This field may not be empty.
+                      maxLength: 1024
+                      minLength: 1
+                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                      type: string
+                    status:
+                      description: status of the condition, one of True, False, Unknown.
+                      enum:
+                      - "True"
+                      - "False"
+                      - Unknown
+                      type: string
+                    type:
+                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                      maxLength: 316
+                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                      type: string
+                  required:
+                  - lastTransitionTime
+                  - message
+                  - reason
+                  - status
+                  - type
+                  type: object
+                type: array
+              phase:
+                description: Phase is the current reconciliation phase.
+                enum:
+                - Pending
+                - Ready
+                - Failed
+                type: string
+              shards:
+                description: |-
+                  Shards lists every shard backing this queue, active or draining
+                  (design.md §2a/§2c).
+                items:
+                  description: ShardStatus describes one shard backing a Queue (design.md
+                    §2a/§2c).
+                  properties:
+                    availabilityZones:
+                      description: |-
+                        AvailabilityZones lists the topology.kubernetes.io/zone values of every
+                        node currently hosting a Kafka replica of this shard's topic, resolved
+                        from the broker pods' node placement each reconcile. Empty until the
+                        first successful resolution (e.g. node lookup failed transiently).
+                      items:
+                        type: string
+                      type: array
+                    createdAt:
+                      description: |-
+                        CreatedAt timestamps when this shard was created, used to enforce
+                        ShardSplitCooldownSeconds.
+                      format: date-time
+                      type: string
+                    hashRangeEnd:
+                      format: int64
+                      type: integer
+                    hashRangeStart:
+                      description: |-
+                        HashRangeStart/HashRangeEnd define the [start, end) murmur2 hash range
+                        this shard owns over the 32-bit key space. Stored as int64 (not uint32)
+                        because controller-gen maps Go uint32 to OpenAPI format:int32, whose max
+                        (2147483647) is smaller than FullHashRangeEnd (0xFFFFFFFF) and the
+                        apiserver rejects the status update.
+                      format: int64
+                      type: integer
+                    id:
+                      description: ID is the shard's identifier, used in its topic
+                        name (kmsvc.{queue}.shard-{id}).
+                      type: string
+                    parentId:
+                      description: |-
+                        ParentID is the shard ID this shard was split from, empty for the
+                        original shard-0.
+                      type: string
+                    phase:
+                      description: Phase is this shard's lifecycle state.
+                      enum:
+                      - Active
+                      - Closing
+                      - Closed
+                      type: string
+                    topic:
+                      description: Topic is the underlying Kafka topic name for this
+                        shard.
+                      type: string
+                  required:
+                  - hashRangeEnd
+                  - hashRangeStart
+                  - id
+                  - phase
+                  - topic
+                  type: object
+                type: array
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

k8s/apps/messaging/queue-crd/templates/operator-deployment.yaml

@@ -0,0 +1,38 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: queue-operator
+  namespace: {{ .Values.namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: queue-operator
+  template:
+    metadata:
+      labels:
+        app: queue-operator
+    spec:
+      serviceAccountName: queue-operator
+      containers:
+        - name: queue-operator
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["/queue-operator"]
+          env:
+            - name: KMSVC_KAFKA_BROKERS
+              value: {{ .Values.kafkaBrokers | quote }}
+            - name: KMSVC_REDIS_ADDR
+              value: {{ .Values.redisAddr | quote }}
+            - name: GOMEMLIMIT
+              value: {{ .Values.goMemLimit | quote }}
+            - name: KMSVC_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: KMSVC_KAFKA_CLUSTER_NAME
+              value: {{ .Values.kafkaClusterName | quote }}
+            - name: KMSVC_KAFKA_POOL_NAME
+              value: {{ .Values.kafkaPoolName | quote }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}

k8s/apps/messaging/queue-crd/templates/rbac.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: queue-operator
+  namespace: {{ .Values.namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: queue-operator
+rules:
+  - apiGroups: ["kmsvc.io"]
+    resources: ["queues"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["queues/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["queues/finalizers"]
+    verbs: ["update"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods", "nodes"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: queue-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: queue-operator
+subjects:
+  - kind: ServiceAccount
+    name: queue-operator
+    namespace: {{ .Values.namespace }}

k8s/apps/messaging/queue-crd/values.yaml

@@ -0,0 +1,26 @@
+namespace: sqs
+
+image:
+  repository: forgejo.riotpiao.com/rock/kmsvc-manage
+  tag: latest
+  pullPolicy: Always
+
+kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
+redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
+
+# Must match kafka-cluster chart's clusterName/derived pool name -- used to
+# resolve "<kafkaClusterName>-<kafkaPoolName>-<brokerID>" broker pod names
+# for AZ-aware Queue status (design.md §2a).
+kafkaClusterName: kmsvc
+kafkaPoolName: kmsvc-pool
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 256Mi
+
+# See management-service/values.yaml's goMemLimit comment -- same reasoning.
+goMemLimit: "230MiB"

k8s/apps/paperless/adapter-configmap.yaml

@@ -0,0 +1,95 @@
+# Overrides paperless-ngx's own paperless/adapter.py at the same import path
+# (mounted via subPath in deployment.yaml) - settings.py hardcodes
+# SOCIALACCOUNT_ADAPTER = "paperless.adapter.CustomSocialAccountAdapter", so
+# no Django setting needs to change, just the file content underneath it.
+#
+# Stock CustomSocialAccountAdapter.populate_user() is a stub ("kept in case
+# global default permissions are implemented in the future" - they aren't),
+# so every OIDC signup lands with zero permissions and 403s on every API
+# endpoint. This adds the actual mapping: Authentik's "permissions" claim
+# (via the permissions scope, requested in PAPERLESS_SOCIALACCOUNT_PROVIDERS,
+# computed server-side from group membership by authentik-provision.py) ->
+# "paperless:write" or "*" (homelab-admins) grants is_staff+is_superuser,
+# same convention already used for MinIO's policy claim and Grafana's
+# role_attribute_path. Checking the permission string rather than a literal
+# group name decouples "what grants access" from which group happens to
+# hold it - same pattern applies to every other service's Role/RoleBinding
+# in k8s/infra/rbac/.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: paperless-adapter
+data:
+  adapter.py: |
+    from urllib.parse import quote
+
+    from allauth.account.adapter import DefaultAccountAdapter
+    from allauth.core import context
+    from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
+    from django.conf import settings
+    from django.forms import ValidationError
+    from django.urls import reverse
+
+    REQUIRED_PERMISSIONS = {"paperless:write", "*"}
+
+
+    class CustomAccountAdapter(DefaultAccountAdapter):
+        def is_open_for_signup(self, request):
+            allow_signups = super().is_open_for_signup(request)
+            return getattr(settings, "ACCOUNT_ALLOW_SIGNUPS", allow_signups)
+
+        def pre_authenticate(self, request, **credentials):
+            if settings.DISABLE_REGULAR_LOGIN:
+                raise ValidationError("Regular login is disabled")
+            return super().pre_authenticate(request, **credentials)
+
+        def is_safe_url(self, url):
+            from django.utils.http import url_has_allowed_host_and_scheme
+
+            allowed_hosts = {context.request.get_host()} | set(settings.ALLOWED_HOSTS)
+            if "*" in allowed_hosts:
+                allowed_hosts.remove("*")
+                allowed_hosts.add(context.request.get_host())
+                return url_has_allowed_host_and_scheme(url, allowed_hosts=allowed_hosts)
+            return url_has_allowed_host_and_scheme(url, allowed_hosts=allowed_hosts)
+
+        def get_reset_password_from_key_url(self, key):
+            if settings.PAPERLESS_URL is None:
+                return super().get_reset_password_from_key_url(key)
+            path = reverse(
+                "account_reset_password_from_key",
+                kwargs={"uidb36": "UID", "key": "KEY"},
+            )
+            path = path.replace("UID-KEY", quote(key))
+            return settings.PAPERLESS_URL + path
+
+
+    class CustomSocialAccountAdapter(DefaultSocialAccountAdapter):
+        def is_open_for_signup(self, request, sociallogin):
+            allow_signups = super().is_open_for_signup(request, sociallogin)
+            return getattr(settings, "SOCIALACCOUNT_ALLOW_SIGNUPS", allow_signups)
+
+        def get_connect_redirect_url(self, request, socialaccount):
+            return reverse("base")
+
+        def populate_user(self, request, sociallogin, data):
+            user = super().populate_user(request, sociallogin, data)
+            perms = set(sociallogin.account.extra_data.get("permissions") or [])
+            if perms & REQUIRED_PERMISSIONS:
+                user.is_staff = True
+                user.is_superuser = True
+            return user
+
+        def save_user(self, request, sociallogin, form=None):
+            # populate_user() sets the flags on the in-memory user, but
+            # allauth's default save_user() re-derives is_staff from
+            # ACCOUNT_DEFAULT_HTTP_PROTOCOL-independent defaults and can
+            # overwrite them on save - re-apply after super().save_user()
+            # persists the row, matching the permissions check above exactly.
+            user = super().save_user(request, sociallogin, form)
+            perms = set(sociallogin.account.extra_data.get("permissions") or [])
+            if perms & REQUIRED_PERMISSIONS and not (user.is_staff and user.is_superuser):
+                user.is_staff = True
+                user.is_superuser = True
+                user.save(update_fields=["is_staff", "is_superuser"])
+            return user

k8s/apps/paperless/backup-cronjob.yaml

@@ -0,0 +1,94 @@
+# Nightly: pg_dump the paperless DB + mirror the media PVC into the scoped
+# `paperless` MinIO bucket (see minio-provision-paperless-job.yaml). This is a
+# BACKUP target, not live storage - paperless-ngx has no native S3 backend, it
+# only ever reads/writes the local media PVC directly.
+#
+# Pinned to talos-cp-3, same as deployment.yaml: media is a ReadWriteOnce
+# Longhorn volume with a single replica physically on that node's disk -
+# mounting it read-only here from a different node would conflict with the
+# live webserver's attachment.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: paperless-backup
+spec:
+  schedule: "0 3 * * *"  # 03:00 daily, low-traffic window
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: Never
+          nodeSelector:
+            kubernetes.io/hostname: talos-cp-3
+          initContainers:
+            - name: pg-dump
+              image: postgres:16-alpine
+              env:
+                - name: PGHOST
+                  value: paperless-db-rw
+                - name: PGDATABASE
+                  value: paperless
+                - name: PGUSER
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-db-app
+                      key: username
+                - name: PGPASSWORD
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-db-app
+                      key: password
+              command:
+                - sh
+                - -c
+                - pg_dump --format=custom --file=/backup/paperless-db.dump
+              volumeMounts:
+                - name: backup
+                  mountPath: /backup
+          containers:
+            - name: mc-mirror
+              image: minio/mc:latest
+              env:
+                - name: ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-minio-creds
+                      key: ACCESS_KEY
+                - name: SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-minio-creds
+                      key: SECRET_KEY
+                - name: BUCKET
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-minio-creds
+                      key: BUCKET
+                - name: ENDPOINT
+                  valueFrom:
+                    secretKeyRef:
+                      name: paperless-minio-creds
+                      key: ENDPOINT
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -e
+                  mc alias set b "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY"
+                  mc cp /backup/paperless-db.dump "b/$BUCKET/db/paperless-db-$(date +%Y%m%d).dump"
+                  mc mirror --overwrite /media "b/$BUCKET/media"
+                  echo "Backup done."
+              volumeMounts:
+                - name: backup
+                  mountPath: /backup
+                - name: media
+                  mountPath: /media
+                  readOnly: true
+          volumes:
+            - name: backup
+              emptyDir: {}
+            - name: media
+              persistentVolumeClaim:
+                claimName: paperless-media
+                readOnly: true

k8s/apps/paperless/configmap.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: paperless-config
+data:
+  PAPERLESS_URL: "https://paperless.riotpiao.com"
+  PAPERLESS_TIME_ZONE: "America/Los_Angeles"
+  PAPERLESS_OCR_LANGUAGE: "eng"
+  PAPERLESS_DBHOST: "paperless-db-rw"
+  PAPERLESS_DBNAME: "paperless"
+  PAPERLESS_REDIS: "redis://paperless-redis:6379"
+  # django-allauth generic OIDC provider. The client_id/secret/server_url
+  # bundle itself lives in the paperless-oidc Secret
+  # (SOCIALACCOUNT_PROVIDERS_JSON key, composed by authentik-provision.py) -
+  # env vars can't be split across a ConfigMap + Secret for the same key, so
+  # this whole value is sourced from the Secret in deployment.yaml instead.
+  PAPERLESS_APPS: "allauth.socialaccount.providers.openid_connect"
+  # Authentik already verifies identity via OIDC - a second email-confirmation
+  # step has no SMTP configured to send it anyway, and paperless-ngx doesn't
+  # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
+  # on 'account_confirm_email' without this.
+  PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"

k8s/apps/paperless/deployment.yaml

@@ -0,0 +1,103 @@
+# Single container runs webserver + consumer + scheduler (paperless-ngx's
+# stock entrypoint does this internally) - no need to split into separate
+# Deployments. replicas: 1 only: paperless-media is ReadWriteOnce, and the
+# consumer polling the media dir doesn't benefit from horizontal scaling here.
+#
+# Pinned to talos-cp-3: paperless-media's disk physically lives there. Longhorn
+# RWO volumes can only be attached from one node at a time, and the nightly
+# backup-cronjob.yaml also mounts this same PVC (read-only) to mirror it into
+# MinIO - pinning both to the same node avoids a cross-node attach conflict,
+# and keeps the 3.5Ti read/write path off the network entirely.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperless
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate  # ReadWriteOnce media PVC - avoid two pods fighting over it
+  selector:
+    matchLabels:
+      app: paperless
+  template:
+    metadata:
+      labels:
+        app: paperless
+    spec:
+      # Kubernetes injects legacy Docker-links env vars for every Service in
+      # this namespace (<SVC>_SERVICE_HOST, <SVC>_PORT, ...). The Service here
+      # is named "paperless", so that becomes PAPERLESS_PORT=tcp://<ip>:8000 -
+      # paperless-ngx's own entrypoint reads PAPERLESS_PORT for gunicorn's
+      # bind address, collides, and gunicorn crash-loops on "not a valid port
+      # number". Disable the injection instead of renaming the Service.
+      enableServiceLinks: false
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      containers:
+        - name: paperless
+          image: ghcr.io/paperless-ngx/paperless-ngx:2.20.15
+          ports:
+            - containerPort: 8000
+          envFrom:
+            - configMapRef:
+                name: paperless-config
+          env:
+            - name: PAPERLESS_DBUSER
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-db-app
+                  key: username
+            - name: PAPERLESS_DBPASS
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-db-app
+                  key: password
+            - name: PAPERLESS_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-secrets
+                  key: PAPERLESS_SECRET_KEY
+            - name: PAPERLESS_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-secrets
+                  key: PAPERLESS_ADMIN_USER
+            - name: PAPERLESS_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-secrets
+                  key: PAPERLESS_ADMIN_PASSWORD
+            - name: PAPERLESS_SOCIALACCOUNT_PROVIDERS
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-oidc
+                  key: SOCIALACCOUNT_PROVIDERS_JSON
+          resources:
+            requests: { cpu: "500m", memory: "1Gi" }
+            limits: { cpu: "2", memory: "4Gi" }
+          volumeMounts:
+            - name: media
+              mountPath: /usr/src/paperless/media
+            - name: data
+              mountPath: /usr/src/paperless/data
+            - name: consume
+              mountPath: /usr/src/paperless/consume
+            # Overrides paperless-ngx's own adapter.py in place - settings.py
+            # hardcodes the import path, so no Django setting changes, just
+            # the file content underneath it (see adapter-configmap.yaml).
+            - name: adapter
+              mountPath: /usr/src/paperless/src/paperless/adapter.py
+              subPath: adapter.py
+              readOnly: true
+      volumes:
+        - name: media
+          persistentVolumeClaim:
+            claimName: paperless-media
+        - name: data
+          persistentVolumeClaim:
+            claimName: paperless-data
+        - name: consume
+          emptyDir: {}
+        - name: adapter
+          configMap:
+            name: paperless-adapter

k8s/apps/paperless/ingress.yaml

@@ -0,0 +1,24 @@
+# Direct nginx ingress to the paperless Service - not routed via the Go
+# api-gateway (api.riotpiao.com), which has no WebSocket upgrade support and
+# paperless-ngx keeps a long-lived /ws/ connection open for live task status.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: paperless
+  annotations:
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"  # large scanned PDF uploads
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: paperless.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: paperless
+                port:
+                  number: 8000

k8s/apps/paperless/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: paperless
+resources:
+  - pvc.yaml
+  - configmap.yaml
+  - redis.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
+  - backup-cronjob.yaml
+  - adapter-configmap.yaml
+  - rbac.yaml
+# postgres: paperless-db CNPG Cluster, deployed by k8s/infra/databases (wave 2,
+# before this app at wave 8) - not duplicated here. Same for the paperless-oidc
+# and paperless-minio-creds Secrets, written by PostSync provisioning Jobs in
+# k8s/infra/iam and k8s/infra/minio respectively.

k8s/apps/paperless/pvc.yaml

@@ -0,0 +1,35 @@
+# Two volumes, deliberately separate storage classes:
+#
+# - media: the actual documents (originals + OCR'd archive PDFs + thumbnails).
+#   Lives on the cp-3 USB HDD, single replica (see
+#   k8s/infra/longhorn/longhorn-paperless-storageclass.yaml). Shares the disk
+#   with Immich's immich-media PVC (k8s/apps/immich/pvc.yaml, 2000Gi) - photo
+#   libraries grow much faster than scanned documents, so paperless gets the
+#   smaller 500Gi share.
+# - data: the SQLite classification model + search index. Small (low GB),
+#   frequently rewritten, and disposable (rebuilds from the DB + media on
+#   next consume) - stays on the default 3-replica pool instead of the
+#   single-disk HDD.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-media
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-paperless-media
+  resources:
+    requests:
+      storage: 500Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 5Gi

k8s/apps/paperless/rbac.yaml

@@ -0,0 +1,35 @@
+# Scoped operator access for paperless-admins: restart/config-edit rights on
+# just this service's own resources, nothing CNPG-managed (paperless-db-*)
+# or provisioning-managed (paperless-oidc, paperless-minio-creds). Inert
+# until kube-apiserver's OIDC wiring lands (--oidc-groups-claim=groups,
+# --oidc-groups-prefix=oidc:) - subject name below assumes that prefix.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: paperless-operator
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    resourceNames: ["paperless"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["paperless-config"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["paperless-secrets"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: paperless-admins-binding
+subjects:
+  - kind: Group
+    name: "oidc:paperless-admins"
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: Role
+  name: paperless-operator
+  apiGroup: rbac.authorization.k8s.io

k8s/apps/paperless/redis.yaml

@@ -0,0 +1,37 @@
+# Task queue broker + websocket channel layer for paperless-ngx. No PVC:
+# queued/scheduled task state is disposable - a lost queue on restart just
+# means re-triggering consumption, not data loss (documents themselves live
+# on paperless-media).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: paperless-redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: paperless-redis
+  template:
+    metadata:
+      labels:
+        app: paperless-redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          ports:
+            - containerPort: 6379
+          resources:
+            requests: { cpu: "50m", memory: "64Mi" }
+            limits: { cpu: "250m", memory: "256Mi" }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: paperless-redis
+spec:
+  selector:
+    app: paperless-redis
+  ports:
+    - port: 6379
+      targetPort: 6379

k8s/apps/paperless/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: paperless
+spec:
+  selector:
+    app: paperless
+  ports:
+    - port: 8000
+      targetPort: 8000

k8s/apps/portainer/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: portainer
+resources: []
+# Portainer deployed via Helm chart or existing manifests

k8s/apps/portainer/portainer-values.yaml

@@ -0,0 +1,59 @@
+# k8s/portainer/portainer-values.yaml
+# Portainer — web UI for browsing cluster workloads, exec-ing into pods,
+# and viewing logs without kubectl. Operator-only access (ClusterIP + port-forward).
+#
+# Node failure behaviour:
+#   Portainer is a Deployment (not StatefulSet), so K8s auto-evicts and
+#   reschedules it ~5 min after a node becomes unreachable. Longhorn
+#   reattaches the PVC on the new node in ~1-2 min. Worst case: ~7-10 min.
+#
+#   To cut that down: in Longhorn UI → Settings set
+#     nodeDownPodDeletionPolicy = delete-deployment-pod
+#   Longhorn will force-delete the stuck pod immediately when the node is
+#   fenced rather than waiting for Kubernetes' eviction timeout.
+
+# ── Service ───────────────────────────────────────────────────────────────────
+# ClusterIP — no external exposure. Access via:
+#   kubectl -n dashboard port-forward svc/portainer 9000:9000
+# Portainer holds cluster-admin credentials; never expose as LoadBalancer.
+service:
+  type: ClusterIP
+
+# ── TLS ───────────────────────────────────────────────────────────────────────
+# Portainer by default redirects HTTP → HTTPS using a self-signed cert.
+# force: false disables the redirect so plain HTTP over port-forward works
+# without browser cert warnings. TLS is terminated at the ingress layer
+# if/when an ingress rule is added.
+tls:
+  force: false
+
+# ── Persistence ───────────────────────────────────────────────────────────────
+# Stores Portainer's own config: environment registrations, user accounts,
+# stack definitions, and access control settings. Longhorn provides the
+# RWO block volume. 10Gi is generous for config data but cheap on Longhorn.
+persistence:
+  enabled: true
+  storageClass: "longhorn"
+  size: 10Gi
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+# ── Scheduling ────────────────────────────────────────────────────────────────
+# Allow scheduling on talos-cp-1 (carries NoSchedule taint) so Portainer
+# keeps running even when the worker node is down.
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+# Pin to az-b (talos-cp-2) — sole Longhorn storage node (dedicated disks).
+# Its RWO PVC can only attach there; without this the pod may land on
+# cp-1/cp-3 and fail to mount.
+nodeSelector:
+  topology.kubernetes.io/zone: az-b

k8s/apps/secret-rotation-controller/crd.yaml

@@ -0,0 +1,144 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: secretrotations.homelab.riotpiao.com
+spec:
+  group: homelab.riotpiao.com
+  names:
+    kind: SecretRotation
+    plural: secretrotations
+  scope: Namespaced
+  versions:
+  - name: v1
+    served: true
+    storage: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        properties:
+          metadata:
+            type: object
+          spec:
+            type: object
+            required:
+            - provider
+            - rotationInterval
+            properties:
+              # External system: authentik | forgejo | minio | vault
+              provider:
+                type: string
+                enum: [authentik, forgejo, minio, vault]
+              
+              # How often to rotate (hours)
+              rotationInterval:
+                type: integer
+                minimum: 24
+              
+              # Application ID in external system
+              appId:
+                type: string
+              
+              # k8s Secret to update (name, namespace, key)
+              secretRef:
+                type: object
+                required: [name, namespace]
+                properties:
+                  name:
+                    type: string
+                  namespace:
+                    type: string
+                  key:
+                    type: string
+                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
+              
+              # Path to git file that holds the secret (for .enc.yaml files)
+              gitPath:
+                type: string
+                description: "Path in homelab repo to .enc.yaml file"
+              
+              # Ansible template values to substitute
+              templateValues:
+                type: object
+                additionalProperties:
+                  type: string
+          
+          status:
+            type: object
+            properties:
+              lastRotationTime:
+                type: string
+                format: date-time
+              nextRotationTime:
+                type: string
+                format: date-time
+              lastRotationStatus:
+                type: string
+                enum: [Success, Failed, Pending]
+              lastRotationError:
+                type: string
+              lastCommitHash:
+                type: string
+
+---
+# Example usage:
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160  # 90 days in hours
+  appId: minio
+  secretRef:
+    name: minio-oidc
+    namespace: storage
+    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: portfolio-agent-oidc
+  namespace: secret-rotation
+spec:
+  provider: authentik
+  rotationInterval: 2160
+  appId: portfolio-agent
+  secretRef:
+    name: portfolio-agent-oidc
+    namespace: portfolio
+    key: CLIENT_SECRET
+  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: forgejo-registry-token
+  namespace: secret-rotation
+spec:
+  provider: forgejo
+  rotationInterval: 2160
+  appId: rock/riotpiao.com
+  secretRef:
+    name: forgejo-registry-secret
+    namespace: kube-system
+    key: REGISTRY_TOKEN
+  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
+
+---
+apiVersion: homelab.riotpiao.com/v1
+kind: SecretRotation
+metadata:
+  name: minio-root-credentials
+  namespace: secret-rotation
+spec:
+  provider: minio
+  rotationInterval: 4320  # 180 days in hours
+  appId: root
+  secretRef:
+    name: minio-creds
+    namespace: storage
+  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml

k8s/apps/secret-rotation-controller/deployment.yaml

@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: secret-rotation-controller
+  template:
+    metadata:
+      labels:
+        app: secret-rotation-controller
+    spec:
+      serviceAccountName: secret-rotation-controller
+      containers:
+      - name: controller
+        image: secret-rotation-controller:latest
+        imagePullPolicy: IfNotPresent
+        env:
+        # SOPS reads age key from this file
+        - name: SOPS_AGE_KEY_FILE
+          value: /etc/sops/age/private-key.txt
+        
+        # Vault auth (token in projected volume)
+        - name: VAULT_ADDR
+          value: http://vault.vault.svc.cluster.local:8200
+        - name: VAULT_TOKEN_FILE
+          value: /var/run/secrets/vault/token
+        
+        # Authentik
+        - name: AUTHENTIK_URL
+          value: http://authentik-server.iam.svc.cluster.local
+        - name: AUTHENTIK_BOOTSTRAP_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: authentik-bootstrap
+              key: token
+        
+        # Git
+        - name: GIT_REPO
+          value: https://forgejo.riotpiao.com/rock/homelab.git
+        - name: GIT_AUTHOR_EMAIL
+          value: [email protected]
+        - name: GIT_AUTHOR_NAME
+          value: Secret Rotation Controller
+        - name: FORGEJO_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: forgejo-registry-secret
+              key: REGISTRY_TOKEN
+        
+        volumeMounts:
+        # Age key from ExternalSecret (synced from Vault)
+        - name: age-key
+          mountPath: /etc/sops/age
+          readOnly: true
+        
+        # Vault auth token (projected)
+        - name: vault-token
+          mountPath: /var/run/secrets/vault
+          readOnly: true
+        
+        # Temp working dir
+        - name: tmp
+          mountPath: /tmp
+        
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 500m
+            memory: 512Mi
+      
+      volumes:
+      - name: age-key
+        secret:
+          secretName: sops-age-key
+          defaultMode: 0400
+      
+      - name: vault-token
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              audience: vault
+              expirationSeconds: 3600
+      
+      - name: tmp
+        emptyDir: {}

k8s/apps/secret-rotation-controller/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: secret-rotation
+
+resources:
+  - rbac.yaml
+  - crd.yaml
+  - external-secret.yaml
+  - deployment.yaml
+
+commonLabels:
+  app.kubernetes.io/name: secret-rotation-controller
+  app.kubernetes.io/component: automation
+  managed-by: argocd

k8s/apps/secret-rotation-controller/rbac.yaml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secret-rotation-controller
+rules:
+# Read SecretRotation CRDs
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations"]
+  verbs: ["get", "list", "watch"]
+
+# Update status
+- apiGroups: ["homelab.riotpiao.com"]
+  resources: ["secretrotations/status"]
+  verbs: ["get", "patch", "update"]
+
+# Read k8s secrets that will be rotated
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+
+# For recording events
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secret-rotation-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secret-rotation-controller
+subjects:
+- kind: ServiceAccount
+  name: secret-rotation-controller
+  namespace: secret-rotation
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: secret-rotation
+  labels:
+    kubernetes.io/metadata.name: secret-rotation

k8s/apps/sms/deployment-macos.yaml

@@ -0,0 +1,115 @@
+# macOS VM (Docker-OSX) hosting the BlueBubbles server.
+#
+# ── Why a VM and not a container ────────────────────────────────────────────
+# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
+# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
+# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
+# container — a VM in a box, not a macOS container.
+#
+# ── Why this works on worker-2 ──────────────────────────────────────────────
+# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
+# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
+# module). Bare metal, so no nested virtualisation needed.
+#
+# ── Read this before relying on it ──────────────────────────────────────────
+# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
+#    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
+#    into iMessage, THEN install BlueBubbles inside the guest. This manifest
+#    only provides the machine; it does not provision macOS.
+# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
+#    own guidance: "test sending an iMessage to yourself. If it does not
+#    succeed, it's likely best to restart from the beginning."
+# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
+#    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
+# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
+#    warranty".
+#
+# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
+# inside the guest and is NOT required for plain send/receive. Skip it.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: macos-bluebubbles
+  labels:
+    app.kubernetes.io/name: macos-bluebubbles
+    app.kubernetes.io/part-of: sms
+spec:
+  replicas: 1
+  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
+  # concurrently — two QEMU processes on one image corrupts it.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: macos-bluebubbles
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: macos-bluebubbles
+        app.kubernetes.io/part-of: sms
+    spec:
+      # Dedicated node. The taint keeps everything else off worker-2; this
+      # toleration is what lets the VM on. Both halves are required.
+      nodeSelector:
+        workload: imessage
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: imessage
+          effect: NoSchedule
+      containers:
+        - name: macos
+          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
+          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
+          # only schedules because the sms namespace is labelled privileged.
+          securityContext:
+            privileged: true
+          env:
+            # Generates a unique serial / board-serial / UUID / MAC and persists
+            # them to bootdisk.qcow2. This synthetic identity is what iMessage
+            # activates against — it must stay stable across restarts, which is
+            # why the PVC matters.
+            - name: GENERATE_UNIQUE
+              value: "true"
+            # Identity is only plausible if it matches a real product line.
+            - name: DEVICE_MODEL
+              value: "iMacPro1,1"
+            - name: RAM
+              value: "12"
+            - name: CORES
+              value: "6"
+            - name: EXTRA
+              # Expose the BlueBubbles server port from the guest to the pod.
+              # Guest :1234 (BlueBubbles default) -> pod :1234.
+              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
+          ports:
+            - name: vnc
+              containerPort: 5999
+              protocol: TCP
+            - name: bluebubbles
+              containerPort: 1234
+              protocol: TCP
+          resources:
+            requests:
+              cpu: "6"
+              memory: 14Gi
+            limits:
+              cpu: "12"
+              memory: 20Gi
+          volumeMounts:
+            - name: macos-disk
+              mountPath: /home/arch/OSX-KVM/disk
+            - name: kvm
+              mountPath: /dev/kvm
+          # No readiness probe on purpose. The guest takes many minutes to boot,
+          # and until macOS + BlueBubbles are installed BY HAND there is nothing
+          # listening on 1234. A probe here would crash-loop the pod through the
+          # entire interactive install.
+      volumes:
+        - name: macos-disk
+          persistentVolumeClaim:
+            claimName: macos-disk
+        - name: kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice

k8s/apps/sms/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: sms
+resources:
+  - namespace.yaml
+  - storageclass.yaml
+  - pvc-macos.yaml
+  - deployment-macos.yaml
+  - service.yaml
+  - networkpolicy.yaml

k8s/apps/sms/namespace.yaml

@@ -0,0 +1,19 @@
+# iMessage delivery for the cluster.
+#
+# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
+# that hooks Messages.app private APIs — it cannot be containerised on Linux,
+# because containers share the host kernel and macOS needs XNU + Cocoa. The only
+# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
+# needs a privileged pod with /dev/kvm.
+#
+# Hence privileged PodSecurity: the cluster default from the Talos controlplane
+# is `enforce: baseline`, which forbids privileged containers and host devices.
+# Scope is limited to this namespace.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sms
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged

k8s/apps/sms/networkpolicy.yaml

@@ -0,0 +1,22 @@
+# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
+# ID and exposes an unauthenticated VNC console; nothing should reach it except
+# opted-in clients.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: sms-default-deny
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: sms
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              sms-client: "true"
+      ports:
+        - protocol: TCP
+          port: 1234

k8s/apps/sms/pvc-macos.yaml

@@ -0,0 +1,25 @@
+# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
+#
+# This volume is NOT disposable: it holds the VM's serial number, board serial,
+# UUID and MAC, which together form the identity iMessage was activated against.
+# Losing it means re-running activation, which is the least reliable step of the
+# whole setup.
+#
+# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
+# the installer, the base system, and qcow2 growth.
+#
+# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
+# Losing worker-2's disk therefore means losing the activated identity and
+# redoing iMessage activation. Once the guest is installed and activated, take
+# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: macos-disk
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-imessage-local
+  resources:
+    requests:
+      storage: 200Gi

k8s/apps/sms/service.yaml

@@ -0,0 +1,31 @@
+# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
+# it is an unauthenticated console onto a machine holding a live Apple ID
+# session. Reach it with `kubectl port-forward`, never an Ingress.
+apiVersion: v1
+kind: Service
+metadata:
+  name: macos-vnc
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: vnc
+      port: 5999
+      targetPort: vnc
+---
+# The BlueBubbles REST API, once installed inside the guest. This is the stable
+# name cluster services use, so callers never depend on the pod IP or on whether
+# the backend is this VM or a real Mac mini later.
+apiVersion: v1
+kind: Service
+metadata:
+  name: bluebubbles
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: http
+      port: 1234
+      targetPort: bluebubbles

k8s/apps/sms/storageclass.yaml

@@ -0,0 +1,32 @@
+# Dedicated StorageClass for the macOS VM disk.
+#
+# The default `longhorn` class does not work here, for two independent reasons:
+#
+# 1. Replica count. Default is 3, and Longhorn schedules against
+#    storageMaximum - storageScheduled with over-provisioning at 100%. Free
+#    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
+#    volume has only one node that can hold even a single replica — a 3-replica
+#    volume fails outright with ReplicaSchedulingFailure.
+# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
+#    created, before any pod is scheduled. Combined with strict-local that
+#    pins the data to an arbitrary node, not the one the VM runs on.
+#
+# So: one replica, kept local to the VM, bound only once the pod has a node.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-imessage-local
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
+# the replica is placed on that node and not somewhere else.
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
+  # Serving it over the network from another node's disk would be the single
+  # worst thing for guest responsiveness, so force it local.
+  numberOfReplicas: "1"
+  dataLocality: "strict-local"
+  staleReplicaTimeout: "30"
+  fsType: "ext4"

k8s/apps/temporal/temporal-values.yaml

@@ -0,0 +1,142 @@
+# k8s/temporal/temporal-values.yaml
+# Temporal — workflow engine
+# Uses external CNPG PostgreSQL for persistence (temporal-db)
+# Visibility via same PostgreSQL instance, separate database.
+#
+# IMPORTANT — chart schema note (root-caused after Postgres never actually
+# taking effect despite looking configured):
+# We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in
+# k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence
+# schema:
+#   server.config.persistence.<default|visibility>.driver: "sql"|"cassandra"
+#   server.config.persistence.<default|visibility>.sql: {...}
+# NOT the newer `datastores:`-wrapped schema
+# (server.config.persistence.datastores.<store>.sql) shown in the current
+# chart's values/values.postgresql.yaml example - that key was introduced in
+# a later major version and doesn't exist in 0.74.0. Helm doesn't validate
+# unknown keys, so a `datastores:` block here is silently a no-op: Temporal
+# would keep defaulting to Cassandra (with empty hosts: []) regardless of
+# anything nested inside it. Verified via `helm template` against the actual
+# 0.74.0 chart before writing this file - see chat history for the
+# side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and
+# temporal-cassandra-tool commands using the old datastores:-based values).
+#
+# Likewise `schema.setup.enabled` / `schema.update.enabled` /
+# `schema.createDatabase.enabled` are the real toggles for the schema-setup
+# Job (all default true) - there is no `jobs.autoSetup` key in this chart.
+
+# ── Disable every bundled/optional sub-chart ─────────────────────────────────
+# postgresql/mysql: never enable - we never want the chart to deploy its own
+# DB, only to know how to talk to our external CNPG instance (which happens
+# via server.config.persistence.*.sql below, independent of these flags).
+postgresql:
+  enabled: false
+mysql:
+  enabled: false
+cassandra:
+  enabled: false
+elasticsearch:
+  enabled: false
+prometheus:
+  enabled: false
+grafana:
+  enabled: false
+
+# ── Schema setup/update Jobs ──────────────────────────────────────────────────
+# The `temporal` DB is created by the dedicated temporal-db cluster's initdb and
+# `temporal_visibility` by a CNPG Database CR — both in
+# k8s/infra/databases/temporal-db.yaml — so createDatabase stays disabled.
+# setup/update run temporal-sql-tool as the `app` owner against those existing
+# DBs to install and migrate the
+# Temporal server schema — without them both DBs have zero tables and the
+# server dies on "no usable database connection found" (no schema_version row).
+schema:
+  createDatabase:
+    enabled: false
+  setup:
+    enabled: true
+  update:
+    enabled: true
+
+# ── Temporal server config (PostgreSQL persistence) ──────────────────────────
+server:
+  replicaCount: 1
+  # temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to
+  # built-in sprig config templating. The chart still defaults to the legacy
+  # configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a
+  # config the 1.30 server never loads — it then falls back to its embedded
+  # env-only template (Cassandra default) and dies with
+  # "Persistence.DataStores[default](value).Cassandra.Hosts: zero value".
+  # Switch to the sprig ConfigMap and point the server at it (chart's own
+  # recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath).
+  configMapsToMount: "sprig"
+  setConfigFilePath: true
+  jobService:
+    enabled: false
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          podAffinityTerm:
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/instance: temporal
+            topologyKey: kubernetes.io/hostname
+  config:
+    logLevel: "info"
+    persistence:
+      defaultStore: default
+      visibilityStore: visibility
+      numHistoryShards: 512
+      default:
+        driver: "sql"
+        sql:
+          driver: "postgres12"
+          host: "temporal-db-rw.temporal.svc.cluster.local"
+          port: 5432
+          database: "temporal"
+          user: "app"
+          # existingSecret + secretKey: point directly at the CNPG-generated
+          # Secret (kubernetes.io/basic-auth, keys: username/password/...)
+          # rather than duplicating the password in git as plaintext. When
+          # existingSecret is set the chart's own server-secret.yaml Secret
+          # template is skipped entirely (see templates/server-secret.yaml:
+          # `not $driverConfig.existingSecret` guards its creation).
+          # Use unified temporal-db-app secret (generated in temporal namespace)
+          existingSecret: "temporal-db-app"
+          secretKey: "password"
+          maxConns: 20
+          maxIdleConns: 10
+          maxConnLifetime: "1h"
+          # NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation
+          # is a MySQL-only connection parameter. The Postgres `pq` driver rejects
+          # it ("unrecognized configuration parameter"), which killed every DB
+          # connection (schema-setup job AND server) with the misleading
+          # "no usable database connection found". Postgres defaults to READ
+          # COMMITTED isolation anyway, so nothing is lost by omitting it.
+      visibility:
+        driver: "sql"
+        sql:
+          driver: "postgres12"
+          host: "temporal-db-rw.temporal.svc.cluster.local"
+          port: 5432
+          database: "temporal_visibility"
+          user: "app"
+          # Use unified temporal-db-app secret (generated in temporal namespace)
+          existingSecret: "temporal-db-app"
+          secretKey: "password"
+          maxConns: 20
+          maxIdleConns: 10
+          maxConnLifetime: "1h"
+  service:
+    type: ClusterIP
+
+# ── Temporal Web UI ────────────────────────────────────────────────────────
+web:
+  replicaCount: 1
+  service:
+    type: ClusterIP
+
+# ── Ingress ────────────────────────────────────────────────────────
+ingress:
+  enabled: false

k8s/apps/temporal/worker-production.yaml

@@ -0,0 +1,18 @@
+apiVersion: kmsvc.io/v1
+kind: TemporalWorker
+metadata:
+  name: worker-production
+  namespace: temporal
+spec:
+  namespace: production
+  taskQueue: worker-production
+  concurrency: 10
+  workflowTypes:
+    - HelloWorldWorkflow
+    - GreeterWorkflow
+    - ProcessOrderWorkflow
+  activityTypes:
+    - GreetActivity
+    - ValidateOrderActivity
+    - ProcessPaymentActivity
+    - NotifyCustomerActivity

k8s/argocd/apps/-1-projects.yaml

@@ -0,0 +1,25 @@
+# Wave -1 — AppProject definitions (must sync before any Application that references them).
+# Syncs k8s/argocd/projects/ which was previously applied by hand.
+# Enabled by Stage 1 (A2).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-projects
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/argocd/projects
+  destination:
+    server: https://kubernetes.default.svc

k8s/argocd/apps/00-secrets.yaml

@@ -0,0 +1,28 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: sops-secrets
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
+    # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the
+    # old sops-secrets-v1.0 CMP whose discover glob silently hijacked kustomize
+    # rendering of any app whose path contained a *.enc.yaml.
+    path: k8s/argocd/secrets
+  destination:
+    server: https://kubernetes.default.svc

k8s/argocd/apps/00-substrate.yaml

@@ -0,0 +1,212 @@
+# Wave 0/1 — cluster substrate: cert-manager, ingress-nginx, reloader, and the
+# Let's Encrypt issuers + wildcard cert. Previously installed by Terraform; now
+# owned by app-of-apps (Pure GitOps). Controllers at wave 0; the ClusterIssuers
+# and wildcard Certificate at wave 1 so cert-manager CRDs exist first.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cert-manager
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://charts.jetstack.io
+      chart: cert-manager
+      targetRevision: "v1.21.0"
+      helm:
+        valueFiles:
+          - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cert-manager
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+# ingress-nginx removed: duplicate of ingress-nginx-bootstrap
+# The bootstrap version (k8s/bootstrap-local/06-ingress-nginx.yaml) is kept
+# to break the circular dependency (ArgoCD needs Forgejo domain access)
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: reloader
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://stakater.github.io/stakater-charts
+    chart: reloader
+    targetRevision: "2.2.14"
+    helm:
+      values: |
+        reloader:
+          # Watch every workload — no per-Deployment reloader annotation needed
+          # (several charts, e.g. homarr, don't expose Deployment-level
+          # annotations). reloadOnCreate rolls a workload when a Secret/ConfigMap
+          # it references is first CREATED, not only updated — so ksops-delivered
+          # secrets landing after a pod started auto-restart it.
+          autoReloadAll: true
+          reloadOnCreate: true
+          deployment:
+            tolerations:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+                effect: NoSchedule
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: reloader
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+# Wave 1 — LE ClusterIssuers + wildcard cert (needs cert-manager CRDs from wave 0).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cert-manager-issuers
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
+    # deterministically. The previous directory.include with bare filenames
+    # rendered EMPTY — ArgoCD's include glob never matched — so this app silently
+    # tracked 0 resources; its ConfigMaps/Issuers only existed from bootstrap
+    # kubectl apply, and an automated prune wiped them.
+    path: k8s/bootstrap/cert-manager
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cert-manager
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+# Consolidated: wildcard-cert + homelab-ingress → ingress-config
+# Manages both the wildcard TLS certificate and all Ingress rules.
+# Certificate must exist before Ingresses (wave 1), but both are in same directory.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingress-config
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/bootstrap/ingress
+  destination:
+    server: https://kubernetes.default.svc
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cluster-maintenance
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/cluster-maintenance
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kyverno
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  source:
+    repoURL: https://kyverno.github.io/kyverno/
+    chart: kyverno
+    targetRevision: "1.14.0"
+    helm:
+      valueFiles:
+        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
+  sources:
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kyverno
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kyverno-policies
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/bootstrap/kyverno
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kyverno
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/apps/01-image-updater.yaml

@@ -0,0 +1,33 @@
+# ArgoCD Image Updater - auto-updates Application images from registry
+# Watches forgejo.riotpiao.com for new image tags and updates Applications
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: argocd-image-updater
+  namespace: argocd
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://argoproj.github.io/argo-helm
+      chart: argocd-image-updater
+      targetRevision: "0.11.2"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/argocd-image-updater/values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: argocd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=false

k8s/argocd/apps/05-networking.yaml

@@ -0,0 +1,8 @@
+# Wave 0 — networking substrate is Talos-owned (terraform inlineManifests), not
+# ArgoCD:
+#   - CoreDNS Corefile + hostname rewrites  -> terraform/files/coredns/Corefile
+#   - Cilium LB-IPAM pool + L2 announcement -> terraform/files/cilium/*.yaml
+# Both were previously ArgoCD apps here whose empty `resources: []`
+# kustomizations never actually applied them (live objects came from manual
+# kubectl). Managing them from ArgoCD too would let two reconcilers fight. This
+# file intentionally defines no Applications now.

k8s/argocd/apps/08-secret-rotation.yaml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: secret-rotation
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: secret-rotation
+spec:
+  project: homelab
+  
+  sources:
+  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    path: k8s/apps/secret-rotation-controller
+    targetRevision: main
+  
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: secret-rotation
+  
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    - RespectIgnoreDifferences=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/10-storage-observability.yaml

@@ -0,0 +1,251 @@
+# Wave 1 — MinIO (operator + tenant), Longhorn policy, Prometheus stack.
+# Helm charts pull from public repos; values come from the git repo via a
+# second "ref: values" source (ArgoCD multi-source pattern).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: minio-operator
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://operator.min.io/
+      chart: operator
+      targetRevision: "5.0.18"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/minio/minio-operator-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: storage
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+# Tenant + buckets + replication are raw CRs (MinIO Tenant CRD from operator).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: minio-tenant
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/minio
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: storage
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+# Longhorn itself is substrate (bootstrap-installed); this app manages only its
+# ServiceMonitor / policy manifests.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: longhorn-config
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/longhorn
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: longhorn-system
+  # Longhorn writes disk state back into its own Node CRs — the disk key it
+  # generates, storageReserved, diskType, evictionRequested. Git declares only
+  # allowScheduling; without this the controller's writes read as drift forever.
+  ignoreDifferences:
+    - group: longhorn.io
+      kind: Node
+      jsonPointers:
+        - /spec/disks
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: prometheus
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://prometheus-community.github.io/helm-charts
+      chart: kube-prometheus-stack
+      targetRevision: "*"
+      helm:
+        skipCrds: true
+        valueFiles:
+          - $values/k8s/infra/monitoring/prometheus-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    managedNamespaceMetadata:
+      # node-exporter needs hostNetwork/hostPID/hostPath/hostPort; blocked by
+      # default baseline PSS (DaemonSet created 0 pods, Prometheus STS stuck).
+      labels:
+        pod-security.kubernetes.io/enforce: privileged
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      # ServerSideApply removed — it conflicts with managedNamespaceMetadata's
+      # forced namespace apply ("--force cannot be used with --server-side").
+      # helm.skipCrds: true above stops ArgoCD from ever managing the CRDs
+      # through this Application (previously it kept re-patching them via
+      # client-side apply and hitting etcd's 262144-byte annotation limit on
+      # kubectl.kubernetes.io/last-applied-configuration, permanently failing
+      # sync). CRDs are applied once via the separate prometheus-crds
+      # Application below, which uses ServerSideApply=true (no namespace-
+      # metadata conflict since CRDs are cluster-scoped).
+---
+# CRDs only, extracted to plain YAML (`helm show crds kube-prometheus-stack`)
+# and committed to git under k8s/infra/monitoring/crds/, applied via Server-
+# Side Apply to avoid the etcd 262144-byte last-applied-configuration
+# annotation limit that client-side apply hits on these very large CRDs
+# (prometheuses, alertmanagers, scrapeconfigs, etc). A plain git path source
+# (not a remote Helm source) is used deliberately so ArgoCD applies exactly
+# these 8 CRD manifests and nothing else — no ambiguity about what "CRDs only"
+# means from a Helm chart. Split out from the main `prometheus` Application
+# (helm.skipCrds: true there) because ServerSideApply conflicts with that
+# app's managedNamespaceMetadata.
+# NOTE: bump k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml
+# whenever the kube-prometheus-stack chart version changes materially
+# (`helm show crds prometheus-community/kube-prometheus-stack > ...`).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: prometheus-crds
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/monitoring/crds
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+---
+# Cluster monitoring config: custom PrometheusRules (per-app namespaces),
+# ServiceMonitors (monitoring ns), and Grafana dashboard ConfigMaps (logging ns,
+# grafana sidecar-discovered). Single source = k8s/infra/monitoring (one
+# kustomization, no namespace transformer so per-app rule namespaces are kept).
+# Wave 2: after prometheus-operator CRDs (wave 0) + stack (wave 1) and grafana
+# (wave 2, logging). ServerSideApply avoids the etcd last-applied annotation
+# limit on the large dashboard ConfigMap JSON.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: monitoring-config
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/monitoring
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: blackbox-exporter
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://prometheus-community.github.io/helm-charts
+      chart: prometheus-blackbox-exporter
+      targetRevision: "~11"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+# Distributed tracing: Tempo + OpenTelemetry Collector.
+# Receives traces from instrumented services, stores in local volume (72h retention).
+# Grafana datasource auto-configured, service graph + latency dashboards included.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: tracing
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/tracing
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: tracing
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

k8s/argocd/apps/20-logging.yaml

@@ -0,0 +1,106 @@
+# Wave 2 — Loki / Grafana / Promtail (Grafana Helm charts).
+# NOTE: loki-values / grafana-values reference secrets (S3 creds, admin password)
+# that helmfile used to inject via --set. Under ArgoCD these come from the
+# *.enc.yaml SOPS files in the same dir via the SOPS plugin — verify the plugin
+# is configured before first sync, or these will render with empty secrets.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: loki
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://grafana.github.io/helm-charts
+      chart: loki
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/logging/loki-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: logging
+  syncPolicy:
+    managedNamespaceMetadata:
+      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
+      # privileged:true) to tail node logs — default baseline PSS blocks it.
+      labels:
+        pod-security.kubernetes.io/enforce: privileged
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: grafana
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://grafana.github.io/helm-charts
+      chart: grafana
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/logging/grafana-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: logging
+  syncPolicy:
+    managedNamespaceMetadata:
+      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
+      # privileged:true) to tail node logs — default baseline PSS blocks it.
+      labels:
+        pod-security.kubernetes.io/enforce: privileged
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: promtail
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "2"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://grafana.github.io/helm-charts
+      chart: promtail
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/logging/promtail-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: logging
+  syncPolicy:
+    managedNamespaceMetadata:
+      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
+      # privileged:true) to tail node logs — default baseline PSS blocks it.
+      labels:
+        pod-security.kubernetes.io/enforce: privileged
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

k8s/argocd/apps/30-security.yaml

@@ -0,0 +1,215 @@
+# Wave 3 — Vault + Authentik (identity), plus IAM raw jobs and the Forgejo
+# runner. Authentik/Vault values reference SOPS-managed secrets (see *.enc.yaml
+# in k8s/infra/iam) resolved by the ArgoCD SOPS plugin at sync time.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: vault
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://helm.releases.hashicorp.com
+      chart: vault
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/iam/vault-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: iam
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: authentik
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://charts.goauthentik.io
+      chart: authentik
+      targetRevision: "*"
+      helm:
+        valueFiles:
+          - $values/k8s/infra/iam/authentik-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: iam
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+# Raw IAM manifests: key-rotation cronjob + authentik migration job.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: iam-jobs
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/iam
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: iam
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+# Forgejo itself. Was a bootstrap Helm release (phase 3) until it was brought
+# under Argo, because values changes there were inert — a proxy-body-size fix
+# sat committed while the live Ingress kept nginx's 1m default and rejected
+# every OCI push with 413.
+#
+# Wave 3: after databases (wave 2) — Forgejo needs CNPG and Redis up first.
+#
+# Retiring the Helm release: Argo adopts the existing objects on first sync.
+# Delete the release secrets afterwards so helm stops claiming ownership:
+#   kubectl -n cicd delete secret -l owner=helm,name=forgejo
+#
+# automated sync is deliberately absent. This chart owns the Forgejo PVC and
+# the git forge itself; the first sync is manual so its diff can be read before
+# anything is applied. Turn on automated+selfHeal once that diff is clean.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://dl.gitea.com/charts/
+      chart: gitea
+      targetRevision: 12.7.0
+      helm:
+        valueFiles:
+          - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  # Reloader injects a STAKATER_* env var carrying a hash of the config Secret,
+  # so the pod rolls when that Secret changes. The chart does not render it, so
+  # Argo would strip it on every sync — and with selfHeal on, Argo and Reloader
+  # would fight over the field and Recreate the forge each round.
+  ignoreDifferences:
+    - group: apps
+      kind: Deployment
+      name: forgejo-gitea
+      jqPathExpressions:
+        - '.spec.template.spec.containers[].env[] | select(.name | startswith("STAKATER_"))'
+  syncPolicy:
+    syncOptions:
+      # Adopt the objects the bootstrap Helm release already created rather
+      # than failing on "already exists".
+      - ServerSideApply=true
+---
+# Forgejo runners (local chart, one instance per language), replacing the
+# single generic "docker"-labeled runner. Each instance is a full standalone
+# Deployment with its own dind sidecar, own PVCs (registration + layer
+# cache) and own registered label -- there is no shared generic runner
+# anymore, so each instance also builds and pushes images for the repos it
+# serves (the chart's ConfigMap/NetworkPolicy fixes for that -- valid_volumes,
+# network: host, egress to ingress-nginx -- apply identically to all three).
+#
+# `values.yaml` is the chart's default and doubles as the golang instance's
+# config; node and rust layer a small values-<lang>.yaml override on top for
+# just runner.name/runner.labels. All three share one runner-token Secret
+# (Forgejo registration tokens are reusable across multiple runners, unlike
+# GitHub's one-time tokens) -- if that assumption is ever wrong, registration
+# will fail loudly in the register initContainer's logs, not silently.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo-runner-golang
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/forgejo-runner
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo-runner-node
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/forgejo-runner
+    helm:
+      valueFiles:
+        - values-node.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo-runner-rust
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/forgejo-runner
+    helm:
+      valueFiles:
+        - values-rust.yaml
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true