fix: remove namespace from rbac Application destination

- RBAC kustomization contains cluster-scoped (ClusterRoleBinding) and namespace-scoped (Role/RoleBinding) resources - Each resource has explicit metadata.namespace, so Application shouldn't force a default namespace - Fixes: ClusterRoleBinding gets namespace=default, causing sync failure with 'unsupported role reference kind: ""'
fix: remove backslash line continuations from YAML multiline string
2026-09-05 14:53:59 -07:00 · 2026-09-05 14:48:21 -07:00 · 2026-09-05 14:42:17 -07:00 · 2026-09-05 14:33:27 -07:00 · 2026-09-05 14:33:13 -07:00 · 2026-09-05 14:30:41 -07:00
279 changed files with 94878 additions and 675 deletions
@@ -4,8 +4,8 @@
 # ── Cluster Configuration ──────────────────────────────────────────────────────
 # Base domain for external services (Authentik, MinIO, Forgejo, etc.)
-#   Example: riotpiao.homelab.com
+#   Example: riotpiao.com
-CLUSTER_DOMAIN=riotpiao.homelab.com
+CLUSTER_DOMAIN=riotpiao.com
 # Internal Kubernetes DNS names (svc.cluster.local)
 # Only change these if your cluster domain differs
@@ -68,3 +68,14 @@ AUTHENTIK_TEMPORAL_CLIENT_ID=
 # ── CI/CD ──────────────────────────────────────────────────────────────────────
 # Forgejo Personal Access Token (from rock user) for pushing images to registry
 FORGEJO_RIOTPIAO_PAT=
 # ── Cloudflare Tunnel (remote off-LAN access to kubectl/talosctl) ──────────────
 # From Cloudflare Zero Trust dashboard → Networks → Tunnels
 #   CLOUDFLARE_CONNECTOR_TOKEN: full tunnel token (JWT-like base64 string)
 #   CLOUDFLARE_ACCOUNT_ID: your account ID (hex string)
 #   CLOUDFLARE_TUNNEL_ID: tunnel UUID
 #   CLOUDFLARE_API_TOKEN: API token for programmatic tunnel config (optional)
 CLOUDFLARE_CONNECTOR_TOKEN=
 CLOUDFLARE_ACCOUNT_ID=
 CLOUDFLARE_TUNNEL_ID=
 CLOUDFLARE_API_TOKEN=
@@ -0,0 +1,269 @@
 name: Cluster CI Pipeline
 on:
  push:
    branches:
      - main
      - develop
    paths:
      - 'k8s/**'
      - '.forgejo/workflows/cluster-ci.yaml'
  pull_request:
    paths:
      - 'k8s/**'
 jobs:
  ci:
    runs-on: docker
    steps:
      # === Checkout ===
      - name: Checkout
        run: |
          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
          git clone --depth 1 "$CLONE_URL" .
          git fetch origin main
          git checkout main
      # === Install Tools ===
      - name: Install Tools
        run: |
          unset GITHUB_TOKEN
          apt-get update && apt-get install -y \
            yamllint \
            python3-pip \
            curl \
            jq
          # kubeval
          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
          mv -f kubeval /usr/local/bin/
          # kustomize
          rm -f kustomize
          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
          mv -f kustomize /usr/local/bin/
          # argocd
          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
          chmod +x /usr/local/bin/argocd
          # trivy
          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
          # polaris
          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
          chmod +x /usr/local/bin/polaris
      # === YAML Lint ===
      - name: YAML Lint
        run: |
          echo "=== Linting YAML files ==="
          yamllint k8s/ -c .yamllint.yaml || true
      # === Kubeval - Validate K8s Syntax ===
      - name: Kubeval - Validate K8s Syntax
        run: |
          echo "=== Validating Kubernetes manifests ==="
          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
            echo "Validating $file..."
            kubeval "$file" -d 2>/dev/null || true
          done
      # === Kustomize Build - All overlays ===
      - name: Kustomize Build - Infrastructure
        run: |
          echo "=== Building k8s/infrastructure/ ==="
          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
          echo "✓ Infrastructure built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
      - name: Kustomize Build - Bootstrap
        run: |
          echo "=== Building k8s/bootstrap/ ==="
          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
          echo "✓ Bootstrap built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
      - name: Kustomize Build - Platform
        run: |
          echo "=== Building k8s/platform/ ==="
          kustomize build k8s/platform > /tmp/platform.yaml
          echo "✓ Platform built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
      - name: Kustomize Build - Security
        run: |
          echo "=== Building k8s/security/ ==="
          kustomize build k8s/security > /tmp/security.yaml
          echo "✓ Security built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
      - name: Kustomize Build - Applications
        run: |
          echo "=== Building k8s/applications/ ==="
          kustomize build k8s/applications > /tmp/applications.yaml
          echo "✓ Applications built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
      - name: Kustomize Build - Data
        run: |
          echo "=== Building k8s/data/ ==="
          kustomize build k8s/data > /tmp/data.yaml
          echo "✓ Data built successfully"
          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
      - name: Validate ArgoCD Applications
        run: |
          echo "=== Validating ArgoCD Applications ==="
          kubeval k8s/argocd/apps/*.yaml
      # === Trivy - Scan Dockerfile ===
      - name: Trivy - Scan Dockerfile
        run: |
          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
            echo "=== Scanning Dockerfiles with Trivy ==="
            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
          else
            echo "No Dockerfiles found"
          fi
      # === Trivy - Scan Helm Charts ===
      - name: Trivy - Scan Helm Charts
        run: |
          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
            echo "=== Scanning Helm charts with Trivy ==="
            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
              echo "Scanning $chart..."
              trivy config "$chart" || true
            done
          else
            echo "No Helm charts found"
          fi
      # === Polaris - K8s Security Audit ===
      - name: Polaris - K8s Security Audit
        run: |
          echo "=== Running Polaris K8s security audit ==="
          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
          if [ -f /tmp/polaris-audit.json ]; then
            echo "Security issues found:"
            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
          fi
      # === Check for Secrets in Code ===
      - name: Check for Secrets in Code
        run: |
          echo "=== Scanning for hardcoded secrets ==="
          # BLOCKING. This step used to only count findings and then exit 0, so a
          # plaintext deploy key rode through it into a public remote. Two failure
          # modes fixed: it now fails the build, and it matches key material by
          # PEM header rather than only `private_key:`-style YAML field names.
          # Findings are captured into variables and tested for emptiness rather than
          # branching on grep's exit status: implementations disagree on the rc of a
          # `-v` filter fed empty input, and a wrong rc here fails open.
          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
          # and silently scans nothing.
          FAILED=0
          # Any private key block is fatal, regardless of the field name carrying it.
          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
                 | grep -v "\.enc\.yaml" || true)
          if [ -n "$KEYS" ]; then
            echo "❌ Unencrypted private key material found:"
            echo "$KEYS"
            FAILED=1
          fi
          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
          # so encrypted files never trip this.
          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
          if [ -n "$VALS" ]; then
            echo "❌ Plaintext secret value found:"
            echo "$VALS"
            FAILED=1
          fi
          if [ "$FAILED" -ne 0 ]; then
            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
            exit 1
          fi
          echo "✓ No hardcoded secrets found"
      # === Check K8s Security Best Practices ===
      - name: Check K8s Security Best Practices
        run: |
          echo "=== Checking K8s security best practices ==="
          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
            echo "⚠️  Found privileged containers"
          fi
          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
            echo "⚠️  Found hostNetwork usage"
          fi
          echo "Checking for missing resource limits..."
          MISSING=0
          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
              if ! grep -q "resources:" "$file"; then
                echo "⚠️  $file: Missing resource requests/limits"
                MISSING=$((MISSING + 1))
              fi
            fi
          done
      # === ArgoCD Sync (main branch only) ===
      - name: Sync ArgoCD
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== Syncing homelab-root ==="
          argocd app sync homelab-root --force
          argocd app wait homelab-root --timeout 5m
      - name: Check Sync Status
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== ArgoCD Applications Status ==="
          argocd app list -o table
          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
          if [ "$STATUS" != "Synced" ]; then
            echo "❌ Root app sync failed: $STATUS"
            exit 1
          fi
          echo "✓ Root app synced successfully"
      - name: Health Check
        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
        env:
          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
        run: |
          echo "=== Checking Application Health ==="
          argocd app get homelab-root -o wide
      # === Summary ===
      - name: Summary
        if: always()
        run: |
          echo "=== CI Pipeline Summary ==="
          echo "✓ YAML linted"
          echo "✓ Manifests validated"
          echo "✓ Kustomizations built"
          echo "✓ Security scans completed"
          echo "✓ Secrets check passed"
          echo "✓ Best practices verified"
          echo ""
          echo "✓ All checks passed"
@@ -1,5 +1,6 @@
 # Environment files — real values must never be committed
 .env
 .env.terraform.sh
 # Private CA key and generated TLS certs — ca.key must never enter the cluster or git.
 # Only ca.crt is safe to share, but we exclude the whole dir to avoid accidents.
@@ -8,6 +9,9 @@ forge/pki/
 # Talos machine configs — contain WireGuard private keys, bootstrap tokens, PKI
 cluster-config/controlplane.yaml
 cluster-config/worker*.yaml
 cluster-config/talos-worker*.yaml
 cluster-config/cp-*.yaml
 cluster-config/talos-cp-*.yaml
 cluster-config/secrets.yaml
 cluster-config/talosconfig
 talos-forge-trust.yaml
@@ -34,7 +38,31 @@ k8s/storage/test/test
 *.key
 *.conf
-# Allowed markdown: CLAUDE.example.md, README.md, TROUBLESHOOTING.md
+# CLAUDE.md is now version-controlled (was previously excluded as a
-CLAUDE.md
+# private-notes file; contains no secrets - just architecture, IPs
 # [private RFC1918 space], and operational lessons, same bar as README.md).
 # Terraform state and cache (local files, remote state in MinIO)
 .terraform/
 terraform/.terraform/
 terraform/*.tfstate
 terraform/*.tfstate.*
 terraform.tfvars.local
 skills-lock.json
 secrets-plaintext.yaml
 # Saved plan files — binary, environment-specific, may embed resource attributes
 terraform/tfplan
 terraform/tfplan-*
 .DS_Store
 CLAUDE.md
 docs/
 bootstrap-argocd.log
 # Any plaintext (non-SOPS) secret manifest. Encrypted ones are *.enc.yaml and
 # ARE committed — see .sops.yaml. A missing newline once merged two patterns on
 # one line here, which is how a plaintext deploy key reached a public remote.
 k8s/**/*-secret.yaml
 !k8s/**/*.enc.yaml
@@ -0,0 +1,5 @@
 creation_rules:
  # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
  # and was committed in plaintext to a public remote.
  - path_regex: k8s/.*secrets?.*\.ya?ml
    age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
@@ -0,0 +1,33 @@
 ---
 extends: default
 rules:
  line-length:
    max: 120
    level: warning
  indentation:
    spaces: 2
  brackets:
    min-spaces-inside: 0
    max-spaces-inside: 0
  braces:
    min-spaces-inside: 0
    max-spaces-inside: 0
  comments:
    min-spaces-from-content: 2
  comments-indentation: {}
  document-end: disable
  document-start: disable
  empty-lines:
    max: 3
  empty-values:
    forbid-in-block-mappings: true
    forbid-in-flow-mappings: true
  key-duplicates: enable
  key-ordering: disable
  new-line-at-end-of-file: enable
  new-lines:
    type: unix
  trailing-spaces: enable
  truthy:
    level: warning
@@ -0,0 +1,206 @@
 # Authentik Auth Integration for NextJS
 ## Current State
 ### Gateway Auth Status
 | Endpoint | Auth Status | Notes |
 |----------|-------------|-------|
 | `/v1/chat/completions` | ❌ **OFF** | LLM routes have no auth middleware |
 | `/v1/embeddings` | ❌ **OFF** | Same - no auth |
 | `/v1/rerank` | ❌ **OFF** | Same - no auth |
 | `X-Service: sqs` | ✅ **ON** | JWT validated via `internal/auth/jwt.go` |
 | `/workflow` | ❌ **OFF** | Pass-through to Temporal |
 **Auth module exists** at `homelab-frontend/internal/auth/jwt.go` but only wired for SQS.
 LLM routes in `internal/proxy/proxy.go` have no auth middleware.
 ### Authentik App
 Authentik app `local-llm` exists for LLM API auth:
 - **Client ID**: `local-llm`
 - **Client Secret**: `kubectl -n llm-serving get secret local-llm-jwt -o jsonpath='{.data.client-secret}' | base64 -d`
 - **Token endpoint**: `https://authentik.riotpiao.com/application/o/token/`
 - **Userinfo endpoint**: `https://authentik.riotpiao.com/application/o/userinfo/`
 - **OIDC discovery**: `https://authentik.riotpiao.com/application/o/local-llm/.well-known/openid-configuration`
 ## Sign-in Methods
 ### 1. Resource Owner Password Credentials (ROPC)
 Direct username/password login. Server-side only (needs client_secret).
 ```typescript
 // API Route: app/api/auth/login/route.ts
 const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
  method: 'POST',
  headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
  body: new URLSearchParams({
    grant_type: 'password',
    client_id: 'local-llm',
    client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
    username: '[email protected]',
    password: 'userpassword',
    scope: 'openid email profile groups',
  }),
 });
 const tokens = await response.json();
 // { access_token, refresh_token, expires_in, token_type }
 ```
 ### 2. Authorization Code Flow (Browser Redirect)
 Requires adding redirect URIs to `local-llm` Authentik app:
 ```python
 # In k8s/infra/iam/scripts/authentik-provision.py, update:
 "local-llm": {
    ...
    "redirect_uris": [
        "http://localhost:3000/api/auth/callback",  # dev
        "https://your-nextjs-app.com/api/auth/callback",  # prod
    ],
 }
 ```
 Then standard OIDC flow:
 1. Redirect to `https://authentik.riotpiao.com/application/o/authorize/?client_id=local-llm&redirect_uri=...&response_type=code&scope=openid email profile groups`
 2. User logs in via Authentik UI
 3. Callback receives `code`, exchange for tokens
 ## JWT Token Persistence
 ### Browser (localStorage)
 ```typescript
 const TOKEN_KEY = 'llm_auth_token';
 // Save
 localStorage.setItem(TOKEN_KEY, JSON.stringify({
  access_token: tokens.access_token,
  refresh_token: tokens.refresh_token,
  expires_at: Date.now() + tokens.expires_in * 1000,
 }));
 // Load
 const stored = JSON.parse(localStorage.getItem(TOKEN_KEY) || 'null');
 if (stored && stored.expires_at > Date.now()) {
  // Token valid
 }
 // Clear (logout)
 localStorage.removeItem(TOKEN_KEY);
 ```
 ### Server-side (HTTP-only cookies)
 ```typescript
 // app/api/auth/login/route.ts
 import { cookies } from 'next/headers';
 // After successful login
 cookies().set('llm_auth_token', JSON.stringify(tokens), {
  httpOnly: true,
  secure: process.env.NODE_ENV === 'production',
  sameSite: 'lax',
  maxAge: tokens.expires_in,
  path: '/',
 });
 // Read in middleware or API routes
 const tokenCookie = cookies().get('llm_auth_token');
 const tokens = JSON.parse(tokenCookie?.value || 'null');
 ```
 ## Token Refresh
 ```typescript
 async function refreshAccessToken(refresh_token: string) {
  const response = await fetch('https://authentik.riotpiao.com/application/o/token/', {
    method: 'POST',
    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
    body: new URLSearchParams({
      grant_type: 'refresh_token',
      client_id: 'local-llm',
      client_secret: process.env.AUTHENTIK_CLIENT_SECRET,
      refresh_token,
    }),
  });
  return response.json();
 }
 ```
 ## Environment Variables
 ```bash
 # .env.local
 AUTHENTIK_URL=https://authentik.riotpiao.com
 AUTHENTIK_CLIENT_ID=local-llm
 AUTHENTIK_CLIENT_SECRET=<from-secret>
 # For client-side (public)
 NEXT_PUBLIC_AUTHENTIK_URL=https://authentik.riotpiao.com
 NEXT_PUBLIC_AUTHENTIK_CLIENT_ID=local-llm
 ```
 ## Using Token with LLM API
 ```typescript
 const token = await getValidToken(); // from localStorage or cookie
 const response = await fetch('https://api.riotpiao.com/v1/chat/completions', {
  method: 'POST',
  headers: {
    'Content-Type': 'application/json',
    'Authorization': `Bearer ${token}`,  // JWT from Authentik
  },
  body: JSON.stringify({
    model: 'reasoning',
    messages: [{ role: 'user', content: 'Hello' }],
  }),
 });
 ```
 ## TODO
 ### Gateway-side (homelab-frontend)
 - [ ] Wire `internal/auth/jwt.go` into LLM proxy handler (`internal/proxy/proxy.go`)
 - [ ] Add `authRequired: true` to model config or create LLM-specific middleware
 - [ ] Example pattern from SQS (in `internal/serviceadapter/router.go`):
 ```go
 // In proxy.go ServeHTTP, before dispatching to LLM upstream:
 if strings.HasPrefix(r.URL.Path, "/v1/") {
    authHeader := r.Header.Get("Authorization")
    claims, err := llmJWTAuth.ValidateBearerToken(authHeader)
    if err != nil {
        // Return 401/403
    }
    if !llmJWTAuth.CheckPermissions(claims, "llm:inference", "*") {
        // Return 403 insufficient permissions
    }
 }
 ```
 ### Authentik-side
 - [ ] Enable ROPC grant in Authentik provider settings (if not already)
 - [ ] Add redirect URIs to `local-llm` app if browser OAuth flow needed:
 ```python
 # k8s/infra/iam/scripts/authentik-provision.py
 "local-llm": {
    ...
    "redirect_uris": [
        "http://localhost:3000/api/auth/callback",
        "https://your-app.com/api/auth/callback",
    ],
 }
 ```
 ### NextJS-side
 - [ ] Until gateway auth is wired, LLM API works without token
 - [ ] Once wired, add `Authorization: Bearer <token>` to all LLM requests
@@ -1,72 +1,210 @@
-# CLAUDE.md — Homelab Integration Guide
+# CLAUDE.md — Homelab Integration Guide (Example / Reusable Template)
-**Homelab:** A bare-metal three-node Kubernetes cluster running Talos Linux with a full observability stack, SSO via Authentik, secret management via Vault, and CI/CD infrastructure (Forgejo + Argo CD, deployed).
+> **This is a sanitized template.** Copy to `CLAUDE.md`, fill in your own
 > node IPs/hostnames/secrets, and delete this notice. Nothing in this file
 > should contain real credentials, real IPs beyond illustrative examples, or
 > anything that would matter if this file became public. It's meant to be
 > shared across homelabs running similar hardware/topology (3-node bare-metal
 > Talos Kubernetes + ArgoCD GitOps), not just this one.
 **Homelab:** A bare-metal N-node Kubernetes cluster running Talos Linux with a
 full observability stack, SSO via Authentik, secret management via Vault, and
 CI/CD infrastructure (self-hosted git forge + Argo CD).
 ## Cluster Topology (adjust to your hardware)
 | Node | IP | Zone | Scheduling | Storage |
 |------|----|----|-----------|---------|
 | `<node-1>` | `<ip>` | az-a | schedulable (all workloads) | sole storage node (if single-node storage) |
 | `<node-2>` | `<ip>` | az-b | dedicated (`NoSchedule`) | none |
 | `<node-3>` | `<ip>` | az-c | dedicated (`NoSchedule`) | none |
 If your storage layer (Longhorn, local-path, etc.) only runs on one node,
 every stateful workload pinned to that storage class is effectively
 single-instance regardless of your control-plane HA count — document that
 explicitly here, it changes your failure-mode assumptions everywhere else.
 ## Deployment Model: ArgoCD GitOps (app-of-apps)
 ```
 git commit → git push (your git forge) → ArgoCD auto-sync → cluster
 ```
 Structure:
 - One root `Application` (`k8s/argocd/root/`) pointing at a directory of
  child `Application` manifests (`k8s/argocd/apps/*.yaml`)
 - Each child Application is either:
  - A remote Helm chart + a **second** git source (`ref: values`) supplying
    just the values file — lets you pin an upstream chart version while
    keeping your values under normal git history/review
  - A plain git directory of raw manifests (optionally with a
    `kustomization.yaml`)
 - `argocd.argoproj.io/sync-wave` annotations control ordering across
  Applications (lower number syncs first)
 **Never `kubectl apply`/`patch`/`delete` a resource ArgoCD manages**, except:
 - Pure cleanup of stuck/dead state (e.g. deleting a permanently-failed hook
  Job so the next real sync can create a fresh one) — this is not a config
  change, just clearing wreckage that GitOps itself won't clean up
  automatically (see Gotchas below)
 - Genuine one-time bootstrap circular dependencies (e.g. Vault
  `operator init`/unseal — nothing can configure Vault's own unseal keys
  before Vault has generated them)
 ## Hard Rules (adapt freely, but keep something like these)
 🔴 **Identify and document your storage/stateful-singleton node explicitly.**
 Whatever node holds your CSI driver's data (Longhorn, local-path, etc.),
 renaming or wiping it orphans every PVC pinned there. Name it here, in
 caps, so nobody "cleans up" it by accident.
 🔴 **If you run multi-member etcd across a LAN + VPN/WireGuard overlay,
 pin the advertised subnet explicitly** (e.g. Talos's
 `cluster.etcd.advertisedSubnets`). Without it, etcd may advertise on the
 wrong interface and new members hang as non-promoting learners.
 🔴 **Run your IaC formatter (terraform fmt, etc.) before every commit
 that touches infra code.** Wire this into CI as a hard gate, not a
 suggestion.
 🔴 **Whatever your source of truth is (Terraform, ArgoCD, both) — never
 manually mutate resources it manages.** State drift is the single most
 common cause of "why did my last apply undo my manual fix" confusion.
 Fix the source, re-apply/re-sync, never bypass.
 🔴 **Never delete a PVC without confirming replica count / backup
 freshness first.** This is always a one-way door.
 🔴 **Decide your commit message convention up front and enforce it.**
 (This template's origin project uses: no co-authored-by footers, single-line
 commit summarizing what/why, solo-authorship assumption — adjust to your
 team's norms.)
 🔴 **Decide your git workflow (rebase vs merge) up front and stick to it**
 cluster-wide, across every contributor/agent working in the repo.
 🔴 **Long-running commands should not block a synchronous session** — run
 them in the background and poll, especially anything that waits on a
 Kubernetes rollout, an image pull, or a Terraform apply.
 ## GitOps / ArgoCD Gotchas (transferable to any ArgoCD-based homelab)
 🟠 **A `kustomization.yaml` with an explicit `resources:` allowlist
 silently drops anything you forget to list.** No error, no drift shown in
 ArgoCD's UI — it just reports `Synced/Healthy` against a manifest set that
 never included your new file. Always run `kubectl kustomize <dir>/`
 locally before pushing to confirm exactly what ArgoCD will build.
 🟠 **A top-level `namespace:` transformer in `kustomization.yaml` rewrites
 `metadata.namespace` on every resource it builds** — including RBAC
 bindings deliberately targeting a *different* namespace (e.g. granting a
 ServiceAccount in namespace A read access to Secrets in namespace B). If
 any manifest needs cross-namespace RBAC, either drop the transformer
 (safe if every resource already sets its own explicit namespace) or give
 that manifest its own Application/directory.
 🟠 **PreSync hooks run before an Application's own normal resources are
 synced.** A PreSync Job that depends on RBAC/ServiceAccounts defined as
 plain (non-hook) resources in the *same* Application will deadlock — it
 tries to start before its own permissions exist. Use PostSync instead if
 the hook needs resources from its own Application, or move the
 prerequisite RBAC into an earlier sync-wave Application.
 🟠 **ArgoCD hooks are not continuously reconciled by `selfHeal`.** Once a
 hook Job finishes (success, or exhausts `backoffLimit`), it's only
 deleted+recreated during an *actual new Sync operation* — not by passive
 drift detection, even with `automated.selfHeal: true` on. If you fix a
 broken hook's spec and push, the Application's `status.sync.revision` can
 show "caught up" while the live hook resource is still the old, broken
 one, because no new operation actually re-ran it. To force it: delete the
 stuck hook (clear `argocd.argoproj.io/hook-finalizer` manually if it's
 stuck `Terminating`), and if that alone doesn't trigger a fresh full sync,
 delete + re-`kubectl apply -f` the Application object itself.
 🟠 **If you route ArgoCD's own `repoURL` through an ingress/reverse-proxy
 hostname that only listens on 80/443, don't use a non-standard port in the
 URL** — it'll silently time out trying to reach a port the proxy never
 opened, and depending on your setup this can block *every* Application's
 sync simultaneously (repo-server can't fetch git refs for anything).
 🟠 **Don't pin exact version tags for images from registries that don't
 guarantee tag retention** (Bitnami stopped publishing versioned tags for
 free-tier images in 2025 — only `latest` + sha256 digests remain). Verify
 a tag actually exists before pinning it, or prefer minimal base images +
 a stdlib-only runtime download (e.g. Python's `urllib.request` to fetch a
 static binary) to avoid depending on any third party's tagging policy.
 🟠 **Non-root containers can't `apk add`/`apt install` in most default
 base images** — package manager directories are root-owned. Use a
 world-writable scratch dir (`/tmp`) for anything you need to
 download/install at runtime instead.
 🟠 **Helm does not validate unknown `values.yaml` keys.** A typo, or a
 values schema copied from the wrong chart *version's* docs/examples, is
 silently a no-op — not an error. Before concluding "this chart doesn't
 support X," clone the chart at your exact pinned version/tag and run
 `helm template` against your real values file, then diff the rendered
 output. Don't trust a chart's current `main`-branch example values file
 if you're pinned to an older release — schemas do change between major
 versions without warning in your own values file.
 ## Service Integration Routes
-**New service? Pick your stack below:**
+**New service? Pick your stack below** (adjust doc paths to match your repo):
 | Need | Doc | Example |
 |------|-----|---------|
 | **Authentication** | `project-usage/authentik-oidc.md` | OAuth2 login, RBAC groups, JWT tokens |
-| **Async messaging** | `project-usage/sqs-messaging.md` | Kafka topic consumers, fire-and-forget, DLQ |
+| **Async messaging** | `project-usage/sqs-messaging.md` | Queue consumers, fire-and-forget, DLQ |
 | **Object storage** | `project-usage/minio-s3.md` | File uploads, backups, log backend |
-| **CI/CD pipeline** | `project-usage/cicd-workflow.md` | GitHub Actions syntax, image push, Argo CD sync |
+| **CI/CD pipeline** | `project-usage/cicd-workflow.md` | Pipeline syntax, image push, ArgoCD sync |
 | **Workflows** | `project-usage/temporal-workflows.md` | Long-running jobs, retries, state machines |
 | **Database** | `project-usage/database-postgres.md` | CloudNativePG setup, schema migrations, replicas |
 | **Monitoring** | `project-usage/monitoring-metrics.md` | Prometheus scrape, Grafana dashboard, alerts |
 | **Secrets** | `project-usage/vault-secrets.md` | Store credentials, rotate tokens, seal/unseal |
 | **Networking** | `project-usage/networking-ingress.md` | Public HTTPS, hostname routing, TLS |
-## Cluster Essentials
+## Cluster Essentials (fill in your own inventory)
-**22 namespaces, 18 releases:**
+**Architecture principles (adjust to taste, but these travel well):**
-```
+- Immutable OS (Talos, or similar — no SSH, fully declarative config)
-Core: cert-manager, ingress-nginx, kube-system, cilium
+- Secrets in a proper secrets backend (Vault) + SOPS-encrypted manifests in
-Storage: longhorn-system, storage (MinIO)
+  git (`*.enc.yaml`, age-encrypted); never commit plaintext secrets or `.env`
-Data: ddb (PostgreSQL), iam (Authentik + Vault)
+- ArgoCD app-of-apps as the single CD source of truth; two-phase bootstrap
-Observability: logging (Loki + Grafana), monitoring (Prometheus)
+  documented separately (chicken-and-egg: ArgoCD needs to exist before it
-Apps: cicd (Forgejo + Argo CD), sqs (Kafka + kmsvc), temporal, story-crater-backend
+  can deploy itself declaratively — document your exact bootstrap steps)
-```
+- Pull-based GitOps — no kubeconfig/cluster credentials ever touch your CI
  runner; the runner only needs push access to git, ArgoCD does the rest
 - Federated OIDC (one identity provider fronting every service that
  supports it)
-**Architecture principles:**
+## Deployment Checklist (per new service)
 - Immutable OS (Talos — no SSH, declarative config)
 - Secrets in Vault (never commit `.env`, credentials in Secret volumes)
 - Helmfile = single source of truth (`helmfile.yaml.gotmpl`)
 - Pull-based GitOps (Argo CD, no kubeconfig in CI)
 - Federated OIDC (Authentik provider for all services)
-
+- [ ] Prometheus `/metrics` endpoint or ServiceMonitor, if it exposes metrics
-## Deployment Checklist
+- [ ] All credentials in your secrets backend (never in plain values.yaml,
-
+      pod env directly, or committed anywhere in cleartext)
- [ ] Service has Prometheus `/metrics` endpoint or ServiceMonitor
+- [ ] Ingress rule with TLS, if externally reachable
- [ ] All credentials in Vault (never in pod env, ConfigMap, or code)
+- [ ] Dashboard + alert rules, if metrics are exposed
- [ ] Ingress rule in `k8s/ingress/` with TLS cert
+- [ ] ArgoCD `Application` manifest added to the appropriate sync-wave file,
- [ ] Grafana dashboard in `k8s/monitoring/dashboards/svc-<name>.yaml`
+      **not** a standalone `helm install`/`kubectl apply` run by hand
- [ ] Alert rules in `k8s/monitoring/alerts/svc-<name>-rules.yaml` (if needed)
+- [ ] Validated locally before push: `kubectl apply --dry-run=client -f`,
- [ ] Helm release in `helmfile.yaml.gotmpl` with correct `needs:` dependencies
+      `kubectl kustomize <dir>/` (if applicable), or `helm template` against
-
+      the exact pinned chart version (if Helm-sourced)
-## Hard Rules
+- [ ] After push: confirmed ArgoCD's `status.sync.revision` actually matches
-
+      your new commit — not just that `status.sync.status` says `Synced`
-1. **No kubeconfig in CI** — Argo CD bridges gap (pull-based, never push secrets to runner)
+      (see Gotchas — a stale hook can hide behind an otherwise-current app)
 2. **Field name = variable name** — In Vault: `talos put cluster/KAFKA_BOOTSTRAP KAFKA_BOOTSTRAP="..."`
 3. **Secrets via volumes** — Never `--env` flag in pod specs (exposes in `kubectl describe`)
 4. **External services via Ingress** — All public endpoints via TLS (homelab-ca)
 5. **Never commit `.env`** — Only `.env.example` in git; real secrets in Vault
 ## Git & Release
-**Multi-remote push:**
+**Incremental commits (service-layer grouped) tend to age well:**
 ```bash
 git push origin main
 ```
 **Incremental commits (service-layer grouped):**
 - Foundation & Docs
- Helmfile & Core Infra
+- Core Infra (CNI, ingress, cert management, storage)
 - Storage Layer
 - Observability Stack
 - IAM & Secrets
 - CI/CD & GitOps
- Messaging Infrastructure
+- Messaging / Data Infrastructure
 - Applications & Utilities
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
@@ -1,21 +1,21 @@
-# ── Node IPs ──────────────────────────────────────────────────────────────────
+# ── Node IPs (3-CP HA topology) ───────────────────────────────────────────────
-# CP_IP has a default. All W{N}_IP variables are expected to be exported from
+CP1_IP  := 192.168.1.166  # talos-cp-1
-# ~/.zshrc (e.g. export W1_IP=192.168.1.162). No guards — assumed always set.
+CP2_IP  := 192.168.1.214  # talos-cp-2 (storage: 3 disks)
-CP_IP ?= 192.168.1.213
+CP3_IP  := 192.168.1.162  # talos-cp-3
-
+CP_VIP  := 192.168.1.166  # controlplane VIP (currently .166)
 export CP_IP
 # ── Paths ─────────────────────────────────────────────────────────────────────
-TALOSCONFIG := cluster-config/coreconfig
+TALOSCONFIG := cluster-config/talosconfig
-CP_CONFIG   := cluster-config/controlplane.yaml
+CP1_CONFIG  := cluster-config/talos-cp-1.yaml
-SECRETS     := cluster-config/secrets.yaml
+CP2_CONFIG  := cluster-config/talos-cp-2.yaml
 CP3_CONFIG  := cluster-config/talos-cp-3.yaml
 KUBECONFIG  := cluster-config/kubeconfig
-CLUSTER_NAME := homelab-cluster
+CLUSTER_NAME := homelab
-CP_ENDPOINT  := https://$(CP_IP):6443
+CP_ENDPOINT  := https://$(CP_VIP):6443
 TALOS_IMAGE  := factory.core.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.13.3
-TALOSCTL := corectl --coreconfig $(TALOSCONFIG)
+# Use talosctl (not corectl). Needs TALOSCONFIG env var pointing to talosconfig file.
 TALOSCTL := talosctl
 KUBECTL  := kubectl --kubeconfig $(KUBECONFIG)
 # Derive IP and config from worker number N (used by generic targets).
@@ -28,236 +28,157 @@ W_CONFIG = cluster-config/worker-$(N).yaml
 # ── Help ──────────────────────────────────────────────────────────────────────
 .PHONY: help
 help:
-	@echo "Homelab cluster — available targets"
+	@echo "Homelab cluster (3-CP HA: .166/.214/.162) — available targets"
 	@echo ""
-	@echo "  Status"
+	@echo "  Status & Services"
 	@echo "    nodes             kubectl get nodes"
-	@echo "    status-cp         core node overview  (control plane)"
+	@echo "    status-all        etcd members on all 3 CPs"
-	@echo "    status-w1         core node overview  (worker-1)"
+	@echo "    status-cp1/2/3    etcd members on specific CP"
-	@echo "    services-cp       list core services  (control plane)"
+	@echo "    services-cp1/2/3  list Talos services on specific CP"
 	@echo "    services-w1       list core services  (worker-1)"
 	@echo ""
 	@echo "  Logs"
-	@echo "    logs-cp           stream kubelet logs  (control plane)"
+	@echo "    logs-cp1/2/3      stream kubelet logs from CP{1,2,3}"
-	@echo "    logs-w1           stream kubelet logs  (worker-1)"
+	@echo "    dmesg-cp1/2/3     stream kernel dmesg from CP{1,2,3}"
-	@echo "    dmesg-cp          kernel dmesg         (control plane)"
+	@echo "    log-svc-cp1/2/3   stream service logs (SVC=<name>)"
 	@echo "    dmesg-w1          kernel dmesg         (worker-1)"
 	@echo "    log-svc-cp        stream a service log (control plane)  SVC=<name>"
 	@echo "    log-svc-w1        stream a service log (worker-1)       SVC=<name>"
 	@echo ""
-	@echo "  Config"
+	@echo "  Config Apply"
-	@echo "    gen-config        regenerate controlplane.yaml + worker-N.yaml from secrets"
+	@echo "    apply-all         apply configs to all 3 CPs (talos-cp-{1,2,3}.yaml)"
-	@echo "    apply-cp          apply controlplane.yaml to CP node (live cluster)"
+	@echo "    apply-cp1/2/3     apply config to specific CP"
 	@echo "    apply-w1          apply cluster-config/worker-1.yaml to worker-1"
 	@echo "    apply-w1-insecure first-time apply to worker-1 (no certs yet)"
 	@echo "    apply-worker      apply cluster-config/worker-N.yaml   N=<num> W<N>_IP=<ip>"
 	@echo "    apply-worker-new  first-time apply (--insecure)        N=<num> W<N>_IP=<ip>"
 	@echo ""
-	@echo "  Upgrade"
+	@echo "  Reboot"
-	@echo "    upgrade-cp        upgrade Talos on control plane"
+	@echo "    reboot-all        reboot all 3 CPs"
-	@echo "    upgrade-w1        upgrade Talos on worker-1"
+	@echo "    reboot-cp1/2/3    reboot specific CP"
 	@echo "    upgrade-worker    upgrade any worker                    N=<num> W<N>_IP=<ip>"
 	@echo ""
 	@echo "  Shutdown / Reboot"
 	@echo "    shutdown-cluster  graceful full shutdown (drain w1 → off w1 → off cp)"
 	@echo "    shutdown-cp       shut down control plane only"
 	@echo "    shutdown-w1       shut down worker-1 only"
 	@echo "    shutdown-worker   shut down any worker                  N=<num> W<N>_IP=<ip>"
 	@echo "    reboot-cp         reboot control plane"
 	@echo "    reboot-w1         reboot worker-1"
 	@echo "    reboot-worker     reboot any worker                     N=<num> W<N>_IP=<ip>"
 	@echo ""
 	@echo "  Inspect (node filesystem)"
 	@echo "    node-ls   <ip> <path>   list files on a node"
 	@echo "    node-read <ip> <path>   read a file on a node"
 	@echo ""
 	@echo "  Maintenance"
 	@echo "    clean-pods        delete Evicted/Failed/Terminating pods cluster-wide"
 	@echo ""
 	@echo "  Port-forwards"
 	@echo "    pf-grafana        localhost:3000 → Grafana"
 	@echo "    pf-minio          localhost:9001 → MinIO console  /  localhost:9000 → S3 API"
 	@echo "    pf-loki           localhost:3100 → Loki HTTP API"
 	@echo "    pf-portainer      localhost:9000 → Portainer UI (dashboard ns)"
 	@echo "    pf-prometheus     localhost:9090 → Prometheus UI (monitoring ns)"
 	@echo "    pf-longhorn       localhost:8080 → Longhorn UI"
-	@echo "    pf-iam            localhost:7000 → Authentik IAM (when deployed)"
+	@echo "    pf-prometheus     localhost:9090 → Prometheus UI"
 	@echo ""
-	@echo "  CLI"
+	@echo "  IPs"
-	@echo "    cli               build core-cli and install to ~/.local/bin/core"
+	@echo "    CP1 (talos-cp-1): $(CP1_IP)  — NVMe, wg0/wg1, VIP"
-	@echo ""
+	@echo "    CP2 (talos-cp-2): $(CP2_IP)  — 3 Longhorn disks"
-	@echo "  Variables"
+	@echo "    CP3 (talos-cp-3): $(CP3_IP)  — NVMe"
 	@echo "    CP_IP   (default: 192.168.1.160)"
 	@echo "    W1_IP   (export from ~/.zshrc — e.g. export W1_IP=192.168.1.162)"
 	@echo "    N       (required for generic targets — worker number, e.g. N=2)"
 	@echo "    W<N>_IP (export from ~/.zshrc — e.g. export W2_IP=192.168.1.163)"
 	@echo "    SVC     (required for log-svc-* targets, e.g. SVC=kubelet)"
 # ── Status ────────────────────────────────────────────────────────────────────
 .PHONY: nodes
 nodes:
 	$(KUBECTL) get nodes -o wide
 .PHONY: status-all
 status-all: status-cp1 status-cp2 status-cp3
 .PHONY: status-cp1
 status-cp1:
 	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) etcd members
 .PHONY: status-cp2
 status-cp2:
 	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) etcd members
 .PHONY: status-cp3
 status-cp3:
 	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) etcd members
 .PHONY: status-cp
-status-cp:
+status-cp: status-all
 	$(TALOSCTL) --nodes $(CP_IP) get members
-.PHONY: status-w1
+.PHONY: services-cp1
-status-w1:
+services-cp1:
-	$(TALOSCTL) --nodes $(W1_IP) get members
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) service
-.PHONY: services-cp
+.PHONY: services-cp2
-services-cp:
+services-cp2:
-	$(TALOSCTL) --nodes $(CP_IP) service
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) service
-.PHONY: services-w1
+.PHONY: services-cp3
-services-w1:
+services-cp3:
-	$(TALOSCTL) --nodes $(W1_IP) service
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) service
-# ── Logs ──────────────────────────────────────────────────────────────────────
+# ── Logs (3-CP) ───────────────────────────────────────────────────────────────
-.PHONY: logs-cp
+.PHONY: logs-cp1
-logs-cp:
+logs-cp1:
-	$(TALOSCTL) --nodes $(CP_IP) logs kubelet -f
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs kubelet -f
-.PHONY: logs-w1
+.PHONY: logs-cp2
-logs-w1:
+logs-cp2:
-	$(TALOSCTL) --nodes $(W1_IP) logs kubelet -f
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs kubelet -f
-.PHONY: dmesg-cp
+.PHONY: logs-cp3
-dmesg-cp:
+logs-cp3:
-	$(TALOSCTL) --nodes $(CP_IP) dmesg --follow
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs kubelet -f
-.PHONY: dmesg-w1
+.PHONY: dmesg-cp1
-dmesg-w1:
+dmesg-cp1:
-	$(TALOSCTL) --nodes $(W1_IP) dmesg --follow
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) dmesg --follow
-# Usage: make log-svc-cp SVC=etcd
+.PHONY: dmesg-cp2
-.PHONY: log-svc-cp
+dmesg-cp2:
-log-svc-cp:
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) dmesg --follow
 .PHONY: dmesg-cp3
 dmesg-cp3:
 	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) dmesg --follow
 # Usage: make log-svc-cp1 SVC=etcd
 .PHONY: log-svc-cp1
 log-svc-cp1:
 ifndef SVC
-	$(error SVC is not set — run: make log-svc-cp SVC=<service-name>)
+	$(error SVC is not set — run: make log-svc-cp1 SVC=<service-name>)
 endif
-	$(TALOSCTL) --nodes $(CP_IP) logs $(SVC) -f
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs $(SVC) -f
-.PHONY: log-svc-w1
+.PHONY: log-svc-cp2
-log-svc-w1:
+log-svc-cp2:
 ifndef SVC
-	$(error SVC is not set — run: make log-svc-w1 SVC=<service-name>)
+	$(error SVC is not set — run: make log-svc-cp2 SVC=<service-name>)
 endif
-	$(TALOSCTL) --nodes $(W1_IP) logs $(SVC) -f
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs $(SVC) -f
-# ── Config generation ─────────────────────────────────────────────────────────
+.PHONY: log-svc-cp3
-.PHONY: gen-config
+log-svc-cp3:
-gen-config:
+ifndef SVC
-	corectl gen config $(CLUSTER_NAME) $(CP_ENDPOINT) \
+	$(error SVC is not set — run: make log-svc-cp3 SVC=<service-name>)
-	  --with-secrets $(SECRETS) \
+endif
-	  --output-dir cluster-config/ \
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs $(SVC) -f
-	  --force
+
 # ── Config Apply (3-CP) ───────────────────────────────────────────────────────
 .PHONY: apply-all
 apply-all: apply-cp1 apply-cp2 apply-cp3
 	@echo "✓ All 3 control planes configured"
 .PHONY: apply-cp1
 apply-cp1:
 	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) apply-config -f $(CP1_CONFIG)
 .PHONY: apply-cp2
 apply-cp2:
 	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) apply-config -f $(CP2_CONFIG)
 .PHONY: apply-cp3
 apply-cp3:
 	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) apply-config -f $(CP3_CONFIG)
 # ── Config apply ──────────────────────────────────────────────────────────────
 .PHONY: apply-cp
-apply-cp:
+apply-cp: apply-all
 	$(TALOSCTL) apply-config \
 	  --nodes $(CP_IP) \
 	  --file $(CP_CONFIG)
-.PHONY: apply-w1
+.PHONY: reboot-all
-apply-w1:
+reboot-all: reboot-cp1 reboot-cp2 reboot-cp3
-	$(TALOSCTL) apply-config \
+	@echo "✓ All 3 control planes rebooting"
 	  --nodes $(W1_IP) \
 	  --file cluster-config/worker-1.yaml
-# First-time apply to worker-1 (no certs yet)
+.PHONY: reboot-cp1
-.PHONY: apply-w1-insecure
+reboot-cp1:
-apply-w1-insecure:
+	$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) reboot
 	$(TALOSCTL) apply-config \
 	  --nodes $(W1_IP) \
 	  --file cluster-config/worker-1.yaml \
 	  --insecure
-# Generic targets — derive both IP and config from N.
+.PHONY: reboot-cp2
-# Usage: make apply-worker N=2 W2_IP=192.168.1.162
+reboot-cp2:
-#        make apply-worker N=3 W3_IP=192.168.1.163
+	$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) reboot
 .PHONY: apply-worker
 apply-worker:
 ifndef N
 	$(error N is not set — run: make apply-worker N=<num> W<N>_IP=<ip>)
 endif
 	$(TALOSCTL) apply-config \
 	  --nodes $(W_IP) \
 	  --file $(W_CONFIG)
-.PHONY: apply-worker-new
+.PHONY: reboot-cp3
-apply-worker-new:
+reboot-cp3:
-ifndef N
+	$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) reboot
 	$(error N is not set — run: make apply-worker-new N=<num> W<N>_IP=<ip>)
 endif
 	$(TALOSCTL) apply-config \
 	  --nodes $(W_IP) \
 	  --file $(W_CONFIG) \
 	  --insecure
 # ── Upgrade ───────────────────────────────────────────────────────────────────
 .PHONY: upgrade-cp
 upgrade-cp:
 	$(TALOSCTL) upgrade \
 	  --nodes $(CP_IP) \
 	  --image $(TALOS_IMAGE) \
 	  --preserve
 .PHONY: upgrade-w1
 upgrade-w1:
 	$(TALOSCTL) upgrade \
 	  --nodes $(W1_IP) \
 	  --image $(TALOS_IMAGE) \
 	  --preserve
 # Usage: make upgrade-worker N=2 W2_IP=192.168.1.162
 .PHONY: upgrade-worker
 upgrade-worker:
 ifndef N
 	$(error N is not set — run: make upgrade-worker N=<num> W<N>_IP=<ip>)
 endif
 	$(TALOSCTL) upgrade \
 	  --nodes $(W_IP) \
 	  --image $(TALOS_IMAGE) \
 	  --preserve
 # ── Shutdown / Reboot ─────────────────────────────────────────────────────────
 # Full cluster: drain workers first so pods stop cleanly, then workers off,
 # then CP last (etcd must be the final process to stop).
 .PHONY: shutdown-cluster
 shutdown-cluster:
 	@echo "--- draining core-worker-1 ---"
 	$(KUBECTL) drain core-worker-1 --ignore-daemonsets --delete-emptydir-data
 	@echo "--- shutting down worker-1 ---"
 	$(TALOSCTL) --nodes $(W1_IP) shutdown
 	@echo "--- shutting down control plane (last) ---"
 	$(TALOSCTL) --nodes $(CP_IP) shutdown
 .PHONY: shutdown-cp
 shutdown-cp:
 	$(TALOSCTL) --nodes $(CP_IP) shutdown
 .PHONY: shutdown-w1
 shutdown-w1:
 	$(TALOSCTL) --nodes $(W1_IP) shutdown
 # Usage: make shutdown-worker N=2 W2_IP=192.168.1.162
 .PHONY: shutdown-worker
 shutdown-worker:
 ifndef N
 	$(error N is not set — run: make shutdown-worker N=<num> W<N>_IP=<ip>)
 endif
 	$(TALOSCTL) --nodes $(W_IP) shutdown
 .PHONY: reboot-cp
-reboot-cp:
+reboot-cp: reboot-all
 	$(TALOSCTL) --nodes $(CP_IP) reboot
 .PHONY: reboot-w1
 reboot-w1:
 	$(TALOSCTL) --nodes $(W1_IP) reboot
 # Usage: make reboot-worker N=2 W2_IP=192.168.1.162
 .PHONY: reboot-worker
@@ -268,18 +189,16 @@ endif
 	$(TALOSCTL) --nodes $(W_IP) reboot
 # ── Inspect ───────────────────────────────────────────────────────────────────
-# Positional args: make node-ls 192.168.1.160 /etc/kubernetes/manifests
+# Positional args: make node-ls 192.168.1.166 /etc/kubernetes/manifests
 # $(word 2/3, $(MAKECMDGOALS)) captures the extra words; the % rule absorbs
 # them so Make doesn't error with "No rule to make target".
 .PHONY: node-ls
 node-ls:
-	$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
+	$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
 .PHONY: node-read
 node-read:
-	$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
+	$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
-# Absorb positional arguments passed to node-ls / node-read
+# Absorb positional arguments
 %:
 	@:
@@ -42,6 +42,86 @@ All logs + metrics centralized in Grafana for debugging
 - **Secrets at rest** — Vault + encrypted etcd; credentials never in logs or ConfigMaps
 - **Infrastructure-as-code** — Every service deployed via Helmfile; one `helmfile apply` recovers from total failure
 ## ArgoCD — GitOps Deployment Flow
 **ArgoCD** pulls infrastructure changes from git and syncs the cluster automatically. 
 No manual `kubectl apply` — push to git, ArgoCD detects the change, and deploys within ~3 minutes.
 ```
 Developer pushes to git
       ↓
 ArgoCD detects change (every 3 min or webhook)
       ↓
 Syncs manifests to cluster
       ↓
 Workloads reconcile automatically
 ```
 Applications are deployed in waves (numbered 00, 10, 20, 30, ...) to respect dependencies — 
 storage deploys before databases, databases before applications.
 ### Tracked Git Repositories
 ArgoCD monitors these repos for changes:
 | Repository | Purpose |
 |------------|----------|
 | `https://github.com/Riotpiaole/riotpiao.homelab.com` | Main infrastructure repo (all manifests in `k8s/argocd/apps/`) |
 | `https://forgejo.riotpiao.com/rock/*` | Any `rock/*` repo in in-cluster Forgejo (apps + configs) |
 | `https://github.com/Riotpiaole/Poimen-*` | External Poimen services (memory, workflows) |
 To deploy a new application: create a git repo, add an Application manifest to the homelab repo's 
 `k8s/argocd/apps/`, commit + push, and ArgoCD syncs within 3 minutes.
 ## Management Planes — Talos vs Kubernetes
 This cluster has **two separate management planes**, each with different workflows:
 | Plane | What it manages | Workflow | Tool |
 |-------|-----------------|----------|------|
 | **Talos (OS)** | Node configuration, kernel params, networking, CoreDNS, machine state | Edit `terraform/` → `terraform apply` → `make apply-cp` | `terraform` + `talosctl` |
 | **Kubernetes (workloads)** | All pods, services, deployments, ingresses, databases | Edit `k8s/argocd/apps/` → `git push` → ArgoCD syncs | `git` + ArgoCD |
 **Critical distinction:**
 - **Kubernetes resources** (`k8s/**`) flow through **git → ArgoCD** — never use `kubectl apply`
 - **Talos machine config** (`terraform/**`) uses **local `terraform apply`** (sanctioned exception — CI can't hold node credentials)
 Example: To add a CoreDNS hostname rewrite, you edit `terraform/files/coredns/Corefile`, then:
 ```bash
 cd terraform && terraform apply -var-file=terraform.tfvars.local
 cd .. && make apply-cp     # talosctl apply-config to all 3 control planes
 ```
 But to add a new Kubernetes Deployment or update an Ingress, you only `git push` — **never `kubectl apply`**.
 ### CoreDNS ConfigMap Ownership — Critical
 ⚠️ **Warning:** The `coredns` ConfigMap in `kube-system` namespace is **owned by Talos**, not ArgoCD or kubectl. 
 It is rendered from `terraform/files/coredns/Corefile` into Talos's machine config at bootstrap time.
 **Do not `kubectl apply` or `kubectl edit` this ConfigMap directly.** Doing so transfers field ownership to kubectl's 
 client-side-apply mechanism, and Talos's inline-manifest controller will silently no-op on every future reconcile 
 (server-side-apply conflict, no error surfaced).
 **To update CoreDNS (e.g., add a hostname rewrite):**
 1. Edit `terraform/files/coredns/Corefile`
 2. Commit + push
 3. Run `cd terraform && terraform apply -var-file=terraform.tfvars.local`
 4. Run `make apply-cp` to push config to all control planes
 5. CoreDNS picks up changes via its `reload` plugin — no pod restart needed
 **If you accidentally edited the ConfigMap directly and broke Talos's ownership:**
 ```bash
 kubectl delete configmap coredns -n kube-system
 # Wait ~30s for Talos's k8s.ManifestApplyController to recreate it
 kubectl get configmap coredns -n kube-system -w
 ```
 Or as a stopgap, apply the correct content yourself:
 ```bash
 kubectl apply --server-side -f <(terraform output coredns_config)
 ```
 ## Quick Start — Deploying the Cluster
 ### 1. Bootstrap Talos Nodes
@@ -60,7 +140,7 @@ Edit `.env` and fill in cluster configuration. See `.env.example` for all option
 ```bash
 # Cluster configuration
-CLUSTER_DOMAIN=riotpiao.homelab.com           # Your cluster domain
+CLUSTER_DOMAIN=riotpiao.com           # Your cluster domain
 POSTGRES_HOST=ddb-cluster-rw.ddb.svc.cluster.local
 MINIO_ENDPOINT=minio.storage.svc.cluster.local:9000
 KAFKA_BOOTSTRAP=kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092
@@ -285,26 +365,39 @@ Add to `/etc/hosts` on every client machine (Mac/Linux):
 ```
 # WireGuard access (remote — via talos-cp-1)
-10.6.0.1  grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com
+10.6.0.1  grafana.riotpiao.com authentik.riotpiao.com vault.riotpiao.com minio.riotpiao.com prometheus.riotpiao.com portainer.riotpiao.com longhorn.riotpiao.com loki.riotpiao.com forgejo.riotpiao.com temporal.riotpiao.com temporal-grpc.riotpiao.com kmsvc.riotpiao.com
 # LAN access (on the home network — use actual LoadBalancer IP from above)
-192.168.1.160  grafana.riotpiao.homelab.com authentik.riotpiao.homelab.com vault.riotpiao.homelab.com minio.riotpiao.homelab.com prometheus.riotpiao.homelab.com portainer.riotpiao.homelab.com longhorn.riotpiao.homelab.com loki.riotpiao.homelab.com forgejo.riotpiao.homelab.com
+192.168.1.160  grafana.riotpiao.com authentik.riotpiao.com vault.riotpiao.com minio.riotpiao.com prometheus.riotpiao.com portainer.riotpiao.com longhorn.riotpiao.com loki.riotpiao.com forgejo.riotpiao.com temporal.riotpiao.com temporal-grpc.riotpiao.com kmsvc.riotpiao.com
 ```
 **Note:** `192.168.1.160` is an example Cilium LB-IPAM assignment. Verify with `kubectl get svc -n ingress-nginx ingress-nginx`.
 **There is no real DNS wildcard for `*.riotpiao.com`** — every hostname must be added to `/etc/hosts` explicitly (as above) before it resolves. Adding a new Ingress host doesn't make it reachable by itself; add the line too.
 ### kubectl Context
 Two contexts exist in `cluster-config/kubeconfig`, pointed at the same cluster over different paths:
 | Context | Server | Use when |
 |---|---|---|
 | `admin@homelab-cluster` | `192.168.1.213:6443` (LAN) | On the home network |
 | `admin@homelab-cluster-1` | `10.6.0.1:6443` (WireGuard) | Remote / off-LAN |
 If `kubectl` commands hang or refuse the connection, switch: `kubectl config use-context admin@homelab-cluster-1`.
 Then access services at:
 | Service | URL | Credentials |
 |---------|-----|-------------|
-| Grafana | http://grafana.riotpiao.homelab.com | admin / `GRAFANA_ADMIN_PASSWORD` or Authentik SSO |
+| Grafana | http://grafana.riotpiao.com | admin / `GRAFANA_ADMIN_PASSWORD` or Authentik SSO |
-| Authentik | http://authentik.riotpiao.homelab.com | akadmin / see `.env` |
+| Authentik | http://authentik.riotpiao.com | akadmin / see `.env` |
-| Vault | http://vault.riotpiao.homelab.com | root token / see `setup_vault.sh` output |
+| Vault | http://vault.riotpiao.com | root token / see `setup_vault.sh` output |
-| MinIO console | http://minio.riotpiao.homelab.com | `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD` |
+| MinIO console | http://minio.riotpiao.com | `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD` |
-| Prometheus | http://prometheus.riotpiao.homelab.com | no auth |
+| Prometheus | http://prometheus.riotpiao.com | no auth |
-| Portainer | http://portainer.riotpiao.homelab.com | set on first visit |
+| Portainer | http://portainer.riotpiao.com | set on first visit |
-| Longhorn | http://longhorn.riotpiao.homelab.com | no auth |
+| Longhorn | http://longhorn.riotpiao.com | no auth |
-| Forgejo *(planned)* | https://forgejo.forge.riotpiao.homelab.com | `rock` / `FORGEJO_ADMIN_PASSWORD`, or Authentik SSO |
+| Forgejo *(planned)* | https://forgejo.forge.riotpiao.com | `rock` / `FORGEJO_ADMIN_PASSWORD`, or Authentik SSO |
 | Argo CD *(planned)* | `kubectl port-forward -n argocd svc/argocd-server 8080:443` | Authentik SSO (admins only) |
 Grafana → "Homelab" folder has the operator dashboards (sidecar-loaded from `k8s/monitoring/dashboards/`, no restart needed on change):
@@ -366,7 +459,7 @@ Authentik is the central OIDC identity provider. Vault stores secrets and delega
       │
       │  OAuth2 / OIDC
       ▼
-  Authentik  (authentik.riotpiao.homelab.com)
+  Authentik  (authentik.riotpiao.com)
  ├── grafana app      → Grafana OIDC login (group → Admin/Viewer role)
  ├── minio app        → MinIO OIDC login (group → readwrite/readonly policy)
  ├── vault-browser    → Vault UI OIDC login / `vault login -method=oidc`
@@ -374,13 +467,13 @@ Authentik is the central OIDC identity provider. Vault stores secrets and delega
       │
       │  JWKS endpoint for JWT validation
       ▼
-  HashiCorp Vault  (vault.riotpiao.homelab.com)
+  HashiCorp Vault  (vault.riotpiao.com)
  ├── auth/jwt   — core-cli authenticates with device code JWT
  ├── auth/oidc  — browser/UI login via Authentik
  └── secret/    — KV v2: mcp/*, cluster/*, cloud/*
 ```
-**talos-cli device code login:**
+**core CLI device code login:**
 ```bash
 core secrets login        # prints URL + code → approve in browser → Vault token cached
 core put cluster/DUCKDNS_TOKEN DUCKDNS_TOKEN="abc"            # field name = variable name, never `value`
@@ -389,8 +482,8 @@ core put cluster/DUCKDNS_TOKEN DUCKDNS_TOKEN="abc"            # field name = var
 **One-time IAM setup (after `helmfile apply`):**
 ```bash
 # 1. Provision OIDC apps and groups in Authentik
-GRAFANA_URL=http://grafana.riotpiao.homelab.com \
+GRAFANA_URL=http://grafana.riotpiao.com \
-MINIO_URL=http://minio.riotpiao.homelab.com \
+MINIO_URL=http://minio.riotpiao.com \
 python3 k8s/talos-iam/provision_oidc.py
 # 2. Init Vault, wire JWT + OIDC auth, seed secrets
@@ -474,7 +567,7 @@ Pods / Talos journal (both nodes)
       │
  Grafana (logging ns)              queries Loki + Prometheus via dashboards
       │
-  Nginx Ingress → grafana.riotpiao.homelab.com   browser access
+  Nginx Ingress → grafana.riotpiao.com   browser access
 ```
 ## Example Applications & Workloads
@@ -1,6 +1,18 @@
 ## Cluster Architecture at a Glance
-**Homelab** is a 2-node bare-metal Kubernetes cluster deployed with Talos Linux, designed for self-hosted services, observability, and GitOps-ready CI/CD.
+**Homelab** is a 3-node bare-metal Kubernetes cluster deployed with Talos Linux, designed for self-hosted services, observability, and GitOps-ready CI/CD.
 ### Node Topology (3 control-plane HA, since 2026-07-20)
 | Node | IP | Zone | Role | Scheduling | Storage |
 |------|----|----|------|-----------|---------|
 | `talos-cp-1` | 192.168.1.213 | az-a | control-plane | **schedulable** (runs all workloads) | sole Longhorn node (sdb/sdc/sdd) |
 | `talos-cp-2` | 192.168.1.163 | az-b | control-plane | dedicated (`NoSchedule`) | none |
 | `talos-cp-3` | 192.168.1.166 | az-c | control-plane | dedicated (`NoSchedule`) | none |
 - **etcd** has 3 voting members peering over the LAN (`cluster.etcd.advertisedSubnets: 192.168.1.0/24` — without it Talos may advertise on the WireGuard IP and new members hang as learners). Tolerates 1 node loss.
 - Only `talos-cp-1` runs workloads and holds storage, so stateful services are **single-instance** (e.g. CNPG `ddb-cluster` = 1 instance). The kube-apiserver endpoint is single-homed to `.213` (no VIP yet).
 - Scheduling is declarative: `allowSchedulingOnControlPlanes: true` + per-node `machine.nodeTaints` re-adds the control-plane taint on the dedicated nodes only.
 ### Deployment Stack (18 Helm releases)
@@ -12,7 +24,7 @@
 | **Certificates** | cert-manager + homelab-ca | cert-manager | Self-signed CA, auto-renewal |
 | **Storage (Block)** | Longhorn v1.7.0 | longhorn-system | Persistent volumes, default StorageClass |
 | **Storage (Object)** | MinIO (3-node, site-repl) | storage | S3-compatible, multi-AZ replication |
-| **Database** | CloudNativePG (3 replicas) | ddb | PostgreSQL 16 + pgvector |
+| **Database** | CloudNativePG (1 instance) | ddb | PostgreSQL 16 + pgvector (single-node; see topology) |
 | **IAM / OIDC** | Authentik | iam | Federated OIDC provider for all services |
 | **Secrets** | HashiCorp Vault | iam | KV secrets backend, JWT auth |
 | **Logs** | Loki (SingleBinary) | logging | 10-day retention, MinIO backend |
@@ -73,101 +85,252 @@ sqs (Kafka + Message Queue)
 ---
-## Custom CLI — `talos`
+## Custom CLI — `core`
-Homelab cluster control CLI (`core/`). Manages cluster nodes and Vault secrets.
+Homelab cluster control CLI (source: `~/workplace/core/`). Manages cluster nodes, Authentik IAM, and Vault secrets.
-### Secret path convention
+**Setup:**
-
+```bash
-All secrets live under `cluster/<VARIABLE_NAME>`. The field name is always the variable name itself (SCREAMING_SNAKE_CASE), matching the `.env` key. Example paths:
+make cli                     # builds + installs to ~/.local/bin/core
-
+core auth login-oob         # authenticate with Authentik (OOB flow)
-```
+core nodes                  # verify cluster access
 cluster/ANTHROPIC_API_KEY
 cluster/AUTHENTIK_FORGEJO_CLIENT_ID
 cluster/AUTHENTIK_ARGOCD_CLIENT_SECRET
 ```
-### `talos put` — write a secret to Vault
+**Full documentation:** See [~/workplace/core/USAGE.md](../core/USAGE.md)
 ---
 ### Authentication
 #### OAuth2 Out-of-Band (OOB) Login
 Browser-based login with manual code entry (recommended).
 ```bash
-talos put cluster/VARIABLE_NAME VARIABLE_NAME="secret-value"
+export CORE_CLI_SECRET="cyoVr96FB9oeq3o64DUG0BmoVzMPsOTIWxVEd8ZdTezKrEZYwrKpIRkOwrDQNEtF6QJyNUPH4mjr9jWokQY7KVBWX1KVUXFyyhgAHTqWRRWAYUJ8r3H35wLFiTfn5KsV"
-talos put cluster/FORGEJO_ADMIN_PASSWORD FORGEJO_ADMIN_PASSWORD="$FORGEJO_ADMIN_PASSWORD"
+core auth login-oob
 ```
-Field name = variable name — never `value`.
+Token expires in 8 hours. Check status:
 ```bash
 core auth status   # show expiry
 core auth clear    # force re-auth on next command
 ```
-### `talos get` — fetch a secret from Vault
+---
 ### Cluster Management
 ```bash
-talos get cluster/VARIABLE_NAME --key VARIABLE_NAME        # always specify --key
+core nodes                    # list cluster nodes
-talos get cluster/FORGEJO_ADMIN_PASSWORD --key FORGEJO_ADMIN_PASSWORD
+core status <ip>              # Talos node overview
-talos get cluster/VARIABLE_NAME --json                     # full secret as JSON
+core services <ip>            # list Talos services
 core logs <ip>                # stream kubelet logs
 core log-svc <ip> <svc>       # logs for specific service (etcd, kubelet, etc.)
 core pods clean               # delete Failed/Evicted/Terminating pods
 ```
-Note: `talos get` uses `--key` (long flag), not a positional arg — unlike `talos secrets get`.
+#### kubectl Context (LAN vs. WireGuard)
-### `vsource` — load a `.env` into the shell
+`cluster-config/kubeconfig` has two contexts pointed at the same cluster:
-
+`admin@homelab-cluster` (LAN, `192.168.1.213:6443`) and `admin@homelab-cluster-1`
-zsh function (lives in `~/.zshrc`, not in the repo — can reference but cannot run directly).
+(WireGuard, `10.6.0.1:6443`). If `kubectl`/`core nodes` hangs or refuses the
-Empty `.env` values are fetched from Vault at `cluster/<KEY>`; hardcoded values pass through.
+connection, you're likely off-LAN — switch contexts:
 ```zsh
 vsource            # loads .env in current directory
 vsource .env.local # loads a specific file
 ```
 `.env` format — leave secrets empty, vsource resolves them from Vault:
 ```bash
-ANTHROPIC_API_KEY=          # fetched from cluster/ANTHROPIC_API_KEY
+core config kube-list             # list contexts
-AUTHENTIK_ARGOCD_CLIENT_ID= # fetched from cluster/AUTHENTIK_ARGOCD_CLIENT_ID
+core config kube-use admin@homelab-cluster-1   # switch to WireGuard path
 DEBUG=true                  # hardcoded, passed through as-is
 ```
-### Typical workflow for a generated secret
+**Known gap:** `core config use <talos-context>` (the combined talosctl+kubectl
 switch) only maps to `admin@homelab-cluster` today — its WireGuard mapping
 (`home-cluster-wire-guard`) is stale, that kubectl context doesn't exist. Use
 `core config kube-use admin@homelab-cluster-1` directly until that's fixed.
 ---
 ### Secret Management (Vault)
 All secrets live under `cluster/<VARIABLE_NAME>`. Field name = variable name (SCREAMING_SNAKE_CASE).
 #### Write Secret to Vault
 ```bash
-# 1. Store immediately after generation (keeps secrets out of shell history)
+core put cluster/ANTHROPIC_API_KEY ANTHROPIC_API_KEY="sk-ant-..."
-talos put cluster/AUTHENTIK_FORGEJO_CLIENT_SECRET AUTHENTIK_FORGEJO_CLIENT_SECRET="<paste>"
+core put cluster/FORGEJO_ADMIN_PASSWORD FORGEJO_ADMIN_PASSWORD="secret123"
 # 2. Use via subshell when creating K8s secrets
 kubectl create secret generic my-secret \
  --from-literal=client-secret="$(talos get cluster/AUTHENTIK_FORGEJO_CLIENT_SECRET --key AUTHENTIK_FORGEJO_CLIENT_SECRET)"
 # 3. Or load into shell via vsource for helmfile/env-driven tools
 vsource .env && helmfile apply
 ```
-### IAM Management (Federated OIDC, Phases 1–6 Complete)
+**Key rule:** Field name must match variable name — never `value=`.
-**Status:** ✅ Fully deployed (2026-07-02). Single federated OIDC provider (`talos-federation`) handles all service auth.
+#### Fetch Secret from Vault
 **Quick reference:**
 ```bash
-# View roles and capabilities
+core get cluster/ANTHROPIC_API_KEY              # default field
-talos iam roles list && talos iam roles describe admin
+core get cluster/AUTHENTIK_ARGOCD_CLIENT_SECRET # full value
 # Service registry (Grafana, MinIO, Forgejo, etc.)
 talos iam services list && talos iam services describe grafana
 # Agents (admin-bot, ci-bot with auto-rotation)
 talos iam agents list && talos iam agents rotate ci-bot
 # Role bindings (user → role with TTL)
 talos iam bindings grant [email protected] devops --expires 2026-12-31
 talos iam bindings list
 # Audit trail (90-day retention, 12 event types)
 talos iam audit list && talos iam audit export --format json
 # OIDC provider sync with Authentik
 talos iam providers sync-authentik
 ```
-**See `homelab/CLAUDE.md` § IAM Management for full reference** (roles, services, agents, bindings, audit, providers).
+#### List All Secrets
-**Vault paths:** All IAM state stored under `cluster/iam/{federation,roles,services,agents,bindings}`.
+```bash
 core secrets list
 ```
 #### Load into Shell (vsource)
 zsh function in `~/.zshrc` — fetches empty `.env` values from Vault:
 ```bash
 # .env format
 ANTHROPIC_API_KEY=              # fetched from Vault
 AUTHENTIK_ARGOCD_CLIENT_ID=     # fetched from Vault
 DEBUG=true                      # hardcoded, passed through
 # Usage
 vsource                         # loads .env in current dir
 vsource .env.prod               # loads specific file
 eval "$(vsource .env)" && helmfile apply   # inject + deploy
 ```
 ---
 ### IAM Management (Authentik)
 Manage OAuth2 applications, groups, and user access via Authentik.
 #### List Groups
 ```bash
 core iam list-groups
 ```
 Output:
 ```
 authentik Admins (id: 9d72cbf2-9d52-4d3c-bba9-0068525d7a91)
 grafana-admins (id: 5f2e1e79-7d7e-4ef0-9b99-3c6df19c0b88)
 minio-admins (id: e640d887-eb85-431b-b5b2-5b6a8a7e0a44)
 argocd-admins (id: 22a3c296-0d98-433f-8456-ebc2f1c8d489)
 forgejo-admins (id: 4084c8d6-0c12-46af-acf8-7372172b9016)
 ```
 #### List OAuth2 Applications
 ```bash
 core iam list-apps
 ```
 #### Create New Application
 ```bash
 core iam create-app "my-service" --slug my-service --redirect-uri "https://my-service.riotpiao.com/callback"
 ```
 Returns client ID and secret (save immediately).
 #### Describe Application
 ```bash
 core iam describe-app grafana
 ```
 Shows:
 - Client ID
 - Client Secret
 - Redirect URIs
 - Scope claims
 #### Bind Group to Application
 ```bash
 core iam bind-app grafana grafana-admins
 ```
 Members of `grafana-admins` can log in to Grafana via OIDC.
 #### Create Group
 ```bash
 core iam create-group "developers"
 ```
 #### Add User to Group
 ```bash
 core iam add-member grafana-admins newuser
 ```
 #### Rotate Application Secret
 ```bash
 core iam rotate-secret grafana
 ```
 ⚠️ Must update deployment after rotating.
 ---
 ### Port Forwarding
 ```bash
 core pf grafana         # localhost:3000 → Grafana
 core pf prometheus      # localhost:9090 → Prometheus
 core pf minio           # localhost:9001 → MinIO console
 core pf iam             # localhost:7000 → Authentik
 ```
 ---
 ### Workflow: Rotate OAuth2 Secret
 ```bash
 # 1. Rotate in Authentik
 SECRET=$(core iam rotate-secret grafana | jq -r '.client_secret')
 # 2. Update deployment
 vi k8s/logging/grafana-values.yaml
 # Set: GRAFANA_OIDC_CLIENT_SECRET="$SECRET"
 # 3. Redeploy
 helmfile apply -l app=grafana
 # 4. Verify
 core iam describe-app grafana
 ```
 ---
 ### Workflow: Add User to Service
 ```bash
 # 1. Create or verify group exists
 core iam list-groups | grep minio-admins
 # 2. Add user to group
 core iam add-member minio-admins alice
 # 3. Verify
 # (User will have access next login via OIDC)
 ```
 ---
 ### Vault Integration (Advanced)
 Vault paths for IAM state (if using federated OIDC):
 ```
 cluster/iam/federation/
 cluster/iam/roles/
 cluster/iam/services/
 cluster/iam/agents/
 cluster/iam/bindings/
 ```
 Query via:
 ```bash
 core get cluster/iam/roles/admin --key roles
 ```
 **See `CLAUDE.md` § IAM for full architecture** (roles, services, agents, audit).
 ### CI/CD Image Registry Authentication (Forgejo + Runner)
@@ -177,16 +340,16 @@ talos iam providers sync-authentik
 ```bash
 # 1. Get ci-bot JWT token (runner has this injected via ServiceAccount)
-export REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token)
+export REGISTRY_TOKEN=$(core get cluster/iam/agents/ci-bot --key token)
 # 2. Authenticate docker/podman to Forgejo registry
-docker login forgejo.riotpiao.homelab.com \
+docker login forgejo.riotpiao.com \
  --username ci-bot \
  --password "$REGISTRY_TOKEN"
 # 3. Tag and push image
-docker tag myapp:latest forgejo.riotpiao.homelab.com/rock/myapp:latest
+docker tag myapp:latest forgejo.riotpiao.com/rock/myapp:latest
-docker push forgejo.riotpiao.homelab.com/rock/myapp:latest
+docker push forgejo.riotpiao.com/rock/myapp:latest
 ```
 **Pull images in runner (automatic):**
@@ -194,7 +357,7 @@ docker push forgejo.riotpiao.homelab.com/rock/myapp:latest
 ```bash
 # Inside .forgejo/workflows/*.yml, runner pulls via K8s ServiceAccount
 # No explicit login needed — imagePullSecrets injected by runner pod
-image: forgejo.riotpiao.homelab.com/rock/myapp:latest
+image: forgejo.riotpiao.com/rock/myapp:latest
 ```
 **Runner pod setup:**
@@ -1,324 +0,0 @@
 # Flux CD Integration Planning — START HERE
 ## What Just Happened?
 Your subagent completed **comprehensive planning documentation** for integrating Flux CD v2 with your homelab's helmfile-based infrastructure.
 **Three complete documents created:**
 1. **FLUX_INTEGRATION_PLAN.md** (1,810 lines)
   - Full technical specification with code examples
   - Phase-by-phase implementation roadmap
   - Conflict resolution & safety procedures
   - Testing strategy & risk assessment
 2. **FLUX_PLANNING_SUMMARY.md** (351 lines)
   - Executive overview for stakeholders
   - Decision matrices & quick reference
   - Timeline & effort estimates
   - Success metrics
 3. **FLUX_PLANNING_INDEX.md** (356 lines)
   - Navigation guide across all documents
   - Quick start for different audiences
   - FAQ & next steps
 **Total:** 2,517 lines of planning documentation
 ---
 ## The Plan in 60 Seconds
 ### What Problem Are We Solving?
 Current helmfile workflow:
 - Manual `helmfile apply` required
 - No automatic drift detection
 - No Git audit trail for changes
 - No approval gates
 - Hard to scale to multi-cluster
 ### What's the Solution?
 Deploy **Flux CD v2** (GitOps) to:
 - Continuously reconcile cluster state from Git
 - Auto-detect & correct drift
 - Maintain full audit trail
 - Support staged rollouts with approval gates
 - Keep helmfile.yaml.gotmpl as fallback during transition
 ### How Do We Do It?
 **3 phases, 6–8 weeks, ~99 hours:**
 | Phase | Timeline | Work | Goal |
 |-------|----------|------|------|
 | **1** | Weeks 1–2 | Bootstrap Flux + helmfile bridge | Zero breaking changes |
 | **2** | Weeks 3–6 | Migrate 23 releases to HelmRelease CRDs | Parallel migration (4 streams) |
 | **3** | Weeks 7–8 | Enable auto-sync, metrics, runbooks | Full GitOps readiness |
 **Key:** No downtime. Helmfile stays functional as fallback throughout.
 ---
 ## Architecture Simplified
 ```
 Git (Forgejo) ← Source of Truth
  └─→ Flux Reconciliation Loop (every 5 min)
      └─→ Kubernetes Cluster
          └─→ 23 Helm Releases (reconciled state)
 ```
 That's it. Flux watches Git. When you push changes, Flux applies them. If someone manually changes the cluster (kubectl), Flux auto-corrects on next reconciliation.
 ---
 ## Key Decisions (No Surprises)
 | Decision | Choice | Reasoning |
 |----------|--------|-----------|
 | **Controller** | Flux v2 | Stable, battle-tested; v3 still beta |
 | **Helm** | HelmRelease CRDs | Preserves values-based workflow |
 | **Secrets** | SOPS + age | Git-stored, audited, simple |
 | **Rollout** | Phased (3×8 weeks) | Lower risk, easier debugging |
 All decisions explained in detail in FLUX_INTEGRATION_PLAN.md §3 (Architecture Decision Matrix).
 ---
 ## What You Get
 ### By End of Phase 1 (Week 2)
 - ✅ Flux running in cluster
 - ✅ Git syncing every 60 seconds
 - ✅ Helmfile still works as fallback
 - ✅ Zero disruption to running workloads
 ### By End of Phase 2 (Week 6)
 - ✅ All 23 releases migrated to Git-based HelmRelease CRDs
 - ✅ Helmfile no longer used for deployments
 - ✅ Every release tested & verified
 - ✅ Full test suite in place
 ### By End of Phase 3 (Week 8)
 - ✅ Automatic reconciliation enabled
 - ✅ Drift detection + alerting working
 - ✅ Metrics flowing to Prometheus
 - ✅ Team trained on GitOps workflows
 - ✅ RTO < 2 hours (restore from Git if needed)
 ---
 ## How to Read the Documentation
 ### Quick Overview (10 min)
 → **Read:** FLUX_PLANNING_SUMMARY.md
 Start here to understand what we're doing and why. Tables, diagrams, high-level summary. Perfect for stakeholder presentations.
 ### Getting Ready to Build (1 hour)
 → **Read:** FLUX_PLANNING_INDEX.md + FLUX_INTEGRATION_PLAN.md (Executive Summary)
 Learn the full architecture, decision rationale, and how phases fit together.
 ### Phase 1 Implementation (Week 1–2)
 → **Reference:** FLUX_INTEGRATION_PLAN.md §5.1 (Phase 1: Flux Bootstrap)
 Detailed tasks:
 - 1.1: Bootstrap Flux into cluster
 - 1.2: Create Git repo structure
 - 1.3: HelmRepository CRDs (13 repos)
 - 1.4: SOPS + age setup
 - 1.5: Helmfile-bridge CronJob
 ### Phase 2 Migration (Weeks 3–6)
 → **Reference:** FLUX_INTEGRATION_PLAN.md §5.2 (Phase 2: HelmRelease Migration)
 Four parallel streams:
 - Stream A: Low-risk (reloader, prometheus)
 - Stream B: Medium-risk (cert-manager, ingress)
 - Stream C: High-risk secrets (authentik, vault)
 - Stream D: Complex stateful (minio, forgejo)
 Per-release process: generate CRD → validate → deploy → test → commit
 ### Phase 3 Production Readiness (Weeks 7–8)
 → **Reference:** FLUX_INTEGRATION_PLAN.md §5.3 (Phase 3: Continuous Reconciliation)
 Auto-sync, metrics, runbooks, team training.
 ### Troubleshooting & Rollback
 → **Reference:** FLUX_INTEGRATION_PLAN.md §7 (Rollback & Safety Guardrails)
 How to recover if something breaks:
 - Suspend Flux + manual rollback
 - Git revert + auto-reconciliation
 - Disaster recovery from Git
 ### Testing Strategy
 → **Reference:** FLUX_INTEGRATION_PLAN.md §8 (Testing Strategy)
 Unit tests, integration tests, chaos tests, production deployment strategy.
 ---
 ## Risk Summary
 ### Main Risks & How We Handle Them
 | Risk | Mitigation |
 |------|-----------|
 | **Flux + helmfile conflict** | Stagger reconciliation (helmfile 30min, Flux 5min) |
 | **Secret injection breaks** | Three-tier approach (SOPS + ConfigMaps + .env fallback) |
 | **Secrets leak in Git** | SOPS encryption from start + pre-commit hooks |
 | **Cluster recovery fails** | Keep helmfile as fallback; test quarterly |
 All risks detailed with specific mitigations in FLUX_INTEGRATION_PLAN.md §9 (Risk Assessment).
 ---
 ## Timeline Reality Check
 ```
 Week 1–2:  Phase 1 bootstrap (20 hrs)
           ├─ 1 DevOps engineer + 1 Security engineer
           └─ 0 downtime to running workloads
 Week 3–6:  Phase 2 migration (40 hrs)
           ├─ 4 parallel streams (DevOps + Ops + Security)
           └─ Release-by-release (low risk)
 Week 7–8:  Phase 3 hardening (16 hrs)
           ├─ DevOps + QA
           └─ Runbooks + training
 Total:     ~99 hours (~2.5 FTE-weeks)
           6–8 calendar weeks (with parallelization)
 ```
 Actual timeline depends on:
 - Team size (4 engineers = 8 weeks; 2 engineers = 12 weeks)
 - Experience with Flux (learning curve ~40 hours)
 - Testing rigor (each phase adds 1–2 weeks)
 ---
 ## Next Actions
 ### Immediately (Today)
 1. **Review FLUX_PLANNING_SUMMARY.md** (15 min)
   - Understand the approach
   - Check decision matrix
   - Confirm timeline is acceptable
 2. **Share with stakeholders**
   - Security team: review SOPS approach
   - Ops team: review rollback procedures
   - Management: confirm timeline & resources
 3. **Get approval** for:
   - Phased approach (6–8 weeks)
   - Flux v2 + HelmRelease CRDs
   - SOPS encryption for secrets
   - ~99 hours effort
 ### Week 1 (Phase 1 Kickoff)
 1. **Assign team members**
   - DevOps lead
   - Security engineer (SOPS)
   - Ops engineer (testing)
 2. **Bootstrap Flux**
   - `flux bootstrap git` command
   - Set up Git repo structure
   - Deploy HelmRepository CRDs
 3. **Start helmfile-bridge development**
   - CronJob to run `helmfile apply` every 30 min
   - Test alongside Flux (staggered intervals)
 ### Weeks 3–8 (Phases 2 & 3)
 Follow the phase roadmap in FLUX_INTEGRATION_PLAN.md with weekly syncs.
 ---
 ## Files Created
 All in `/Users/rockliang/workplace/homelab/`:
 1. **FLUX_INTEGRATION_PLAN.md** (55 KB)
   - Complete technical specification
   - Phase-by-phase breakdown
   - Code examples & detailed procedures
 2. **FLUX_PLANNING_SUMMARY.md** (13 KB)
   - Executive overview
   - Decision matrices
   - Quick reference tables
 3. **FLUX_PLANNING_INDEX.md** (13 KB)
   - Navigation guide
   - Quick start by audience
   - FAQ & related docs
 4. **_FLUX_START_HERE.md** (this file)
   - Quick orientation
   - Next actions
 ---
 ## Questions to Ask
 Before Phase 1 starts, clarify:
 1. **Team capacity?** How many FTE can we dedicate?
   - 4 FTE → 8 weeks
   - 2 FTE → 12 weeks
 2. **Timeline flexibility?** Hard deadline or can we adjust?
   - If hard: compress with more parallel streams
   - If flexible: add more testing/validation
 3. **Flux experience on team?** Anyone used Flux before?
   - If no: add 1–2 weeks for learning curve
   - If yes: can reduce onboarding time
 4. **Multi-cluster plans?** Will you add more clusters after homelab?
   - If yes: design for portability from start
   - If no: homelab-specific is fine
 5. **SOPS comfort?** Any concerns about secret encryption in Git?
   - If yes: alternative is store in Vault (referenced from HelmRelease)
   - If no: SOPS is recommended
 ---
 ## Document Quality Checklist
 The planning documentation includes:
 - ✅ **Executive summary** — problem & solution in 1 page
 - ✅ **Current state analysis** — what we're migrating from
 - ✅ **Architecture decisions** — Flux v2, HelmRelease, SOPS (with reasoning)
 - ✅ **Detailed design** — GitRepository, Kustomization, HelmRelease CRDs
 - ✅ **3-phase roadmap** — specific tasks, timelines, deliverables, success criteria
 - ✅ **Conflict resolution** — helmfile + Flux, .env → SOPS, kubectl drift
 - ✅ **Rollback procedures** — what to do if something breaks
 - ✅ **Safety guardrails** — RBAC, audit logging, validation webhooks, approval gates
 - ✅ **Testing strategy** — unit, integration, chaos, production deployment
 - ✅ **Risk assessment** — probability, impact, mitigation for each risk
 - ✅ **Timeline & effort** — 99 hours, 6-8 weeks, team composition
 - ✅ **Useful commands** — Flux CLI cheatsheet
 - ✅ **FAQ** — downtime, rollback, recovery, cost
 Ready for review and implementation kickoff.
 ---
 **Status:** Planning phase complete. Ready for team discussion & approval.
 **Next:** Review FLUX_PLANNING_SUMMARY.md, approve approach, assign Phase 1 team.
@@ -0,0 +1,269 @@
 #!/usr/bin/env bash
 #
 # Phase-0 bootstrap — bring a bare Talos cluster to a self-hosted GitOps control
 # plane, breaking the ArgoCD <-> Forgejo circle via a GitHub seed + cutover.
 # See docs/adr/0001-gitops-bootstrap-and-cd.md (Part A) and docs/plans/0001-EXECUTION.md.
 #
 # Order (all manual, once): Cilium -> Longhorn -> CNPG operator -> forgejo-db
 # (wait Ready) -> Forgejo -> ArgoCD (seeded from GitHub) -> cutover to Forgejo.
 # Everything ELSE is deployed by ArgoCD from the seed repo, in sync-wave order.
 #
 # Prereqs:
 #   - Talos cluster up; kubectl context points at it
 #   - helm 3, kubectl
 #   - SOPS age key at $SOPS_KEY (for the ArgoCD SOPS CMP plugin)
 #
 # The GitHub seed repo is public, so it is cloned anonymously over HTTPS — no
 # deploy key, no repository Secret, one less thing to bootstrap before ArgoCD.
 #
 set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 BOOT="$SCRIPT_DIR/k8s/bootstrap"
 SOPS_KEY="${SOPS_KEY:-$HOME/.sops/key.txt}"
 log()  { echo "[$(date +%H:%M:%S)] $*"; }
 die()  { echo "ERROR: $*" >&2; exit 1; }
 phase(){ echo; echo "━━━ $* ━━━"; echo; }
 # Idempotent helm repo setup
 ensure_helm_repo() {
  local name=$1 url=$2
  helm repo list 2>/dev/null | grep -q "^$name" || helm repo add "$name" "$url" >/dev/null
  helm repo update "$name" >/dev/null 2>&1 || true
 }
 preflight() {
  log "preflight…"
  kubectl cluster-info >/dev/null || die "kubectl not configured / cluster unreachable"
  command -v helm >/dev/null || die "helm 3 not found"
  [[ -f "$SOPS_KEY" ]]   || die "SOPS age key missing at $SOPS_KEY"
  log "✅ preflight ok"
 }
 p1_cilium() {
  phase "PHASE 1a: CNI (Cilium)"
  if kubectl -n kube-system get ds cilium >/dev/null 2>&1; then log "cilium present, skip"; return; fi
  ensure_helm_repo cilium https://helm.cilium.io
  helm install cilium cilium/cilium -n kube-system \
    --set ipam.mode=kubernetes --set kubeProxyReplacement=true --wait --timeout 10m
  log "✅ cilium installed"
 }
 p1_longhorn() {
  phase "PHASE 1b: STORAGE (Longhorn)"
  # Always ensure namespace + StorageClasses (idempotent, resumable)
  kubectl apply -f "$BOOT/phase1-storage/namespace.yaml"
  # Install Longhorn if not present
  if ! helm -n longhorn-system list 2>/dev/null | grep -q longhorn; then
    ensure_helm_repo longhorn https://charts.longhorn.io
    log "Installing Longhorn storage (this may take 5-10 minutes)..."
    if helm install longhorn longhorn/longhorn -n longhorn-system \
      --values "$BOOT/phase1-storage/longhorn-values.yaml" --wait --timeout 10m; then
      log "✅ Longhorn installed"
    else
      log "⚠️  Helm install failed, but continuing to ensure resources..."
    fi
  fi
  # Always apply StorageClasses (even if helm install partially failed)
  kubectl apply -f "$BOOT/phase1-storage/storageclasses.yaml"
  # Verify critical components (resumable check)
  if kubectl -n longhorn-system wait --for=condition=available --timeout=300s deploy/longhorn-manager 2>/dev/null; then
    log "✅ longhorn installed"
  else
    log "⚠️  longhorn-manager not ready yet, but StorageClasses applied. Re-run to verify."
  fi
 }
 p1_ingress() {
  phase "PHASE 1c: INGRESS (Nginx Ingress Controller)"
  # Install nginx-ingress if not present
  if kubectl get ingressclass nginx >/dev/null 2>&1; then
    log "nginx IngressClass present, skip install"
    return
  fi
  # Always ensure namespace with PodSecurity labels (idempotent)
  kubectl apply -f "$BOOT/ingress/namespace.yaml"
  ensure_helm_repo ingress-nginx https://kubernetes.github.io/ingress-nginx
  log "Installing nginx-ingress controller (this may take 2-3 minutes)..."
  if helm install ingress-nginx ingress-nginx/ingress-nginx -n ingress-nginx \
    --values "$BOOT/ingress/nginx-values.yaml" --timeout 5m; then
    log "✅ nginx-ingress installed"
  else
    log "❌ nginx-ingress install failed"
    return 1
  fi
  # Apply additional ingress resources (cert, ingress rules)
  log "Applying ingress manifests (ignoring cert-manager CRD errors)..."
  kubectl apply -k "$BOOT/ingress/" 2>&1 | grep -v "no matches for kind" || true
  log "✅ Ingress resources applied (cert-manager resources will be created by ArgoCD)"
 }
 p2_cnpg() {
  phase "PHASE 2: CNPG OPERATOR"
  if kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then log "cnpg CRD present, skip install"; return; fi
  ensure_helm_repo cnpg https://cloudnative-pg.github.io/charts
  log "Installing CloudNativePG operator (this may take 2-3 minutes)..."
  if helm install cnpg cnpg/cloudnative-pg -n cnpg-system --create-namespace \
    --values "$BOOT/phase2-cnpg/cnpg-values.yaml" --wait --timeout 5m; then
    log "✅ CNPG operator installed"
  else
    log "❌ CNPG operator install failed"
    return 1
  fi
  kubectl get crd clusters.postgresql.cnpg.io >/dev/null || die "CNPG CRD not registered"
  log "✅ cnpg operator installed"
 }
 p3_forgejo() {
  phase "PHASE 3: forgejo-db + Forgejo (ns cicd)"
  # Always ensure namespace + NetworkPolicy + Secrets (idempotent)
  kubectl apply -f "$BOOT/phase3-forgejo/namespace.yaml"
  # Clean up old Valkey NetworkPolicy if it exists (from bundled chart)
  kubectl delete networkpolicy forgejo-valkey-cluster -n cicd 2>/dev/null || true
  # Apply CNPG-specific NetworkPolicy
  kubectl apply -f "$BOOT/phase3-forgejo/cnpg-networkpolicy.yaml"
  # Create Forgejo admin secret (bootstrap-time only, before ArgoCD exists)
  # In GitOps mode, ArgoCD will sync the SOPS-encrypted version from git
  if ! kubectl get secret forgejo-admin -n cicd >/dev/null 2>&1; then
    log "Creating forgejo-admin secret from .env (bootstrap mode)"
    [ -f "$HOME/workplace/homelab/.env" ] && source "$HOME/workplace/homelab/.env"
    kubectl -n cicd create secret generic forgejo-admin \
      --from-literal=username=rock \
      --from-literal=password="${FORGEJO_ADMIN_PASSWORD}" \
      --from-literal=email=[email protected]
  else
    log "forgejo-admin secret exists, skip (managed by ArgoCD in GitOps mode)"
  fi
  # Check if forgejo-db cluster exists and is Ready
  if kubectl get cluster forgejo-db -n cicd >/dev/null 2>&1; then
    if kubectl get cluster forgejo-db -n cicd -o jsonpath='{.status.phase}' 2>/dev/null | grep -q "Cluster in healthy state"; then
      log "forgejo-db already Ready, skip wait"
    else
      log "forgejo-db exists but not Ready, waiting for all 3 instances (up to 30 min)…"
      if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
        log "✅ forgejo-db cluster is Ready"
      else
        log "❌ forgejo-db cluster failed to become Ready"
        return 1
      fi
    fi
  else
    log "creating forgejo-db cluster (3 instances)"
    kubectl apply -f "$BOOT/phase3-forgejo/forgejo-db.yaml"
    log "Waiting for all 3 CNPG instances to be Ready (up to 30 min)…"
    if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
      log "✅ forgejo-db cluster is Ready"
    else
      log "❌ forgejo-db cluster failed to become Ready"
      return 1
    fi
  fi
  kubectl -n cicd get secret forgejo-db-app >/dev/null || die "CNPG did not create forgejo-db-app secret"
  # Install Forgejo if not present
  if helm -n cicd list 2>/dev/null | grep -q forgejo; then log "forgejo helm release present, skip"; return; fi
  ensure_helm_repo forgejo https://dl.gitea.io/charts/
  log "Installing Forgejo (this may take 10-15 minutes on slow nodes)..."
  if helm install forgejo forgejo/gitea -n cicd \
    --values "$BOOT/phase3-forgejo/forgejo-values.yaml" --wait --timeout 10m; then
    log "✅ Forgejo installed"
  else
    log "❌ Forgejo install failed"
    return 1
  fi
  log "Forgejo is up — now push this repo to Forgejo and configure the GitHub pull-mirror"
 }
 p4_argocd() {
  phase "PHASE 4: ArgoCD (seeded from GitHub)"
  # Always ensure namespace (idempotent). The seed repo is public — ArgoCD clones
  # it anonymously over HTTPS, so there is no repository Secret to create.
  kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
  # Decrypt and apply any encrypted secrets from bootstrap dir (local SOPS)
  if command -v sops &> /dev/null; then
    export SOPS_AGE_KEY_FILE="$SOPS_KEY"
    log "Decrypting encrypted secrets with local SOPS..."
    local decrypted_count=0
    for enc_file in "$BOOT"/phase*/**.enc.yaml; do
      [ -f "$enc_file" ] || continue
      log "  → Decrypting $(basename "$enc_file")..."
      if sops -d "$enc_file" | kubectl apply -f -; then
        decrypted_count=$((decrypted_count + 1))
        log "    ✅ Applied"
      else
        log "    ⚠️  Failed (may already exist)"
      fi
    done
    log "Decrypted and applied $decrypted_count secret(s)"
  else
    log "⚠️  SOPS not installed, skipping encrypted secret decryption"
  fi
  # Install ArgoCD if not present
  if ! helm -n argocd list 2>/dev/null | grep -q argocd; then
    ensure_helm_repo argo https://argoproj.github.io/argo-helm
    log "Installing ArgoCD via Helm (installing chart, pods will start afterward)..."
    if helm install argocd argo/argo-cd -n argocd \
      --values "$BOOT/phase4-argocd/argocd-values.yaml" --timeout 10m; then
      log "✅ ArgoCD Helm release installed (pods starting...)"
    else
      log "❌ ArgoCD Helm install failed"
      return 1
    fi
  else
    log "ArgoCD Helm release already exists, skipping install"
  fi
  # Wait for server ready (resumable - slow on talos-cp-2)
  log "Waiting for argocd-server deployment to be available (max 10 minutes)..."
  if kubectl -n argocd wait --for=condition=available --timeout=600s deploy/argocd-server; then
    log "✅ argocd-server is available"
  else
    log "❌ argocd-server failed to become available within 10 minutes"
    log "Check pods: kubectl get pods -n argocd"
    return 1
  fi
  # Always apply root app (idempotent)
  kubectl apply -f "$BOOT/phase4-argocd/root-app-github.yaml"
  log "✅ ArgoCD syncing from GitHub seed. Watch: kubectl get applications -n argocd"
  log "NOTE: SOPS CMP plugin not installed yet (bootstrap uses local SOPS decryption)."
  log "      To add SOPS plugin for GitOps, see k8s/bootstrap/phase4-argocd/argocd-cmp-cm.yaml"
 }
 p5_cutover() {
  phase "PHASE 5: CUTOVER GitHub -> Forgejo"
  read -rp "Forgejo healthy AND mirroring GitHub? (y/N) " r; [[ $r =~ ^[Yy]$ ]] || die "push+mirror to Forgejo first"
  kubectl apply -f "$BOOT/phase5-cutover/root-app-forgejo.yaml"
  log "✅ root app now sourced from Forgejo. GitHub mirror = DR seed. Circle dead."
 }
 case "${1:-all}" in
  all)      preflight; p1_cilium; p1_longhorn; p1_ingress; p2_cnpg; p3_forgejo; p4_argocd
            log "Phases 1-4 done. Push repo to Forgejo + set up pull-mirror, then: $0 cutover" ;;
  cilium)   preflight; p1_cilium ;;
  storage)  preflight; p1_longhorn ;;
  ingress)  preflight; p1_ingress ;;
  cnpg)     preflight; p2_cnpg ;;
  forgejo)  preflight; p3_forgejo ;;
  argocd)   preflight; p4_argocd ;;
  cutover)  preflight; p5_cutover ;;
  *) echo "usage: $0 {all|cilium|storage|ingress|cnpg|forgejo|argocd|cutover}"; exit 1 ;;
 esac
@@ -0,0 +1,86 @@
 # cluster-config/cilium-values.yaml
 # Cilium CNI — installed via talosctl (not helmfile) during cluster bootstrap.
 # Applied once: `helm install cilium cilium/cilium -n kube-system -f cilium-values.yaml`
 #
 # Why Cilium: Talos Linux does not ship kube-proxy. Cilium's eBPF dataplane
 # replaces it entirely (kubeProxyReplacement: true) and also handles LB-IPAM
 # so LoadBalancer services get real IPs without MetalLB.
 # ── cgroup ────────────────────────────────────────────────────────────────────
 # Talos mounts cgroups at boot before any container runtime starts.
 # autoMount: false tells Cilium to use the existing mount rather than trying
 # to mount its own — double-mounting on Talos causes init failures.
 cgroup:
  autoMount:
    enabled: false
  hostRoot: /sys/fs/cgroup   # where Talos exposes the cgroup v2 hierarchy
 # ── IPAM ──────────────────────────────────────────────────────────────────────
 # kubernetes mode: Cilium allocates pod IPs from the pod CIDR that Talos
 # configured for each node (--pod-cidr in the kubelet). Alternative is
 # Cilium's own cluster-pool IPAM, but that requires extra config and
 # conflicts with the Talos node CIDR assignment.
 ipam:
  mode: kubernetes
 # ── Operator ──────────────────────────────────────────────────────────────────
 # Single replica is fine for a 3-node homelab. The operator manages CiliumNode
 # objects and LB-IPAM pools — it does not sit in the data path.
 operator:
  replicas: 1
 # ── kube-proxy replacement ────────────────────────────────────────────────────
 # Talos is deliberately installed without kube-proxy (machineConfig
 # install.extensions excludes it). Cilium must replace it completely —
 # partial replacement would leave Service ClusterIPs unreachable.
 kubeProxyReplacement: true
 # ── L2 announcements ──────────────────────────────────────────────────────────
 # Without this, LB-IPAM (k8s/cilium/lb-ipam-pool.yaml) assigns real IPs to
 # LoadBalancer Services, but nothing ARPs for them on the LAN — the IP shows
 # up in `kubectl get svc` but is 100% unreachable from outside the cluster
 # (confirmed: forgejo's .165 and shadowsocks' .166 both had incomplete ARP
 # entries and 100% ping loss before this). This flag is what actually makes
 # k8s/cilium/l2-announcement-policy.yaml take effect instead of being inert.
 l2announcements:
  enabled: true
 # ── API server endpoint ───────────────────────────────────────────────────────
 # Cilium needs to talk to the Kubernetes API to watch Nodes/Services/Endpoints.
 # On Talos the API server listens on 127.0.0.1:7445 locally (the external
 # port 6443 requires the node's external cert, which may not be available
 # during early bootstrap). This is the standard Talos Cilium bootstrap config.
 k8sServiceHost: 127.0.0.1
 k8sServicePort: 7445
 # ── Security context / capabilities ──────────────────────────────────────────
 # Cilium's eBPF programs run in the kernel and require elevated capabilities.
 # These are the minimum set needed — removing any of them breaks networking.
 #
 #   NET_ADMIN / NET_RAW  — manipulate iptables/nftables and raw sockets
 #   IPC_LOCK             — lock eBPF maps in memory (prevents paging out BPF state)
 #   SYS_ADMIN            — call bpf() syscall and mount BPF filesystem
 #   SYS_RESOURCE         — raise RLIMIT_MEMLOCK for BPF map memory
 #   DAC_OVERRIDE / FOWNER / SETGID / SETUID — file permission ops during init
 #   CHOWN / KILL         — container lifecycle management
 #
 # cleanCiliumState runs as a one-shot init container to wipe stale eBPF state
 # on upgrades — it needs NET_ADMIN, SYS_ADMIN, SYS_RESOURCE only.
 securityContext:
  capabilities:
    ciliumAgent:
    - CHOWN
    - KILL
    - NET_ADMIN
    - NET_RAW
    - IPC_LOCK
    - SYS_ADMIN
    - SYS_RESOURCE
    - DAC_OVERRIDE
    - FOWNER
    - SETGID
    - SETUID
    cleanCiliumState:
    - NET_ADMIN
    - SYS_ADMIN
    - SYS_RESOURCE
@@ -0,0 +1,19 @@
 # bootstrap.sh
 #!/bin/bash
 # Wait for cluster to be ready
 kubectl wait --for=condition=Ready nodes --all --timeout=300s
 # Longhorn requires privileged pods and hostPath volumes
 kubectl create namespace longhorn-system --dry-run=client -o yaml | kubectl apply -f -
 kubectl label namespace longhorn-system \
  pod-security.kubernetes.io/enforce=privileged \
  pod-security.kubernetes.io/enforce-version=latest \
  --overwrite
 # Install Longhorn
 kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.7.0/deploy/longhorn.yaml
 # Set as default StorageClass
 kubectl patch storageclass longhorn \
  -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
@@ -0,0 +1,10 @@
 # Trust the homelab-ca CA for pulling from the Forgejo container registry.
 # Without this, containerd fails: x509 certificate signed by unknown authority
 # (nginx terminates forgejo.riotpiao.com TLS with a homelab-ca cert).
 # Apply: talosctl -n <node> patch mc --patch @cluster-config/patches/forgejo-registry-ca.yaml
 machine:
  registries:
    config:
      forgejo.riotpiao.com:
        tls:
          ca: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJiVENDQVJTZ0F3SUJBZ0lVYTBVaGs3Rm81d3BiWjZsRzVEWWJUVkFic1k4d0NnWUlLb1pJemowRUF3SXcKRlRFVE1CRUdBMVVFQXhNS2FHOXRaV3hoWWkxallUQWVGdzB5TmpBM01UQXdORFF3TXpaYUZ3MHpOakEzTURjdwpORFF3TXpaYU1CVXhFekFSQmdOVkJBTVRDbWh2YldWc1lXSXRZMkV3V1RBVEJnY3Foa2pPUFFJQkJnZ3Foa2pPClBRTUJCd05DQUFTc1pNU2piUWI0YzNiUk00MjMxVEVrRXVLTnFLUUhaaW5uYnUzbWZGbStRc0wweTF3cjg1Uk0KUWJ6ZEZ2N01JZmN4REpMbHFqQTY1bEJ6TE9pdHRZZHRvMEl3UURBT0JnTlZIUThCQWY4RUJBTUNBcVF3RHdZRApWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVUVmcGJQL3FnYWsxaXUvQzdaQi9uZk5zc0hpd3dDZ1lJCktvWkl6ajBFQXdJRFJ3QXdSQUlnR2ltdnJiWU1xZjhGYThCeTBBM0M1ak1VL0d3dGU0NHgzOU4rRDRyaTJ1a0MKSUNkOEtIQXhhV0s2ZkVJcEFYZGdUQ1FxQmFiZjVZUDdhQzNDVzkzYkNsTjIKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
@@ -0,0 +1,33 @@
 # helmfile.yaml — DEPRECATED
 #
 # This file is kept for historical reference only.
 # All Helm releases have been migrated to:
 #   1. Terraform (bootstrap releases: cert-manager, reloader, ingress-nginx)
 #   2. ArgoCD Applications (all workload releases)
 #
 # Deployment now uses:
 #   - terraform apply (for core infrastructure)
 #   - ArgoCD auto-sync (for all workloads)
 #
 # To view or manage releases:
 #   kubectl get applications -n argocd
 #
 # To modify releases, update k8s/argocd/apps/*.yaml files and commit to git.
 #
 # This file remains as a marker to prevent accidental `helmfile apply` usage.
 # DELETE if no longer needed after full migration verification.
 # Historical note:
 # - Phases 0-3 migrated: cert-manager, reloader, ingress-nginx, strimzi-operator,
 #   kmsvc-redis, kafka-cluster, queue-crd, management-service, promtail, portainer,
 #   cloudnative-pg, loki, grafana, prometheus, forgejo, forgejo-runner, authentik
 # - Phase 4 cutover: 2026-07-15 (helmfile stubbed, ArgoCD becomes sole convergence engine)
 # ── DO NOT USE ────────────────────────────────────────────────────────────────
 # helmfile apply                  # FORBIDDEN (use ArgoCD)
 # helmfile diff                   # FORBIDDEN (use argocd app diff)
 # helmfile destroy                # FORBIDDEN (use kubectl delete)
 # For drift detection (CI only):
 #   argocd app diff <app-name>    # Check what ArgoCD would change
 #   terraform plan                # Check what Terraform would change
@@ -0,0 +1,18 @@
 apiVersion: v1
 data:
  settings.json: |
    {
      "defaultProvider": "homelab-ornith",
      "defaultModel": "ornith:35b",
      "defaultThinkingLevel": "medium",
      "theme": "light",
      "compaction": {
        "enabled": true,
        "reserveTokens": 16000,
        "keepRecentTokens": 6000
      }
    }
 kind: ConfigMap
 metadata:
  name: pi-config
  namespace: agent-pod
@@ -0,0 +1,49 @@
 # Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
 # (trigger a new session) -- both are routes on the same hub.js service.
 #
 # Was ingressClassName: kong until Kong was retired on 2026-08-19. Pointed
 # straight at nginx rather than through the replacement Go gateway because that
 # gateway has no WebSocket upgrade support yet -- routing /console through it
 # would break the console outright. nginx handles the upgrade natively.
 #
 # Path precedence: the nginx Ingress api/api catch-alls `/` on this same host
 # to the gateway. nginx matches longest prefix first, so these three paths win
 # over `/` and the rest of the host still reaches the gateway.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: console
  namespace: agent-pod
  annotations:
    # A console WebSocket stays open across a whole agent session; nginx's 60s
    # default read timeout would drop it mid-run.
    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-buffering: "off"
 spec:
  ingressClassName: nginx
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /console
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
          - path: /run
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
          - path: /sessions
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
@@ -0,0 +1,982 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: coordinator-src
  namespace: agent-pod
 data:
  coordinator.js: |
    #!/usr/bin/env node
    // coordinator: local CLI that drives a multi-phase, multi-task pipeline of
    // planner/investigator/implementer/judge stages, same state machine as
    // hub.js's old /pipeline handler. Every interactive stage runs through the
    // patched agent-manager fork's `spawn` subcommand, so it is a real tracked
    // session (tmux pane on agent-manager's private socket + a state.db row)
    // from the moment it exists -- attachable and visible in agent-manager's
    // own TUI the whole time it runs.
    //
    // Per repo, each role (planner/investigator/implementer/judge) is ONE
    // persistent agent-manager session, not a fresh spawn per task: the first
    // task to need a role spawns it, every later task for that role reuses the
    // same tmux pane via `tmux send-keys` (see runOnPool) -- the same nudge
    // mechanism that used to only fire on a stall now doubles as "give this
    // agent its next task." Every role reads everything it needs fresh off disk
    // each call, so every pane gets a `/new` before every reuse instead of
    // accumulating history that degrades and eventually errors out task after
    // task -- same pane, same agent-manager session, zero memory of the last
    // task it handled.
    // Because only one implementer/judge/etc. exists per repo, tasks within a
    // phase run strictly sequentially against the pool -- no per-task worktree,
    // no per-task branch, no merge-back step; every task commits straight onto
    // the phase branch in the repo's one shared clone.
    //
    // The unit of concurrency is now the REPO, not the task: runCoordinator
    // takes a list of repos and runs up to REPO_CONCURRENCY of them at once,
    // each with its own clone (under WORK_DIR/<repoId>) and its own 4-agent
    // pool. The coordinator never kills a role's session; it rests at an idle
    // prompt between tasks, and agent-manager's session list becomes the audit
    // trail of everything every repo's pipeline ran. Completion is signaled by
    // sentinel files under the repo's clone (unchanged convention), waited on
    // with fs.watch instead of polling.
    const fs = require("node:fs");
    const path = require("node:path");
    const { spawn } = require("node:child_process");
    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
    // Never rely on a bare `pi`/`agent-manager` on $PATH -- see PI_BIN's own
    // comment below; the same collision risk applies to any CLI name. Always
    // invoke explicit pinned paths.
    const PI_BIN =
      process.env.PI_BIN ||
      path.join(__dirname, "..", ".pi-cli", "node_modules", "@earendil-works", "pi-coding-agent", "dist", "cli.js");
    const AGENT_MANAGER_BIN = process.env.AGENT_MANAGER_BIN || path.join(__dirname, "..", ".bin", "agent-manager-fork");
    // agent-manager's own session-state DB -- used to detect a session that has
    // actually died (process crashed/exited, status flips to "errored"/"dead")
    // instead of one that's merely slow. Read-only introspection plus the one
    // UPDATE in killDeadSession below, same class of operation as the tmux
    // nudges already done directly against agent-manager's internals.
    const AGENT_MANAGER_DB =
      process.env.AGENT_MANAGER_DB || path.join(require("node:os").homedir(), ".config", "agent-manager", "state.db");
    // Empty means "let pi fall back to ~/.pi/agent/settings.json's default"
    // (currently anthropic/claude-sonnet-4-5, real paid usage). Set both to
    // route every stage -- headless (spawnPi) and interactive (runOnPool) --
    // at the homelab model instead, e.g. AGENT_PROVIDER=homelab-ornith
    // AGENT_MODEL=ornith:35b.
    const AGENT_PROVIDER = process.env.AGENT_PROVIDER || "";
    const AGENT_MODEL = process.env.AGENT_MODEL || "";
    // judge can run a different model than the rest of the chain, e.g.
    // homelab-reasoning instead of homelab-ornith now that verifier/PRM is
    // retired. Falls back to AGENT_PROVIDER/AGENT_MODEL when unset, so a run
    // that doesn't care keeps one uniform model everywhere.
    const JUDGE_PROVIDER = process.env.JUDGE_PROVIDER || AGENT_PROVIDER;
    const JUDGE_MODEL = process.env.JUDGE_MODEL || AGENT_MODEL;
    function providerModelFor(role) {
      return role === "judge" ? { provider: JUDGE_PROVIDER, model: JUDGE_MODEL } : { provider: AGENT_PROVIDER, model: AGENT_MODEL };
    }
    // agent-manager's private tmux server and session-naming scheme
    // (internal/tmux/tmux.go: defaultSocket = "agentmgr", sessionName(id) =
    // "am_"+id) -- stable, documented internals of the fork, used here only
    // for read-only introspection (pane capture) and role nudges, exactly the
    // class of operation hub.js already ran directly against its own sessions
    // rather than asking a model to do it.
    const AM_SOCKET = "agentmgr";
    function amSessionName(id) {
      return `am_${id}`;
    }
    function runAmTmux(args) {
      return runCmd("tmux", ["-L", AM_SOCKET, ...args]);
    }
    const ROLE_SKILLS = new Set(["planner", "investigator", "info-collector", "implementer", "judge", "resolver"]);
    // Every role reads everything it needs fresh off disk each call -- PLAN.md,
    // the task spec, judge's verdict file, `git diff` against baseBranch --
    // nothing depends on remembering earlier tasks. Left to accumulate, a
    // pooled session's conversation grows without bound across every task in a
    // repo and both correctness and reliability degrade hard once it does
    // (observed: a planner session at ~1.5M cumulative tokens started erroring
    // out every call, an investigator session that far gone started narrating a
    // different codebase entirely). So every role gets reset to a clean
    // conversation before every reuse instead of just being nudged with the
    // next prompt -- same pane, same agent-manager session (still
    // visible/attachable), zero history carried between tasks.
    function sleep(ms) {
      return new Promise((resolve) => setTimeout(resolve, ms));
    }
    const HARD_RULES =
      "Read and follow ~/.pi/agent/skills/karpathy-guidelines/SKILL.md and " +
      "~/.pi/agent/skills/caveman/SKILL.md as hard rules for this entire task, before anything else. ";
    // judge (routed to homelab-reasoning) has been observed narrating an
    // entire review in prose -- "I should run git diff, then check X..." --
    // and then writing a verdict based on that narration without ever calling
    // a real tool. Live example: a phase-judge call produced a page of
    // "I would check..." reasoning, declared VERDICT: PASS, and showed the
    // touch command as a fenced code block IN ITS OWN TEXT rather than
    // executing it. Coordinator just timed out waiting on a sentinel that was
    // never going to appear, since nothing was ever actually run. Spelled out
    // explicitly since "use the judge skill" alone apparently isn't enough to
    // rule this out.
    const REQUIRE_REAL_TOOL_CALLS =
      "Do not narrate what you would check -- actually run the commands via a real tool call and read their real " +
      "output before writing anything. A verdict based on describing checks instead of executing them is invalid. " +
      "Writing the verdict file and touching the sentinel are themselves tool calls you must execute, not text to " +
      "display in your response. ";
    function parseVerdictLine(text, label) {
      if (!text) return null;
      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
      const m = text.match(re);
      return m ? m[1].toUpperCase() : null;
    }
    function runCmd(bin, args, cwd) {
      return new Promise((resolve) => {
        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
        let out = "";
        child.stdout.on("data", (c) => (out += c));
        child.stderr.on("data", (c) => (out += c));
        child.on("close", (code) => resolve({ code, out: out.trim() }));
      });
    }
    function runGit(cwd, args) {
      return runCmd("git", args, cwd);
    }
    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
    // in practice: a stage writes a real file and simply never runs `git add`/
    // `git commit`, leaving it untracked and invisible to every later `git diff`.
    // Sweep and commit anything left dirty after every stage, deterministically.
    async function commitPending(cwd, message) {
      await runGit(cwd, ["add", "-A"]);
      const status = await runGit(cwd, ["status", "--porcelain"]);
      if (!status.out) return { committed: false };
      const commit = await runGit(cwd, ["commit", "-m", message]);
      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
    }
    // Headless one-shot pi call (`pi -p --mode json <prompt>`), used only for
    // quick diagnostic/mechanical calls that don't need to be a watchable
    // session: resolver's crash/stall diagnosis, and the initial clone. Kept
    // exactly as before -- only the interactive per-role stages (runOnPool,
    // below) go through agent-manager.
    //
    // Bounded by SPAWN_PI_TIMEOUT_MS -- unlike runOnPool's pooled sessions
    // (which now have status polling to catch a dead session fast, see
    // waitForSentinel/killDeadSession), this is a raw child_process with no
    // equivalent escape hatch. Observed live: a resolver call shared the
    // default backend with a concurrently-busy repo's implementer and sat for
    // 6+ minutes producing nothing -- with no timeout here, that blocks the
    // entire calling repo's pipeline forever, since askResolver is always
    // awaited before the next stage can run.
    const SPAWN_PI_TIMEOUT_MS = 5 * 60 * 1000;
    function spawnPi({ agent, prompt, cwd }) {
      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${HARD_RULES}${prompt}` : prompt;
      const args = ["-p", "--mode", "json"];
      if (AGENT_PROVIDER) args.push("--provider", AGENT_PROVIDER);
      if (AGENT_MODEL) args.push("--model", AGENT_MODEL);
      args.push(finalPrompt);
      const child = spawn(PI_BIN, args, { stdio: ["ignore", "pipe", "pipe"], cwd });
      let lastText = "";
      let stderrTail = "";
      let buf = "";
      child.stdout.on("data", (chunk) => {
        buf += chunk;
        let idx;
        while ((idx = buf.indexOf("\n")) !== -1) {
          const line = buf.slice(0, idx);
          buf = buf.slice(idx + 1);
          if (!line.trim()) continue;
          try {
            const event = JSON.parse(line);
            if (event.type === "message_end" && event.message && Array.isArray(event.message.content)) {
              const text = event.message.content
                .filter((c) => c.type === "text")
                .map((c) => c.text)
                .join("\n");
              if (text) lastText = text;
            }
          } catch {
            // non-JSON stdout noise, ignore
          }
        }
      });
      child.stderr.on("data", (chunk) => {
        process.stderr.write(chunk);
        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
      });
      return new Promise((resolve) => {
        let settled = false;
        const timer = setTimeout(() => {
          if (settled) return;
          settled = true;
          child.kill("SIGKILL");
          resolve({ code: null, lastText, stderrTail, timedOut: true });
        }, SPAWN_PI_TIMEOUT_MS);
        child.on("close", (code) => {
          if (settled) return;
          settled = true;
          clearTimeout(timer);
          resolve({ code, lastText, stderrTail });
        });
      });
    }
    async function askResolver(cwd, repoId, diagnosticPrompt) {
      const result = await spawnPi({ agent: "resolver", prompt: diagnosticPrompt, cwd });
      return parseVerdictLine(result.lastText, "RESOLUTION");
    }
    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
    // Resolves as soon as filePath appears (fs.watch on its directory, same as
    // before), as soon as target's agent-manager status flips to "errored" or
    // "dead" (polled -- state.db has no watch mechanism), or after limitMs with
    // neither. A session that has actually crashed will never touch the
    // sentinel, so without the status poll this just burns the full STAGE_
    // TIMEOUT_MS waiting on a file that was never coming, same as a genuine
    // stall -- polling status catches that in ~pollMs instead.
    function waitForSentinel(filePath, target, limitMs, pollMs = 5000) {
      return new Promise((resolve) => {
        if (fs.existsSync(filePath)) return resolve({ ok: true });
        const dir = path.dirname(filePath);
        const id = target.replace(/^am_/, "");
        let settled = false;
        let watcher;
        let poller;
        let timer;
        const finish = (result) => {
          if (settled) return;
          settled = true;
          clearTimeout(timer);
          clearInterval(poller);
          if (watcher) {
            try {
              watcher.close();
            } catch {
              // already closed
            }
          }
          resolve(result);
        };
        try {
          watcher = fs.watch(dir, () => {
            if (fs.existsSync(filePath)) finish({ ok: true });
          });
        } catch {
          // dir missing at watch time is a real bug elsewhere (cwd should
          // already exist); surface it as a timeout rather than hang forever.
          return finish({ timedOut: true });
        }
        // Closes the race between the existsSync check above and the watcher
        // actually being attached.
        if (fs.existsSync(filePath)) return finish({ ok: true });
        poller = setInterval(async () => {
          const { out } = await runCmd("sqlite3", [AGENT_MANAGER_DB, `SELECT status FROM sessions WHERE id='${id}'`]);
          const status = out.trim();
          if (status === "errored" || status === "dead") finish({ dead: true, status });
        }, pollMs);
        timer = setTimeout(() => finish({ timedOut: true }), limitMs);
      });
    }
    // Kills a session that's actually crashed (not just slow) and archives it
    // in agent-manager's own DB so it stops showing up as a live, unattended
    // pane -- otherwise every crash leaves an orphaned tmux session + state.db
    // row behind permanently, identical to the manually-cleaned-up poiman-
    // planner ghost session found earlier this same run.
    async function killDeadSession(target) {
      await runAmTmux(["kill-session", "-t", target]);
      const id = target.replace(/^am_/, "");
      await runCmd("sqlite3", [AGENT_MANAGER_DB, `UPDATE sessions SET archived=1 WHERE id='${id}'`]);
    }
    // Runs one task's worth of work on a persistent per-role agent: spawns the
    // role's session the first time it's ever needed for this repo, sends every
    // later prompt into that same tmux pane via send-keys -- prefixed with a
    // `/new` first, so the pane and agent-manager session stay the same but the
    // model starts that prompt with a clean conversation, no history carried
    // over from whatever task this role last handled. pool is a plain object
    // keyed by role name ("planner"/"investigator"/"implementer"/"judge"),
    // shared across every task in a repo's pipeline (see runRepoPipeline) -- it
    // IS the 4-agent pool, one entry per role, filled in lazily as each role
    // gets its first task.
    // A dead/errored session gets one respawn-and-retry (same prompt, fresh
    // session) before this stage is abandoned -- matches resolver-SKILL.md's
    // own documented contract of retrying a failed stage at most once.
    const DEAD_SESSION_RETRIES = 1;
    async function runOnPool(pool, cwd, repoId, role, prompt, sentinelFile) {
      fs.rmSync(sentinelFile, { force: true });
      const label = `${repoId}-${role}`;
      // A pooled session's shell cwd drifts as it explores the repo (e.g. cd
      // into a Rust workspace subdirectory to read source) and nothing resets
      // it back between turns. Seen in practice: a repo whose own internal
      // workspace folder is one letter off from the repo's own directory name
      // ("poiman" the repo vs. "poimen" the crate workspace inside it) was
      // enough for the agent to touch its sentinel one level off from where
      // this function is watching for it -- coordinator waits out the full
      // STAGE_TIMEOUT_MS for a file that already exists, just in the wrong
      // place. State the absolute target directory and use absolute paths for
      // every filesystem instruction, so there's nothing for the agent to get
      // wrong by reasoning about a relative "current directory."
      const cwdReminder = `Your working directory for this task is ${cwd} -- if your shell isn't already there, run: cd ${cwd}\n\n`;
      const spawnFresh = async () => {
        const spawnArgs = ["spawn", "--tool", "pi", "--cwd", cwd, "--name", label, "--group", repoId, "--prompt", cwdReminder + HARD_RULES + prompt];
        const { provider, model } = providerModelFor(role);
        if (provider) spawnArgs.push("--provider", provider);
        if (model) spawnArgs.push("--model", model);
        const spawned = await runCmd(AGENT_MANAGER_BIN, spawnArgs);
        return spawned.code === 0 ? amSessionName(spawned.out) : null;
      };
      let target = pool[role];
      if (!target) {
        target = await spawnFresh();
        if (!target) return { ok: false, crashed: true, error: "spawn failed", sessionName: label };
        pool[role] = target;
      } else {
        await runAmTmux(["send-keys", "-t", target, "/new", "Enter"]);
        await sleep(1000);
        await runAmTmux(["send-keys", "-t", target, cwdReminder + HARD_RULES + prompt, "Enter"]);
      }
      for (let deadRetries = 0; ; deadRetries++) {
        const outcome = await waitForSentinel(sentinelFile, target, STAGE_TIMEOUT_MS);
        if (outcome.ok) return { ok: true, sessionName: label };
        if (outcome.dead) {
          await killDeadSession(target);
          if (pool[role] === target) delete pool[role];
          if (deadRetries >= DEAD_SESSION_RETRIES) {
            return { ok: false, crashed: true, error: `session died (status: ${outcome.status})`, sessionName: label };
          }
          target = await spawnFresh();
          if (!target) return { ok: false, crashed: true, error: "respawn after death failed", sessionName: label };
          pool[role] = target;
          continue;
        }
        // Plain stall -- session still alive, just slow. Ask resolver once,
        // nudge if it says worth it, and stop here either way (this is not
        // the death path, so no respawn/retry loop).
        const pane = await runAmTmux(["capture-pane", "-t", target, "-p", "-S", "-200"]);
        const resolution = await askResolver(
          cwd,
          repoId,
          `Repo ${repoId}'s "${role}" agent hasn't finished its current task after 10 minutes. Its pane tail:\n${pane.out.slice(-3000)}\n\n` +
            `Decide: is it still making real progress and worth nudging to wrap up, or stuck and worth abandoning?`
        );
        let ok = false;
        if (resolution === "RETRY") {
          await runAmTmux(["send-keys", "-t", target, `Please wrap up now and run: touch ${sentinelFile}`, "Enter"]);
          const nudged = await waitForSentinel(sentinelFile, target, NUDGE_TIMEOUT_MS);
          ok = nudged.ok === true;
          if (nudged.dead) {
            await killDeadSession(target);
            if (pool[role] === target) delete pool[role];
          }
        }
        return { ok, sessionName: label };
      }
    }
    function plannerPrompt(task, specHint, judgeOnly, cwd) {
      // judgeOnly (auto-discovered tasks only, see parseTaskBoard): planner
      // itself decides whether the task is already done before planning it,
      // reading tasks/INDEX.md's own status notes plus git log/current code --
      // replaces what used to be a separate judge pre-check call. One LLM round
      // trip instead of two, and the same agent that's about to plan the task
      // is the one deciding whether planning it is even necessary.
      const resultFile = path.join(cwd, `.task-result-${task}`);
      const decideStep = judgeOnly
        ? `First, decide whether task ${task} is already fully implemented on this branch: check ` +
          `\`git log --oneline --grep '${task}'\`, tasks/INDEX.md's own status notes for this task, and the current ` +
          `code directly against its spec (${specHint})'s acceptance criteria. Write your decision to ` +
          `${resultFile} as a single "VERDICT: PASS" (already done, no further work needed) or ` +
          `"VERDICT: FAIL" (needs work) line plus one line of rationale. If VERDICT is FAIL, continue below and ` +
          `draft the plan in this same turn; if VERDICT is PASS, skip the rest and go straight to the touch step.\n\n`
        : "";
      return (
        `${decideStep}Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). ` +
        `PLAN.md is scratch state for this harness, not a deliverable -- do NOT commit it or add it to git. ` +
        `Then run: touch ${path.join(cwd, `.stage-done-${task}-planner`)}`
      );
    }
    function investigatorPrompt(task, cwd) {
      return (
        `Use the investigator skill to confirm PLAN.md against real sources for task ${task}, append findings. ` +
        `PLAN.md is scratch state for this harness, not a deliverable -- do NOT commit it or add it to git. ` +
        `Then run: touch ${path.join(cwd, `.stage-done-${task}-investigator`)}`
      );
    }
    function implementerPrompt(task, attempt, feedbackHint, cwd) {
      return (
        `Use the implementer skill to implement what the current PLAN.md specifies for task ${task} (commit as you go). ` +
        `${feedbackHint} Then run: touch ${path.join(cwd, `.stage-done-${task}-implementer-${attempt}`)}`
      );
    }
    function judgePrompt(task, baseBranch, attempt, cwd) {
      return (
        `${REQUIRE_REAL_TOOL_CALLS}Use the judge skill to review the diff against ${baseBranch}...HEAD for task ${task}. ` +
        `Write your verdict to ${path.join(cwd, `.task-result-${task}`)} as a single "VERDICT: PASS" or "VERDICT: FAIL" ` +
        `line plus one line of rationale, then run: touch ${path.join(cwd, `.stage-done-${task}-judge-${attempt}`)}`
      );
    }
    const MAX_IMPLEMENT_ATTEMPTS = 5;
    const MAX_PLAN_REVISIONS = 3;
    // Runs one task against the repo's shared role pool: planner drafts
    // PLAN.md (for auto-discovered tasks, first deciding off tasks/INDEX.md and
    // the repo's own state whether the task is already done -- see
    // plannerPrompt's judgeOnly branch; judge never does this pre-check),
    // investigator confirms it, then implementer and judge go back and
    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
    // next implementer attempt is told to read and address. After
    // MAX_IMPLEMENT_ATTEMPTS straight fails, the planner role is asked to judge
    // whether the plan itself is wrong -- fresh conversation, same as any other
    // planner call, reading PLAN.md/the judge feedback/the
    // diff off disk rather than remembering having drafted the original plan.
    // If it decides the approach is wrong it revises PLAN.md and the implementer
    // gets a fresh attempt budget.
    // MAX_PLAN_REVISIONS caps this from looping forever on a task that's
    // genuinely stuck. All work happens directly in cwd (the repo's one shared
    // clone, currently checked out to the phase branch) -- no worktree, since
    // only one implementer/judge exist per repo and tasks run strictly one at a
    // time (see runPhase).
    async function runTaskOnPool(cwd, baseBranch, task, pool, repoId, pipelineSession, judgeOnly) {
      const resultFile = path.join(cwd, `.task-result-${task}`);
      fs.rmSync(resultFile, { force: true });
      const specHint = `the file under tasks/ starting with "${task}-"`;
      const stage = async (role, prompt, sentinel, displayLabel) => {
        const label = displayLabel || role;
        pipelineSession.activeTasks[task] = { stage: label, startedAt: new Date().toISOString() };
        logProgress(pipelineSession);
        const result = await runOnPool(pool, cwd, repoId, role, prompt, sentinel);
        await commitPending(cwd, `task: ${task} (${label})`);
        return result;
      };
      const abandon = (stageLabel, result, attempt) => {
        delete pipelineSession.activeTasks[task];
        logProgress(pipelineSession);
        return {
          task,
          status: result.crashed ? "spawn-crashed" : "timed-out",
          error: result.error,
          stoppedAt: stageLabel,
          ...(attempt !== undefined ? { attempt } : {}),
        };
      };
      // PLAN.md is scratch state for this one task, not a deliverable (see
      // plannerPrompt/investigatorPrompt -- it's gitignored too, as a backstop
      // in case an agent commits it anyway). Discard it once the task is done,
      // whatever the outcome, so it never bleeds into the next task's planner
      // call or sits around as stale harness clutter in the shared clone.
      try {
        let result = await stage("planner", plannerPrompt(task, specHint, judgeOnly, cwd), path.join(cwd, `.stage-done-${task}-planner`));
        if (!result.ok) return abandon("planner", result);
        if (judgeOnly) {
          const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
          if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
            delete pipelineSession.activeTasks[task];
            logProgress(pipelineSession);
            return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
          }
        }
        result = await stage("investigator", investigatorPrompt(task, cwd), path.join(cwd, `.stage-done-${task}-investigator`));
        if (!result.ok) return abandon("investigator", result);
        let planRevisions = 0;
        let implementAttempt = 0;
        let verdict = null;
        let resultText = "";
        let justRevisedPlan = false;
        while (true) {
          implementAttempt++;
          const feedbackHint = fs.existsSync(resultFile)
            ? justRevisedPlan
              ? `${resultFile} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
                `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
              : `A previous judge review exists at ${resultFile} -- read it and address every issue it raises.`
            : "";
          justRevisedPlan = false;
          result = await stage(
            "implementer",
            implementerPrompt(task, implementAttempt, feedbackHint, cwd),
            path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
          );
          if (!result.ok) return abandon("implementer", result, implementAttempt);
          result = await stage("judge", judgePrompt(task, baseBranch, implementAttempt, cwd), path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`));
          if (!result.ok) return abandon("judge", result, implementAttempt);
          resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
          verdict = parseVerdictLine(resultText, "VERDICT");
          if (verdict === "PASS") break;
          if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
            if (planRevisions >= MAX_PLAN_REVISIONS) break;
            planRevisions++;
            result = await stage(
              "planner",
              `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
                `the judge's feedback in ${resultFile}, and the current diff against ${baseBranch}...HEAD. Decide ` +
                `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md. If ` +
                `you change the approach, also use the investigator skill to confirm the new approach against real ` +
                `sources. If the plan is sound, note why in PLAN.md and leave it as-is. PLAN.md is scratch state for ` +
                `this harness, not a deliverable -- do NOT commit it or add it to git. Then run: ` +
                `touch ${path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)}`,
              path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`),
              "planner-revise"
            );
            if (!result.ok) return abandon("planner-revise", result, planRevisions);
            implementAttempt = 0;
            justRevisedPlan = true;
          }
        }
        delete pipelineSession.activeTasks[task];
        logProgress(pipelineSession);
        if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
          return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
        }
        return { task, status: verdict === "PASS" ? "done" : "done-with-concerns", judgeRationale: resultText };
      } finally {
        fs.rmSync(path.join(cwd, "PLAN.md"), { force: true });
      }
    }
    // Committed (never gitignored) so it survives a resumed phase branch --
    // one task id per line, appended as each task resolves. This is what lets
    // a resumed run skip straight past already-resolved tasks instead of
    // re-running planner's judgeOnly decision on every one of them again:
    // resuming the git branch alone only recovers the CODE, not "which tasks
    // are already settled," and re-deciding that from scratch for every task
    // burns a full LLM call per already-done task before ever reaching the
    // first one that actually needs work.
    function progressLedgerPath(cwd) {
      return path.join(cwd, ".agent-progress");
    }
    function readCompletedTasks(cwd) {
      const file = progressLedgerPath(cwd);
      if (!fs.existsSync(file)) return new Set();
      return new Set(
        fs
          .readFileSync(file, "utf8")
          .split("\n")
          .map((line) => line.trim())
          .filter(Boolean)
      );
    }
    async function recordTaskComplete(cwd, task) {
      fs.appendFileSync(progressLedgerPath(cwd), `${task}\n`);
      await runGit(cwd, ["add", path.basename(progressLedgerPath(cwd))]);
      await runGit(cwd, ["commit", "-m", `chore: mark ${task} complete in progress ledger`]);
    }
    // Runs every task in a phase (no declared dependency between them) strictly
    // one at a time against the repo's shared role pool -- only one implementer/
    // judge/etc. exists per repo, so there is no per-task concurrency to have
    // here anymore (see REPO_CONCURRENCY below for where concurrency now
    // lives). No worktrees: every task commits directly onto phaseBranch in the
    // one shared cwd. baseBranch here is the TRUE base (e.g. "main") -- judge
    // reviews `git diff baseBranch...HEAD`, not phaseBranch...HEAD, which would
    // always be empty since HEAD *is* phaseBranch while it's checked out.
    //
    // Pushes phaseBranch after every task, not just once at full-phase-end: the
    // pod is ephemeral and every restart re-clones baseBranch fresh (see
    // runRepoPipeline) -- without this, a redeploy mid-phase silently discards
    // every task committed so far, and the next run re-decides "is this done?"
    // from a clone that never saw any of that work.
    async function runPhase(cwd, baseBranch, phaseBranch, phaseTasks, pool, repoId, pipelineSession) {
      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
      const completed = readCompletedTasks(cwd);
      for (const entry of entries) {
        if (completed.has(entry.id)) {
          const result = { task: entry.id, status: "done", resumed: true };
          pipelineSession.taskResults.push(result);
          logProgress(pipelineSession);
          continue;
        }
        const result = await runTaskOnPool(cwd, baseBranch, entry.id, pool, repoId, pipelineSession, entry.judgeOnly);
        pipelineSession.taskResults.push(result);
        if (result.status === "done" || result.status === "done-with-concerns") {
          await recordTaskComplete(cwd, entry.id);
        }
        await runGit(cwd, ["push", "-u", "origin", phaseBranch]);
        logProgress(pipelineSession);
      }
    }
    // Discovers phases/tasks from the repo's own tasks/INDEX.md instead of
    // requiring the caller to pass --tasks. Matches this convention's board
    // shape (see e.g. Poimen/agent-rust's tasks/INDEX.md): a numbered phase
    // heading ("## 1 — Foundations · T0.x"), followed by a markdown table
    // whose rows link to each task's own spec file ("| [T0.1](T0.1-....md) |
    // ... |"). Headings that aren't a numbered phase (prose sections like
    // "## Ordering — declared, never derived", "## Progress") are skipped --
    // only "## <digits> — ..." starts a new phase. Returns null if
    // tasks/INDEX.md doesn't exist; an empty array if it exists but no phase
    // yielded any task rows.
    function parseTaskBoard(cwd) {
      const indexPath = path.join(cwd, "tasks", "INDEX.md");
      if (!fs.existsSync(indexPath)) return null;
      const phaseHeaderRe = /^##\s+\d+\s+—/;
      const taskRowRe = /^\|\s*\[([A-Za-z0-9.]+)\]\(/;
      const phases = [];
      let current = null;
      for (const line of fs.readFileSync(indexPath, "utf8").split("\n")) {
        if (phaseHeaderRe.test(line)) {
          current = [];
          phases.push(current);
          continue;
        }
        const m = line.match(taskRowRe);
        if (m && current) current.push(m[1]);
      }
      return phases.filter((phase) => phase.length > 0);
    }
    function phaseLabelFor(phaseTasks, index) {
      const first = phaseTasks[0];
      const id = typeof first === "string" ? first : first.id;
      const dot = id.indexOf(".");
      return dot === -1 ? `phase-${index}` : id.slice(0, dot);
    }
    function logProgress(pipelineSession) {
      console.log(`[repo ${pipelineSession.id}] ${JSON.stringify(pipelineSession)}`);
    }
    // Runs one repo's full pipeline: clone, then phases strictly sequentially.
    // tasks: array of phases, each phase an array of task ids with no declared
    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]).
    // A flat array of ids is also accepted and treated as one single phase. If
    // omitted, phases are discovered from the repo's own tasks/INDEX.md and run
    // judgeOnly first (a cheap "is this already done" check against the
    // board's possibly-stale checkmarks). Each phase gets its own branch
    // (agent-run/<repoId>/<phaseLabel>, e.g. .../T1); once every task in that
    // phase lands "done" or "done-with-concerns" AND the phase judge (the same
    // pooled judge agent that reviewed each task) passes the integration
    // review, the phase branch is squash-merged into baseBranch and pushed,
    // then the next phase branches off that updated base. Any failure halts
    // this repo's pipeline before merging -- it does not affect other repos
    // running concurrently (see runCoordinator).
    async function runRepoPipeline({ repoId, repo, baseBranch, tasks, branchName }, pipelineSession) {
      const cwd = path.join(WORK_DIR, repoId);
      // repoId is a slug derived from the repo URL now (see slugFor), not a
      // fresh UUID -- reusable across separate `runCoordinator` invocations
      // against the same repo, so a stale clone from a prior run has to be
      // wiped before this one starts, not merged into.
      fs.rmSync(cwd, { recursive: true, force: true });
      fs.mkdirSync(cwd, { recursive: true });
      const pool = {};
      const finish = (status) => {
        pipelineSession.status = status;
        pipelineSession.endedAt = new Date().toISOString();
        logProgress(pipelineSession);
        return pipelineSession;
      };
      // Deterministic, not routed through an LLM -- clone is 100% mechanical
      // (same reasoning as commitPending/the squash-merge sequence below), and
      // was the one place left that broke that pattern: a headless spawnPi
      // call here meant a crash gave zero diagnostic output, just a silent
      // exit code with nothing to debug from.
      const clone = await runGit(cwd, ["clone", "--branch", baseBranch, repo, "."]);
      if (clone.code !== 0) {
        pipelineSession.gitError = clone.out;
        return finish("clone-crashed");
      }
      if (!fs.existsSync(path.join(cwd, ".git"))) return finish("clone-missing");
      let phases = tasks ? (Array.isArray(tasks[0]) ? tasks : [tasks]) : parseTaskBoard(cwd);
      if (!phases || phases.length === 0) {
        pipelineSession.gitError = "no tasks given and tasks/INDEX.md not found or empty";
        return finish("no-tasks-found");
      }
      if (!tasks) {
        phases = phases.map((phase) => phase.map((id) => ({ id, judgeOnly: true })));
      }
      pipelineSession.totalTasks = phases.flat().length;
      for (let i = 0; i < phases.length; i++) {
        const phaseTasks = phases[i];
        const phaseLabel = phaseLabelFor(phaseTasks, i);
        const phaseBranch = branchName ? `${branchName}/${phaseLabel}` : `agent-run/${repoId}/${phaseLabel}`;
        // Resume a phase branch a prior (since-restarted) run already pushed,
        // instead of always branching fresh off baseBranch -- otherwise every
        // redeploy silently discards whatever tasks that prior run already
        // committed and pushed (see runPhase's per-task push below).
        const fetchExisting = await runGit(cwd, ["fetch", "origin", phaseBranch]);
        const resuming = fetchExisting.code === 0;
        const branchResult = resuming
          ? await runGit(cwd, ["checkout", "-b", phaseBranch, "FETCH_HEAD"])
          : await runGit(cwd, ["checkout", "-b", phaseBranch]);
        if (branchResult.code !== 0) {
          pipelineSession.gitError = branchResult.out;
          return finish("branch-crashed");
        }
        // Idempotent and run every phase, NOT gated on a fresh (non-resumed)
        // start -- every run this session was a resume, so the old i===0 &&
        // !resuming gate meant this setup permanently never ran on poiman's
        // branch, and portfolio's PLAN.md stayed tracked from before this rule
        // ever existed (gitignore has no effect on an already-tracked file --
        // observed live: it kept getting swept back in by every `git add -A`
        // regardless of the ignore rule). Check-and-fix on every phase instead
        // of once-at-genesis so a repo that's missing either self-heals on its
        // very next run rather than carrying the gap forever.
        const gitignorePath = path.join(cwd, ".gitignore");
        const currentGitignore = fs.existsSync(gitignorePath) ? fs.readFileSync(gitignorePath, "utf8").split("\n") : [];
        const requiredGitignoreLines = [
          "*.tar.gz",
          "*.tgz",
          "*.crate",
          "*.zip",
          "*.bin",
          "*.whl",
          "vendor/",
          "node_modules/",
          ".task-result-*",
          ".phase-result-*",
          ".stage-done-*",
          "PLAN.md",
        ];
        const missingGitignoreLines = requiredGitignoreLines.filter((line) => !currentGitignore.includes(line));
        if (missingGitignoreLines.length > 0) {
          fs.appendFileSync(
            gitignorePath,
            "\n# agent-harness: build artifacts, vendored archives, and harness bookkeeping never belong in source control\n" +
              missingGitignoreLines.join("\n") +
              "\n"
          );
          await runGit(cwd, ["add", ".gitignore"]);
          await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
        }
        const trackedFiles = await runGit(cwd, ["ls-tree", "-r", "HEAD", "--name-only"]);
        if (trackedFiles.out.split("\n").includes("PLAN.md")) {
          await runGit(cwd, ["rm", "--cached", "PLAN.md"]);
          await runGit(cwd, ["commit", "-m", "chore: untrack PLAN.md (already gitignored, was committed pre-rule)"]);
        }
        await runPhase(cwd, baseBranch, phaseBranch, phaseTasks, pool, repoId, pipelineSession);
        const phaseTaskIds = new Set(phaseTasks.map((t) => (typeof t === "string" ? t : t.id)));
        const phaseResults = pipelineSession.taskResults.filter((r) => phaseTaskIds.has(r.task));
        const phaseClean =
          phaseResults.length === phaseTaskIds.size && phaseResults.every((r) => r.status === "done" || r.status === "done-with-concerns");
        if (!phaseClean) {
          pipelineSession.haltedAt = phaseLabel;
          return finish("halted-phase-failed");
        }
        const phaseResultFile = path.join(cwd, `.phase-result-${phaseLabel}`);
        fs.rmSync(phaseResultFile, { force: true });
        const phaseJudge = await runOnPool(
          pool,
          cwd,
          repoId,
          "judge",
          `${REQUIRE_REAL_TOOL_CALLS}Use the judge skill to review the full phase diff for phase ${phaseLabel} against ` +
            `${baseBranch}...HEAD (covers every task in this phase: ${[...phaseTaskIds].join(", ")}). Every ` +
            `individual task already passed its own judge review -- your job here is different: confirm the ` +
            `tasks integrate correctly as one coherent narrative, and that real integration tests (not just ` +
            `each task's isolated unit checks) exist and actually exercise the phase's intended use case end ` +
            `to end. Write your verdict to ${phaseResultFile} as a single "VERDICT: PASS" or ` +
            `"VERDICT: FAIL" line plus rationale, then run: touch ${path.join(cwd, `.stage-done-phase-${phaseLabel}-judge`)}`,
          path.join(cwd, `.stage-done-phase-${phaseLabel}-judge`)
        );
        await commitPending(cwd, `phase: ${phaseLabel} integration review`);
        if (!phaseJudge.ok) {
          pipelineSession.haltedAt = phaseLabel;
          pipelineSession.gitError = phaseJudge.error;
          return finish("phase-judge-crashed");
        }
        const phaseJudgeText = fs.existsSync(phaseResultFile) ? fs.readFileSync(phaseResultFile, "utf8") : "";
        if (parseVerdictLine(phaseJudgeText, "VERDICT") !== "PASS") {
          pipelineSession.haltedAt = phaseLabel;
          pipelineSession.phaseJudgeRationale = phaseJudgeText;
          return finish("halted-phase-judge-failed");
        }
        const checkoutBase = await runGit(cwd, ["checkout", baseBranch]);
        if (checkoutBase.code !== 0) {
          pipelineSession.gitError = checkoutBase.out;
          return finish("squash-crashed");
        }
        const squash = await runGit(cwd, ["merge", "--squash", phaseBranch]);
        if (squash.code !== 0) {
          await runGit(cwd, ["merge", "--abort"]);
          pipelineSession.gitError = squash.out;
          return finish("squash-crashed");
        }
        const commit = await runGit(cwd, ["commit", "-m", `feat: ${phaseLabel} (${[...phaseTaskIds].join(", ")})`]);
        if (commit.code !== 0) {
          pipelineSession.gitError = commit.out;
          return finish("squash-crashed");
        }
        const push = await runGit(cwd, ["push", "origin", baseBranch]);
        if (push.code !== 0) {
          pipelineSession.gitError = push.out;
          return finish("squash-push-crashed");
        }
        // Milestone's content now lives in baseBranch as one squashed commit --
        // the phase branch (and whatever a prior restart already pushed of it)
        // has no further reason to exist. Delete it both places so a future run
        // never tries to resume a phase that's already done, and so origin
        // doesn't accumulate one dangling branch per completed phase forever.
        await runGit(cwd, ["branch", "-D", phaseBranch]);
        await runGit(cwd, ["push", "origin", "--delete", phaseBranch]);
        logProgress(pipelineSession);
      }
      return finish("completed");
    }
    async function runConcurrent(items, limit, worker) {
      const results = new Array(items.length);
      let i = 0;
      async function next() {
        while (i < items.length) {
          const idx = i++;
          results[idx] = await worker(items[idx], idx);
        }
      }
      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
      return results;
    }
    // How many repos can be mid-flight at once. Each repo gets its own clone
    // and its own 4-agent pool (planner/investigator/implementer/judge), so
    // this is now the real concurrency knob -- tasks within one repo are
    // already serialized against that repo's pool (see runPhase). The backend
    // (homelab-ornith) actually runs 2 GPU replicas behind one Kubernetes
    // Service, each with its own copy of the model loaded (see homelab's
    // k8s/apps/llm-serving/ornith.yaml) -- so up to 2 concurrent LLM calls get
    // real independent instances; a 3rd+ concurrent call queues inside
    // whichever replica the Service's own load-balancing lands it on (each
    // replica runs OLLAMA_NUM_PARALLEL=1). REPO_CONCURRENCY above 2 is still
    // useful (more repos in flight overlaps git/file work, not just LLM calls)
    // but past 2 simultaneous LLM calls, extra concurrency mostly means queueing
    // rather than added throughput -- bump the backend's replica count to
    // change that, not this constant.
    const REPO_CONCURRENCY = Number(process.env.REPO_CONCURRENCY) || 3;
    // repoId is the repo's own name, not a random id -- it's what every role
    // session's --name is built from (see runOnPool: `${repoId}-${role}`), so
    // agent-manager's own session list groups naturally by repo ("portfolio-
    // planner", "portfolio-judge", "poiman-planner", ...) instead of by opaque
    // UUID. Takes the last path segment of the URL, strips a trailing `.git`,
    // and sanitizes anything that isn't safe in a tmux session name / directory
    // name / git branch name. Two different repos that happen to share a
    // basename (e.g. two orgs' "portfolio") would collide -- not handled, since
    // nothing about this harness's usage has needed more than one org per run.
    function slugFor(repoUrl) {
      const last = repoUrl.replace(/\/+$/, "").split("/").pop() || repoUrl;
      return last.replace(/\.git$/, "").replace(/[^a-zA-Z0-9._-]/g, "-");
    }
    // Top-level entry point: runs every repo in `repos` to completion, up to
    // REPO_CONCURRENCY at a time. Returns a map of repoId -> final
    // pipelineSession, one per repo, independent of how the others fared.
    async function runCoordinator({ repos, base, tasks, branchName }) {
      const sessions = {};
      await runConcurrent(repos, REPO_CONCURRENCY, async (repoUrl) => {
        const repoId = slugFor(repoUrl);
        const pipelineSession = {
          id: repoId,
          repo: repoUrl,
          status: "running",
          taskResults: [],
          activeTasks: {},
          totalTasks: 0,
          startedAt: new Date().toISOString(),
        };
        sessions[repoId] = pipelineSession;
        await runRepoPipeline({ repoId, repo: repoUrl, baseBranch: base, tasks, branchName }, pipelineSession);
      });
      return sessions;
    }
    function parseArgs(argv) {
      const opts = { base: "main" };
      for (let i = 0; i < argv.length; i++) {
        const a = argv[i];
        if (a === "--repo") opts.repo = argv[++i];
        else if (a === "--repos") opts.repos = argv[++i];
        else if (a === "--base") opts.base = argv[++i];
        else if (a === "--tasks") opts.tasks = argv[++i];
        else if (a === "--branch") opts.branch = argv[++i];
      }
      return opts;
    }
    async function main() {
      const opts = parseArgs(process.argv.slice(2));
      const repos = opts.repos ? opts.repos.split(",") : opts.repo ? [opts.repo] : null;
      if (!repos || repos.length === 0) {
        console.error(
          "usage: coordinator.js --repos <url1,url2,...> [--tasks T0.1,T0.2;T1.1,T1.2,...] [--base main] [--branch <name>]\n" +
            "  --repo <url> also accepted for a single repo\n" +
            "  --tasks applies to every repo listed; omitted: each repo discovers its own phases from tasks/INDEX.md\n" +
            "  REPO_CONCURRENCY env var (default 3): how many repos run at once"
        );
        // process.exitCode + natural exit, not process.exit() -- stdout piped
        // through kubectl exec (not a TTY) can drop buffered console.log/
        // console.error output if the process exits before it flushes. Setting
        // exitCode and letting the event loop drain naturally is the
        // documented-safe way to exit with a specific code without racing it.
        process.exitCode = 1;
        return;
      }
      const phases = opts.tasks ? opts.tasks.split(";").map((phase) => phase.split(",")) : null;
      const sessions = await runCoordinator({ repos, base: opts.base, tasks: phases, branchName: opts.branch });
      process.exitCode = Object.values(sessions).every((s) => s.status === "completed") ? 0 : 1;
    }
    if (require.main === module) {
      main();
    }
    module.exports = { runCoordinator, runRepoPipeline, runOnPool, parseTaskBoard };
@@ -0,0 +1,168 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: agent-pod
  namespace: agent-pod
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: agent-pod
  template:
    metadata:
      labels:
        app: agent-pod
    spec:
      # api.riotpiao.com has no in-cluster DNS record (only resolves from the
      # home network's own resolver) -- pin it to ingress-nginx-controller's
      # ClusterIP so pi's models.json baseUrl works unchanged. TLS still
      # terminates correctly since SNI/Host still say api.riotpiao.com.
      hostAliases:
        - ip: "10.101.128.185"
          hostnames:
            - "api.riotpiao.com"
      containers:
        # hub.js runs in the same container as pi (not a sidecar) so it can
        # spawn `pi -p --mode json` directly via child_process -- a separate
        # container can't exec into another container's filesystem/PATH.
        # It IS the container's long-running process now; no more `sleep
        # infinity` placeholder.
        #
        # Also builds the agent-manager fork (github.com/Riotpiaole/
        # agent-manager, add-headless-spawn branch) from source and drops
        # coordinator.js in beside hub.js -- neither is the container's
        # foreground process. hub.js keeps that role unchanged; coordinator.js
        # itself now owns multi-repo concurrency (REPO_CONCURRENCY env,
        # default 3), so one invocation handles every repo:
        # `kubectl exec <pod> -- node /root/coordinator.js --repos
        # repoA,repoB,... --tasks ...`. Each repo gets its own clone and its
        # own persistent 4-agent pool (planner/investigator/implementer/
        # judge, one agent-manager session per role, reused across every
        # task in that repo) on the container's local tmux server --
        # `kubectl exec -it <pod> -- agent-manager` attaches its TUI live
        # against those same sessions, no cross-machine visibility problem
        # since spawner, tmux server, and viewer are all colocated here.
        #
        # No prebuilt Linux binary is shipped for agent-manager: the local
        # .bin/ build is macOS arm64 (wrong OS/arch for this container
        # anyway) and it's 27MB, well over a ConfigMap's ~1MiB cap. Debian's
        # `apt-get golang-go` is far too old for this fork's go 1.26.5
        # requirement, so the real Go toolchain is fetched directly from
        # go.dev instead.
        - name: pi
          image: node:22-slim
          command:
            - sh
            - -c
            - |
              set -e
              apt-get update && apt-get install -y git curl jq openssh-client tmux python3 sqlite3 gcc build-essential
              ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub
              eval "$(ssh-agent -s)"
              ssh-add /root/.ssh/id_forgejo
              npm install -g @earendil-works/[email protected]
              npm install --prefix /root ws
              curl -fsSL "https://go.dev/dl/go1.26.5.linux-$(dpkg --print-architecture).tar.gz" | tar -C /usr/local -xz
              export PATH="$PATH:/usr/local/go/bin"
              git clone --branch add-headless-spawn --depth 1 \
                https://github.com/Riotpiaole/agent-manager.git /root/agent-manager-src
              (cd /root/agent-manager-src && go build -o /usr/local/bin/agent-manager .)
              # Language toolchains for whatever repos the implementer/investigator/
              # judge roles actually build and test -- go was already fetched above
              # only for building agent-manager itself, and its PATH export above is
              # local to this script, invisible to `kubectl exec` sessions into the
              # already-running container. Symlinking both into /usr/local/bin (on
              # PATH for every exec session, interactive or not) instead of relying
              # on shell rc sourcing, which pi's non-interactive tool calls don't do.
              ln -sf /usr/local/go/bin/go /usr/local/bin/go
              curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
              ln -sf /root/.cargo/bin/cargo /usr/local/bin/cargo
              ln -sf /root/.cargo/bin/rustc /usr/local/bin/rustc
              ln -sf /root/.cargo/bin/rustup /usr/local/bin/rustup
              node /root/hub.js
          env:
            - name: PI_BIN
              value: pi
            - name: AGENT_MANAGER_BIN
              value: /usr/local/bin/agent-manager
            - name: HUB_WORK_DIR
              value: /root/agent-harness-work
            # planner/investigator/implementer stay on the default
            # (homelab-ornith/ornith:35b, pi's settings.json default). Judge
            # moves to the separate homelab-reasoning backend (DeepSeek-R1,
            # its own 2 GPU replicas) so judge calls stop contending with the
            # other 3 roles for the 2 ornith pods -- an entire role's worth
            # of traffic moves onto otherwise-idle capacity instead.
            - name: JUDGE_PROVIDER
              value: homelab-reasoning
            - name: JUDGE_MODEL
              value: reasoning
          ports:
            - containerPort: 9090
          resources:
            requests:
              cpu: "4"
              memory: 8Gi
            limits:
              cpu: "8"
              memory: 16Gi
          volumeMounts:
            - name: pi-config
              mountPath: /root/.pi/agent/settings.json
              subPath: settings.json
            - name: pi-models
              mountPath: /root/.pi/agent/models.json
              subPath: models.json
            - name: pi-skills
              mountPath: /root/.pi/agent/skills
            - name: hub-src
              mountPath: /root/hub.js
              subPath: hub.js
            - name: coordinator-src
              mountPath: /root/coordinator.js
              subPath: coordinator.js
            - name: ssh-key
              mountPath: /root/.ssh/id_forgejo
              subPath: id_forgejo
            - name: ssh-config
              mountPath: /root/.ssh/config
              subPath: config
      volumes:
        - name: pi-config
          configMap:
            name: pi-config
        - name: pi-models
          secret:
            secretName: pi-models
        - name: pi-skills
          configMap:
            name: pi-skills
            items:
              - key: planner-SKILL.md
                path: planner/SKILL.md
              - key: investigator-SKILL.md
                path: investigator/SKILL.md
              - key: info-collector-SKILL.md
                path: info-collector/SKILL.md
              - key: implementer-SKILL.md
                path: implementer/SKILL.md
              - key: judge-SKILL.md
                path: judge/SKILL.md
              - key: resolver-SKILL.md
                path: resolver/SKILL.md
        - name: hub-src
          configMap:
            name: hub-src
        - name: coordinator-src
          configMap:
            name: coordinator-src
        - name: ssh-key
          secret:
            secretName: agent-pod-ssh-key
            defaultMode: 0600
        - name: ssh-config
          configMap:
            name: agent-pod-ssh-config
@@ -0,0 +1,700 @@
 apiVersion: v1
 data:
  hub.js: |
    #!/usr/bin/env node
    // agent-hub: lives inside the pi container (not a sidecar) so it can spawn
    // `pi` directly, and control the pod's own tmux server. One persistent
    // in-cluster service -- POST /run to trigger a single ad-hoc headless agent
    // run, POST /pipeline to run an ordered list of task phases against a
    // repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within
    // a phase run concurrently, each in its own git worktree). Within one task,
    // planner/investigator/implementer/judge are separate agents in separate
    // named tmux sessions (task-<id>-<role>, attachable via `kubectl exec -it --
    // tmux attach -t <name>` while running), coordinating only through what's on
    // disk in that task's worktree -- not one shared conversation. GET /console
    // (WebSocket) watches every concurrent headless run live, relaying pi's own
    // session protocol verbatim (same event shape Claude Code sessions use).
    const http = require("node:http");
    const crypto = require("node:crypto");
    const fs = require("node:fs");
    const path = require("node:path");
    const { spawn } = require("node:child_process");
    const readline = require("node:readline");
    const { WebSocketServer } = require("ws");
    const PORT = process.env.HUB_PORT || 9090;
    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
    // Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide
    // with unrelated tools on this machine (a Rust CLI and a Datadog TUI,
    // respectively, discovered the hard way this session). Always invoke the
    // exact pinned @earendil-works/[email protected] installed locally under
    // .pi-cli/, by explicit path.
    const PI_BIN =
      process.env.PI_BIN ||
      path.join(
        __dirname,
        "..",
        ".pi-cli",
        "node_modules",
        "@earendil-works",
        "pi-coding-agent",
        "dist",
        "cli.js"
      );
    // Job-type skills under pi/skills/<name>/SKILL.md (mounted at
    // ~/.pi/agent/skills/<name>/ in agent-pod). When `agent` matches one of
    // these, the prompt is forced through pi's `/skill:<name> <args>` mechanism
    // instead of being sent bare -- see pi's skills.md docs on single-shot skill
    // loading. `resolver` is never dispatched directly by a caller; only the
    // pipeline driver invokes it, on stage crashes.
    const ROLE_SKILLS = new Set([
      "planner",
      "investigator",
      "info-collector",
      "implementer",
      "judge",
      "resolver",
    ]);
    const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?}
    const viewers = new Set(); // WebSocket connections watching /console
    function broadcast(type, session) {
      const msg = JSON.stringify({ type, session });
      for (const ws of viewers) {
        if (ws.readyState === ws.OPEN) ws.send(msg);
      }
    }
    function startSession(agent, extra = {}) {
      const id = extra.id || crypto.randomUUID();
      const session = {
        ...extra,
        id,
        agent,
        status: "running",
        events: [],
        startedAt: new Date().toISOString(),
      };
      sessions.set(id, session);
      broadcast("start", session);
      return session;
    }
    function addEvent(session, rawLine) {
      const event = JSON.parse(rawLine);
      session.events.push(event);
      broadcast("event", session);
      return event;
    }
    function endSession(session, status) {
      session.status = status;
      session.endedAt = new Date().toISOString();
      broadcast("end", session);
    }
    // Extracts the plain-text content of a message_end event, if any -- used to
    // find the VERDICT:/RESOLUTION: line judge/resolver skills are required to
    // end their final message with.
    function textOf(event) {
      if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) {
        return null;
      }
      return event.message.content
        .filter((c) => c.type === "text")
        .map((c) => c.text)
        .join("\n");
    }
    // Core spawn primitive. Spawns `pi -p --mode json <extraArgs> <prompt>`,
    // relays every line as a session event exactly like before. Returns
    // { session, done } -- `session` is available synchronously (so an HTTP
    // handler can respond with its id right away, same as the old runAgent),
    // `done` is a Promise resolving once the process exits, for callers that
    // need to wait on a stage (the pipeline driver) rather than fire-and-forget.
    function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) {
      const session = startSession(agent, sessionExtra);
      const args = ["-p", "--mode", "json"];
      if (provider) args.push("--provider", provider);
      if (model) args.push("--model", model);
      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt;
      args.push(finalPrompt);
      const child = spawn(PI_BIN, args, {
        stdio: ["ignore", "pipe", "pipe"],
        cwd,
      });
      const rl = readline.createInterface({ input: child.stdout });
      let lastText = "";
      let stderrTail = "";
      rl.on("line", (line) => {
        if (!line.trim()) return;
        try {
          const event = addEvent(session, line);
          const text = textOf(event);
          if (text) lastText = text;
        } catch {
          // non-JSON stdout noise, ignore
        }
      });
      child.stderr.on("data", (chunk) => {
        process.stderr.write(chunk);
        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
      });
      const done = new Promise((resolve) => {
        child.on("close", (code) => {
          endSession(session, code === 0 ? "done" : "error");
          resolve({ code, session, lastText, stderrTail });
        });
      });
      return { session, done };
    }
    function runAgent(agent, prompt, extraArgs = {}) {
      // Fire-and-forget: caller (the /run handler) doesn't await `done`.
      return spawnPi({ agent, prompt, ...extraArgs }).session;
    }
    function parseVerdictLine(text, label) {
      if (!text) return null;
      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
      const m = text.match(re);
      return m ? m[1].toUpperCase() : null;
    }
    // Deterministic git operations, run directly by hub.js rather than left to
    // the model -- branch creation and pushing after each task are mechanical,
    // not judgment calls, and need to happen reliably every time regardless of
    // what a task's stages did or didn't remember to do.
    function runCmd(bin, args, cwd) {
      return new Promise((resolve) => {
        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
        let out = "";
        child.stdout.on("data", (c) => (out += c));
        child.stderr.on("data", (c) => (out += c));
        child.on("close", (code) => resolve({ code, out: out.trim() }));
      });
    }
    function runGit(cwd, args) {
      return runCmd("git", args, cwd);
    }
    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
    // in practice: a stage writes a real file and simply never runs `git add`/
    // `git commit`, leaving it untracked and invisible to every later `git diff`.
    // Sweep and commit anything left dirty after every stage, deterministically.
    async function commitPending(cwd, message) {
      await runGit(cwd, ["add", "-A"]);
      const status = await runGit(cwd, ["status", "--porcelain"]);
      if (!status.out) return { committed: false };
      const commit = await runGit(cwd, ["commit", "-m", message]);
      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
    }
    // Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide
    // RETRY vs ABORT. Shared by both crash-recovery paths below (headless
    // exit-code failures and interactive sentinel-file timeouts) -- the
    // diagnostic prompt differs per caller, but "ask resolver, parse the
    // RESOLUTION: line" is identical either way.
    async function askResolver(pipelineId, cwd, task, diagnosticPrompt) {
      const resolverResult = await spawnPi({
        agent: "resolver",
        prompt: diagnosticPrompt,
        cwd,
        sessionExtra: { pipelineId, stage: "resolver", task },
      }).done;
      return parseVerdictLine(resolverResult.lastText, "RESOLUTION");
    }
    // Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic
    // judge FAIL, which is handled separately), asks the resolver to diagnose
    // and decide RETRY vs ABORT. Retries the failed stage at most once,
    // regardless of what resolver recommends a second time -- a hard cap, not
    // indefinite trust in the model's judgment.
    async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) {
      let result = await spawnPi({
        agent: stage,
        prompt,
        cwd,
        sessionExtra: { pipelineId, stage, task },
      }).done;
      if (result.code === 0) return result;
      const resolution = await askResolver(
        pipelineId,
        cwd,
        task,
        `Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}`
      );
      if (resolution === "RETRY") {
        result = await spawnPi({
          agent: stage,
          prompt,
          cwd,
          sessionExtra: { pipelineId, stage, task },
        }).done;
      }
      return result;
    }
    // Deterministic tmux operations -- same rationale as runGit: mechanical,
    // not a judgment call, run directly rather than trusted to a prompt.
    function runTmux(args) {
      return runCmd("tmux", args);
    }
    function tmuxSessionName(task) {
      return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`;
    }
    // Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in
    // flight at once. No external dep; a plain in-order index cursor shared by
    // `limit` runner loops.
    async function runConcurrent(items, limit, worker) {
      const results = new Array(items.length);
      let i = 0;
      async function next() {
        while (i < items.length) {
          const idx = i++;
          results[idx] = await worker(items[idx], idx);
        }
      }
      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
      return results;
    }
    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
    const POLL_MS = 10 * 1000;
    async function waitForFile(filePath, limitMs) {
      const start = Date.now();
      while (!fs.existsSync(filePath)) {
        if (Date.now() - start > limitMs) return false;
        await new Promise((r) => setTimeout(r, POLL_MS));
      }
      return true;
    }
    const MAX_IMPLEMENT_ATTEMPTS = 5;
    const MAX_PLAN_REVISIONS = 3;
    // Runs one role as its own fresh interactive pi session in its own named
    // tmux session -- planner, investigator, implementer, and judge are
    // separate agents with separate context, not turns in one shared
    // conversation. They coordinate only through what's on disk in the task's
    // worktree: PLAN.md, committed code, judge's result file. Each session is
    // attachable while it runs (kubectl exec -it -- tmux attach -t <name>) and
    // killed once its sentinel file lands or it's abandoned after resolver
    // escalation.
    async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) {
      const sessionName = `${tmuxSessionName(task)}-${stageLabel}`;
      fs.rmSync(sentinelFile, { force: true });
      const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]);
      if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName };
      let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS);
      if (!ok) {
        const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]);
        const resolution = await askResolver(
          pipelineId,
          cwd,
          task,
          `Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` +
            `Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` +
            `real progress and worth nudging to wrap up, or stuck and worth abandoning?`
        );
        if (resolution === "RETRY") {
          await runTmux([
            "send-keys",
            "-t",
            sessionName,
            `Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`,
            "Enter",
          ]);
          ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS);
        }
      }
      await runTmux(["kill-session", "-t", sessionName]);
      return { ok, sessionName };
    }
    // Runs one task in its own git worktree (see runPhase): planner drafts
    // PLAN.md, investigator confirms it, then implementer and judge go back and
    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
    // next implementer attempt is told to read and address. After
    // MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to
    // judge whether the *plan* itself is wrong, not just the implementation; if
    // so it revises PLAN.md and the implementer gets a fresh attempt budget
    // against the new plan. MAX_PLAN_REVISIONS caps this from looping forever
    // on a task that's genuinely stuck.
    async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) {
      const resultFile = path.join(cwd, `.task-result-${task}`);
      fs.rmSync(resultFile, { force: true });
      const specHint = `the file under tasks/ starting with "${task}-"`;
      const runRole = async (stageLabel, prompt, sentinel) => {
        pipelineSession.activeTasks[task] = {
          stage: stageLabel,
          sessionName: `${tmuxSessionName(task)}-${stageLabel}`,
          startedAt: new Date().toISOString(),
        };
        broadcast("event", pipelineSession);
        const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel);
        await commitPending(cwd, `task: ${task} (${stageLabel})`);
        return result;
      };
      const abandon = (stageLabel, result, attempt) => {
        delete pipelineSession.activeTasks[task];
        broadcast("event", pipelineSession);
        return {
          task,
          status: result.crashed ? "spawn-crashed" : "timed-out",
          error: result.error,
          stoppedAt: stageLabel,
          ...(attempt !== undefined ? { attempt } : {}),
        };
      };
      // Task's implementation is inherited already-committed (e.g. from a base
      // branch of prior work) -- try one judge pass against the spec directly
      // (no PLAN.md exists yet) before paying for a full planner/investigator
      // redo. PASS ends the task here; FAIL falls through into the normal flow
      // below, so planner/implementer pick up with the judge's real feedback.
      if (judgeOnly) {
        const quick = await runRole(
          "judge",
          `Task ${task} may already be implemented on this branch -- check ` +
            `\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` +
            `(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` +
            `verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` +
            `one line of rationale, then run: touch .stage-done-${task}-judge-0`,
          path.join(cwd, `.stage-done-${task}-judge-0`)
        );
        if (!quick.ok) return abandon("judge", quick, 0);
        const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
        if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
          delete pipelineSession.activeTasks[task];
          broadcast("event", pipelineSession);
          return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
        }
      }
      let result = await runRole(
        "planner",
        `Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`,
        path.join(cwd, `.stage-done-${task}-planner`)
      );
      if (!result.ok) return abandon("planner", result);
      result = await runRole(
        "investigator",
        `Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`,
        path.join(cwd, `.stage-done-${task}-investigator`)
      );
      if (!result.ok) return abandon("investigator", result);
      let planRevisions = 0;
      let implementAttempt = 0;
      let verdict = null;
      let resultText = "";
      let justRevisedPlan = false;
      while (true) {
        implementAttempt++;
        const feedbackHint = fs.existsSync(resultFile)
          ? justRevisedPlan
            ? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
              `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
            : `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.`
          : "";
        justRevisedPlan = false;
        result = await runRole(
          "implementer",
          `Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`,
          path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
        );
        if (!result.ok) return abandon("implementer", result, implementAttempt);
        result = await runRole(
          "judge",
          `Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` +
            `.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` +
            `rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`,
          path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`)
        );
        if (!result.ok) return abandon("judge", result, implementAttempt);
        resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
        verdict = parseVerdictLine(resultText, "VERDICT");
        if (verdict === "PASS") break;
        if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
          if (planRevisions >= MAX_PLAN_REVISIONS) break;
          planRevisions++;
          result = await runRole(
            "planner-revise",
            `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
              `the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` +
              `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` +
              `commit. If you change the approach, also use the investigator skill to confirm the new approach against ` +
              `real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` +
              `touch .stage-done-${task}-planner-revise-${planRevisions}`,
            path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)
          );
          if (!result.ok) return abandon("planner-revise", result, planRevisions);
          implementAttempt = 0;
          justRevisedPlan = true;
        }
      }
      delete pipelineSession.activeTasks[task];
      broadcast("event", pipelineSession);
      if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
        return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
      }
      return {
        task,
        status: verdict === "PASS" ? "done" : "done-with-concerns",
        judgeRationale: resultText,
      };
    }
    const PHASE_CONCURRENCY = 3;
    // Runs one phase (a batch of tasks with no declared dependency on each
    // other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task
    // gets its own git worktree off workBranch -- concurrent pi sessions writing
    // into one shared working tree would corrupt the index; worktrees share the
    // same object database but give each task an isolated checkout. After a
    // task's session ends, its branch is merged back into workBranch and pushed,
    // one merge at a time (git ref updates aren't safe to run concurrently even
    // though the worktrees themselves are isolated).
    async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) {
      // Each entry is either a plain task id, or { id, judgeOnly: true } when
      // the task's implementation already exists (e.g. inherited from a base
      // branch) and just needs a real judge pass rather than a full
      // planner/investigator/implementer redo.
      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
      const worktrees = {};
      for (const entry of entries) {
        const task = entry.id;
        const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`);
        const taskBranch = `task/${task}`;
        const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]);
        if (add.code !== 0) {
          pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out });
          continue;
        }
        worktrees[task] = { wtDir, taskBranch };
      }
      const runnable = entries.filter((e) => worktrees[e.id]);
      await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => {
        const { wtDir } = worktrees[entry.id];
        const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly);
        pipelineSession.taskResults.push(result);
        return result;
      });
      // Merge + push sequentially -- ref updates on the shared repo, one at a
      // time, in the declared task order for this phase.
      for (const entry of runnable) {
        const task = entry.id;
        const { wtDir, taskBranch } = worktrees[task];
        const result = pipelineSession.taskResults.find((r) => r.task === task);
        const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]);
        if (merge.code !== 0) {
          await runGit(cwd, ["merge", "--abort"]);
          if (result) {
            result.status = "merge-conflict";
            result.mergeError = merge.out;
          }
        } else {
          const push = await runGit(cwd, ["push", "-u", "origin", workBranch]);
          if (result) {
            result.pushed = push.code === 0;
            if (!result.pushed) result.pushError = push.out;
          }
        }
        await runGit(cwd, ["worktree", "remove", wtDir, "--force"]);
        await runGit(cwd, ["branch", "-D", taskBranch]);
        broadcast("event", pipelineSession);
      }
    }
    // tasks: array of phases, each phase an array of task ids with no declared
    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) --
    // caller's responsibility to supply real phase grouping (see tasks/INDEX.md;
    // filename/numeric sort does NOT match execution order on boards like this).
    // A flat array of ids is also accepted and treated as one single phase.
    // Phases run strictly sequentially (a phase boundary is a real dependency
    // gate); tasks within a phase run concurrently, each in its own worktree --
    // see runPhase.
    function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) {
      const cwd = path.join(WORK_DIR, pipelineId);
      fs.mkdirSync(cwd, { recursive: true });
      const phases = Array.isArray(tasks[0]) ? tasks : [tasks];
      const pipelineSession = startSession("pipeline", {
        id: pipelineId,
        pipelineId,
        stage: "pipeline",
        taskResults: [],
        activeTasks: {},
        totalTasks: phases.flat().length,
      });
      (async () => {
        const clone = await runStageWithResolver(
          pipelineId,
          cwd,
          "planner",
          `Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`,
          "clone"
        );
        if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed");
        if (!fs.existsSync(path.join(cwd, ".git"))) {
          // The model deciding to `cd` elsewhere before cloning (instead of
          // cloning into the assigned cwd) is a real failure mode seen in
          // practice, not a hypothetical -- exit code 0 doesn't mean the clone
          // landed where every later stage's cwd assumes it did.
          return endSession(pipelineSession, "clone-missing");
        }
        // Dedicated branch, never main -- and pushed after every single task
        // (not just at the end) so a pod restart mid-run loses at most the
        // in-progress task's work, not everything since the start.
        const workBranch = branchName || `agent-run/${pipelineId}`;
        const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]);
        if (branchResult.code !== 0) {
          pipelineSession.gitError = branchResult.out;
          return endSession(pipelineSession, "branch-crashed");
        }
        // Seen in practice: a task manually downloads a dependency tarball
        // (crates.io registry access isn't guaranteed from every sandboxed
        // checkout) and it lands at repo root, outside whatever .gitignore
        // already covers -- then git add -A (ours or the model's own) commits
        // it. Append broad build-artifact/archive patterns before any task
        // runs, so it's excluded regardless of who stages files later.
        const gitignoreAdditions = [
          "",
          "# agent-harness: build artifacts and vendored archives never belong in source control",
          "*.tar.gz",
          "*.tgz",
          "*.crate",
          "*.zip",
          "*.bin",
          "*.whl",
          "vendor/",
          "node_modules/",
          "",
          "# agent-harness: task completion sentinel files, harness bookkeeping only",
          ".task-result-*",
          ".stage-done-*",
        ].join("\n");
        fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
        await runGit(cwd, ["add", ".gitignore"]);
        await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
        for (const phaseTasks of phases) {
          await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession);
        }
        const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed"));
        endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed");
      })();
      return pipelineSession;
    }
    const server = http.createServer((req, res) => {
      const url = new URL(req.url, "http://localhost");
      if (url.pathname === "/healthz") {
        res.writeHead(200).end();
        return;
      }
      if (url.pathname === "/sessions" && req.method === "GET") {
        res.writeHead(200, { "Content-Type": "application/json" });
        res.end(JSON.stringify([...sessions.values()]));
        return;
      }
      if (url.pathname === "/run" && req.method === "POST") {
        let body = "";
        req.on("data", (chunk) => (body += chunk));
        req.on("end", () => {
          try {
            const { agent, prompt, provider, model } = JSON.parse(body);
            if (!agent || !prompt) throw new Error("agent and prompt are required");
            const session = runAgent(agent, prompt, { provider, model });
            res.writeHead(200, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ id: session.id }));
          } catch (err) {
            res.writeHead(400, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ error: err.message }));
          }
        });
        return;
      }
      if (url.pathname === "/pipeline" && req.method === "POST") {
        let body = "";
        req.on("data", (chunk) => (body += chunk));
        req.on("end", () => {
          try {
            const { repo, baseBranch, tasks, branchName } = JSON.parse(body);
            if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) {
              throw new Error("repo, baseBranch, and a non-empty tasks array are required");
            }
            const pipelineId = crypto.randomUUID();
            runPipeline({ pipelineId, repo, baseBranch, tasks, branchName });
            res.writeHead(200, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ id: pipelineId }));
          } catch (err) {
            res.writeHead(400, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ error: err.message }));
          }
        });
        return;
      }
      res.writeHead(404).end();
    });
    const wss = new WebSocketServer({ server, path: "/console" });
    wss.on("connection", (ws) => {
      for (const session of sessions.values()) {
        ws.send(JSON.stringify({ type: "snapshot", session }));
      }
      viewers.add(ws);
      ws.on("close", () => viewers.delete(ws));
    });
    server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
 kind: ConfigMap
 metadata:
  name: hub-src
  namespace: agent-pod
@@ -0,0 +1,11 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: agent-hub
  namespace: agent-pod
 spec:
  selector:
    app: agent-pod
  ports:
    - port: 9090
      targetPort: 9090
@@ -0,0 +1,12 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: agent-pod
 resources:
  - deployment.yaml
  - configmap.yaml
  - hub-configmap.yaml
  - coordinator-configmap.yaml
  - pi-skills-configmap.yaml
  - ssh-configmap.yaml
  - hub-service.yaml
  - console-ingress.yaml
@@ -0,0 +1,122 @@
 apiVersion: v1
 data:
  implementer-SKILL.md: |
    ---
    name: implementer
    description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run.
    allowed-tools: Read Grep Find Ls Write Edit Bash
    ---
    Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions.
    - Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history.
    - Push only if the task explicitly asks for it.
    Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch.
    **UI/frontend changes:** don't trust that the code compiles as proof it works. Start the app (or its dev server) and use `npx playwright` via Bash to actually load the page and look — screenshot the affected view before and after your change, and click through the golden path the plan describes. `npx playwright screenshot <url> out.png` for a quick visual check; for interaction (clicks, form fills, navigation), write a small throwaway script under a scratch path (e.g. `/tmp/`, never committed) using `playwright` the library, run it with `node`, then delete it. This doesn't apply to non-UI work (a Rust library, a CLI, a backend-only change) — use judgment.
    **Hard rules:**
    - Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing.
    - Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it.
    **Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it.
  info-collector-SKILL.md: |
    ---
    name: info-collector
    description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill).
    allowed-tools: Read Bash
    ---
    **Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there.
    **Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job).
    **Modes:**
    - **Collect mode** (default) — search the web (`curl` against `https://api.search.brave.com/res/v1/web/search`, header `X-Subscription-Token: $BRAVE_API_KEY`, `--data-urlencode "q=<query>"`) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill.
    - If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response.
  investigator-SKILL.md: |
    ---
    name: investigator
    description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct.
    allowed-tools: Read Bash
    ---
    Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one.
    - Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), search the web (`curl` against `https://api.search.brave.com/res/v1/web/search`, header `X-Subscription-Token: $BRAVE_API_KEY`) to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`.
    - No `PLAN.md`? Just answer the question asked, citing sources.
    Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next.
    **Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not.
  judge-SKILL.md: |
    ---
    name: judge
    description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria.
    allowed-tools: Read Bash
    ---
    Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp.
    - Run `git diff <base-branch>...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection?
    - **UI/frontend changes:** a diff that reads correctly can still render broken. Start the app and use `npx playwright` via Bash to actually look — screenshot the affected view, click through the golden path the plan/spec describes. FAIL on a visual defect the diff alone wouldn't show (broken layout, a control that doesn't do what its code claims, a state the plan promised that never renders). Doesn't apply to non-UI work — use judgment.
    - FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale.
    - No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly.
    You MUST end your final message with a literal verdict line, exactly one of:
    ```
    VERDICT: PASS
    ```
    ```
    VERDICT: FAIL
    ```
    followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline.
  planner-SKILL.md: |
    ---
    name: planner
    description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it.
    allowed-tools: Read Grep Find Ls Write Bash
    ---
    Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms.
    - If cwd is empty: `git clone` the given repo/branch first (only at pipeline start).
    - Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity.
    - Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/<name>.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: <summary>"`.
    Don't touch any other file.
  resolver-SKILL.md: |
    ---
    name: resolver
    description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion.
    allowed-tools: Read Bash
    ---
    **Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again.
    **Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work.
    **Modes:**
    - **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix).
    You MUST end your final message with a literal resolution line, exactly one of:
    ```
    RESOLUTION: RETRY
    ```
    ```
    RESOLUTION: ABORT
    ```
    followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`.
 kind: ConfigMap
 metadata:
  name: pi-skills
  namespace: agent-pod
@@ -0,0 +1,12 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: agent-pod-ssh-config
  namespace: agent-pod
 data:
  config: |
    Host git.riotpiao.com
        IdentityFile /root/.ssh/id_forgejo
        Port 2222
        User git
        StrictHostKeyChecking accept-new
@@ -0,0 +1,46 @@
 # Edge route for the API gateway.
 #
 # Lives here rather than in the central k8s/bootstrap/ingress/ingress.yaml
 # because that Application syncs in wave 1, before namespace `api` exists.
 #
 # nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its
 # default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards
 # plain HTTP to the gateway.
 #
 # Backend was kong-proxy:80 until Kong was retired on 2026-08-19; it is now the
 # Go gateway's Service, api-gateway:8080, deployed from rock/homelab-frontend.
 # Reverting the cutover is a change to these two lines and nothing else.
 #
 # Catch-all `/` on purpose: everything under this host belongs to the gateway.
 # Listing per-API paths here would duplicate the gateway's routing table inside
 # nginx, and the two copies would drift.
 #
 # In-cluster callers should prefer http://api-gateway.api.svc.cluster.local:8080
 # directly. Resolving api.riotpiao.com sends them out to nginx and back in,
 # which is a pointless hairpin unless they need TLS or the public hostname.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: api
  namespace: api
  annotations:
    # An API gateway carries streaming responses (SSE, gRPC-web, LLM token
    # streams). nginx's 60s default read timeout and its response buffering
    # would truncate or stall those.
    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
    nginx.ingress.kubernetes.io/proxy-buffering: "off"
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
 spec:
  ingressClassName: nginx
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: api-gateway
                port:
                  number: 8080
@@ -0,0 +1,14 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # Explicit allowlist. Anything added to this directory and not listed here is
 # silently dropped — no error, no drift shown.
 #
 # Down to a single Ingress since Kong was retired (2026-08-19). The Kong Helm
 # values, the KongClusterPlugin for Prometheus, the six KongPlugin CRs behind
 # the path-per-model LLM surface, the KongConsumer and the key-auth plan all
 # went with it.
 resources:
  - ingress.yaml
 # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
 # namespace, and the transformer rewrites metadata.namespace on every resource
 # it builds, which is a trap for anything cross-namespace added later.
@@ -0,0 +1,37 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: cloudflared
  namespace: cloudflared
 spec:
  replicas: 2
  selector:
    matchLabels:
      app: cloudflared
  template:
    metadata:
      labels:
        app: cloudflared
    spec:
      containers:
        - name: cloudflared
          image: cloudflare/cloudflared:latest
          args:
            - tunnel
            - --no-autoupdate
            - run
            - --token
            - $(TUNNEL_TOKEN)
          env:
            - name: TUNNEL_TOKEN
              valueFrom:
                secretKeyRef:
                  name: cloudflared-token
                  key: token
          resources:
            requests:
              cpu: 10m
              memory: 32Mi
            limits:
              cpu: 100m
              memory: 128Mi
@@ -0,0 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: cloudflared
 resources:
  - deployment.yaml
@@ -0,0 +1,78 @@
 # Homarr landing page with Authentik SSO
 # Probe tuning (chart DOES expose these — the old PostSync patch-job was
 # unnecessary and fragile: it only patched one Deployment revision, so any later
 # rollout reverted to the chart's aggressive defaults). Homarr's first-boot icon
 # updater blocks the event loop for ~50s ("icons updater took 49553ms"), during
 # which /api/health/live can't answer within the default 10s×3 window → kubelet
 # SIGTERMs the pod → CrashLoopBackOff (247 restarts, 503 at the ingress). Give
 # liveness a wide window so the icon import can finish without a kill.
 livenessProbe:
  initialDelaySeconds: 60
  periodSeconds: 30
  timeoutSeconds: 5
  failureThreshold: 10
 readinessProbe:
  initialDelaySeconds: 30
  periodSeconds: 15
  timeoutSeconds: 5
  failureThreshold: 6
 image:
  repository: ghcr.io/homarr-labs/homarr
  tag: "latest"
  pullPolicy: Always
 replicaCount: 1
 # Configure SSO via environment variables
 # Chart supports these via top-level env dict (not array)
 env:
  AUTH_PROVIDERS: "oidc,credentials"
  AUTH_OIDC_ISSUER: "https://authentik.riotpiao.com/application/o/homarr/"
  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
  # hides the "Sign in with Authentik" button entirely when it's absent (per the
  # authentik Homarr integration + homarr SSO docs). This was the missing var.
  AUTH_OIDC_URI: "https://authentik.riotpiao.com/application/o/authorize/"
  AUTH_OIDC_CLIENT_NAME: "Authentik"
  AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
  AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
  AUTH_OIDC_AUTO_LOGIN: "false"
  # Link the OIDC identity to an existing homarr account with the same email.
  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
  # per run ("callback took longer than expected"), compounding CPU pressure.
  DISABLE_ANALYTICS: "true"
  BASE_URL: "https://homarr.riotpiao.com"
  NEXTAUTH_URL: "https://homarr.riotpiao.com"
 # Client credentials from homarr-oidc secret
 # Chart doesn't support envFrom, so we add via extraEnv
 extraEnv:
  - name: AUTH_OIDC_CLIENT_ID
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-id
  - name: AUTH_OIDC_CLIENT_SECRET
    valueFrom:
      secretKeyRef:
        name: homarr-oidc
        key: client-secret
 tolerations:
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule
 resources:
  requests:
    cpu: 250m
    memory: 384Mi
  limits:
    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
    # real CPU headroom.
    cpu: "2"
    memory: 1Gi
@@ -0,0 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: dashboard
 # Probes are now tuned via homarr-values.yaml (chart-native); the old
 # fix-probes-job PostSync hook is removed. homarr-secrets/auth-oidc/db-encryption
 # Secrets are delivered by the sops-secrets (ksops) Application.
 resources: []
@@ -0,0 +1,14 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: immich-config
 data:
  DB_HOSTNAME: "immich-db-rw"
  DB_DATABASE_NAME: "immich"
  # Only pgvector is installed (see db.yaml) - no vectorchord extension image
  # exists for pg18 in CNPG's catalog yet. Explicit instead of relying on
  # auto-detect's vectorchord-first preference order.
  DB_VECTOR_EXTENSION: "pgvector"
  REDIS_HOSTNAME: "immich-redis"
  IMMICH_MACHINE_LEARNING_URL: "http://immich-machine-learning:3003"
  TZ: "America/Los_Angeles"
@@ -0,0 +1,58 @@
 # Dedicated CNPG Postgres for Immich. Same recipe as paperless-db/authentik-db
 # (2 instances, default longhorn storage class) except the operand is
 # PostgreSQL 18, not 16.2 - the official CNPG pgvector extension image
 # (ghcr.io/cloudnative-pg/pgvector) is only published for pg18, no pg16 tags
 # exist in that registry. Immich itself supports pg18 fine (immich-app's own
 # postgres image already ships 18-vectorchord builds).
 #
 # pgvector loaded via CNPG's ImageVolume extension mechanism (CNPG 1.27+,
 # k8s ImageVolume feature - both present here: operator is 1.30.0, cluster is
 # v1.36.1). No shared_preload_libraries needed - pgvector doesn't require
 # preload, just CREATE EXTENSION, which immich-server issues itself at
 # startup. Distro/pg-major must match between the operand image and the
 # extension image (both "18"+"trixie" here) - CNPG's own compatibility rule.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: immich-db
  annotations:
    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
  instances: 2
  imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie
  postgresql:
    extensions:
      - name: pgvector
        image:
          reference: ghcr.io/cloudnative-pg/pgvector:0.8.1-18-trixie
  bootstrap:
    initdb:
      database: immich
      owner: app
      encoding: UTF8
      localeCollate: C
      localeCType: C
      # CREATE EXTENSION vector requires superuser (pgvector's control file
      # isn't marked trusted) and the "app" owner role isn't one
      # (enableSuperuserAccess: false, repo convention) - postInitApplicationSQL
      # runs as superuser during initdb, before the app ever connects. Only
      # fires on a fresh bootstrap; the live cluster already had this run
      # manually once (kubectl exec ... psql -U postgres -c 'CREATE EXTENSION').
      postInitApplicationSQL:
        - "CREATE EXTENSION IF NOT EXISTS vector;"
        - "CREATE EXTENSION IF NOT EXISTS cube;"
        - "CREATE EXTENSION IF NOT EXISTS earthdistance;"
  enableSuperuserAccess: false
  resources:
    requests: { memory: "512Mi", cpu: "250m" }
    limits: { memory: "2Gi", cpu: "1" }
  storage:
    size: 20Gi
    storageClass: longhorn
  affinity:
    podAntiAffinityType: preferred
    topologyKey: kubernetes.io/hostname
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
@@ -0,0 +1,100 @@
 # immich-server: pinned to talos-cp-3, same reasoning as paperless
 # (deployment.yaml comment there) - immich-media is a ReadWriteOnce Longhorn
 # volume with a single replica physically on that node's disk (shared with
 # paperless-media on the same 4TB HDD). Recreate strategy for the same
 # reason: two pods can't both attach an RWO volume.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: immich-server
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: immich-server
  template:
    metadata:
      labels:
        app: immich-server
    spec:
      serviceAccountName: immich
      nodeSelector:
        kubernetes.io/hostname: talos-cp-3
      containers:
        - name: immich-server
          image: ghcr.io/immich-app/immich-server:release
          ports:
            - containerPort: 2283
          envFrom:
            - configMapRef:
                name: immich-config
          env:
            - name: DB_USERNAME
              valueFrom:
                secretKeyRef:
                  name: immich-db-app
                  key: username
            - name: DB_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: immich-db-app
                  key: password
            # Composed by k8s/infra/iam's provisioning script (system-config
            # JSON, oauth section) - see immich-oidc Secret.
            - name: IMMICH_CONFIG_FILE
              value: /config/immich.json
          resources:
            requests: { cpu: "500m", memory: "1Gi" }
            limits: { cpu: "2", memory: "4Gi" }
          volumeMounts:
            - name: media
              mountPath: /usr/src/app/upload
            - name: oidc-config
              mountPath: /config
              readOnly: true
      volumes:
        - name: media
          persistentVolumeClaim:
            claimName: immich-media
        - name: oidc-config
          secret:
            secretName: immich-oidc
            items:
              - key: config.json
                path: immich.json
 ---
 # CPU-only for now - the cluster's one GPU node (worker-1) is already
 # dedicated to llm-serving predictors. Not node-pinned: its cache PVC is on
 # the default 3-replica pool, not the single-disk cp-3 HDD.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: immich-machine-learning
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: immich-machine-learning
  template:
    metadata:
      labels:
        app: immich-machine-learning
    spec:
      serviceAccountName: immich
      containers:
        - name: immich-machine-learning
          image: ghcr.io/immich-app/immich-machine-learning:release
          ports:
            - containerPort: 3003
          resources:
            requests: { cpu: "500m", memory: "1Gi" }
            limits: { cpu: "2", memory: "4Gi" }
          volumeMounts:
            - name: ml-cache
              mountPath: /cache
      volumes:
        - name: ml-cache
          persistentVolumeClaim:
            claimName: immich-ml-cache
@@ -0,0 +1,24 @@
 # Direct nginx ingress, same reasoning as paperless: large uploads (photos/
 # videos) and long-lived operations (video transcode, big batch uploads) need
 # proxy-body-size/timeouts raised past nginx's defaults.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: immich
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "0"
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
 spec:
  ingressClassName: nginx
  rules:
    - host: img.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: immich-server
                port:
                  number: 2283
@@ -0,0 +1,14 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: immich
 resources:
  - db.yaml
  - pvc.yaml
  - configmap.yaml
  - redis.yaml
  - deployment.yaml
  - service.yaml
  - ingress.yaml
  - rbac.yaml
 # immich-oidc Secret written by the PostSync provisioning Job in
 # k8s/infra/iam (same as paperless-oidc) - not duplicated here.
@@ -0,0 +1,38 @@
 # Two volumes:
 #
 # - media: original photos/videos + generated thumbnails/encoded videos.
 #   Shares the cp-3 USB HDD with paperless-media, same StorageClass/disk tag,
 #   single replica (single disk, no redundancy possible - same tradeoff
 #   paperless already accepts). Sized 1400Gi, not 2000Gi: the disk's real
 #   usable capacity (~3724GiB, formatting overhead) minus paperless-media's
 #   2000Gi and ~231GiB of other apps' default-class replicas that Longhorn
 #   placed here anyway (disk tags only pull matching volumes in, they don't
 #   exclude non-matching ones when the untagged pool elsewhere is full) only
 #   leaves ~1493Gi of real scheduling headroom right now.
 # - ml-cache: downloaded ML model weights for immich-machine-learning
 #   (face detection / CLIP embeddings). Small, disposable (re-downloads on
 #   loss), but persisted so a pod restart doesn't re-pull multi-GB models -
 #   default 3-replica pool, not node-pinned.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: immich-media
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn-paperless-media
  resources:
    requests:
      storage: 1400Gi
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: immich-ml-cache
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 5Gi
@@ -0,0 +1,43 @@
 # Scoped operator access for immich-admins: restart/config-edit rights on
 # just this service's own resources, nothing CNPG-managed (immich-db-*) or
 # provisioning-managed (immich-oidc). Same pattern as
 # k8s/apps/paperless/rbac.yaml. Inert until kube-apiserver's OIDC wiring
 # lands (--oidc-groups-claim=groups, --oidc-groups-prefix=oidc:).
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: immich
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: immich-operator
 rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    resourceNames: ["immich-server", "immich-machine-learning"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["immich-config"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["immich-oidc"]
    verbs: ["get", "list", "watch", "update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: immich-admins-binding
 subjects:
  - kind: Group
    name: "oidc:immich-admins"
    apiGroup: rbac.authorization.k8s.io
  - kind: ServiceAccount
    name: immich
    namespace: immich
 roleRef:
  kind: Role
  name: immich-operator
  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,37 @@
 # Job queue broker for immich-server. No PVC: queue state is disposable - a
 # lost queue on restart just re-triggers the affected background jobs
 # (thumbnail generation, ML jobs, etc.), no photo data loss since originals
 # live on immich-media.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: immich-redis
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: immich-redis
  template:
    metadata:
      labels:
        app: immich-redis
    spec:
      containers:
        - name: redis
          image: redis:7-alpine
          ports:
            - containerPort: 6379
          resources:
            requests: { cpu: "50m", memory: "64Mi" }
            limits: { cpu: "250m", memory: "256Mi" }
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: immich-redis
 spec:
  selector:
    app: immich-redis
  ports:
    - port: 6379
      targetPort: 6379
@@ -0,0 +1,21 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: immich-server
 spec:
  selector:
    app: immich-server
  ports:
    - port: 2283
      targetPort: 2283
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: immich-machine-learning
 spec:
  selector:
    app: immich-machine-learning
  ports:
    - port: 3003
      targetPort: 3003
@@ -0,0 +1,56 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-embeddings
    app.kubernetes.io/part-of: llm-serving
  name: embeddings
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model-id=nomic-ai/nomic-embed-text-v2-moe
      - --port=8080
      - --hostname=0.0.0.0
      - --auto-truncate
      env:
      - name: HUGGINGFACE_HUB_CACHE
        value: /mnt/models
      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 8Gi
        requests:
          cpu: '8'
          memory: 4Gi
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,18 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # Explicit allowlist, matching k8s/apps/api. Anything added to this directory
 # and not listed here is silently dropped — no error, no drift shown.
 #
 # These five were adopted from live state on 2026-08-15; they had been applied
 # by hand and carried no ArgoCD ownership. Each was exported and verified with
 # `kubectl diff -f <file>` returning empty before the Application below was
 # created, so the first sync was a no-op rather than a redeploy. Re-verify that
 # way after any edit here: a GPU predictor restart is a weights reload measured
 # in tens of seconds, not a rolling update.
 resources:
  - embeddings.yaml
  - ornith.yaml
  - reasoning.yaml
  - reranker.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.
@@ -0,0 +1,108 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # The konghq.com/{connect,read,write}-timeout annotations that used to live
    # here went with Kong (retired 2026-08-19). They existed because Kong read
    # its upstream timeouts off the Kubernetes Service, and its 60s default cut
    # off the first request after any pod restart — a restart flushes VRAM and
    # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
    # problem in steady state.
    #
    # The equivalent budget now belongs to the Go gateway's per-route timeout
    # config in rock/homelab-frontend, not to an annotation on this object.
  labels:
    app.kubernetes.io/name: llm-ornith
    app.kubernetes.io/part-of: llm-serving
  name: ornith
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - command:
      - /bin/sh
      - -c
      - 'set -e
        ollama serve &
        SERVE_PID=$!
        until ollama list >/dev/null 2>&1; do sleep 2; done
        ollama pull ornith:35b
        ollama pull qwen2.5:3b-instruct
        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
        wait $SERVE_PID
        '
      env:
      - name: OLLAMA_HOST
        value: 0.0.0.0:8080
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      - name: OLLAMA_MAX_LOADED_MODELS
        value: '2'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    deploymentStrategy:
      type: Recreate
    # 2 replicas -- each its own GPU, each loading both ornith:35b and
    # qwen2.5:3b-instruct -- so 2 concurrent implementer-style calls each
    # get an independent instance instead of contending on one, at the
    # cost of judge/qwen traffic still sharing whichever replica an
    # implementer call also lands on.
    maxReplicas: 2
    minReplicas: 2
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,122 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      # bnb-4bit retired: no int4 tensor cores on sm70/V100, dequant-then-
      # matmul is two slow kernel launches instead of one fused int4 GEMM,
      # decode crawled at 2.5-10 tok/s regardless of TP/PP. Switched to
      # JunHowie/Qwen3-32B-GPTQ-Int4 -- same dense Qwen3-32B weights, same
      # hermes/qwen3 parser stack (no narration-bug risk, same as before),
      # only the quant format changes. Plain (non-Marlin) GPTQ kernel is
      # confirmed Volta-compatible; Marlin needs sm80+ and vLLM would try
      # to auto-upgrade to it, so --quantization is pinned explicitly to
      # `gptq` to force the plain kernel. Verified checkpoint size: 19.34GB
      # (summed from the real safetensors index, not bits-per-param math).
      # max-model-len=131072 is Qwen3-32B's real ceiling (config.json YaRN:
      # factor=4.0, original_max_position_embeddings=32768) -- 200k was
      # asked for but exceeds this architecturally regardless of VRAM.
      # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128
      # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so
      # each GPU carries ~9.67GB weights + ~128KB/token KV. At
      # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves
      # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably
      # above the 131072 target with room to spare -- the old
      # OffloadingConnector CPU-DRAM spillover (tuned for the previous
      # model's much smaller 16384 context) is no longer needed and is
      # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed,
      # this checkpoint only requires vllm>=0.9.2) -- plain GPTQ has no
      # TP>1 restriction unlike bnb, so tensor-parallel-size=2 is worth
      # trying later, but not risking a parallelism-strategy change in the
      # same rollout as the quant+context-length change.
      # This GPTQ requant's own config.json ships max_position_embeddings=
      # 40960 and rope_scaling=None -- confirmed directly (curl'd the raw
      # config.json), the base Qwen3-32B repo's YaRN block did NOT carry
      # over during quantization. Re-applying it explicitly here restores
      # the same math the base model documents (32768 * 4.0 = 131072);
      # without this, --max-model-len=131072 fails ModelConfig validation
      # against the checkpoint's own (unscaled) 40960 ceiling.
      - --model=JunHowie/Qwen3-32B-GPTQ-Int4
      - --served-model-name=reasoning
      - --quantization=gptq
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --rope-scaling={"rope_type":"yarn","factor":4.0,"original_max_position_embeddings":32768}
      - --tensor-parallel-size=1
      - --pipeline-parallel-size=2
      - --max-model-len=131072
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=4
      - --enable-chunked-prefill
      - --enable-prefix-caching
      # qwen3 is vLLM's dedicated reasoning parser for this family's <think>
      # blocks.
      - --reasoning-parser=qwen3
      # hermes is the documented tool-call parser for general (non-Coder)
      # Qwen3 models -- native chat template support, not narrated text.
      - --enable-auto-tool-choice
      - --tool-call-parser=hermes
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 36Gi
          nvidia.com/gpu: '2'
        requests:
          cpu: '8'
          memory: 12Gi
          nvidia.com/gpu: '2'
      startupProbe:
        failureThreshold: 80
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm
@@ -0,0 +1,56 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reranker
    app.kubernetes.io/part-of: llm-serving
  name: reranker
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model-id=BAAI/bge-reranker-base
      - --port=8080
      - --hostname=0.0.0.0
      - --auto-truncate
      env:
      - name: HUGGINGFACE_HUB_CACHE
        value: /mnt/models
      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 8Gi
        requests:
          cpu: '8'
          memory: 4Gi
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,5 @@
 apiVersion: v2
 name: kafka-cluster
 description: Strimzi Kafka/KafkaNodePool CRs for the kmsvc Kafka cluster (design.md §7)
 type: application
 version: 0.1.0
@@ -0,0 +1,30 @@
 apiVersion: kafka.strimzi.io/v1beta2
 kind: Kafka
 metadata:
  name: {{ .Values.clusterName }}
  namespace: {{ .Values.namespace }}
  annotations:
    strimzi.io/node-pools: enabled
    strimzi.io/kraft: enabled
 spec:
  kafka:
    version: 4.0.0
    metadataVersion: 4.0-IV3
    listeners:
      - name: plain
        port: 9092
        type: internal
        tls: false
      - name: tls
        port: 9093
        type: internal
        tls: true
    config:
      default.replication.factor: {{ .Values.kafka.replicationFactor }}
      min.insync.replicas: {{ .Values.kafka.minInsyncReplicas }}
      offsets.topic.replication.factor: {{ .Values.kafka.replicationFactor }}
      transaction.state.log.replication.factor: {{ .Values.kafka.replicationFactor }}
      transaction.state.log.min.isr: {{ .Values.kafka.minInsyncReplicas }}
  entityOperator:
    topicOperator: {}
    userOperator: {}
@@ -0,0 +1,39 @@
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaNodePool
 metadata:
  name: {{ .Values.clusterName }}-pool
  namespace: {{ .Values.namespace }}
  labels:
    strimzi.io/cluster: {{ .Values.clusterName }}
 spec:
  replicas: {{ .Values.nodePool.replicas }}
  roles:
    - controller
    - broker
  storage:
    type: persistent-claim
    size: {{ .Values.nodePool.storage.sizeGi }}Gi
    class: {{ .Values.nodePool.storage.class }}
    deleteClaim: false
  resources:
    limits:
      memory: {{ .Values.nodePool.resources.memory }}
      cpu: {{ .Values.nodePool.resources.cpu | quote }}
    requests:
      memory: {{ .Values.nodePool.resources.memory }}
      cpu: {{ .Values.nodePool.resources.cpu | quote }}
  template:
    pod:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                topologyKey: {{ .Values.nodePool.antiAffinityTopologyKey }}
                labelSelector:
                  matchLabels:
                    strimzi.io/cluster: {{ .Values.clusterName }}
    kafkaContainer:
      env:
        - name: KAFKA_HEAP_OPTS
          value: {{ .Values.nodePool.heapOpts | quote }}
@@ -0,0 +1,26 @@
 clusterName: kmsvc
 namespace: sqs
 nodePool:
  replicas: 3
  storage:
    class: longhorn
    # Longhorn's per-node scheduling budget on the current 2-node cluster has
    # only ~36Gi of headroom left (other PVCs already reserve the rest), and
    # each node hosts one replica of all 3 broker volumes -- so 3 * sizeGi
    # must fit in that headroom. Revisit once the 3rd node joins.
    sizeGi: 10
  resources:
    memory: 5Gi
    cpu: "2"
  heapOpts: "-Xms2g -Xmx2g"
  # design.md §7: 3 real zones now exist (talos-cp-1=az-a, talos-worker-1=az-b,
  # talos-worker-2=az-c), so anti-affinity keys off zone instead of hostname —
  # spreads the 3 broker pods one-per-zone/one-per-node (equivalent today,
  # but zone is the correct long-term key if a node ever gets replaced within
  # the same zone).
  antiAffinityTopologyKey: topology.kubernetes.io/zone
 kafka:
  replicationFactor: 3
  minInsyncReplicas: 2
@@ -0,0 +1,5 @@
 apiVersion: v2
 name: management-service
 description: kmsvc message-plane gRPC+REST server (design.md §1, §7a, §9)
 type: application
 version: 0.1.0
@@ -0,0 +1,12 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: management-service-config
  namespace: {{ .Values.namespace }}
 data:
  KMSVC_KAFKA_BROKERS: {{ .Values.env.kafkaBrokers | quote }}
  KMSVC_REDIS_ADDR: {{ .Values.env.redisAddr | quote }}
  KMSVC_AUTHENTIK_ISSUER_URL: {{ .Values.env.authentikIssuerURL | quote }}
  KMSVC_AUTHENTIK_AUDIENCE: {{ .Values.env.authentikAudience | quote }}
  KMSVC_GRPC_LISTEN_ADDR: ":{{ .Values.grpcPort }}"
  KMSVC_HTTP_LISTEN_ADDR: ":{{ .Values.httpPort }}"
@@ -0,0 +1,50 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: management-service
  namespace: {{ .Values.namespace }}
 spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      app: management-service
  template:
    metadata:
      labels:
        app: management-service
    spec:
      serviceAccountName: kmsvc
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway
          labelSelector:
            matchLabels:
              app: management-service
      containers:
        - name: management-service
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          ports:
            - name: grpc
              containerPort: {{ .Values.grpcPort }}
            - name: http
              containerPort: {{ .Values.httpPort }}
          env:
            - name: GOMEMLIMIT
              value: {{ .Values.goMemLimit | quote }}
          envFrom:
            - configMapRef:
                name: management-service-config
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
          readinessProbe:
            tcpSocket:
              port: {{ .Values.httpPort }}
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: {{ .Values.httpPort }}
            initialDelaySeconds: 10
            periodSeconds: 20
@@ -0,0 +1,27 @@
 {{- if .Values.hpa.enabled }}
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
  name: management-service
  namespace: {{ .Values.namespace }}
 spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: management-service
  minReplicas: {{ .Values.hpa.minReplicas }}
  maxReplicas: {{ .Values.hpa.maxReplicas }}
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: {{ .Values.hpa.targetCPUUtilizationPercentage }}
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: {{ .Values.hpa.targetMemoryUtilizationPercentage }}
 {{- end }}
@@ -0,0 +1,27 @@
 {{- if and .Values.ingress.enabled .Values.ingress.grpcEnabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: management-service-grpc
  namespace: {{ .Values.namespace }}
  annotations:
    cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
 spec:
  ingressClassName: {{ .Values.ingress.className }}
  tls:
    - hosts:
        - {{ .Values.ingress.host }}
      secretName: {{ .Values.ingress.tlsSecretName }}
  rules:
    - host: {{ .Values.ingress.host }}
      http:
        paths:
          - path: {{ .Values.ingress.grpcPathPrefix }}
            pathType: Prefix
            backend:
              service:
                name: management-service
                port:
                  number: {{ .Values.grpcPort }}
 {{- end }}
@@ -0,0 +1,26 @@
 {{- if .Values.ingress.enabled }}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: management-service
  namespace: {{ .Values.namespace }}
  annotations:
    cert-manager.io/cluster-issuer: {{ .Values.ingress.clusterIssuer }}
 spec:
  ingressClassName: {{ .Values.ingress.className }}
  tls:
    - hosts:
        - {{ .Values.ingress.host }}
      secretName: {{ .Values.ingress.tlsSecretName }}
  rules:
    - host: {{ .Values.ingress.host }}
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: management-service
                port:
                  number: {{ .Values.httpPort }}
 {{- end }}
@@ -0,0 +1,16 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: management-service
  namespace: {{ .Values.namespace }}
 spec:
  selector:
    app: management-service
  ports:
    - name: grpc
      port: {{ .Values.grpcPort }}
      targetPort: {{ .Values.grpcPort }}
    - name: http
      port: {{ .Values.httpPort }}
      targetPort: {{ .Values.httpPort }}
  type: ClusterIP
@@ -0,0 +1,50 @@
 namespace: sqs
 replicaCount: 3
 image:
  repository: forgejo.riotpiao.com/rock/kmsvc-manage
  tag: latest
  pullPolicy: Always
 grpcPort: 9090
 httpPort: 8080
 env:
  kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
  redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
  authentikIssuerURL: ""
  authentikAudience: ""
 resources:
  requests:
    cpu: 200m
    memory: 256Mi
  limits:
    cpu: "1"
    memory: 512Mi
 # Go's GC only reacts to GOGC by default and has no idea about the cgroup
 # memory limit above -- it'll happily grow heap until the kernel OOMKills it.
 # Setting GOMEMLIMIT to ~90% of the container limit makes the GC self-throttle
 # before that happens. Keep this in sync with resources.limits.memory.
 goMemLimit: "460MiB"
 hpa:
  enabled: true
  minReplicas: 3
  maxReplicas: 9
  targetCPUUtilizationPercentage: 70
  targetMemoryUtilizationPercentage: 80
 ingress:
  enabled: true
  className: nginx
  clusterIssuer: homelab-ca
  host: kmsvc.riotpiao.com
  tlsSecretName: kmsvc-tls
  # kmsvc-cli connects via gRPC directly to --server/KMSVC_SERVER (default
  # kmsvc.riotpiao.com:443, see kmsvc-cli's README), so raw gRPC needs an
  # external path too — scoped to the gRPC service's own path prefix on the
  # same host/port, rather than opening the whole host to gRPC passthrough.
  grpcEnabled: true
  grpcPathPrefix: /kafkamgmt.v1.QueueService/
@@ -0,0 +1,6 @@
 apiVersion: v2
 name: memory-queues
 description: Kafka queues (DLQ) for Poimen Memory service (Phase 6.6)
 type: application
 version: 0.1.0
 appVersion: "1.0"
@@ -0,0 +1,20 @@
 {{- range .Values.queues }}
 ---
 apiVersion: kmsvc.io/v1alpha1
 kind: Queue
 metadata:
  name: {{ .name }}
  namespace: {{ $.Values.namespace }}
  labels:
    app: memory-service
    queue: dlq
 spec:
  name: {{ .name }}
  description: {{ .description }}
  partitions: {{ .partitions }}
  replicationFactor: {{ .replicationFactor }}
  config:
    retention.ms: "{{ .config.retention.ms }}"
    message.retention.seconds: "{{ .config.message.retention.seconds }}"
    visibility.timeout.seconds: "{{ .config.visibility.timeout.seconds }}"
 {{- end }}
@@ -0,0 +1,25 @@
 # Poimen Memory Service Kafka Queues (kmsvc)
 # Phase 6.6: DLQ topics for webhook + metrics failures
 queues:
  # DLQ for extraction, webhook, and agent failures
  - name: poimen-memory-dlq
    description: "DLQ for extraction, webhook, and agent failures"
    partitions: 3
    replicationFactor: 1
    config:
      retention.ms: "1209600000"  # 14 days
      message.retention.seconds: "1209600"
      visibility.timeout.seconds: "300"
  # DLQ for metrics persistence failures  
  - name: poimen-memory-metric-dlq
    description: "DLQ for metrics persistence failures"
    partitions: 3
    replicationFactor: 1
    config:
      retention.ms: "1209600000"  # 14 days
      message.retention.seconds: "1209600"
      visibility.timeout.seconds: "300"
 namespace: sqs
@@ -0,0 +1,5 @@
 apiVersion: v2
 name: queue-crd
 description: Queue CRD definition + queue-operator Deployment/RBAC (design.md §2a)
 type: application
 version: 0.1.0
@@ -0,0 +1,274 @@
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  annotations:
    controller-gen.kubebuilder.io/version: v0.21.0
  name: queues.kmsvc.io
 spec:
  group: kmsvc.io
  names:
    kind: Queue
    listKind: QueueList
    plural: queues
    shortNames:
    - queue
    - queues
    singular: queue
  scope: Namespaced
  versions:
  - additionalPrinterColumns:
    - jsonPath: .spec.fifoQueue
      name: FIFO
      type: boolean
    - jsonPath: .status.phase
      name: Phase
      type: string
    name: v1
    schema:
      openAPIV3Schema:
        description: Queue is the Schema for the queues API — see design.md §2a.
        properties:
          apiVersion:
            description: |-
              APIVersion defines the versioned schema of this representation of an object.
              Servers should convert recognized schemas to the latest internal value, and
              may reject unrecognized values.
              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
            type: string
          kind:
            description: |-
              Kind is a string value representing the REST resource this object represents.
              Servers may infer this from the endpoint the client submits requests to.
              Cannot be updated.
              In CamelCase.
              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
            type: string
          metadata:
            type: object
          spec:
            description: QueueSpec defines the desired state of a Queue (design.md
              §2a).
            properties:
              deadLetterTargetQueue:
                description: |-
                  DeadLetterTargetQueue is the name of another Queue to route exhausted
                  messages to. Must not point at itself or at another DLQ (design.md §5).
                type: string
              delaySeconds:
                description: DelaySeconds is the default delivery delay applied to
                  sent messages.
                format: int32
                maximum: 900
                minimum: 0
                type: integer
              fifoQueue:
                default: false
                description: FIFOQueue enables per-MessageGroupId ordering and deduplication
                  semantics.
                type: boolean
              isDLQ:
                description: |-
                  IsDLQ marks this queue as itself a dead-letter queue, used to enforce
                  the no-DLQ-chaining validation rule in design.md §5.
                type: boolean
              maxReceiveCount:
                default: 5
                description: |-
                  MaxReceiveCount is how many times a message may be redelivered before
                  being routed to DeadLetterTargetQueue.
                format: int32
                minimum: 1
                type: integer
              maxShards:
                default: 8
                description: MaxShards is the ceiling on shard count the operator
                  may split up to (design.md §2c).
                format: int32
                minimum: 1
                type: integer
              messageRetentionPeriodSeconds:
                default: 345600
                description: MessageRetentionPeriodSeconds maps to the underlying
                  Kafka topic's retention.ms.
                format: int32
                maximum: 1209600
                minimum: 60
                type: integer
              minShards:
                default: 1
                description: MinShards is the floor on shard count; the operator never
                  merges below this.
                format: int32
                minimum: 1
                type: integer
              partitionsPerShard:
                default: 6
                description: PartitionsPerShard is the Kafka partition count on each
                  shard's topic.
                format: int32
                minimum: 1
                type: integer
              shardSplitCooldownSeconds:
                default: 300
                description: |-
                  ShardSplitCooldownSeconds is the minimum age a shard must reach before it
                  is eligible to be split again, preventing rapid re-splitting of a child
                  that hasn't yet absorbed its share of traffic.
                format: int32
                minimum: 0
                type: integer
              shardSplitThresholdBytesPerSec:
                default: 5242880
                description: |-
                  ShardSplitThresholdBytesPerSec is the sustained per-shard throughput that
                  triggers a split into two child shards (design.md §2c).
                format: int64
                minimum: 1
                type: integer
              visibilityTimeoutSeconds:
                default: 30
                description: |-
                  VisibilityTimeoutSeconds is how long a received-but-unacked message stays
                  invisible to other consumers before being redelivered.
                format: int32
                maximum: 43200
                minimum: 0
                type: integer
            type: object
          status:
            description: QueueStatus defines the observed state of a Queue.
            properties:
              conditions:
                description: Conditions hold detailed status information.
                items:
                  description: Condition contains details for one aspect of the current
                    state of this API Resource.
                  properties:
                    lastTransitionTime:
                      description: |-
                        lastTransitionTime is the last time the condition transitioned from one status to another.
                        This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
                      format: date-time
                      type: string
                    message:
                      description: |-
                        message is a human readable message indicating details about the transition.
                        This may be an empty string.
                      maxLength: 32768
                      type: string
                    observedGeneration:
                      description: |-
                        observedGeneration represents the .metadata.generation that the condition was set based upon.
                        For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
                        with respect to the current state of the instance.
                      format: int64
                      minimum: 0
                      type: integer
                    reason:
                      description: |-
                        reason contains a programmatic identifier indicating the reason for the condition's last transition.
                        Producers of specific condition types may define expected values and meanings for this field,
                        and whether the values are considered a guaranteed API.
                        The value should be a CamelCase string.
                        This field may not be empty.
                      maxLength: 1024
                      minLength: 1
                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
                      type: string
                    status:
                      description: status of the condition, one of True, False, Unknown.
                      enum:
                      - "True"
                      - "False"
                      - Unknown
                      type: string
                    type:
                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
                      maxLength: 316
                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
                      type: string
                  required:
                  - lastTransitionTime
                  - message
                  - reason
                  - status
                  - type
                  type: object
                type: array
              phase:
                description: Phase is the current reconciliation phase.
                enum:
                - Pending
                - Ready
                - Failed
                type: string
              shards:
                description: |-
                  Shards lists every shard backing this queue, active or draining
                  (design.md §2a/§2c).
                items:
                  description: ShardStatus describes one shard backing a Queue (design.md
                    §2a/§2c).
                  properties:
                    availabilityZones:
                      description: |-
                        AvailabilityZones lists the topology.kubernetes.io/zone values of every
                        node currently hosting a Kafka replica of this shard's topic, resolved
                        from the broker pods' node placement each reconcile. Empty until the
                        first successful resolution (e.g. node lookup failed transiently).
                      items:
                        type: string
                      type: array
                    createdAt:
                      description: |-
                        CreatedAt timestamps when this shard was created, used to enforce
                        ShardSplitCooldownSeconds.
                      format: date-time
                      type: string
                    hashRangeEnd:
                      format: int64
                      type: integer
                    hashRangeStart:
                      description: |-
                        HashRangeStart/HashRangeEnd define the [start, end) murmur2 hash range
                        this shard owns over the 32-bit key space. Stored as int64 (not uint32)
                        because controller-gen maps Go uint32 to OpenAPI format:int32, whose max
                        (2147483647) is smaller than FullHashRangeEnd (0xFFFFFFFF) and the
                        apiserver rejects the status update.
                      format: int64
                      type: integer
                    id:
                      description: ID is the shard's identifier, used in its topic
                        name (kmsvc.{queue}.shard-{id}).
                      type: string
                    parentId:
                      description: |-
                        ParentID is the shard ID this shard was split from, empty for the
                        original shard-0.
                      type: string
                    phase:
                      description: Phase is this shard's lifecycle state.
                      enum:
                      - Active
                      - Closing
                      - Closed
                      type: string
                    topic:
                      description: Topic is the underlying Kafka topic name for this
                        shard.
                      type: string
                  required:
                  - hashRangeEnd
                  - hashRangeStart
                  - id
                  - phase
                  - topic
                  type: object
                type: array
            type: object
        type: object
    served: true
    storage: true
    subresources:
      status: {}
@@ -0,0 +1,38 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: queue-operator
  namespace: {{ .Values.namespace }}
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: queue-operator
  template:
    metadata:
      labels:
        app: queue-operator
    spec:
      serviceAccountName: queue-operator
      containers:
        - name: queue-operator
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          imagePullPolicy: {{ .Values.image.pullPolicy }}
          command: ["/queue-operator"]
          env:
            - name: KMSVC_KAFKA_BROKERS
              value: {{ .Values.kafkaBrokers | quote }}
            - name: KMSVC_REDIS_ADDR
              value: {{ .Values.redisAddr | quote }}
            - name: GOMEMLIMIT
              value: {{ .Values.goMemLimit | quote }}
            - name: KMSVC_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: KMSVC_KAFKA_CLUSTER_NAME
              value: {{ .Values.kafkaClusterName | quote }}
            - name: KMSVC_KAFKA_POOL_NAME
              value: {{ .Values.kafkaPoolName | quote }}
          resources:
            {{- toYaml .Values.resources | nindent 12 }}
@@ -0,0 +1,42 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: queue-operator
  namespace: {{ .Values.namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: queue-operator
 rules:
  - apiGroups: ["kmsvc.io"]
    resources: ["queues"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
    resources: ["queues/status"]
    verbs: ["get", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
    resources: ["queues/finalizers"]
    verbs: ["update"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "patch"]
  - apiGroups: [""]
    resources: ["pods", "nodes"]
    verbs: ["get"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: queue-operator
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: queue-operator
 subjects:
  - kind: ServiceAccount
    name: queue-operator
    namespace: {{ .Values.namespace }}
@@ -0,0 +1,26 @@
 namespace: sqs
 image:
  repository: forgejo.riotpiao.com/rock/kmsvc-manage
  tag: latest
  pullPolicy: Always
 kafkaBrokers: "kmsvc-kafka-bootstrap.sqs.svc.cluster.local:9092"
 redisAddr: "kmsvc-redis-master.sqs.svc.cluster.local:6379"
 # Must match kafka-cluster chart's clusterName/derived pool name -- used to
 # resolve "<kafkaClusterName>-<kafkaPoolName>-<brokerID>" broker pod names
 # for AZ-aware Queue status (design.md §2a).
 kafkaClusterName: kmsvc
 kafkaPoolName: kmsvc-pool
 resources:
  requests:
    cpu: 100m
    memory: 128Mi
  limits:
    cpu: 500m
    memory: 256Mi
 # See management-service/values.yaml's goMemLimit comment -- same reasoning.
 goMemLimit: "230MiB"
@@ -0,0 +1,95 @@
 # Overrides paperless-ngx's own paperless/adapter.py at the same import path
 # (mounted via subPath in deployment.yaml) - settings.py hardcodes
 # SOCIALACCOUNT_ADAPTER = "paperless.adapter.CustomSocialAccountAdapter", so
 # no Django setting needs to change, just the file content underneath it.
 #
 # Stock CustomSocialAccountAdapter.populate_user() is a stub ("kept in case
 # global default permissions are implemented in the future" - they aren't),
 # so every OIDC signup lands with zero permissions and 403s on every API
 # endpoint. This adds the actual mapping: Authentik's "permissions" claim
 # (via the permissions scope, requested in PAPERLESS_SOCIALACCOUNT_PROVIDERS,
 # computed server-side from group membership by authentik-provision.py) ->
 # "paperless:write" or "*" (homelab-admins) grants is_staff+is_superuser,
 # same convention already used for MinIO's policy claim and Grafana's
 # role_attribute_path. Checking the permission string rather than a literal
 # group name decouples "what grants access" from which group happens to
 # hold it - same pattern applies to every other service's Role/RoleBinding
 # in k8s/infra/rbac/.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: paperless-adapter
 data:
  adapter.py: |
    from urllib.parse import quote
    from allauth.account.adapter import DefaultAccountAdapter
    from allauth.core import context
    from allauth.socialaccount.adapter import DefaultSocialAccountAdapter
    from django.conf import settings
    from django.forms import ValidationError
    from django.urls import reverse
    REQUIRED_PERMISSIONS = {"paperless:write", "*"}
    class CustomAccountAdapter(DefaultAccountAdapter):
        def is_open_for_signup(self, request):
            allow_signups = super().is_open_for_signup(request)
            return getattr(settings, "ACCOUNT_ALLOW_SIGNUPS", allow_signups)
        def pre_authenticate(self, request, **credentials):
            if settings.DISABLE_REGULAR_LOGIN:
                raise ValidationError("Regular login is disabled")
            return super().pre_authenticate(request, **credentials)
        def is_safe_url(self, url):
            from django.utils.http import url_has_allowed_host_and_scheme
            allowed_hosts = {context.request.get_host()} | set(settings.ALLOWED_HOSTS)
            if "*" in allowed_hosts:
                allowed_hosts.remove("*")
                allowed_hosts.add(context.request.get_host())
                return url_has_allowed_host_and_scheme(url, allowed_hosts=allowed_hosts)
            return url_has_allowed_host_and_scheme(url, allowed_hosts=allowed_hosts)
        def get_reset_password_from_key_url(self, key):
            if settings.PAPERLESS_URL is None:
                return super().get_reset_password_from_key_url(key)
            path = reverse(
                "account_reset_password_from_key",
                kwargs={"uidb36": "UID", "key": "KEY"},
            )
            path = path.replace("UID-KEY", quote(key))
            return settings.PAPERLESS_URL + path
    class CustomSocialAccountAdapter(DefaultSocialAccountAdapter):
        def is_open_for_signup(self, request, sociallogin):
            allow_signups = super().is_open_for_signup(request, sociallogin)
            return getattr(settings, "SOCIALACCOUNT_ALLOW_SIGNUPS", allow_signups)
        def get_connect_redirect_url(self, request, socialaccount):
            return reverse("base")
        def populate_user(self, request, sociallogin, data):
            user = super().populate_user(request, sociallogin, data)
            perms = set(sociallogin.account.extra_data.get("permissions") or [])
            if perms & REQUIRED_PERMISSIONS:
                user.is_staff = True
                user.is_superuser = True
            return user
        def save_user(self, request, sociallogin, form=None):
            # populate_user() sets the flags on the in-memory user, but
            # allauth's default save_user() re-derives is_staff from
            # ACCOUNT_DEFAULT_HTTP_PROTOCOL-independent defaults and can
            # overwrite them on save - re-apply after super().save_user()
            # persists the row, matching the permissions check above exactly.
            user = super().save_user(request, sociallogin, form)
            perms = set(sociallogin.account.extra_data.get("permissions") or [])
            if perms & REQUIRED_PERMISSIONS and not (user.is_staff and user.is_superuser):
                user.is_staff = True
                user.is_superuser = True
                user.save(update_fields=["is_staff", "is_superuser"])
            return user
@@ -0,0 +1,94 @@
 # Nightly: pg_dump the paperless DB + mirror the media PVC into the scoped
 # `paperless` MinIO bucket (see minio-provision-paperless-job.yaml). This is a
 # BACKUP target, not live storage - paperless-ngx has no native S3 backend, it
 # only ever reads/writes the local media PVC directly.
 #
 # Pinned to talos-cp-3, same as deployment.yaml: media is a ReadWriteOnce
 # Longhorn volume with a single replica physically on that node's disk -
 # mounting it read-only here from a different node would conflict with the
 # live webserver's attachment.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
  name: paperless-backup
 spec:
  schedule: "0 3 * * *"  # 03:00 daily, low-traffic window
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          restartPolicy: Never
          nodeSelector:
            kubernetes.io/hostname: talos-cp-3
          initContainers:
            - name: pg-dump
              image: postgres:16-alpine
              env:
                - name: PGHOST
                  value: paperless-db-rw
                - name: PGDATABASE
                  value: paperless
                - name: PGUSER
                  valueFrom:
                    secretKeyRef:
                      name: paperless-db-app
                      key: username
                - name: PGPASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: paperless-db-app
                      key: password
              command:
                - sh
                - -c
                - pg_dump --format=custom --file=/backup/paperless-db.dump
              volumeMounts:
                - name: backup
                  mountPath: /backup
          containers:
            - name: mc-mirror
              image: minio/mc:latest
              env:
                - name: ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: paperless-minio-creds
                      key: ACCESS_KEY
                - name: SECRET_KEY
                  valueFrom:
                    secretKeyRef:
                      name: paperless-minio-creds
                      key: SECRET_KEY
                - name: BUCKET
                  valueFrom:
                    secretKeyRef:
                      name: paperless-minio-creds
                      key: BUCKET
                - name: ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: paperless-minio-creds
                      key: ENDPOINT
              command:
                - /bin/sh
                - -c
                - |
                  set -e
                  mc alias set b "$ENDPOINT" "$ACCESS_KEY" "$SECRET_KEY"
                  mc cp /backup/paperless-db.dump "b/$BUCKET/db/paperless-db-$(date +%Y%m%d).dump"
                  mc mirror --overwrite /media "b/$BUCKET/media"
                  echo "Backup done."
              volumeMounts:
                - name: backup
                  mountPath: /backup
                - name: media
                  mountPath: /media
                  readOnly: true
          volumes:
            - name: backup
              emptyDir: {}
            - name: media
              persistentVolumeClaim:
                claimName: paperless-media
                readOnly: true
@@ -0,0 +1,22 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: paperless-config
 data:
  PAPERLESS_URL: "https://paperless.riotpiao.com"
  PAPERLESS_TIME_ZONE: "America/Los_Angeles"
  PAPERLESS_OCR_LANGUAGE: "eng"
  PAPERLESS_DBHOST: "paperless-db-rw"
  PAPERLESS_DBNAME: "paperless"
  PAPERLESS_REDIS: "redis://paperless-redis:6379"
  # django-allauth generic OIDC provider. The client_id/secret/server_url
  # bundle itself lives in the paperless-oidc Secret
  # (SOCIALACCOUNT_PROVIDERS_JSON key, composed by authentik-provision.py) -
  # env vars can't be split across a ConfigMap + Secret for the same key, so
  # this whole value is sourced from the Secret in deployment.yaml instead.
  PAPERLESS_APPS: "allauth.socialaccount.providers.openid_connect"
  # Authentik already verifies identity via OIDC - a second email-confirmation
  # step has no SMTP configured to send it anyway, and paperless-ngx doesn't
  # wire up allauth's confirm-email view, so signup 500s with NoReverseMatch
  # on 'account_confirm_email' without this.
  PAPERLESS_ACCOUNT_EMAIL_VERIFICATION: "none"
@@ -0,0 +1,103 @@
 # Single container runs webserver + consumer + scheduler (paperless-ngx's
 # stock entrypoint does this internally) - no need to split into separate
 # Deployments. replicas: 1 only: paperless-media is ReadWriteOnce, and the
 # consumer polling the media dir doesn't benefit from horizontal scaling here.
 #
 # Pinned to talos-cp-3: paperless-media's disk physically lives there. Longhorn
 # RWO volumes can only be attached from one node at a time, and the nightly
 # backup-cronjob.yaml also mounts this same PVC (read-only) to mirror it into
 # MinIO - pinning both to the same node avoids a cross-node attach conflict,
 # and keeps the 3.5Ti read/write path off the network entirely.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: paperless
 spec:
  replicas: 1
  strategy:
    type: Recreate  # ReadWriteOnce media PVC - avoid two pods fighting over it
  selector:
    matchLabels:
      app: paperless
  template:
    metadata:
      labels:
        app: paperless
    spec:
      # Kubernetes injects legacy Docker-links env vars for every Service in
      # this namespace (<SVC>_SERVICE_HOST, <SVC>_PORT, ...). The Service here
      # is named "paperless", so that becomes PAPERLESS_PORT=tcp://<ip>:8000 -
      # paperless-ngx's own entrypoint reads PAPERLESS_PORT for gunicorn's
      # bind address, collides, and gunicorn crash-loops on "not a valid port
      # number". Disable the injection instead of renaming the Service.
      enableServiceLinks: false
      nodeSelector:
        kubernetes.io/hostname: talos-cp-3
      containers:
        - name: paperless
          image: ghcr.io/paperless-ngx/paperless-ngx:2.20.15
          ports:
            - containerPort: 8000
          envFrom:
            - configMapRef:
                name: paperless-config
          env:
            - name: PAPERLESS_DBUSER
              valueFrom:
                secretKeyRef:
                  name: paperless-db-app
                  key: username
            - name: PAPERLESS_DBPASS
              valueFrom:
                secretKeyRef:
                  name: paperless-db-app
                  key: password
            - name: PAPERLESS_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: paperless-secrets
                  key: PAPERLESS_SECRET_KEY
            - name: PAPERLESS_ADMIN_USER
              valueFrom:
                secretKeyRef:
                  name: paperless-secrets
                  key: PAPERLESS_ADMIN_USER
            - name: PAPERLESS_ADMIN_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: paperless-secrets
                  key: PAPERLESS_ADMIN_PASSWORD
            - name: PAPERLESS_SOCIALACCOUNT_PROVIDERS
              valueFrom:
                secretKeyRef:
                  name: paperless-oidc
                  key: SOCIALACCOUNT_PROVIDERS_JSON
          resources:
            requests: { cpu: "500m", memory: "1Gi" }
            limits: { cpu: "2", memory: "4Gi" }
          volumeMounts:
            - name: media
              mountPath: /usr/src/paperless/media
            - name: data
              mountPath: /usr/src/paperless/data
            - name: consume
              mountPath: /usr/src/paperless/consume
            # Overrides paperless-ngx's own adapter.py in place - settings.py
            # hardcodes the import path, so no Django setting changes, just
            # the file content underneath it (see adapter-configmap.yaml).
            - name: adapter
              mountPath: /usr/src/paperless/src/paperless/adapter.py
              subPath: adapter.py
              readOnly: true
      volumes:
        - name: media
          persistentVolumeClaim:
            claimName: paperless-media
        - name: data
          persistentVolumeClaim:
            claimName: paperless-data
        - name: consume
          emptyDir: {}
        - name: adapter
          configMap:
            name: paperless-adapter
@@ -0,0 +1,24 @@
 # Direct nginx ingress to the paperless Service - not routed via the Go
 # api-gateway (api.riotpiao.com), which has no WebSocket upgrade support and
 # paperless-ngx keeps a long-lived /ws/ connection open for live task status.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: paperless
  annotations:
    nginx.ingress.kubernetes.io/proxy-body-size: "0"  # large scanned PDF uploads
    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
 spec:
  ingressClassName: nginx
  rules:
    - host: paperless.riotpiao.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: paperless
                port:
                  number: 8000
@@ -0,0 +1,17 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: paperless
 resources:
  - pvc.yaml
  - configmap.yaml
  - redis.yaml
  - deployment.yaml
  - service.yaml
  - ingress.yaml
  - backup-cronjob.yaml
  - adapter-configmap.yaml
  - rbac.yaml
 # postgres: paperless-db CNPG Cluster, deployed by k8s/infra/databases (wave 2,
 # before this app at wave 8) - not duplicated here. Same for the paperless-oidc
 # and paperless-minio-creds Secrets, written by PostSync provisioning Jobs in
 # k8s/infra/iam and k8s/infra/minio respectively.
@@ -0,0 +1,35 @@
 # Two volumes, deliberately separate storage classes:
 #
 # - media: the actual documents (originals + OCR'd archive PDFs + thumbnails).
 #   Lives on the cp-3 USB HDD, single replica (see
 #   k8s/infra/longhorn/longhorn-paperless-storageclass.yaml). Shares the disk
 #   with Immich's immich-media PVC (k8s/apps/immich/pvc.yaml, 2000Gi) - photo
 #   libraries grow much faster than scanned documents, so paperless gets the
 #   smaller 500Gi share.
 # - data: the SQLite classification model + search index. Small (low GB),
 #   frequently rewritten, and disposable (rebuilds from the DB + media on
 #   next consume) - stays on the default 3-replica pool instead of the
 #   single-disk HDD.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-media
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn-paperless-media
  resources:
    requests:
      storage: 500Gi
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-data
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn
  resources:
    requests:
      storage: 5Gi
@@ -0,0 +1,35 @@
 # Scoped operator access for paperless-admins: restart/config-edit rights on
 # just this service's own resources, nothing CNPG-managed (paperless-db-*)
 # or provisioning-managed (paperless-oidc, paperless-minio-creds). Inert
 # until kube-apiserver's OIDC wiring lands (--oidc-groups-claim=groups,
 # --oidc-groups-prefix=oidc:) - subject name below assumes that prefix.
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
  name: paperless-operator
 rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    resourceNames: ["paperless"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["configmaps"]
    resourceNames: ["paperless-config"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["paperless-secrets"]
    verbs: ["get", "list", "watch", "update", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
  name: paperless-admins-binding
 subjects:
  - kind: Group
    name: "oidc:paperless-admins"
    apiGroup: rbac.authorization.k8s.io
 roleRef:
  kind: Role
  name: paperless-operator
  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,37 @@
 # Task queue broker + websocket channel layer for paperless-ngx. No PVC:
 # queued/scheduled task state is disposable - a lost queue on restart just
 # means re-triggering consumption, not data loss (documents themselves live
 # on paperless-media).
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: paperless-redis
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: paperless-redis
  template:
    metadata:
      labels:
        app: paperless-redis
    spec:
      containers:
        - name: redis
          image: redis:7-alpine
          ports:
            - containerPort: 6379
          resources:
            requests: { cpu: "50m", memory: "64Mi" }
            limits: { cpu: "250m", memory: "256Mi" }
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: paperless-redis
 spec:
  selector:
    app: paperless-redis
  ports:
    - port: 6379
      targetPort: 6379
@@ -0,0 +1,10 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: paperless
 spec:
  selector:
    app: paperless
  ports:
    - port: 8000
      targetPort: 8000
@@ -0,0 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: portainer
 resources: []
 # Portainer deployed via Helm chart or existing manifests
@@ -0,0 +1,59 @@
 # k8s/portainer/portainer-values.yaml
 # Portainer — web UI for browsing cluster workloads, exec-ing into pods,
 # and viewing logs without kubectl. Operator-only access (ClusterIP + port-forward).
 #
 # Node failure behaviour:
 #   Portainer is a Deployment (not StatefulSet), so K8s auto-evicts and
 #   reschedules it ~5 min after a node becomes unreachable. Longhorn
 #   reattaches the PVC on the new node in ~1-2 min. Worst case: ~7-10 min.
 #
 #   To cut that down: in Longhorn UI → Settings set
 #     nodeDownPodDeletionPolicy = delete-deployment-pod
 #   Longhorn will force-delete the stuck pod immediately when the node is
 #   fenced rather than waiting for Kubernetes' eviction timeout.
 # ── Service ───────────────────────────────────────────────────────────────────
 # ClusterIP — no external exposure. Access via:
 #   kubectl -n dashboard port-forward svc/portainer 9000:9000
 # Portainer holds cluster-admin credentials; never expose as LoadBalancer.
 service:
  type: ClusterIP
 # ── TLS ───────────────────────────────────────────────────────────────────────
 # Portainer by default redirects HTTP → HTTPS using a self-signed cert.
 # force: false disables the redirect so plain HTTP over port-forward works
 # without browser cert warnings. TLS is terminated at the ingress layer
 # if/when an ingress rule is added.
 tls:
  force: false
 # ── Persistence ───────────────────────────────────────────────────────────────
 # Stores Portainer's own config: environment registrations, user accounts,
 # stack definitions, and access control settings. Longhorn provides the
 # RWO block volume. 10Gi is generous for config data but cheap on Longhorn.
 persistence:
  enabled: true
  storageClass: "longhorn"
  size: 10Gi
 resources:
  requests:
    cpu: 100m
    memory: 128Mi
  limits:
    cpu: 500m
    memory: 512Mi
 # ── Scheduling ────────────────────────────────────────────────────────────────
 # Allow scheduling on talos-cp-1 (carries NoSchedule taint) so Portainer
 # keeps running even when the worker node is down.
 tolerations:
  - key: node-role.kubernetes.io/control-plane
    operator: Exists
    effect: NoSchedule
 # Pin to az-b (talos-cp-2) — sole Longhorn storage node (dedicated disks).
 # Its RWO PVC can only attach there; without this the pod may land on
 # cp-1/cp-3 and fail to mount.
 nodeSelector:
  topology.kubernetes.io/zone: az-b
@@ -0,0 +1,144 @@
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
  name: secretrotations.homelab.riotpiao.com
 spec:
  group: homelab.riotpiao.com
  names:
    kind: SecretRotation
    plural: secretrotations
  scope: Namespaced
  versions:
  - name: v1
    served: true
    storage: true
    schema:
      openAPIV3Schema:
        type: object
        properties:
          metadata:
            type: object
          spec:
            type: object
            required:
            - provider
            - rotationInterval
            properties:
              # External system: authentik | forgejo | minio | vault
              provider:
                type: string
                enum: [authentik, forgejo, minio, vault]
              # How often to rotate (hours)
              rotationInterval:
                type: integer
                minimum: 24
              # Application ID in external system
              appId:
                type: string
              # k8s Secret to update (name, namespace, key)
              secretRef:
                type: object
                required: [name, namespace]
                properties:
                  name:
                    type: string
                  namespace:
                    type: string
                  key:
                    type: string
                    description: "Secret key to update (e.g., MINIO_IDENTITY_OPENID_CLIENT_SECRET)"
              # Path to git file that holds the secret (for .enc.yaml files)
              gitPath:
                type: string
                description: "Path in homelab repo to .enc.yaml file"
              # Ansible template values to substitute
              templateValues:
                type: object
                additionalProperties:
                  type: string
          status:
            type: object
            properties:
              lastRotationTime:
                type: string
                format: date-time
              nextRotationTime:
                type: string
                format: date-time
              lastRotationStatus:
                type: string
                enum: [Success, Failed, Pending]
              lastRotationError:
                type: string
              lastCommitHash:
                type: string
 ---
 # Example usage:
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: minio-oidc
  namespace: secret-rotation
 spec:
  provider: authentik
  rotationInterval: 2160  # 90 days in hours
  appId: minio
  secretRef:
    name: minio-oidc
    namespace: storage
    key: MINIO_IDENTITY_OPENID_CLIENT_SECRET
  gitPath: k8s/argocd/secrets/minio-oidc.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: portfolio-agent-oidc
  namespace: secret-rotation
 spec:
  provider: authentik
  rotationInterval: 2160
  appId: portfolio-agent
  secretRef:
    name: portfolio-agent-oidc
    namespace: portfolio
    key: CLIENT_SECRET
  gitPath: k8s/argocd/secrets/portfolio-agent-oidc.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: forgejo-registry-token
  namespace: secret-rotation
 spec:
  provider: forgejo
  rotationInterval: 2160
  appId: rock/riotpiao.com
  secretRef:
    name: forgejo-registry-secret
    namespace: kube-system
    key: REGISTRY_TOKEN
  gitPath: k8s/argocd/secrets/forgejo-registry-secret.enc.yaml
 ---
 apiVersion: homelab.riotpiao.com/v1
 kind: SecretRotation
 metadata:
  name: minio-root-credentials
  namespace: secret-rotation
 spec:
  provider: minio
  rotationInterval: 4320  # 180 days in hours
  appId: root
  secretRef:
    name: minio-creds
    namespace: storage
  gitPath: k8s/argocd/secrets/minio-secrets.enc.yaml
@@ -0,0 +1,92 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: secret-rotation-controller
  namespace: secret-rotation
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: secret-rotation-controller
  template:
    metadata:
      labels:
        app: secret-rotation-controller
    spec:
      serviceAccountName: secret-rotation-controller
      containers:
      - name: controller
        image: secret-rotation-controller:latest
        imagePullPolicy: IfNotPresent
        env:
        # SOPS reads age key from this file
        - name: SOPS_AGE_KEY_FILE
          value: /etc/sops/age/private-key.txt
        # Vault auth (token in projected volume)
        - name: VAULT_ADDR
          value: http://vault.vault.svc.cluster.local:8200
        - name: VAULT_TOKEN_FILE
          value: /var/run/secrets/vault/token
        # Authentik
        - name: AUTHENTIK_URL
          value: http://authentik-server.iam.svc.cluster.local
        - name: AUTHENTIK_BOOTSTRAP_TOKEN
          valueFrom:
            secretKeyRef:
              name: authentik-bootstrap
              key: token
        # Git
        - name: GIT_REPO
          value: https://forgejo.riotpiao.com/rock/homelab.git
        - name: GIT_AUTHOR_EMAIL
          value: [email protected]
        - name: GIT_AUTHOR_NAME
          value: Secret Rotation Controller
        - name: FORGEJO_TOKEN
          valueFrom:
            secretKeyRef:
              name: forgejo-registry-secret
              key: REGISTRY_TOKEN
        volumeMounts:
        # Age key from ExternalSecret (synced from Vault)
        - name: age-key
          mountPath: /etc/sops/age
          readOnly: true
        # Vault auth token (projected)
        - name: vault-token
          mountPath: /var/run/secrets/vault
          readOnly: true
        # Temp working dir
        - name: tmp
          mountPath: /tmp
        resources:
          requests:
            cpu: 100m
            memory: 256Mi
          limits:
            cpu: 500m
            memory: 512Mi
      volumes:
      - name: age-key
        secret:
          secretName: sops-age-key
          defaultMode: 0400
      - name: vault-token
        projected:
          sources:
          - serviceAccountToken:
              path: token
              audience: vault
              expirationSeconds: 3600
      - name: tmp
        emptyDir: {}
@@ -0,0 +1,15 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: secret-rotation
 resources:
  - rbac.yaml
  - crd.yaml
  - external-secret.yaml
  - deployment.yaml
 commonLabels:
  app.kubernetes.io/name: secret-rotation-controller
  app.kubernetes.io/component: automation
  managed-by: argocd
@@ -0,0 +1,53 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
  name: secret-rotation-controller
  namespace: secret-rotation
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
  name: secret-rotation-controller
 rules:
 # Read SecretRotation CRDs
 - apiGroups: ["homelab.riotpiao.com"]
  resources: ["secretrotations"]
  verbs: ["get", "list", "watch"]
 # Update status
 - apiGroups: ["homelab.riotpiao.com"]
  resources: ["secretrotations/status"]
  verbs: ["get", "patch", "update"]
 # Read k8s secrets that will be rotated
 - apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get", "list"]
 # For recording events
 - apiGroups: [""]
  resources: ["events"]
  verbs: ["create", "patch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: secret-rotation-controller
 roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: secret-rotation-controller
 subjects:
 - kind: ServiceAccount
  name: secret-rotation-controller
  namespace: secret-rotation
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
  name: secret-rotation
  labels:
    kubernetes.io/metadata.name: secret-rotation
@@ -0,0 +1,115 @@
 # macOS VM (Docker-OSX) hosting the BlueBubbles server.
 #
 # ── Why a VM and not a container ────────────────────────────────────────────
 # Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
 # Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
 # or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
 # container — a VM in a box, not a macOS container.
 #
 # ── Why this works on worker-2 ──────────────────────────────────────────────
 # Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
 # /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
 # module). Bare metal, so no nested virtualisation needed.
 #
 # ── Read this before relying on it ──────────────────────────────────────────
 # 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
 #    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
 #    into iMessage, THEN install BlueBubbles inside the guest. This manifest
 #    only provides the machine; it does not provision macOS.
 # 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
 #    own guidance: "test sending an iMessage to yourself. If it does not
 #    succeed, it's likely best to restart from the beginning."
 # 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
 #    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
 # 4. BlueBubbles labels this path "not for beginners", "no guarantees or
 #    warranty".
 #
 # Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
 # inside the guest and is NOT required for plain send/receive. Skip it.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: macos-bluebubbles
  labels:
    app.kubernetes.io/name: macos-bluebubbles
    app.kubernetes.io/part-of: sms
 spec:
  replicas: 1
  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
  # concurrently — two QEMU processes on one image corrupts it.
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: macos-bluebubbles
  template:
    metadata:
      labels:
        app.kubernetes.io/name: macos-bluebubbles
        app.kubernetes.io/part-of: sms
    spec:
      # Dedicated node. The taint keeps everything else off worker-2; this
      # toleration is what lets the VM on. Both halves are required.
      nodeSelector:
        workload: imessage
      tolerations:
        - key: workload
          operator: Equal
          value: imessage
          effect: NoSchedule
      containers:
        - name: macos
          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
          # only schedules because the sms namespace is labelled privileged.
          securityContext:
            privileged: true
          env:
            # Generates a unique serial / board-serial / UUID / MAC and persists
            # them to bootdisk.qcow2. This synthetic identity is what iMessage
            # activates against — it must stay stable across restarts, which is
            # why the PVC matters.
            - name: GENERATE_UNIQUE
              value: "true"
            # Identity is only plausible if it matches a real product line.
            - name: DEVICE_MODEL
              value: "iMacPro1,1"
            - name: RAM
              value: "12"
            - name: CORES
              value: "6"
            - name: EXTRA
              # Expose the BlueBubbles server port from the guest to the pod.
              # Guest :1234 (BlueBubbles default) -> pod :1234.
              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
          ports:
            - name: vnc
              containerPort: 5999
              protocol: TCP
            - name: bluebubbles
              containerPort: 1234
              protocol: TCP
          resources:
            requests:
              cpu: "6"
              memory: 14Gi
            limits:
              cpu: "12"
              memory: 20Gi
          volumeMounts:
            - name: macos-disk
              mountPath: /home/arch/OSX-KVM/disk
            - name: kvm
              mountPath: /dev/kvm
          # No readiness probe on purpose. The guest takes many minutes to boot,
          # and until macOS + BlueBubbles are installed BY HAND there is nothing
          # listening on 1234. A probe here would crash-loop the pod through the
          # entire interactive install.
      volumes:
        - name: macos-disk
          persistentVolumeClaim:
            claimName: macos-disk
        - name: kvm
          hostPath:
            path: /dev/kvm
            type: CharDevice
@@ -0,0 +1,10 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: sms
 resources:
  - namespace.yaml
  - storageclass.yaml
  - pvc-macos.yaml
  - deployment-macos.yaml
  - service.yaml
  - networkpolicy.yaml
@@ -0,0 +1,19 @@
 # iMessage delivery for the cluster.
 #
 # BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
 # that hooks Messages.app private APIs — it cannot be containerised on Linux,
 # because containers share the host kernel and macOS needs XNU + Cocoa. The only
 # way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
 # needs a privileged pod with /dev/kvm.
 #
 # Hence privileged PodSecurity: the cluster default from the Talos controlplane
 # is `enforce: baseline`, which forbids privileged containers and host devices.
 # Scope is limited to this namespace.
 apiVersion: v1
 kind: Namespace
 metadata:
  name: sms
  labels:
    pod-security.kubernetes.io/enforce: privileged
    pod-security.kubernetes.io/audit: privileged
    pod-security.kubernetes.io/warn: privileged
@@ -0,0 +1,22 @@
 # Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
 # ID and exposes an unauthenticated VNC console; nothing should reach it except
 # opted-in clients.
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
 metadata:
  name: sms-default-deny
 spec:
  podSelector:
    matchLabels:
      app.kubernetes.io/part-of: sms
  policyTypes:
    - Ingress
  ingress:
    - from:
        - namespaceSelector: {}
          podSelector:
            matchLabels:
              sms-client: "true"
      ports:
        - protocol: TCP
          port: 1234
@@ -0,0 +1,25 @@
 # Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
 #
 # This volume is NOT disposable: it holds the VM's serial number, board serial,
 # UUID and MAC, which together form the identity iMessage was activated against.
 # Losing it means re-running activation, which is the least reliable step of the
 # whole setup.
 #
 # Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
 # the installer, the base system, and qcow2 growth.
 #
 # ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
 # Losing worker-2's disk therefore means losing the activated identity and
 # redoing iMessage activation. Once the guest is installed and activated, take
 # a Longhorn snapshot/backup of this volume; that is the only redundancy here.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: macos-disk
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: longhorn-imessage-local
  resources:
    requests:
      storage: 200Gi
@@ -0,0 +1,31 @@
 # VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
 # it is an unauthenticated console onto a machine holding a live Apple ID
 # session. Reach it with `kubectl port-forward`, never an Ingress.
 apiVersion: v1
 kind: Service
 metadata:
  name: macos-vnc
 spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: macos-bluebubbles
  ports:
    - name: vnc
      port: 5999
      targetPort: vnc
 ---
 # The BlueBubbles REST API, once installed inside the guest. This is the stable
 # name cluster services use, so callers never depend on the pod IP or on whether
 # the backend is this VM or a real Mac mini later.
 apiVersion: v1
 kind: Service
 metadata:
  name: bluebubbles
 spec:
  type: ClusterIP
  selector:
    app.kubernetes.io/name: macos-bluebubbles
  ports:
    - name: http
      port: 1234
      targetPort: bluebubbles
@@ -0,0 +1,32 @@
 # Dedicated StorageClass for the macOS VM disk.
 #
 # The default `longhorn` class does not work here, for two independent reasons:
 #
 # 1. Replica count. Default is 3, and Longhorn schedules against
 #    storageMaximum - storageScheduled with over-provisioning at 100%. Free
 #    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
 #    volume has only one node that can hold even a single replica — a 3-replica
 #    volume fails outright with ReplicaSchedulingFailure.
 # 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
 #    created, before any pod is scheduled. Combined with strict-local that
 #    pins the data to an arbitrary node, not the one the VM runs on.
 #
 # So: one replica, kept local to the VM, bound only once the pod has a node.
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
  name: longhorn-imessage-local
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 reclaimPolicy: Delete
 # The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
 # the replica is placed on that node and not somewhere else.
 volumeBindingMode: WaitForFirstConsumer
 parameters:
  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
  # Serving it over the network from another node's disk would be the single
  # worst thing for guest responsiveness, so force it local.
  numberOfReplicas: "1"
  dataLocality: "strict-local"
  staleReplicaTimeout: "30"
  fsType: "ext4"
@@ -0,0 +1,142 @@
 # k8s/temporal/temporal-values.yaml
 # Temporal — workflow engine
 # Uses external CNPG PostgreSQL for persistence (temporal-db)
 # Visibility via same PostgreSQL instance, separate database.
 #
 # IMPORTANT — chart schema note (root-caused after Postgres never actually
 # taking effect despite looking configured):
 # We're pinned to temporalio/helm-charts @ 0.74.0 (see targetRevision in
 # k8s/argocd/apps/60-applications.yaml), which uses the OLD flat persistence
 # schema:
 #   server.config.persistence.<default|visibility>.driver: "sql"|"cassandra"
 #   server.config.persistence.<default|visibility>.sql: {...}
 # NOT the newer `datastores:`-wrapped schema
 # (server.config.persistence.datastores.<store>.sql) shown in the current
 # chart's values/values.postgresql.yaml example - that key was introduced in
 # a later major version and doesn't exist in 0.74.0. Helm doesn't validate
 # unknown keys, so a `datastores:` block here is silently a no-op: Temporal
 # would keep defaulting to Cassandra (with empty hosts: []) regardless of
 # anything nested inside it. Verified via `helm template` against the actual
 # 0.74.0 chart before writing this file - see chat history for the
 # side-by-side proof (rendered manifest showed CASSANDRA_HOST env vars and
 # temporal-cassandra-tool commands using the old datastores:-based values).
 #
 # Likewise `schema.setup.enabled` / `schema.update.enabled` /
 # `schema.createDatabase.enabled` are the real toggles for the schema-setup
 # Job (all default true) - there is no `jobs.autoSetup` key in this chart.
 # ── Disable every bundled/optional sub-chart ─────────────────────────────────
 # postgresql/mysql: never enable - we never want the chart to deploy its own
 # DB, only to know how to talk to our external CNPG instance (which happens
 # via server.config.persistence.*.sql below, independent of these flags).
 postgresql:
  enabled: false
 mysql:
  enabled: false
 cassandra:
  enabled: false
 elasticsearch:
  enabled: false
 prometheus:
  enabled: false
 grafana:
  enabled: false
 # ── Schema setup/update Jobs ──────────────────────────────────────────────────
 # The `temporal` DB is created by the dedicated temporal-db cluster's initdb and
 # `temporal_visibility` by a CNPG Database CR — both in
 # k8s/infra/databases/temporal-db.yaml — so createDatabase stays disabled.
 # setup/update run temporal-sql-tool as the `app` owner against those existing
 # DBs to install and migrate the
 # Temporal server schema — without them both DBs have zero tables and the
 # server dies on "no usable database connection found" (no schema_version row).
 schema:
  createDatabase:
    enabled: false
  setup:
    enabled: true
  update:
    enabled: true
 # ── Temporal server config (PostgreSQL persistence) ──────────────────────────
 server:
  replicaCount: 1
  # temporalio/server:1.30.0+ dropped the `dockerize` binary and switched to
  # built-in sprig config templating. The chart still defaults to the legacy
  # configMapsToMount: "dockerize" + setConfigFilePath: false, which produces a
  # config the 1.30 server never loads — it then falls back to its embedded
  # env-only template (Cassandra default) and dies with
  # "Persistence.DataStores[default](value).Cassandra.Hosts: zero value".
  # Switch to the sprig ConfigMap and point the server at it (chart's own
  # recommendation for 1.30.0+ images; sprig mode requires setConfigFilePath).
  configMapsToMount: "sprig"
  setConfigFilePath: true
  jobService:
    enabled: false
  affinity:
    podAntiAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchLabels:
                app.kubernetes.io/instance: temporal
            topologyKey: kubernetes.io/hostname
  config:
    logLevel: "info"
    persistence:
      defaultStore: default
      visibilityStore: visibility
      numHistoryShards: 512
      default:
        driver: "sql"
        sql:
          driver: "postgres12"
          host: "temporal-db-rw.temporal.svc.cluster.local"
          port: 5432
          database: "temporal"
          user: "app"
          # existingSecret + secretKey: point directly at the CNPG-generated
          # Secret (kubernetes.io/basic-auth, keys: username/password/...)
          # rather than duplicating the password in git as plaintext. When
          # existingSecret is set the chart's own server-secret.yaml Secret
          # template is skipped entirely (see templates/server-secret.yaml:
          # `not $driverConfig.existingSecret` guards its creation).
          # Use unified temporal-db-app secret (generated in temporal namespace)
          existingSecret: "temporal-db-app"
          secretKey: "password"
          maxConns: 20
          maxIdleConns: 10
          maxConnLifetime: "1h"
          # NOTE: no `connectAttributes: { tx_isolation: ... }` here — tx_isolation
          # is a MySQL-only connection parameter. The Postgres `pq` driver rejects
          # it ("unrecognized configuration parameter"), which killed every DB
          # connection (schema-setup job AND server) with the misleading
          # "no usable database connection found". Postgres defaults to READ
          # COMMITTED isolation anyway, so nothing is lost by omitting it.
      visibility:
        driver: "sql"
        sql:
          driver: "postgres12"
          host: "temporal-db-rw.temporal.svc.cluster.local"
          port: 5432
          database: "temporal_visibility"
          user: "app"
          # Use unified temporal-db-app secret (generated in temporal namespace)
          existingSecret: "temporal-db-app"
          secretKey: "password"
          maxConns: 20
          maxIdleConns: 10
          maxConnLifetime: "1h"
  service:
    type: ClusterIP
 # ── Temporal Web UI ────────────────────────────────────────────────────────
 web:
  replicaCount: 1
  service:
    type: ClusterIP
 # ── Ingress ────────────────────────────────────────────────────────
 ingress:
  enabled: false
@@ -0,0 +1,18 @@
 apiVersion: kmsvc.io/v1
 kind: TemporalWorker
 metadata:
  name: worker-production
  namespace: temporal
 spec:
  namespace: production
  taskQueue: worker-production
  concurrency: 10
  workflowTypes:
    - HelloWorldWorkflow
    - GreeterWorkflow
    - ProcessOrderWorkflow
  activityTypes:
    - GreetActivity
    - ValidateOrderActivity
    - ProcessPaymentActivity
    - NotifyCustomerActivity
@@ -0,0 +1,25 @@
 # Wave -1 — AppProject definitions (must sync before any Application that references them).
 # Syncs k8s/argocd/projects/ which was previously applied by hand.
 # Enabled by Stage 1 (A2).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: argocd-projects
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "-1"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/argocd/projects
  destination:
    server: https://kubernetes.default.svc
@@ -0,0 +1,28 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: sops-secrets
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
    # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the
    # old sops-secrets-v1.0 CMP whose discover glob silently hijacked kustomize
    # rendering of any app whose path contained a *.enc.yaml.
    path: k8s/argocd/secrets
  destination:
    server: https://kubernetes.default.svc
@@ -0,0 +1,212 @@
 # Wave 0/1 — cluster substrate: cert-manager, ingress-nginx, reloader, and the
 # Let's Encrypt issuers + wildcard cert. Previously installed by Terraform; now
 # owned by app-of-apps (Pure GitOps). Controllers at wave 0; the ClusterIssuers
 # and wildcard Certificate at wave 1 so cert-manager CRDs exist first.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: cert-manager
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  sources:
    - repoURL: https://charts.jetstack.io
      chart: cert-manager
      targetRevision: "v1.21.0"
      helm:
        valueFiles:
          - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: cert-manager
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true
 # ingress-nginx removed: duplicate of ingress-nginx-bootstrap
 # The bootstrap version (k8s/bootstrap-local/06-ingress-nginx.yaml) is kept
 # to break the circular dependency (ArgoCD needs Forgejo domain access)
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: reloader
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
    repoURL: https://stakater.github.io/stakater-charts
    chart: reloader
    targetRevision: "2.2.14"
    helm:
      values: |
        reloader:
          # Watch every workload — no per-Deployment reloader annotation needed
          # (several charts, e.g. homarr, don't expose Deployment-level
          # annotations). reloadOnCreate rolls a workload when a Secret/ConfigMap
          # it references is first CREATED, not only updated — so ksops-delivered
          # secrets landing after a pod started auto-restart it.
          autoReloadAll: true
          reloadOnCreate: true
          deployment:
            tolerations:
              - key: node-role.kubernetes.io/control-plane
                operator: Exists
                effect: NoSchedule
  destination:
    server: https://kubernetes.default.svc
    namespace: reloader
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Wave 1 — LE ClusterIssuers + wildcard cert (needs cert-manager CRDs from wave 0).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: cert-manager-issuers
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
    # deterministically. The previous directory.include with bare filenames
    # rendered EMPTY — ArgoCD's include glob never matched — so this app silently
    # tracked 0 resources; its ConfigMaps/Issuers only existed from bootstrap
    # kubectl apply, and an automated prune wiped them.
    path: k8s/bootstrap/cert-manager
  destination:
    server: https://kubernetes.default.svc
    namespace: cert-manager
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Consolidated: wildcard-cert + homelab-ingress → ingress-config
 # Manages both the wildcard TLS certificate and all Ingress rules.
 # Certificate must exist before Ingresses (wave 1), but both are in same directory.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: ingress-config
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/bootstrap/ingress
  destination:
    server: https://kubernetes.default.svc
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: cluster-maintenance
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/cluster-maintenance
  destination:
    server: https://kubernetes.default.svc
    namespace: kube-system
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: kyverno
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://kyverno.github.io/kyverno/
    chart: kyverno
    targetRevision: "1.14.0"
    helm:
      valueFiles:
        - $values/k8s/bootstrap/kyverno/kyverno-values.yaml
  sources:
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: kyverno
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: kyverno-policies
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/bootstrap/kyverno
  destination:
    server: https://kubernetes.default.svc
    namespace: kyverno
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -0,0 +1,33 @@
 # ArgoCD Image Updater - auto-updates Application images from registry
 # Watches forgejo.riotpiao.com for new image tags and updates Applications
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: argocd-image-updater
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  sources:
    - repoURL: https://argoproj.github.io/argo-helm
      chart: argocd-image-updater
      targetRevision: "0.11.2"
      helm:
        valueFiles:
          - $values/k8s/infra/argocd-image-updater/values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=false
@@ -0,0 +1,8 @@
 # Wave 0 — networking substrate is Talos-owned (terraform inlineManifests), not
 # ArgoCD:
 #   - CoreDNS Corefile + hostname rewrites  -> terraform/files/coredns/Corefile
 #   - Cilium LB-IPAM pool + L2 announcement -> terraform/files/cilium/*.yaml
 # Both were previously ArgoCD apps here whose empty `resources: []`
 # kustomizations never actually applied them (live objects came from manual
 # kubectl). Managing them from ArgoCD too would let two reconcilers fight. This
 # file intentionally defines no Applications now.
@@ -0,0 +1,32 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: secret-rotation
  namespace: argocd
  labels:
    app.kubernetes.io/name: secret-rotation
 spec:
  project: homelab
  sources:
  - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    path: k8s/apps/secret-rotation-controller
    targetRevision: main
  destination:
    server: https://kubernetes.default.svc
    namespace: secret-rotation
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
    - CreateNamespace=true
    - RespectIgnoreDifferences=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
@@ -0,0 +1,251 @@
 # Wave 1 — MinIO (operator + tenant), Longhorn policy, Prometheus stack.
 # Helm charts pull from public repos; values come from the git repo via a
 # second "ref: values" source (ArgoCD multi-source pattern).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: minio-operator
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  sources:
    - repoURL: https://operator.min.io/
      chart: operator
      targetRevision: "5.0.18"
      helm:
        valueFiles:
          - $values/k8s/infra/minio/minio-operator-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: storage
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Tenant + buckets + replication are raw CRs (MinIO Tenant CRD from operator).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: minio-tenant
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/minio
  destination:
    server: https://kubernetes.default.svc
    namespace: storage
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # Longhorn itself is substrate (bootstrap-installed); this app manages only its
 # ServiceMonitor / policy manifests.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: longhorn-config
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/longhorn
  destination:
    server: https://kubernetes.default.svc
    namespace: longhorn-system
  # Longhorn writes disk state back into its own Node CRs — the disk key it
  # generates, storageReserved, diskType, evictionRequested. Git declares only
  # allowScheduling; without this the controller's writes read as drift forever.
  ignoreDifferences:
    - group: longhorn.io
      kind: Node
      jsonPointers:
        - /spec/disks
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: prometheus
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  sources:
    - repoURL: https://prometheus-community.github.io/helm-charts
      chart: kube-prometheus-stack
      targetRevision: "*"
      helm:
        skipCrds: true
        valueFiles:
          - $values/k8s/infra/monitoring/prometheus-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    managedNamespaceMetadata:
      # node-exporter needs hostNetwork/hostPID/hostPath/hostPort; blocked by
      # default baseline PSS (DaemonSet created 0 pods, Prometheus STS stuck).
      labels:
        pod-security.kubernetes.io/enforce: privileged
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      # ServerSideApply removed — it conflicts with managedNamespaceMetadata's
      # forced namespace apply ("--force cannot be used with --server-side").
      # helm.skipCrds: true above stops ArgoCD from ever managing the CRDs
      # through this Application (previously it kept re-patching them via
      # client-side apply and hitting etcd's 262144-byte annotation limit on
      # kubectl.kubernetes.io/last-applied-configuration, permanently failing
      # sync). CRDs are applied once via the separate prometheus-crds
      # Application below, which uses ServerSideApply=true (no namespace-
      # metadata conflict since CRDs are cluster-scoped).
 ---
 # CRDs only, extracted to plain YAML (`helm show crds kube-prometheus-stack`)
 # and committed to git under k8s/infra/monitoring/crds/, applied via Server-
 # Side Apply to avoid the etcd 262144-byte last-applied-configuration
 # annotation limit that client-side apply hits on these very large CRDs
 # (prometheuses, alertmanagers, scrapeconfigs, etc). A plain git path source
 # (not a remote Helm source) is used deliberately so ArgoCD applies exactly
 # these 8 CRD manifests and nothing else — no ambiguity about what "CRDs only"
 # means from a Helm chart. Split out from the main `prometheus` Application
 # (helm.skipCrds: true there) because ServerSideApply conflicts with that
 # app's managedNamespaceMetadata.
 # NOTE: bump k8s/infra/monitoring/crds/kube-prometheus-stack-crds.yaml
 # whenever the kube-prometheus-stack chart version changes materially
 # (`helm show crds prometheus-community/kube-prometheus-stack > ...`).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: prometheus-crds
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/monitoring/crds
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true
 ---
 # Cluster monitoring config: custom PrometheusRules (per-app namespaces),
 # ServiceMonitors (monitoring ns), and Grafana dashboard ConfigMaps (logging ns,
 # grafana sidecar-discovered). Single source = k8s/infra/monitoring (one
 # kustomization, no namespace transformer so per-app rule namespaces are kept).
 # Wave 2: after prometheus-operator CRDs (wave 0) + stack (wave 1) and grafana
 # (wave 2, logging). ServerSideApply avoids the etcd last-applied annotation
 # limit on the large dashboard ConfigMap JSON.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: monitoring-config
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "2"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/monitoring
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: blackbox-exporter
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  sources:
    - repoURL: https://prometheus-community.github.io/helm-charts
      chart: prometheus-blackbox-exporter
      targetRevision: "~11"
      helm:
        valueFiles:
          - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # Distributed tracing: Tempo + OpenTelemetry Collector.
 # Receives traces from instrumented services, stores in local volume (72h retention).
 # Grafana datasource auto-configured, service graph + latency dashboards included.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: tracing
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/tracing
  destination:
    server: https://kubernetes.default.svc
    namespace: tracing
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
@@ -0,0 +1,106 @@
 # Wave 2 — Loki / Grafana / Promtail (Grafana Helm charts).
 # NOTE: loki-values / grafana-values reference secrets (S3 creds, admin password)
 # that helmfile used to inject via --set. Under ArgoCD these come from the
 # *.enc.yaml SOPS files in the same dir via the SOPS plugin — verify the plugin
 # is configured before first sync, or these will render with empty secrets.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: loki
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "2"
 spec:
  project: homelab
  sources:
    - repoURL: https://grafana.github.io/helm-charts
      chart: loki
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/logging/loki-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: logging
  syncPolicy:
    managedNamespaceMetadata:
      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
      # privileged:true) to tail node logs — default baseline PSS blocks it.
      labels:
        pod-security.kubernetes.io/enforce: privileged
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: grafana
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "2"
 spec:
  project: homelab
  sources:
    - repoURL: https://grafana.github.io/helm-charts
      chart: grafana
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/logging/grafana-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: logging
  syncPolicy:
    managedNamespaceMetadata:
      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
      # privileged:true) to tail node logs — default baseline PSS blocks it.
      labels:
        pod-security.kubernetes.io/enforce: privileged
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: promtail
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "2"
 spec:
  project: homelab
  sources:
    - repoURL: https://grafana.github.io/helm-charts
      chart: promtail
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/logging/promtail-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: logging
  syncPolicy:
    managedNamespaceMetadata:
      # promtail needs privileged (hostPath log/journal, DAC_READ_SEARCH,
      # privileged:true) to tail node logs — default baseline PSS blocks it.
      labels:
        pod-security.kubernetes.io/enforce: privileged
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
@@ -0,0 +1,215 @@
 # Wave 3 — Vault + Authentik (identity), plus IAM raw jobs and the Forgejo
 # runner. Authentik/Vault values reference SOPS-managed secrets (see *.enc.yaml
 # in k8s/infra/iam) resolved by the ArgoCD SOPS plugin at sync time.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: vault
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  sources:
    - repoURL: https://helm.releases.hashicorp.com
      chart: vault
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/iam/vault-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: authentik
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  sources:
    - repoURL: https://charts.goauthentik.io
      chart: authentik
      targetRevision: "*"
      helm:
        valueFiles:
          - $values/k8s/infra/iam/authentik-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # Raw IAM manifests: key-rotation cronjob + authentik migration job.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: iam-jobs
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/iam
  destination:
    server: https://kubernetes.default.svc
    namespace: iam
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 # Forgejo itself. Was a bootstrap Helm release (phase 3) until it was brought
 # under Argo, because values changes there were inert — a proxy-body-size fix
 # sat committed while the live Ingress kept nginx's 1m default and rejected
 # every OCI push with 413.
 #
 # Wave 3: after databases (wave 2) — Forgejo needs CNPG and Redis up first.
 #
 # Retiring the Helm release: Argo adopts the existing objects on first sync.
 # Delete the release secrets afterwards so helm stops claiming ownership:
 #   kubectl -n cicd delete secret -l owner=helm,name=forgejo
 #
 # automated sync is deliberately absent. This chart owns the Forgejo PVC and
 # the git forge itself; the first sync is manual so its diff can be read before
 # anything is applied. Turn on automated+selfHeal once that diff is clean.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: forgejo
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  sources:
    - repoURL: https://dl.gitea.com/charts/
      chart: gitea
      targetRevision: 12.7.0
      helm:
        valueFiles:
          - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
    - repoURL: https://forgejo.riotpiao.com/rock/homelab.git
      targetRevision: main
      ref: values
  destination:
    server: https://kubernetes.default.svc
    namespace: cicd
  # Reloader injects a STAKATER_* env var carrying a hash of the config Secret,
  # so the pod rolls when that Secret changes. The chart does not render it, so
  # Argo would strip it on every sync — and with selfHeal on, Argo and Reloader
  # would fight over the field and Recreate the forge each round.
  ignoreDifferences:
    - group: apps
      kind: Deployment
      name: forgejo-gitea
      jqPathExpressions:
        - '.spec.template.spec.containers[].env[] | select(.name | startswith("STAKATER_"))'
  syncPolicy:
    syncOptions:
      # Adopt the objects the bootstrap Helm release already created rather
      # than failing on "already exists".
      - ServerSideApply=true
 ---
 # Forgejo runners (local chart, one instance per language), replacing the
 # single generic "docker"-labeled runner. Each instance is a full standalone
 # Deployment with its own dind sidecar, own PVCs (registration + layer
 # cache) and own registered label -- there is no shared generic runner
 # anymore, so each instance also builds and pushes images for the repos it
 # serves (the chart's ConfigMap/NetworkPolicy fixes for that -- valid_volumes,
 # network: host, egress to ingress-nginx -- apply identically to all three).
 #
 # `values.yaml` is the chart's default and doubles as the golang instance's
 # config; node and rust layer a small values-<lang>.yaml override on top for
 # just runner.name/runner.labels. All three share one runner-token Secret
 # (Forgejo registration tokens are reusable across multiple runners, unlike
 # GitHub's one-time tokens) -- if that assumption is ever wrong, registration
 # will fail loudly in the register initContainer's logs, not silently.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: forgejo-runner-golang
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
  destination:
    server: https://kubernetes.default.svc
    namespace: cicd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: forgejo-runner-node
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
    helm:
      valueFiles:
        - values-node.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: cicd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: forgejo-runner-rust
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"
 spec:
  project: homelab
  source:
    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
    targetRevision: main
    path: k8s/infra/forgejo-runner
    helm:
      valueFiles:
        - values-rust.yaml
  destination:
    server: https://kubernetes.default.svc
    namespace: cicd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
--- a/Show More
+++ b/Show More