fix(agent-pod): remote tui session for multi-agent

llm-serving-default-deny admits port 8080 only from pods carrying
llm-client=true. Kong lacked it, so every route that actually contacts an
upstream timed out. /v1/models masked the problem: request-termination answers
inside Kong and never touches an upstream, so it returned 200 throughout.

Opting in via podLabels rather than relaxing the policy — it is a compensating
control, not hygiene, since vLLM v0.11.0 is frozen on Volta and will not receive
patches for several remote/unauthenticated advisories.

podLabels land only in the pod template, not spec.selector.matchLabels, so this
is not an immutable-field change.

.forgejo/workflows/cluster-ci.yaml

@@ -156,21 +156,43 @@ jobs:
       - name: Check for Secrets in Code
         run: |
           echo "=== Scanning for hardcoded secrets ==="
-          SECRETS_FOUND=0
+          # BLOCKING. This step used to only count findings and then exit 0, so a
+          # plaintext deploy key rode through it into a public remote. Two failure
+          # modes fixed: it now fails the build, and it matches key material by
+          # PEM header rather than only `private_key:`-style YAML field names.
+          # Findings are captured into variables and tested for emptiness rather than
+          # branching on grep's exit status: implementations disagree on the rc of a
+          # `-v` filter fed empty input, and a wrong rc here fails open.
+          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
+          # and silently scans nothing.
+          FAILED=0
 
-          for pattern in "password:" "secret:" "token:" "api_key:" "apikey:" "private_key:" "privatekey:"; do
-            if grep -r "$pattern" k8s/ --include="*.yaml" --include="*.yml" | grep -v "^Binary"; then
-              echo "⚠️  Found potential secret pattern: $pattern"
-              SECRETS_FOUND=$((SECRETS_FOUND + 1))
+          # Any private key block is fatal, regardless of the field name carrying it.
+          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
+                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
+                 | grep -v "\.enc\.yaml" || true)
+          if [ -n "$KEYS" ]; then
+            echo "❌ Unencrypted private key material found:"
+            echo "$KEYS"
+            FAILED=1
           fi
-          done
 
-          if [ $SECRETS_FOUND -gt 0 ]; then
-            echo "⚠️  Warning: Found $SECRETS_FOUND potential secrets"
-            echo "Secrets should be encrypted with SOPS or stored in ArgoCD Sealed Secrets"
-          else
+          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
+          # so encrypted files never trip this.
+          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
+                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
+                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
+          if [ -n "$VALS" ]; then
+            echo "❌ Plaintext secret value found:"
+            echo "$VALS"
+            FAILED=1
+          fi
+
+          if [ "$FAILED" -ne 0 ]; then
+            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
+            exit 1
+          fi
           echo "✓ No hardcoded secrets found"
-          fi
 
       # === Check K8s Security Best Practices ===
       - name: Check K8s Security Best Practices

.gitignore

@@ -50,10 +50,19 @@ terraform/*.tfstate.*
 terraform.tfvars.local
 skills-lock.json
 secrets-plaintext.yaml
-skills-lock.json
+
+# Saved plan files — binary, environment-specific, may embed resource attributes
+terraform/tfplan
+terraform/tfplan-*
 
 .DS_Store
 CLAUDE.md
 docs/
 
-k8s/argocd/seed-repo-secret.yamlbootstrap-argocd.log
+bootstrap-argocd.log
+
+# Any plaintext (non-SOPS) secret manifest. Encrypted ones are *.enc.yaml and
+# ARE committed — see .sops.yaml. A missing newline once merged two patterns on
+# one line here, which is how a plaintext deploy key reached a public remote.
+k8s/**/*-secret.yaml
+!k8s/**/*.enc.yaml

.sops.yaml

@@ -1,3 +1,5 @@
 creation_rules:
-  - path_regex: k8s/.*secrets.*\.ya?ml
+  # `secrets?` — singular too. A `seed-repo-secret.yaml` once slipped this regex
+  # and was committed in plaintext to a public remote.
+  - path_regex: k8s/.*secrets?.*\.ya?ml
     age: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla

bootstrap.sh

@@ -12,16 +12,15 @@
 #   - Talos cluster up; kubectl context points at it
 #   - helm 3, kubectl
 #   - SOPS age key at $SOPS_KEY (for the ArgoCD SOPS CMP plugin)
-#   - GitHub read-only deploy key private half at $DEPLOY_KEY (public half added
-#     to the GitHub repo's Deploy keys)
+#
+# The GitHub seed repo is public, so it is cloned anonymously over HTTPS — no
+# deploy key, no repository Secret, one less thing to bootstrap before ArgoCD.
 #
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 BOOT="$SCRIPT_DIR/k8s/bootstrap"
 SOPS_KEY="${SOPS_KEY:-$HOME/.sops/key.txt}"
-DEPLOY_KEY="${DEPLOY_KEY:-$HOME/.ssh/argocd_seed}"
-GITHUB_SSH="[email protected]:Riotpiaole/riotpiao.homelab.com.git"
 
 log()  { echo "[$(date +%H:%M:%S)] $*"; }
 die()  { echo "ERROR: $*" >&2; exit 1; }
@@ -39,7 +38,6 @@ preflight() {
   kubectl cluster-info >/dev/null || die "kubectl not configured / cluster unreachable"
   command -v helm >/dev/null || die "helm 3 not found"
   [[ -f "$SOPS_KEY" ]]   || die "SOPS age key missing at $SOPS_KEY"
-  [[ -f "$DEPLOY_KEY" ]] || die "GitHub deploy key missing at $DEPLOY_KEY (see phase4-argocd/seed-repo-secret.example.yaml)"
   log "✅ preflight ok"
 }
 
@@ -194,12 +192,9 @@ p3_forgejo() {
 p4_argocd() {
   phase "PHASE 4: ArgoCD (seeded from GitHub)"
   
-  # Always ensure namespace + repository secret (idempotent)
+  # Always ensure namespace (idempotent). The seed repo is public — ArgoCD clones
+  # it anonymously over HTTPS, so there is no repository Secret to create.
   kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
-  kubectl -n argocd create secret generic seed-github-repo \
-    --from-literal=type=git --from-literal=url="$GITHUB_SSH" \
-    --from-file=sshPrivateKey="$DEPLOY_KEY" --dry-run=client -o yaml | kubectl apply -f -
-  kubectl -n argocd label secret seed-github-repo argocd.argoproj.io/secret-type=repository --overwrite 2>/dev/null || true
 
   # Decrypt and apply any encrypted secrets from bootstrap dir (local SOPS)
   if command -v sops &> /dev/null; then

k8s/apps/agent-pod/configmap.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+data:
+  settings.json: |
+    {
+      "defaultProvider": "homelab-ornith",
+      "defaultModel": "ornith:35b",
+      "defaultThinkingLevel": "medium",
+      "theme": "light",
+      "compaction": {
+        "enabled": true,
+        "reserveTokens": 8192,
+        "keepRecentTokens": 12000
+      }
+    }
+kind: ConfigMap
+metadata:
+  name: pi-config
+  namespace: agent-pod

k8s/apps/agent-pod/console-ingress.yaml

@@ -0,0 +1,36 @@
+# Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
+# (trigger a new session) -- both are routes on the same hub.js service.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: console
+  namespace: agent-pod
+  annotations:
+    konghq.com/strip-path: "false"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /console
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /run
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090
+          - path: /sessions
+            pathType: Prefix
+            backend:
+              service:
+                name: agent-hub
+                port:
+                  number: 9090

k8s/apps/agent-pod/deployment.yaml

@@ -0,0 +1,108 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: agent-pod
+  namespace: agent-pod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: agent-pod
+  template:
+    metadata:
+      labels:
+        app: agent-pod
+    spec:
+      # api.riotpiao.com has no in-cluster DNS record (only resolves from the
+      # home network's own resolver) -- pin it to ingress-nginx-controller's
+      # ClusterIP so pi's models.json baseUrl works unchanged. TLS still
+      # terminates correctly since SNI/Host still say api.riotpiao.com.
+      hostAliases:
+        - ip: "10.101.128.185"
+          hostnames:
+            - "api.riotpiao.com"
+      containers:
+        # hub.js runs in the same container as pi (not a sidecar) so it can
+        # spawn `pi -p --mode json` directly via child_process -- a separate
+        # container can't exec into another container's filesystem/PATH.
+        # It IS the container's long-running process now; no more `sleep
+        # infinity` placeholder.
+        - name: pi
+          image: node:22-slim
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              apt-get update && apt-get install -y git curl jq openssh-client tmux
+              ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub
+              eval "$(ssh-agent -s)"
+              ssh-add /root/.ssh/id_forgejo
+              npm install -g @earendil-works/[email protected]
+              npm install --prefix /root ws
+              node /root/hub.js
+          env:
+            - name: PI_BIN
+              value: pi
+          ports:
+            - containerPort: 9090
+          resources:
+            requests:
+              cpu: "4"
+              memory: 8Gi
+            limits:
+              cpu: "8"
+              memory: 16Gi
+          volumeMounts:
+            - name: pi-config
+              mountPath: /root/.pi/agent/settings.json
+              subPath: settings.json
+            - name: pi-models
+              mountPath: /root/.pi/agent/models.json
+              subPath: models.json
+            - name: pi-skills
+              mountPath: /root/.pi/agent/skills
+            - name: hub-src
+              mountPath: /root/hub.js
+              subPath: hub.js
+            - name: ssh-key
+              mountPath: /root/.ssh/id_forgejo
+              subPath: id_forgejo
+            - name: ssh-config
+              mountPath: /root/.ssh/config
+              subPath: config
+      volumes:
+        - name: pi-config
+          configMap:
+            name: pi-config
+        - name: pi-models
+          secret:
+            secretName: pi-models
+        - name: pi-skills
+          configMap:
+            name: pi-skills
+            items:
+              - key: planner-SKILL.md
+                path: planner/SKILL.md
+              - key: investigator-SKILL.md
+                path: investigator/SKILL.md
+              - key: info-collector-SKILL.md
+                path: info-collector/SKILL.md
+              - key: implementer-SKILL.md
+                path: implementer/SKILL.md
+              - key: judge-SKILL.md
+                path: judge/SKILL.md
+              - key: resolver-SKILL.md
+                path: resolver/SKILL.md
+              - key: brave-search-SKILL.md
+                path: brave-search/SKILL.md
+        - name: hub-src
+          configMap:
+            name: hub-src
+        - name: ssh-key
+          secret:
+            secretName: agent-pod-ssh-key
+            defaultMode: 0600
+        - name: ssh-config
+          configMap:
+            name: agent-pod-ssh-config

k8s/apps/agent-pod/hub-configmap.yaml

@@ -0,0 +1,700 @@
+apiVersion: v1
+data:
+  hub.js: |
+    #!/usr/bin/env node
+    // agent-hub: lives inside the pi container (not a sidecar) so it can spawn
+    // `pi` directly, and control the pod's own tmux server. One persistent
+    // in-cluster service -- POST /run to trigger a single ad-hoc headless agent
+    // run, POST /pipeline to run an ordered list of task phases against a
+    // repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within
+    // a phase run concurrently, each in its own git worktree). Within one task,
+    // planner/investigator/implementer/judge are separate agents in separate
+    // named tmux sessions (task-<id>-<role>, attachable via `kubectl exec -it --
+    // tmux attach -t <name>` while running), coordinating only through what's on
+    // disk in that task's worktree -- not one shared conversation. GET /console
+    // (WebSocket) watches every concurrent headless run live, relaying pi's own
+    // session protocol verbatim (same event shape Claude Code sessions use).
+    const http = require("node:http");
+    const crypto = require("node:crypto");
+    const fs = require("node:fs");
+    const path = require("node:path");
+    const { spawn } = require("node:child_process");
+    const readline = require("node:readline");
+    const { WebSocketServer } = require("ws");
+
+    const PORT = process.env.HUB_PORT || 9090;
+    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
+
+    // Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide
+    // with unrelated tools on this machine (a Rust CLI and a Datadog TUI,
+    // respectively, discovered the hard way this session). Always invoke the
+    // exact pinned @earendil-works/[email protected] installed locally under
+    // .pi-cli/, by explicit path.
+    const PI_BIN =
+      process.env.PI_BIN ||
+      path.join(
+        __dirname,
+        "..",
+        ".pi-cli",
+        "node_modules",
+        "@earendil-works",
+        "pi-coding-agent",
+        "dist",
+        "cli.js"
+      );
+
+    // Job-type skills under pi/skills/<name>/SKILL.md (mounted at
+    // ~/.pi/agent/skills/<name>/ in agent-pod). When `agent` matches one of
+    // these, the prompt is forced through pi's `/skill:<name> <args>` mechanism
+    // instead of being sent bare -- see pi's skills.md docs on single-shot skill
+    // loading. `resolver` is never dispatched directly by a caller; only the
+    // pipeline driver invokes it, on stage crashes.
+    const ROLE_SKILLS = new Set([
+      "planner",
+      "investigator",
+      "info-collector",
+      "implementer",
+      "judge",
+      "resolver",
+    ]);
+
+    const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?}
+    const viewers = new Set(); // WebSocket connections watching /console
+
+    function broadcast(type, session) {
+      const msg = JSON.stringify({ type, session });
+      for (const ws of viewers) {
+        if (ws.readyState === ws.OPEN) ws.send(msg);
+      }
+    }
+
+    function startSession(agent, extra = {}) {
+      const id = extra.id || crypto.randomUUID();
+      const session = {
+        ...extra,
+        id,
+        agent,
+        status: "running",
+        events: [],
+        startedAt: new Date().toISOString(),
+      };
+      sessions.set(id, session);
+      broadcast("start", session);
+      return session;
+    }
+
+    function addEvent(session, rawLine) {
+      const event = JSON.parse(rawLine);
+      session.events.push(event);
+      broadcast("event", session);
+      return event;
+    }
+
+    function endSession(session, status) {
+      session.status = status;
+      session.endedAt = new Date().toISOString();
+      broadcast("end", session);
+    }
+
+    // Extracts the plain-text content of a message_end event, if any -- used to
+    // find the VERDICT:/RESOLUTION: line judge/resolver skills are required to
+    // end their final message with.
+    function textOf(event) {
+      if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) {
+        return null;
+      }
+      return event.message.content
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join("\n");
+    }
+
+    // Core spawn primitive. Spawns `pi -p --mode json <extraArgs> <prompt>`,
+    // relays every line as a session event exactly like before. Returns
+    // { session, done } -- `session` is available synchronously (so an HTTP
+    // handler can respond with its id right away, same as the old runAgent),
+    // `done` is a Promise resolving once the process exits, for callers that
+    // need to wait on a stage (the pipeline driver) rather than fire-and-forget.
+    function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) {
+      const session = startSession(agent, sessionExtra);
+      const args = ["-p", "--mode", "json"];
+      if (provider) args.push("--provider", provider);
+      if (model) args.push("--model", model);
+      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt;
+      args.push(finalPrompt);
+
+      const child = spawn(PI_BIN, args, {
+        stdio: ["ignore", "pipe", "pipe"],
+        cwd,
+      });
+      const rl = readline.createInterface({ input: child.stdout });
+      let lastText = "";
+      let stderrTail = "";
+
+      rl.on("line", (line) => {
+        if (!line.trim()) return;
+        try {
+          const event = addEvent(session, line);
+          const text = textOf(event);
+          if (text) lastText = text;
+        } catch {
+          // non-JSON stdout noise, ignore
+        }
+      });
+      child.stderr.on("data", (chunk) => {
+        process.stderr.write(chunk);
+        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
+      });
+
+      const done = new Promise((resolve) => {
+        child.on("close", (code) => {
+          endSession(session, code === 0 ? "done" : "error");
+          resolve({ code, session, lastText, stderrTail });
+        });
+      });
+
+      return { session, done };
+    }
+
+    function runAgent(agent, prompt, extraArgs = {}) {
+      // Fire-and-forget: caller (the /run handler) doesn't await `done`.
+      return spawnPi({ agent, prompt, ...extraArgs }).session;
+    }
+
+    function parseVerdictLine(text, label) {
+      if (!text) return null;
+      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
+      const m = text.match(re);
+      return m ? m[1].toUpperCase() : null;
+    }
+
+    // Deterministic git operations, run directly by hub.js rather than left to
+    // the model -- branch creation and pushing after each task are mechanical,
+    // not judgment calls, and need to happen reliably every time regardless of
+    // what a task's stages did or didn't remember to do.
+    function runCmd(bin, args, cwd) {
+      return new Promise((resolve) => {
+        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
+        let out = "";
+        child.stdout.on("data", (c) => (out += c));
+        child.stderr.on("data", (c) => (out += c));
+        child.on("close", (code) => resolve({ code, out: out.trim() }));
+      });
+    }
+
+    function runGit(cwd, args) {
+      return runCmd("git", args, cwd);
+    }
+
+    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
+    // in practice: a stage writes a real file and simply never runs `git add`/
+    // `git commit`, leaving it untracked and invisible to every later `git diff`.
+    // Sweep and commit anything left dirty after every stage, deterministically.
+    async function commitPending(cwd, message) {
+      await runGit(cwd, ["add", "-A"]);
+      const status = await runGit(cwd, ["status", "--porcelain"]);
+      if (!status.out) return { committed: false };
+      const commit = await runGit(cwd, ["commit", "-m", message]);
+      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
+    }
+
+    // Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide
+    // RETRY vs ABORT. Shared by both crash-recovery paths below (headless
+    // exit-code failures and interactive sentinel-file timeouts) -- the
+    // diagnostic prompt differs per caller, but "ask resolver, parse the
+    // RESOLUTION: line" is identical either way.
+    async function askResolver(pipelineId, cwd, task, diagnosticPrompt) {
+      const resolverResult = await spawnPi({
+        agent: "resolver",
+        prompt: diagnosticPrompt,
+        cwd,
+        sessionExtra: { pipelineId, stage: "resolver", task },
+      }).done;
+      return parseVerdictLine(resolverResult.lastText, "RESOLUTION");
+    }
+
+    // Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic
+    // judge FAIL, which is handled separately), asks the resolver to diagnose
+    // and decide RETRY vs ABORT. Retries the failed stage at most once,
+    // regardless of what resolver recommends a second time -- a hard cap, not
+    // indefinite trust in the model's judgment.
+    async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) {
+      let result = await spawnPi({
+        agent: stage,
+        prompt,
+        cwd,
+        sessionExtra: { pipelineId, stage, task },
+      }).done;
+      if (result.code === 0) return result;
+
+      const resolution = await askResolver(
+        pipelineId,
+        cwd,
+        task,
+        `Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}`
+      );
+
+      if (resolution === "RETRY") {
+        result = await spawnPi({
+          agent: stage,
+          prompt,
+          cwd,
+          sessionExtra: { pipelineId, stage, task },
+        }).done;
+      }
+      return result;
+    }
+
+    // Deterministic tmux operations -- same rationale as runGit: mechanical,
+    // not a judgment call, run directly rather than trusted to a prompt.
+    function runTmux(args) {
+      return runCmd("tmux", args);
+    }
+
+    function tmuxSessionName(task) {
+      return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`;
+    }
+
+    // Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in
+    // flight at once. No external dep; a plain in-order index cursor shared by
+    // `limit` runner loops.
+    async function runConcurrent(items, limit, worker) {
+      const results = new Array(items.length);
+      let i = 0;
+      async function next() {
+        while (i < items.length) {
+          const idx = i++;
+          results[idx] = await worker(items[idx], idx);
+        }
+      }
+      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
+      return results;
+    }
+
+    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
+    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
+    const POLL_MS = 10 * 1000;
+
+    async function waitForFile(filePath, limitMs) {
+      const start = Date.now();
+      while (!fs.existsSync(filePath)) {
+        if (Date.now() - start > limitMs) return false;
+        await new Promise((r) => setTimeout(r, POLL_MS));
+      }
+      return true;
+    }
+
+    const MAX_IMPLEMENT_ATTEMPTS = 5;
+    const MAX_PLAN_REVISIONS = 3;
+
+    // Runs one role as its own fresh interactive pi session in its own named
+    // tmux session -- planner, investigator, implementer, and judge are
+    // separate agents with separate context, not turns in one shared
+    // conversation. They coordinate only through what's on disk in the task's
+    // worktree: PLAN.md, committed code, judge's result file. Each session is
+    // attachable while it runs (kubectl exec -it -- tmux attach -t <name>) and
+    // killed once its sentinel file lands or it's abandoned after resolver
+    // escalation.
+    async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) {
+      const sessionName = `${tmuxSessionName(task)}-${stageLabel}`;
+      fs.rmSync(sentinelFile, { force: true });
+
+      const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]);
+      if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName };
+
+      let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS);
+      if (!ok) {
+        const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]);
+        const resolution = await askResolver(
+          pipelineId,
+          cwd,
+          task,
+          `Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` +
+            `Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` +
+            `real progress and worth nudging to wrap up, or stuck and worth abandoning?`
+        );
+        if (resolution === "RETRY") {
+          await runTmux([
+            "send-keys",
+            "-t",
+            sessionName,
+            `Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`,
+            "Enter",
+          ]);
+          ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS);
+        }
+      }
+
+      await runTmux(["kill-session", "-t", sessionName]);
+      return { ok, sessionName };
+    }
+
+    // Runs one task in its own git worktree (see runPhase): planner drafts
+    // PLAN.md, investigator confirms it, then implementer and judge go back and
+    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
+    // next implementer attempt is told to read and address. After
+    // MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to
+    // judge whether the *plan* itself is wrong, not just the implementation; if
+    // so it revises PLAN.md and the implementer gets a fresh attempt budget
+    // against the new plan. MAX_PLAN_REVISIONS caps this from looping forever
+    // on a task that's genuinely stuck.
+    async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) {
+      const resultFile = path.join(cwd, `.task-result-${task}`);
+      fs.rmSync(resultFile, { force: true });
+
+      const specHint = `the file under tasks/ starting with "${task}-"`;
+
+      const runRole = async (stageLabel, prompt, sentinel) => {
+        pipelineSession.activeTasks[task] = {
+          stage: stageLabel,
+          sessionName: `${tmuxSessionName(task)}-${stageLabel}`,
+          startedAt: new Date().toISOString(),
+        };
+        broadcast("event", pipelineSession);
+        const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel);
+        await commitPending(cwd, `task: ${task} (${stageLabel})`);
+        return result;
+      };
+
+      const abandon = (stageLabel, result, attempt) => {
+        delete pipelineSession.activeTasks[task];
+        broadcast("event", pipelineSession);
+        return {
+          task,
+          status: result.crashed ? "spawn-crashed" : "timed-out",
+          error: result.error,
+          stoppedAt: stageLabel,
+          ...(attempt !== undefined ? { attempt } : {}),
+        };
+      };
+
+      // Task's implementation is inherited already-committed (e.g. from a base
+      // branch of prior work) -- try one judge pass against the spec directly
+      // (no PLAN.md exists yet) before paying for a full planner/investigator
+      // redo. PASS ends the task here; FAIL falls through into the normal flow
+      // below, so planner/implementer pick up with the judge's real feedback.
+      if (judgeOnly) {
+        const quick = await runRole(
+          "judge",
+          `Task ${task} may already be implemented on this branch -- check ` +
+            `\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` +
+            `(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` +
+            `verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` +
+            `one line of rationale, then run: touch .stage-done-${task}-judge-0`,
+          path.join(cwd, `.stage-done-${task}-judge-0`)
+        );
+        if (!quick.ok) return abandon("judge", quick, 0);
+
+        const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+        if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
+          delete pipelineSession.activeTasks[task];
+          broadcast("event", pipelineSession);
+          return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
+        }
+      }
+
+      let result = await runRole(
+        "planner",
+        `Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`,
+        path.join(cwd, `.stage-done-${task}-planner`)
+      );
+      if (!result.ok) return abandon("planner", result);
+
+      result = await runRole(
+        "investigator",
+        `Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`,
+        path.join(cwd, `.stage-done-${task}-investigator`)
+      );
+      if (!result.ok) return abandon("investigator", result);
+
+      let planRevisions = 0;
+      let implementAttempt = 0;
+      let verdict = null;
+      let resultText = "";
+      let justRevisedPlan = false;
+
+      while (true) {
+        implementAttempt++;
+        const feedbackHint = fs.existsSync(resultFile)
+          ? justRevisedPlan
+            ? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
+              `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
+            : `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.`
+          : "";
+        justRevisedPlan = false;
+
+        result = await runRole(
+          "implementer",
+          `Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`,
+          path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
+        );
+        if (!result.ok) return abandon("implementer", result, implementAttempt);
+
+        result = await runRole(
+          "judge",
+          `Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` +
+            `.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` +
+            `rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`,
+          path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`)
+        );
+        if (!result.ok) return abandon("judge", result, implementAttempt);
+
+        resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
+        verdict = parseVerdictLine(resultText, "VERDICT");
+        if (verdict === "PASS") break;
+
+        if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
+          if (planRevisions >= MAX_PLAN_REVISIONS) break;
+          planRevisions++;
+          result = await runRole(
+            "planner-revise",
+            `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
+              `the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` +
+              `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` +
+              `commit. If you change the approach, also use the investigator skill to confirm the new approach against ` +
+              `real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` +
+              `touch .stage-done-${task}-planner-revise-${planRevisions}`,
+            path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)
+          );
+          if (!result.ok) return abandon("planner-revise", result, planRevisions);
+          implementAttempt = 0;
+          justRevisedPlan = true;
+        }
+      }
+
+      delete pipelineSession.activeTasks[task];
+      broadcast("event", pipelineSession);
+
+      if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
+        return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
+      }
+      return {
+        task,
+        status: verdict === "PASS" ? "done" : "done-with-concerns",
+        judgeRationale: resultText,
+      };
+    }
+
+    const PHASE_CONCURRENCY = 3;
+
+    // Runs one phase (a batch of tasks with no declared dependency on each
+    // other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task
+    // gets its own git worktree off workBranch -- concurrent pi sessions writing
+    // into one shared working tree would corrupt the index; worktrees share the
+    // same object database but give each task an isolated checkout. After a
+    // task's session ends, its branch is merged back into workBranch and pushed,
+    // one merge at a time (git ref updates aren't safe to run concurrently even
+    // though the worktrees themselves are isolated).
+    async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) {
+      // Each entry is either a plain task id, or { id, judgeOnly: true } when
+      // the task's implementation already exists (e.g. inherited from a base
+      // branch) and just needs a real judge pass rather than a full
+      // planner/investigator/implementer redo.
+      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
+
+      const worktrees = {};
+      for (const entry of entries) {
+        const task = entry.id;
+        const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`);
+        const taskBranch = `task/${task}`;
+        const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]);
+        if (add.code !== 0) {
+          pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out });
+          continue;
+        }
+        worktrees[task] = { wtDir, taskBranch };
+      }
+
+      const runnable = entries.filter((e) => worktrees[e.id]);
+      await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => {
+        const { wtDir } = worktrees[entry.id];
+        const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly);
+        pipelineSession.taskResults.push(result);
+        return result;
+      });
+
+      // Merge + push sequentially -- ref updates on the shared repo, one at a
+      // time, in the declared task order for this phase.
+      for (const entry of runnable) {
+        const task = entry.id;
+        const { wtDir, taskBranch } = worktrees[task];
+        const result = pipelineSession.taskResults.find((r) => r.task === task);
+
+        const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]);
+        if (merge.code !== 0) {
+          await runGit(cwd, ["merge", "--abort"]);
+          if (result) {
+            result.status = "merge-conflict";
+            result.mergeError = merge.out;
+          }
+        } else {
+          const push = await runGit(cwd, ["push", "-u", "origin", workBranch]);
+          if (result) {
+            result.pushed = push.code === 0;
+            if (!result.pushed) result.pushError = push.out;
+          }
+        }
+
+        await runGit(cwd, ["worktree", "remove", wtDir, "--force"]);
+        await runGit(cwd, ["branch", "-D", taskBranch]);
+        broadcast("event", pipelineSession);
+      }
+    }
+
+    // tasks: array of phases, each phase an array of task ids with no declared
+    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) --
+    // caller's responsibility to supply real phase grouping (see tasks/INDEX.md;
+    // filename/numeric sort does NOT match execution order on boards like this).
+    // A flat array of ids is also accepted and treated as one single phase.
+    // Phases run strictly sequentially (a phase boundary is a real dependency
+    // gate); tasks within a phase run concurrently, each in its own worktree --
+    // see runPhase.
+    function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) {
+      const cwd = path.join(WORK_DIR, pipelineId);
+      fs.mkdirSync(cwd, { recursive: true });
+
+      const phases = Array.isArray(tasks[0]) ? tasks : [tasks];
+
+      const pipelineSession = startSession("pipeline", {
+        id: pipelineId,
+        pipelineId,
+        stage: "pipeline",
+        taskResults: [],
+        activeTasks: {},
+        totalTasks: phases.flat().length,
+      });
+
+      (async () => {
+        const clone = await runStageWithResolver(
+          pipelineId,
+          cwd,
+          "planner",
+          `Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`,
+          "clone"
+        );
+        if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed");
+        if (!fs.existsSync(path.join(cwd, ".git"))) {
+          // The model deciding to `cd` elsewhere before cloning (instead of
+          // cloning into the assigned cwd) is a real failure mode seen in
+          // practice, not a hypothetical -- exit code 0 doesn't mean the clone
+          // landed where every later stage's cwd assumes it did.
+          return endSession(pipelineSession, "clone-missing");
+        }
+
+        // Dedicated branch, never main -- and pushed after every single task
+        // (not just at the end) so a pod restart mid-run loses at most the
+        // in-progress task's work, not everything since the start.
+        const workBranch = branchName || `agent-run/${pipelineId}`;
+        const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]);
+        if (branchResult.code !== 0) {
+          pipelineSession.gitError = branchResult.out;
+          return endSession(pipelineSession, "branch-crashed");
+        }
+
+        // Seen in practice: a task manually downloads a dependency tarball
+        // (crates.io registry access isn't guaranteed from every sandboxed
+        // checkout) and it lands at repo root, outside whatever .gitignore
+        // already covers -- then git add -A (ours or the model's own) commits
+        // it. Append broad build-artifact/archive patterns before any task
+        // runs, so it's excluded regardless of who stages files later.
+        const gitignoreAdditions = [
+          "",
+          "# agent-harness: build artifacts and vendored archives never belong in source control",
+          "*.tar.gz",
+          "*.tgz",
+          "*.crate",
+          "*.zip",
+          "*.bin",
+          "*.whl",
+          "vendor/",
+          "node_modules/",
+          "",
+          "# agent-harness: task completion sentinel files, harness bookkeeping only",
+          ".task-result-*",
+          ".stage-done-*",
+        ].join("\n");
+        fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
+        await runGit(cwd, ["add", ".gitignore"]);
+        await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
+
+        for (const phaseTasks of phases) {
+          await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession);
+        }
+
+        const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed"));
+        endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed");
+      })();
+
+      return pipelineSession;
+    }
+
+    const server = http.createServer((req, res) => {
+      const url = new URL(req.url, "http://localhost");
+
+      if (url.pathname === "/healthz") {
+        res.writeHead(200).end();
+        return;
+      }
+
+      if (url.pathname === "/sessions" && req.method === "GET") {
+        res.writeHead(200, { "Content-Type": "application/json" });
+        res.end(JSON.stringify([...sessions.values()]));
+        return;
+      }
+
+      if (url.pathname === "/run" && req.method === "POST") {
+        let body = "";
+        req.on("data", (chunk) => (body += chunk));
+        req.on("end", () => {
+          try {
+            const { agent, prompt, provider, model } = JSON.parse(body);
+            if (!agent || !prompt) throw new Error("agent and prompt are required");
+            const session = runAgent(agent, prompt, { provider, model });
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ id: session.id }));
+          } catch (err) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      if (url.pathname === "/pipeline" && req.method === "POST") {
+        let body = "";
+        req.on("data", (chunk) => (body += chunk));
+        req.on("end", () => {
+          try {
+            const { repo, baseBranch, tasks, branchName } = JSON.parse(body);
+            if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) {
+              throw new Error("repo, baseBranch, and a non-empty tasks array are required");
+            }
+            const pipelineId = crypto.randomUUID();
+            runPipeline({ pipelineId, repo, baseBranch, tasks, branchName });
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ id: pipelineId }));
+          } catch (err) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          }
+        });
+        return;
+      }
+
+      res.writeHead(404).end();
+    });
+
+    const wss = new WebSocketServer({ server, path: "/console" });
+    wss.on("connection", (ws) => {
+      for (const session of sessions.values()) {
+        ws.send(JSON.stringify({ type: "snapshot", session }));
+      }
+      viewers.add(ws);
+      ws.on("close", () => viewers.delete(ws));
+    });
+
+    server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
+kind: ConfigMap
+metadata:
+  name: hub-src
+  namespace: agent-pod

k8s/apps/agent-pod/hub-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-hub
+  namespace: agent-pod
+spec:
+  selector:
+    app: agent-pod
+  ports:
+    - port: 9090
+      targetPort: 9090

k8s/apps/agent-pod/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-pod
+resources:
+  - deployment.yaml
+  - configmap.yaml
+  - hub-configmap.yaml
+  - pi-skills-configmap.yaml
+  - ssh-configmap.yaml
+  - hub-service.yaml
+  - console-ingress.yaml

k8s/apps/agent-pod/pi-skills-configmap.yaml

@@ -0,0 +1,156 @@
+apiVersion: v1
+data:
+  brave-search-SKILL.md: |
+    ---
+    name: brave-search
+    description: Web search via the Brave Search API, called directly with curl. Use for searching documentation, facts, or any current web content.
+    allowed-tools: Bash
+    ---
+
+    # Brave Search
+
+    Direct HTTP call to the Brave Search API — no separate script or package, just `curl` (`BRAVE_API_KEY` is already set in the environment).
+
+    ## Search
+
+    ```bash
+    curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
+      --get --data-urlencode "q=<query>" --data-urlencode "count=5" \
+      "https://api.search.brave.com/res/v1/web/search"
+    ```
+
+    Options (add as extra `--data-urlencode` pairs):
+    - `count=<n>` — number of results (max 20, default 5)
+    - `country=<code>` — two-letter country code (default US)
+    - `freshness=pd|pw|pm|py` — past day/week/month/year, or `freshness=YYYY-MM-DDtoYYYY-MM-DD`
+
+    Response is JSON; the results live at `.web.results[]`, each with `title`, `url`, `description`, `age`. Pipe through `jq` if you want a shorter view, e.g.:
+
+    ```bash
+    curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
+      --get --data-urlencode "q=<query>" "https://api.search.brave.com/res/v1/web/search" \
+      | jq -r '.web.results[] | "- \(.title)\n  \(.url)\n  \(.description)\n"'
+    ```
+
+    There's no page-content-extraction helper here — if a result needs reading in full, `curl` the URL directly and read the raw HTML/text; don't expect readability-cleaned markdown.
+
+    ## When to Use
+
+    - Searching for documentation or API references
+    - Looking up facts or current information
+    - Confirming a claim or approach against real sources
+  implementer-SKILL.md: |
+    ---
+    name: implementer
+    description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run.
+    allowed-tools: Read Grep Find Ls Write Edit Bash
+    ---
+
+    Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions.
+
+    - Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history.
+    - Push only if the task explicitly asks for it.
+
+    Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch.
+
+    **Hard rules:**
+    - Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing.
+    - Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it.
+
+    **Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it.
+  info-collector-SKILL.md: |
+    ---
+    name: info-collector
+    description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill).
+    allowed-tools: Read Bash
+    ---
+
+    **Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there.
+
+    **Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job).
+
+    **Modes:**
+
+    - **Collect mode** (default) — use the `brave-search` skill (`curl` against the Brave Search API) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill.
+    - If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response.
+  investigator-SKILL.md: |
+    ---
+    name: investigator
+    description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct.
+    allowed-tools: Read Bash
+    ---
+
+    Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one.
+
+    - Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), use `brave-search` to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`.
+    - No `PLAN.md`? Just answer the question asked, citing sources.
+
+    Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next.
+
+    **Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not.
+  judge-SKILL.md: |
+    ---
+    name: judge
+    description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria.
+    allowed-tools: Read Bash
+    ---
+
+    Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp.
+
+    - Run `git diff <base-branch>...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection?
+    - FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale.
+    - No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly.
+
+    You MUST end your final message with a literal verdict line, exactly one of:
+
+    ```
+    VERDICT: PASS
+    ```
+    ```
+    VERDICT: FAIL
+    ```
+
+    followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline.
+  planner-SKILL.md: |
+    ---
+    name: planner
+    description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it.
+    allowed-tools: Read Grep Find Ls Write Bash
+    ---
+
+    Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms.
+
+    - If cwd is empty: `git clone` the given repo/branch first (only at pipeline start).
+    - Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity.
+    - Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/<name>.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: <summary>"`.
+
+    Don't touch any other file.
+  resolver-SKILL.md: |
+    ---
+    name: resolver
+    description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion.
+    allowed-tools: Read Bash
+    ---
+
+    **Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again.
+
+    **Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work.
+
+    **Modes:**
+
+    - **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix).
+
+    You MUST end your final message with a literal resolution line, exactly one of:
+
+    ```
+    RESOLUTION: RETRY
+    ```
+    ```
+    RESOLUTION: ABORT
+    ```
+
+    followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`.
+kind: ConfigMap
+metadata:
+  name: pi-skills
+  namespace: agent-pod

k8s/apps/agent-pod/ssh-configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-pod-ssh-config
+  namespace: agent-pod
+data:
+  config: |
+    Host git.riotpiao.com
+        IdentityFile /root/.ssh/id_forgejo
+        Port 2222
+        User git
+        StrictHostKeyChecking accept-new

k8s/apps/api/AUTH-PLAN.md

@@ -0,0 +1,83 @@
+# API Auth Layer — Authentik service account + Kong JWT (model invoke)
+
+Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik
+service account holding a valid **client_credentials** JWT can invoke the KServe
+models. "Invoke role" = **possession of a JWT from the dedicated model-invoke
+OAuth2 provider** (only the service account can obtain one).
+
+## Flow
+
+```
+service account ── client_credentials ──▶ Authentik token endpoint
+                    (client_id + secret)   https://authentik.riotpiao.com/application/o/token/
+        │
+        ▼  RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/)
+client ── Authorization: Bearer <jwt> ──▶ Kong (api.riotpiao.com/*)
+                                           jwt plugin: verify RS256 sig via Authentik JWKS,
+                                           check iss/exp → map to KongConsumer → allow
+                                           ▼
+                                        KServe model (reasoning / ornith / ...)
+```
+
+Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in
+**`jwt`** plugin: it validates an RS256 signature against a public key we pin on
+a KongConsumer, keyed by the token's `iss`.
+
+## Changes
+
+### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py)
+- New **service account** user `model-invoker` (type `service_account`, no
+  password; Authentik issues an app-password/token for M2M).
+- New **OAuth2 provider + application** `model-invoke`:
+  - `client_type: confidential`, `grant_types: ["client_credentials"]`
+  - signing key = existing RS256 keypair (same as other providers)
+  - mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes
+    needed for M2M.
+- Client secret written to k8s Secret `api/model-invoke-oidc`
+  (keys `client-id`, `client-secret`), labelled for whoever consumes it.
+- Bind the service account so it (and only it) can use the provider.
+
+### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`)
+- **KongConsumer** `model-invoker` (ns api).
+- **`jwt` credential** on that consumer (a Secret of type
+  `konghq.com/v1/credential`):
+  - `algorithm: RS256`
+  - `key`  = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/`
+  - `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing
+    cert (fetched from Authentik JWKS / cert, stored in git or ksops).
+- **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`).
+
+### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml)
+- Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation
+  (currently e.g. `llm-rewrite-reasoning`) → becomes
+  `llm-rewrite-reasoning,jwt-auth`.
+- Leave `/models` list route open OR protect too (decision).
+
+## Client usage (after build)
+```bash
+TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \
+  -d grant_type=client_credentials \
+  -d client_id=model-invoke \
+  -d client_secret=<secret> \
+  -d scope=openid | jq -r .access_token)
+
+curl https://api.riotpiao.com/v1/chat/completions \
+  -H "Authorization: Bearer $TOKEN" -d '{...}'
+```
+
+## Test plan
+1. No token → Kong returns 401.
+2. Valid client_credentials token → 200, model responds.
+3. Expired/garbage token → 401.
+4. Confirm the `/models` route behaviour matches the decision.
+
+## Open items / risks
+- Authentik `client_credentials` for a *service account* may require an
+  **app-password / JWT-assertion** flow rather than plain client_secret POST —
+  verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs
+  the SA token). Adjust step 1 accordingly before wiring Kong.
+- Pinning `rsa_public_key`: Authentik key rotation would break it — document a
+  rotation runbook, or have the provision script re-export the cert PEM into the
+  Kong credential on each run (keeps them in sync, same idea as ksops secrets).
+- Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's
+  issuer is stable.

k8s/apps/api/ingress.yaml

@@ -0,0 +1,43 @@
+# Edge route for the API gateway.
+#
+# Lives here rather than in the central k8s/bootstrap/ingress/ingress.yaml
+# because that Application syncs in wave 1, before namespace `api` exists.
+#
+# nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its
+# default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards
+# plain HTTP to kong-proxy. Kong then does the real routing, from Ingresses
+# carrying `ingressClassName: kong`.
+#
+# Catch-all `/` on purpose: everything under this host belongs to Kong. Listing
+# per-API paths here would duplicate Kong's routing table inside nginx, and the
+# two copies would drift.
+#
+# In-cluster callers should prefer http://kong-proxy.api.svc.cluster.local
+# directly. Resolving api.riotpiao.com sends them out to nginx and back in,
+# which is a pointless hairpin unless they need TLS or the public hostname.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: api
+  namespace: api
+  annotations:
+    # An API gateway carries streaming responses (SSE, gRPC-web, LLM token
+    # streams). nginx's 60s default read timeout and its response buffering
+    # would truncate or stall those.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: kong-proxy
+                port:
+                  number: 80

k8s/apps/api/kong-metrics.yaml

@@ -0,0 +1,19 @@
+# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the
+# ingress controller apply it to every route on this Kong instance, so all
+# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics
+# without touching llm-routes.yaml. Scraped via kong-values.yaml's
+# serviceMonitor (status listener, already on by chart default at :8100).
+apiVersion: configuration.konghq.com/v1
+kind: KongClusterPlugin
+metadata:
+  name: prometheus
+  annotations:
+    kubernetes.io/ingress.class: kong
+  labels:
+    global: "true"
+plugin: prometheus
+config:
+  status_code_metrics: true
+  latency_metrics: true
+  bandwidth_metrics: true
+  upstream_health_metrics: true

k8s/apps/api/kong-values.yaml

@@ -0,0 +1,140 @@
+# Kong Gateway — cluster-internal API gateway (namespace `api`).
+#
+# Chart: kong/kong 3.4.1 (appVersion 3.9). Only overrides are listed; every key
+# here was checked against `helm show values kong/kong --version 3.4.1`, because
+# Helm silently ignores unknown keys — a typo is a no-op, not an error.
+#
+# ── Topology ────────────────────────────────────────────────────────────────
+#   external:  client -> nginx (TLS, wildcard *.riotpiao.com) -> kong-proxy:80
+#   internal:  pod    -> kong-proxy.api.svc.cluster.local:80
+#
+# nginx stays the single edge and the only LoadBalancer (192.168.1.160). Kong is
+# the policy/routing layer behind it, so it needs no LB IP and no TLS of its own
+# — hence ClusterIP and proxy.tls disabled. Giving Kong its own IP from
+# homelab-pool would mean duplicating cert-manager wiring and diverging from the
+# CoreDNS convention that sends every *.riotpiao.com host to nginx.
+#
+# ── Routing model ───────────────────────────────────────────────────────────
+# Consumers publish an Ingress with `ingressClassName: kong`; the controller
+# turns it into a Kong route. `nginx` remains the default IngressClass, so this
+# is strictly opt-in and no existing Ingress changes behaviour.
+
+# Without this the release name is prefixed onto everything (`kong-kong-proxy`).
+# Pinning it keeps the Service name stable and independent of the release name,
+# which matters because the nginx Ingress in k8s/bootstrap/ingress/ingress.yaml
+# references it by name.
+fullnameOverride: kong
+
+# Two replicas so a node drain or rollout doesn't take the gateway down. Kong is
+# stateless in DB-less mode, so replicas are pure redundancy.
+replicaCount: 2
+
+# Opt in to the `llm-serving-default-deny` NetworkPolicy, which admits port 8080
+# only from pods carrying this label. That policy is a compensating control, not
+# hygiene: vLLM v0.11.0 is frozen on Volta and will never receive patches for
+# several remote/unauthenticated advisories, so it must not be broadly reachable.
+#
+# Without this label Cilium DROPS the packets rather than refusing them, so the
+# symptom is a request that hangs until the client's timeout — not a connection
+# error. /v1/models still worked while this was missing, because
+# request-termination answers inside Kong and never touches an upstream.
+podLabels:
+  llm-client: "true"
+
+env:
+  # DB-less. Config comes from Kubernetes objects via the ingress controller, so
+  # git stays the source of truth. A Postgres-backed Kong would put live routing
+  # config in a database mutated through the Admin API — state outside git, plus
+  # migration Jobs on every upgrade.
+  database: "off"
+  # `nginx_proxy_<directive>` injects a directive into the proxy location block;
+  # this renders `proxy_buffering off;`.
+  #
+  # Required for LLM streaming. With buffering on (the default) nginx accumulates
+  # the upstream response before forwarding, so an SSE stream from
+  # `"stream": true` arrives in lumps or stalls until the generation finishes —
+  # which defeats the point of streaming. The matching setting is already on the
+  # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the
+  # buffered one dominates.
+  nginx_proxy_proxy_buffering: "off"
+  # Any plugin that rewrites the request body — request-transformer on the
+  # llm-chat-* routes — reads it through `kong.request.get_body()`, and that
+  # returns nothing once nginx has spilled the body past
+  # client_body_buffer_size into a temp file. The plugin then re-serializes a
+  # body with no `messages`, and the upstream answers
+  #   HTTP 400 {"error":{"message":"[] is too short - 'messages'"}}
+  # Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400.
+  # An agent request carrying tool schemas clears that in one turn, so the
+  # buffer has to hold a whole conversation, not a chat message.
+  nginx_http_client_body_buffer_size: "16m"
+  nginx_http_client_max_body_size: "16m"
+
+ingressController:
+  enabled: true
+  ingressClass: kong
+  # The chart's ingress-class template is gated on
+  # `.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass"`, so a
+  # bare `helm template` renders nothing. ArgoCD passes --api-versions from the
+  # live cluster, so it does render there — verify `kubectl get ingressclass
+  # kong` after the first sync rather than assuming it.
+  createIngressClass: true
+  # Deliberately empty: setting is-default-class here would hijack every Ingress
+  # in the cluster that omits ingressClassName. nginx keeps that role.
+  ingressClassAnnotations: {}
+
+proxy:
+  enabled: true
+  # Chart default is LoadBalancer, which would claim an IP from homelab-pool.
+  type: ClusterIP
+  http:
+    enabled: true
+    servicePort: 80
+    containerPort: 8000
+  # nginx already terminated TLS; a second handshake to the same cluster buys
+  # nothing and would need Kong to hold its own certificate.
+  tls:
+    enabled: false
+
+# No Service for the Admin API. The controller reaches it over localhost inside
+# the pod, so exposing it would only create an unauthenticated write path to the
+# gateway's entire configuration.
+admin:
+  enabled: false
+
+# Kong Manager UI — chart default is `enabled: true` with type NodePort, which
+# would open a port on every node. Not wanted.
+manager:
+  enabled: false
+
+resources:
+  requests:
+    cpu: 200m
+    memory: 256Mi
+  limits:
+    cpu: "2"
+    memory: 1Gi
+
+podDisruptionBudget:
+  enabled: true
+  minAvailable: 1
+
+# Status listener (metrics/health) is on by default at :8100 (chart default,
+# verified via `helm show values`). This just wires the ServiceMonitor the
+# chart already knows how to generate for it, so kong_http_requests_total /
+# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the
+# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml.
+serviceMonitor:
+  enabled: true
+  labels:
+    release: kube-prometheus-stack
+
+# Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
+# situation degrades to co-location instead of leaving a pod Pending.
+topologySpreadConstraints:
+  - maxSkew: 1
+    topologyKey: kubernetes.io/hostname
+    whenUnsatisfiable: ScheduleAnyway
+    labelSelector:
+      matchLabels:
+        app.kubernetes.io/name: kong
+        app.kubernetes.io/instance: kong

k8s/apps/api/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist so kong-values.yaml in this directory is NOT treated as a
+# manifest — it is Helm input consumed by the chart source of the `kong`
+# Application, not a Kubernetes object. Anything new added here must be listed
+# or it is silently dropped with no error and no drift shown.
+resources:
+  - ingress.yaml
+  - kong-metrics.yaml
+  - llm-routes.yaml
+  - model-auth.yaml
+# No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
+# namespace, and the transformer rewrites metadata.namespace on every resource
+# it builds, which is a trap for anything cross-namespace added later.

k8s/apps/api/llm-routes.yaml

@@ -0,0 +1,317 @@
+# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped.
+#
+# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress
+# can only reference a Service in its own namespace and the predictor Services
+# are there. The Kong ingress controller watches all namespaces, so the routes
+# still land on the gateway. They are synced by the `kong` Application (which
+# has a `path: k8s/apps/api` source) so all gateway config stays in one place.
+#
+# ── Model -> upstream map (verified live) ───────────────────────────────────
+#   reasoning              -> reasoning-predictor   vLLM, DeepSeek-R1-Distill-32B
+#   ornith:35b             -> ornith-predictor      Ollama
+#   qwen2.5:3b-instruct    -> ornith-predictor      Ollama  (same pod!)
+#   nomic-embed-text-v2    -> embeddings-predictor  TEI
+#   bge-reranker-base      -> reranker-predictor    TEI
+#   Qwen2.5-Math-PRM-7B    -> verifier-predictor    vLLM pooling
+#
+# ── Why path-per-model, and why the body is rewritten ───────────────────────
+# Kong matches routes on host, path, method and headers — never on the request
+# body. So a single /v1/chat/completions endpoint that dispatches on the body's
+# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
+# multi-target model routing, is Enterprise-only).
+#
+# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct`
+# share ONE Ollama pod, and Ollama still reads which model to load from the
+# body's `model` field. If only the path selected the route, a client calling
+# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the
+# 35B model. So each chat route force-overwrites `model` in the body, making the
+# path the single source of truth. Callers may omit `model` entirely.
+#
+# ── Timeouts ───────────────────────────────────────────────────────────────
+# Kong's upstream timeouts default to 60000ms. A 32B model generating a long
+# answer on a Volta GPU routinely exceeds that, and the client would see a
+# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds.
+
+# ── GET /v1/models ──────────────────────────────────────────────────────────
+# Served entirely by Kong via request-termination: the plugin short-circuits in
+# the access phase, so the backend below is never contacted. It only exists
+# because an Ingress rule requires a backend.
+#
+# The list is static, which means it can drift from what the engines actually
+# serve — notably if the Ollama pull list in the ornith InferenceService
+# changes. Verify with:
+#   curl -s $SVC/v1/models   (against each *-predictor)
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: llm-models-list
+  namespace: llm-serving
+plugin: request-termination
+config:
+  status_code: 200
+  content_type: application/json
+  body: |
+    {"object":"list","data":[
+      {"id":"reasoning","object":"model","owned_by":"homelab","created":0},
+      {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
+      {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
+      {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
+      {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0},
+      {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0}
+    ]}
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-models
+  namespace: llm-serving
+  annotations:
+    konghq.com/plugins: llm-models-list,model-key-auth
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "GET"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/models
+            pathType: Exact
+            backend:
+              # Never actually called — request-termination answers first.
+              service:
+                name: reasoning-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/reasoning/chat/completions ─────────────────────────────────────
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: llm-rewrite-reasoning
+  namespace: llm-serving
+plugin: request-transformer
+config:
+  # `add` only applies when the field is absent, `replace` only when present.
+  # Both are needed to force the value in either case.
+  add:
+    body:
+      - "model:reasoning"
+  replace:
+    body:
+      - "model:reasoning"
+    # The model lives in the path for routing; the upstream still expects the
+    # canonical OpenAI path.
+    uri: /v1/chat/completions
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-chat-reasoning
+  namespace: llm-serving
+  annotations:
+    konghq.com/plugins: llm-rewrite-reasoning,model-key-auth
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/reasoning/chat/completions
+            pathType: Prefix
+            backend:
+              service:
+                name: reasoning-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/ornith/chat/completions ────────────────────────────────────────
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: llm-rewrite-ornith
+  namespace: llm-serving
+plugin: request-transformer
+config:
+  add:
+    body:
+      - "model:ornith:35b"
+  replace:
+    body:
+      - "model:ornith:35b"
+    uri: /v1/chat/completions
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-chat-ornith
+  namespace: llm-serving
+  annotations:
+    konghq.com/plugins: llm-rewrite-ornith,model-key-auth
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/ornith/chat/completions
+            pathType: Prefix
+            backend:
+              service:
+                name: ornith-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/qwen/chat/completions ──────────────────────────────────────────
+# Same upstream pod as ornith — only the forced body `model` differs. Both stay
+# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and
+# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request.
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: llm-rewrite-qwen
+  namespace: llm-serving
+plugin: request-transformer
+config:
+  add:
+    body:
+      - "model:qwen2.5:3b-instruct"
+  replace:
+    body:
+      - "model:qwen2.5:3b-instruct"
+    uri: /v1/chat/completions
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-chat-qwen
+  namespace: llm-serving
+  annotations:
+    konghq.com/plugins: llm-rewrite-qwen,model-key-auth
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/qwen/chat/completions
+            pathType: Prefix
+            backend:
+              service:
+                name: ornith-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/embeddings ─────────────────────────────────────────────────────
+# No path-per-model and no rewrite: there is exactly one embeddings backend, so
+# there is nothing to disambiguate, and TEI already serves the canonical
+# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists).
+# That makes an OpenAI SDK a drop-in here.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-embeddings
+  namespace: llm-serving
+  annotations:
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "600000"
+    konghq.com/write-timeout: "600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/embeddings
+            pathType: Prefix
+            backend:
+              service:
+                name: embeddings-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/rerank ─────────────────────────────────────────────────────────
+# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing
+# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely
+# needs the rewrite that embeddings does not.
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: llm-rewrite-rerank
+  namespace: llm-serving
+plugin: request-transformer
+config:
+  replace:
+    uri: /rerank
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-rerank
+  namespace: llm-serving
+  annotations:
+    konghq.com/plugins: llm-rewrite-rerank,model-key-auth
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "600000"
+    konghq.com/write-timeout: "600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/rerank
+            pathType: Prefix
+            backend:
+              service:
+                name: reranker-predictor
+                port:
+                  number: 80
+---
+# ── POST /v1/score ──────────────────────────────────────────────────────────
+# The process reward model. Returns scores, not tokens, so it is deliberately
+# not under /chat/completions. vLLM serves /v1/score natively (verified), so no
+# rewrite is needed.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: llm-score
+  namespace: llm-serving
+  annotations:
+    konghq.com/strip-path: "false"
+    konghq.com/methods: "POST"
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "600000"
+    konghq.com/write-timeout: "600000"
+spec:
+  ingressClassName: kong
+  rules:
+    - host: api.riotpiao.com
+      http:
+        paths:
+          - path: /v1/score
+            pathType: Prefix
+            backend:
+              service:
+                name: verifier-predictor
+                port:
+                  number: 80

k8s/apps/api/model-auth.yaml

@@ -0,0 +1,49 @@
+# API auth layer — Kong key-auth on the model routes.
+#
+# The model API (api.riotpiao.com/v1/...) requires a static API key, presented
+# OpenAI-style as `Authorization: Bearer <key>` (or `apikey: <key>`). The key
+# lives in the ksops-managed Secret model-invoke-apikey (labelled
+# konghq.com/credential: key-auth) and is bound to the KongConsumer below.
+#
+# Issue the key to rock; use it as the OpenAI SDK api_key. Rotate by updating the
+# ksops secret. This is self-contained in Kong — the invoke path does not depend
+# on an Authentik token (Authentik still fronts every *human* dashboard SSO).
+---
+apiVersion: configuration.konghq.com/v1
+kind: KongConsumer
+metadata:
+  name: model-invoker
+  namespace: api
+  annotations:
+    kubernetes.io/ingress.class: kong
+username: model-invoker
+credentials:
+  - model-invoke-apikey
+---
+# key-auth: require the API key on the model routes. key_in_header accepts the
+# `apikey` header; key_in_bearer accepts `Authorization: Bearer <key>` so any
+# OpenAI-compatible SDK (api_key=..., base_url=https://api.riotpiao.com/v1) works
+# unchanged.
+#
+# Namespace `llm-serving`, not `api`: the ingress controller resolves a
+# `konghq.com/plugins` annotation against the annotated object's OWN namespace,
+# and all five model routes in llm-routes.yaml live in llm-serving. While this
+# sat in `api` the reference dangled, the plugin never bound, and every model
+# route served traffic with no key at all — verified: an unauthenticated
+# /v1/models and /v1/ornith/chat/completions both returned 200. A dangling
+# plugin reference is silent; it fails open, so re-test without a key after any
+# move rather than trusting that the object exists.
+apiVersion: configuration.konghq.com/v1
+kind: KongPlugin
+metadata:
+  name: model-key-auth
+  namespace: llm-serving
+plugin: key-auth
+config:
+  key_names:
+    - apikey
+    - authorization
+  key_in_header: true
+  key_in_query: false
+  key_in_body: false
+  hide_credentials: true

k8s/apps/homarr/homarr-values.yaml

@@ -30,10 +30,19 @@ replicaCount: 1
 env:
   AUTH_PROVIDERS: "oidc,credentials"
   AUTH_OIDC_ISSUER: "https://authentik.riotpiao.com/application/o/homarr/"
+  # AUTH_OIDC_URI (authorize endpoint) is REQUIRED in addition to ISSUER — homarr
+  # hides the "Sign in with Authentik" button entirely when it's absent (per the
+  # authentik Homarr integration + homarr SSO docs). This was the missing var.
+  AUTH_OIDC_URI: "https://authentik.riotpiao.com/application/o/authorize/"
   AUTH_OIDC_CLIENT_NAME: "Authentik"
   AUTH_OIDC_GROUPS_ATTRIBUTE: "groups"
   AUTH_OIDC_SCOPE_OVERWRITE: "openid email profile groups"
   AUTH_OIDC_AUTO_LOGIN: "false"
+  # Link the OIDC identity to an existing homarr account with the same email.
+  OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING: "true"
+  # The analytics cron blocked the (single-threaded) Next.js event loop for ~16s
+  # per run ("callback took longer than expected"), compounding CPU pressure.
+  DISABLE_ANALYTICS: "true"
   BASE_URL: "https://homarr.riotpiao.com"
   NEXTAUTH_URL: "https://homarr.riotpiao.com"
 
@@ -58,8 +67,12 @@ tolerations:
 
 resources:
   requests:
-    cpu: 100m
-    memory: 256Mi
+    cpu: 250m
+    memory: 384Mi
   limits:
-    cpu: 500m
-    memory: 512Mi
+    # Next.js 16 + bundled redis + the icon-updater (28k icons) saturated the old
+    # 500m limit; CPU throttling made Next.js abort with exit 134 (SIGABRT) and
+    # self-restart in a loop, so nginx saw no upstream and returned 502. Give it
+    # real CPU headroom.
+    cpu: "2"
+    memory: 1Gi

k8s/apps/llm-serving/embeddings.yaml

@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-embeddings
+    app.kubernetes.io/part-of: llm-serving
+  name: embeddings
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=nomic-ai/nomic-embed-text-v2-moe
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/llm-serving/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist, matching k8s/apps/api. Anything added to this directory
+# and not listed here is silently dropped — no error, no drift shown.
+#
+# These five were adopted from live state on 2026-08-15; they had been applied
+# by hand and carried no ArgoCD ownership. Each was exported and verified with
+# `kubectl diff -f <file>` returning empty before the Application below was
+# created, so the first sync was a no-op rather than a redeploy. Re-verify that
+# way after any edit here: a GPU predictor restart is a weights reload measured
+# in tens of seconds, not a rolling update.
+resources:
+  - embeddings.yaml
+  - ornith.yaml
+  - reasoning.yaml
+  - reranker.yaml
+  - verifier.yaml
+# No namespace transformer: every file sets its own, and the transformer would
+# rewrite metadata.namespace on anything cross-namespace added later.

k8s/apps/llm-serving/ornith.yaml

@@ -0,0 +1,108 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+    # Kong reads its timeouts from the Kubernetes Service, not the Ingress —
+    # Ingress annotations configure Route entities (strip-path, methods,
+    # plugins), these configure the Service entity. They were on
+    # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
+    # default in force. KServe propagates InferenceService annotations to the
+    # Service it generates, which is how they reach Kong from here.
+    #
+    # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
+    # request ever waited on a cold load. A pod restart flushes VRAM, and
+    # loading ornith:35b takes longer than 60s, so the first request after any
+    # restart returned 504.
+    konghq.com/connect-timeout: "10000"
+    konghq.com/read-timeout: "3600000"
+    konghq.com/write-timeout: "3600000"
+  labels:
+    app.kubernetes.io/name: llm-ornith
+    app.kubernetes.io/part-of: llm-serving
+  name: ornith
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - command:
+      - /bin/sh
+      - -c
+      - 'set -e
+
+        ollama serve &
+
+        SERVE_PID=$!
+
+        until ollama list >/dev/null 2>&1; do sleep 2; done
+
+        ollama pull ornith:35b
+
+        ollama pull qwen2.5:3b-instruct
+
+        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
+
+        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
+
+        wait $SERVE_PID
+
+        '
+      env:
+      - name: OLLAMA_HOST
+        value: 0.0.0.0:8080
+      - name: OLLAMA_MODELS
+        value: /mnt/models/ollama
+      - name: OLLAMA_CONTEXT_LENGTH
+        value: '32768'
+      - name: OLLAMA_KEEP_ALIVE
+        value: '-1'
+      - name: OLLAMA_NUM_PARALLEL
+        value: '1'
+      - name: OLLAMA_MAX_LOADED_MODELS
+        value: '2'
+      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
+            grep -q qwen2.5
+        failureThreshold: 120
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/llm-serving/reasoning.yaml

@@ -0,0 +1,81 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reasoning
+    app.kubernetes.io/part-of: llm-serving
+  name: reasoning
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
+      - --served-model-name=reasoning
+      - --quantization=bitsandbytes
+      - --dtype=float16
+      - --kv-cache-dtype=auto
+      - --tensor-parallel-size=1
+      - --max-model-len=16384
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
+      - --enable-chunked-prefill
+      - --enable-prefix-caching
+      - --reasoning-parser=deepseek_r1
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: '0'
+      - name: VLLM_ATTENTION_BACKEND
+        value: TRITON_ATTN
+      - name: HF_HOME
+        value: /mnt/models
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        failureThreshold: 80
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 2
+    minReplicas: 2
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        sizeLimit: 2Gi
+      name: shm
+

k8s/apps/llm-serving/reranker.yaml

@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reranker
+    app.kubernetes.io/part-of: llm-serving
+  name: reranker
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=BAAI/bge-reranker-base
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+

k8s/apps/llm-serving/verifier.yaml

@@ -0,0 +1,76 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-verifier
+    app.kubernetes.io/part-of: llm-serving
+  name: verifier
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model=Qwen/Qwen2.5-Math-PRM-7B
+      - --served-model-name=verifier
+      - --runner=pooling
+      - --dtype=float16
+      - --tensor-parallel-size=1
+      - --max-model-len=4096
+      - --max-num-seqs=8
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: '0'
+      - name: VLLM_ATTENTION_BACKEND
+        value: XFORMERS
+      - name: HF_HOME
+        value: /mnt/models
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '4'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        sizeLimit: 1Gi
+      name: shm
+

k8s/apps/messaging/queue-crd/templates/rbac.yaml

@@ -10,13 +10,22 @@ metadata:
   name: queue-operator
 rules:
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues", "temporalworkers"]
+    resources: ["queues"]
     verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues/status", "temporalworkers/status"]
+    resources: ["queues/status"]
     verbs: ["get", "update", "patch"]
   - apiGroups: ["kmsvc.io"]
-    resources: ["queues/finalizers", "temporalworkers/finalizers"]
+    resources: ["queues/finalizers"]
+    verbs: ["update"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/status"]
+    verbs: ["get", "update", "patch"]
+  - apiGroups: ["kmsvc.io"]
+    resources: ["temporalworkers/finalizers"]
     verbs: ["update"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]

k8s/apps/sms/deployment-macos.yaml

@@ -0,0 +1,115 @@
+# macOS VM (Docker-OSX) hosting the BlueBubbles server.
+#
+# ── Why a VM and not a container ────────────────────────────────────────────
+# Containers share the host kernel. macOS binaries are Mach-O and need XNU plus
+# Cocoa/IOKit, which a Linux kernel cannot provide, so no macOS container exists
+# or can exist. Docker-OSX is QEMU running a macOS guest, packaged in a
+# container — a VM in a box, not a macOS container.
+#
+# ── Why this works on worker-2 ──────────────────────────────────────────────
+# Verified on the existing hardware: amd64, `vmx` (Intel VT-x) present, and
+# /dev/kvm exists on Talos nodes (KVM is compiled into Talos' kernel, not a
+# module). Bare metal, so no nested virtualisation needed.
+#
+# ── Read this before relying on it ──────────────────────────────────────────
+# 1. Setup is INTERACTIVE. First boot runs the macOS installer: connect over
+#    VNC (:5999), erase the disk in Disk Utility, install, create a user, sign
+#    into iMessage, THEN install BlueBubbles inside the guest. This manifest
+#    only provides the machine; it does not provision macOS.
+# 2. iMessage activation on non-Apple hardware is a coin flip. BlueBubbles'
+#    own guidance: "test sending an iMessage to yourself. If it does not
+#    succeed, it's likely best to restart from the beginning."
+# 3. Apple's macOS licence permits virtualisation only on Apple hardware. This
+#    is a Hackintosh. Use a throwaway Apple ID, not a primary one.
+# 4. BlueBubbles labels this path "not for beginners", "no guarantees or
+#    warranty".
+#
+# Private API (reactions, typing indicators, edit/unsend) needs SIP disabled
+# inside the guest and is NOT required for plain send/receive. Skip it.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: macos-bluebubbles
+  labels:
+    app.kubernetes.io/name: macos-bluebubbles
+    app.kubernetes.io/part-of: sms
+spec:
+  replicas: 1
+  # Recreate: the qcow2 disk is RWO and a second pod must never attach it
+  # concurrently — two QEMU processes on one image corrupts it.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: macos-bluebubbles
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: macos-bluebubbles
+        app.kubernetes.io/part-of: sms
+    spec:
+      # Dedicated node. The taint keeps everything else off worker-2; this
+      # toleration is what lets the VM on. Both halves are required.
+      nodeSelector:
+        workload: imessage
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: imessage
+          effect: NoSchedule
+      containers:
+        - name: macos
+          image: sickcodes/docker-osx:latest@sha256:3a3c82c79bc4e73531f819ccdfa4053b3084efd7c1f645678dbf8b4b3a24369c
+          # QEMU needs /dev/kvm; Talos enforces `baseline` cluster-wide, so this
+          # only schedules because the sms namespace is labelled privileged.
+          securityContext:
+            privileged: true
+          env:
+            # Generates a unique serial / board-serial / UUID / MAC and persists
+            # them to bootdisk.qcow2. This synthetic identity is what iMessage
+            # activates against — it must stay stable across restarts, which is
+            # why the PVC matters.
+            - name: GENERATE_UNIQUE
+              value: "true"
+            # Identity is only plausible if it matches a real product line.
+            - name: DEVICE_MODEL
+              value: "iMacPro1,1"
+            - name: RAM
+              value: "12"
+            - name: CORES
+              value: "6"
+            - name: EXTRA
+              # Expose the BlueBubbles server port from the guest to the pod.
+              # Guest :1234 (BlueBubbles default) -> pod :1234.
+              value: "-device virtio-net-pci,netdev=net0 -netdev user,id=net0,hostfwd=tcp::1234-:1234"
+          ports:
+            - name: vnc
+              containerPort: 5999
+              protocol: TCP
+            - name: bluebubbles
+              containerPort: 1234
+              protocol: TCP
+          resources:
+            requests:
+              cpu: "6"
+              memory: 14Gi
+            limits:
+              cpu: "12"
+              memory: 20Gi
+          volumeMounts:
+            - name: macos-disk
+              mountPath: /home/arch/OSX-KVM/disk
+            - name: kvm
+              mountPath: /dev/kvm
+          # No readiness probe on purpose. The guest takes many minutes to boot,
+          # and until macOS + BlueBubbles are installed BY HAND there is nothing
+          # listening on 1234. A probe here would crash-loop the pod through the
+          # entire interactive install.
+      volumes:
+        - name: macos-disk
+          persistentVolumeClaim:
+            claimName: macos-disk
+        - name: kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice

k8s/apps/sms/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: sms
+resources:
+  - namespace.yaml
+  - storageclass.yaml
+  - pvc-macos.yaml
+  - deployment-macos.yaml
+  - service.yaml
+  - networkpolicy.yaml

k8s/apps/sms/namespace.yaml

@@ -0,0 +1,19 @@
+# iMessage delivery for the cluster.
+#
+# BlueBubbles' server is a macOS Electron app paired with an Objective-C helper
+# that hooks Messages.app private APIs — it cannot be containerised on Linux,
+# because containers share the host kernel and macOS needs XNU + Cocoa. The only
+# way to run it on Talos is a full macOS VM under QEMU/KVM (Docker-OSX), which
+# needs a privileged pod with /dev/kvm.
+#
+# Hence privileged PodSecurity: the cluster default from the Talos controlplane
+# is `enforce: baseline`, which forbids privileged containers and host devices.
+# Scope is limited to this namespace.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sms
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged

k8s/apps/sms/networkpolicy.yaml

@@ -0,0 +1,22 @@
+# Default-deny. This namespace runs a privileged QEMU VM signed into an Apple
+# ID and exposes an unauthenticated VNC console; nothing should reach it except
+# opted-in clients.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: sms-default-deny
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/part-of: sms
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              sms-client: "true"
+      ports:
+        - protocol: TCP
+          port: 1234

k8s/apps/sms/pvc-macos.yaml

@@ -0,0 +1,25 @@
+# Persistent macOS disk image + generated hardware identity (bootdisk.qcow2).
+#
+# This volume is NOT disposable: it holds the VM's serial number, board serial,
+# UUID and MAC, which together form the identity iMessage was activated against.
+# Losing it means re-running activation, which is the least reliable step of the
+# whole setup.
+#
+# Docker-OSX documents 128GB minimum for the guest image; 200Gi leaves room for
+# the installer, the base system, and qcow2 growth.
+#
+# ⚠️ Single replica (see storageclass.yaml — capacity and IO both rule out 3).
+# Losing worker-2's disk therefore means losing the activated identity and
+# redoing iMessage activation. Once the guest is installed and activated, take
+# a Longhorn snapshot/backup of this volume; that is the only redundancy here.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: macos-disk
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: longhorn-imessage-local
+  resources:
+    requests:
+      storage: 200Gi

k8s/apps/sms/service.yaml

@@ -0,0 +1,31 @@
+# VNC is how you drive the interactive macOS install. Deliberately ClusterIP —
+# it is an unauthenticated console onto a machine holding a live Apple ID
+# session. Reach it with `kubectl port-forward`, never an Ingress.
+apiVersion: v1
+kind: Service
+metadata:
+  name: macos-vnc
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: vnc
+      port: 5999
+      targetPort: vnc
+---
+# The BlueBubbles REST API, once installed inside the guest. This is the stable
+# name cluster services use, so callers never depend on the pod IP or on whether
+# the backend is this VM or a real Mac mini later.
+apiVersion: v1
+kind: Service
+metadata:
+  name: bluebubbles
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: macos-bluebubbles
+  ports:
+    - name: http
+      port: 1234
+      targetPort: bluebubbles

k8s/apps/sms/storageclass.yaml

@@ -0,0 +1,32 @@
+# Dedicated StorageClass for the macOS VM disk.
+#
+# The default `longhorn` class does not work here, for two independent reasons:
+#
+# 1. Replica count. Default is 3, and Longhorn schedules against
+#    storageMaximum - storageScheduled with over-provisioning at 100%. Free
+#    space is cp-1 146Gi / cp-2 8Gi / cp-3 146Gi / worker-1 292Gi, so a 200Gi
+#    volume has only one node that can hold even a single replica — a 3-replica
+#    volume fails outright with ReplicaSchedulingFailure.
+# 2. Binding mode. `Immediate` provisions the volume the moment the PVC is
+#    created, before any pod is scheduled. Combined with strict-local that
+#    pins the data to an arbitrary node, not the one the VM runs on.
+#
+# So: one replica, kept local to the VM, bound only once the pod has a node.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: longhorn-imessage-local
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+# The pod is pinned to worker-2 by nodeSelector; wait for it to be scheduled so
+# the replica is placed on that node and not somewhere else.
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  # A qcow2 backing a live VM is latency-sensitive and rewritten constantly.
+  # Serving it over the network from another node's disk would be the single
+  # worst thing for guest responsiveness, so force it local.
+  numberOfReplicas: "1"
+  dataLocality: "strict-local"
+  staleReplicaTimeout: "30"
+  fsType: "ext4"

k8s/argocd/apps/00-ingress-nginx.yaml

@@ -1,42 +0,0 @@
-# Wave 0 — Nginx Ingress Controller
-# Foundational infrastructure required for all ingress resources and ArgoCD UI access.
-# Must be wave 0 to ensure LoadBalancer IP is available before other apps deploy.
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: ingress-nginx
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  sources:
-    - repoURL: https://kubernetes.github.io/ingress-nginx
-      chart: ingress-nginx
-      targetRevision: "4.15.1"
-      helm:
-        valueFiles:
-          - $values/k8s/bootstrap/ingress/nginx-values.yaml
-    - repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: ingress-nginx
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=true
-      - ServerSideApply=true
-    retry:
-      limit: 3
-      backoff:
-        duration: 10s
-        factor: 2
-        maxDuration: 3m

k8s/argocd/apps/00-secrets.yaml

@@ -17,7 +17,7 @@ spec:
     syncOptions:
       - CreateNamespace=true
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     # ksops decrypts every *.enc.yaml here at kustomize-build time (repo-server
     # runs `kustomize build --enable-alpha-plugins --enable-exec`). Replaces the

k8s/argocd/apps/00-substrate.yaml

@@ -21,7 +21,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/bootstrap/cert-manager/cert-manager-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -93,7 +93,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     # A real kustomization.yaml (resources: the 3 issuer/CA files) renders these
     # deterministically. The previous directory.include with bare filenames
@@ -127,7 +127,7 @@ spec:
   project: homelab
   revisionHistoryLimit: 3
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/bootstrap/ingress
   destination:

k8s/argocd/apps/05-networking.yaml

@@ -1,26 +1,8 @@
-# Wave 0 — networking policies layered on the Cilium CNI + CoreDNS that the
-# cluster bootstrap already installed (substrate). These are raw manifests only.
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: cilium-policy
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "0"
-spec:
-  project: homelab
-  source:
-    repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-    targetRevision: main
-    path: k8s/bootstrap/cilium
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kube-system
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-# coredns-config Application removed: CoreDNS (incl. homelab hostname rewrites)
-# is owned by Talos via an inlineManifest (terraform/files/coredns/Corefile).
-# Managing the coredns ConfigMap from ArgoCD too would let the two reconcilers
-# fight and revert the rewrites.
+# Wave 0 — networking substrate is Talos-owned (terraform inlineManifests), not
+# ArgoCD:
+#   - CoreDNS Corefile + hostname rewrites  -> terraform/files/coredns/Corefile
+#   - Cilium LB-IPAM pool + L2 announcement -> terraform/files/cilium/*.yaml
+# Both were previously ArgoCD apps here whose empty `resources: []`
+# kustomizations never actually applied them (live objects came from manual
+# kubectl). Managing them from ArgoCD too would let two reconcilers fight. This
+# file intentionally defines no Applications now.

k8s/argocd/apps/10-storage-observability.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/minio/minio-operator-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -41,7 +41,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/minio
   destination:
@@ -66,12 +66,20 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/longhorn
   destination:
     server: https://kubernetes.default.svc
     namespace: longhorn-system
+  # Longhorn writes disk state back into its own Node CRs — the disk key it
+  # generates, storageReserved, diskType, evictionRequested. Git declares only
+  # allowScheduling; without this the controller's writes read as drift forever.
+  ignoreDifferences:
+    - group: longhorn.io
+      kind: Node
+      jsonPointers:
+        - /spec/disks
   syncPolicy:
     automated:
       prune: true
@@ -94,7 +102,7 @@ spec:
         skipCrds: true
         valueFiles:
           - $values/k8s/infra/monitoring/prometheus-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -144,7 +152,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/monitoring/crds
   destination:
@@ -175,7 +183,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/monitoring
   destination:
@@ -205,7 +213,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/monitoring/blackbox-exporter-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:

k8s/argocd/apps/20-logging.yaml

@@ -19,7 +19,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/loki-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -53,7 +53,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/grafana-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -87,7 +87,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/logging/promtail-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:

k8s/argocd/apps/30-security.yaml

@@ -17,7 +17,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/vault-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -46,7 +46,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/infra/iam/authentik-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/iam
   destination:
@@ -90,7 +90,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/forgejo-runner
   destination:

k8s/argocd/apps/40-data.yaml

@@ -14,7 +14,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/infra/databases
   destination:

k8s/argocd/apps/50-messaging.yaml

@@ -68,7 +68,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/apps/messaging/kafka-cluster
   destination:
@@ -89,7 +89,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/apps/messaging/queue-crd
   destination:
@@ -110,7 +110,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/apps/messaging/management-service
   destination:

k8s/argocd/apps/55-api-gateway.yaml

@@ -0,0 +1,58 @@
+# Wave 7 — Kong, the cluster's internal API gateway (namespace `api`).
+#
+# Sits between nginx and the backend services: nginx owns the edge and TLS,
+# Kong owns routing policy, auth and rate limiting. Wave 7 puts it after the
+# data/messaging tiers it fronts and before the wave-8 applications that
+# publish routes into it.
+#
+# DB-less: routing config comes from Kubernetes objects (Ingress with
+# `ingressClassName: kong`, plus KongPlugin/KongConsumer CRDs), so git remains
+# the source of truth and there are no migration Jobs on upgrade.
+#
+# CRDs ship in the chart's crds/ directory; ArgoCD applies those by default
+# (helm.skipCrds is left false).
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kong
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "7"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  sources:
+    - repoURL: https://charts.konghq.com
+      chart: kong
+      targetRevision: "3.4.1"
+      helm:
+        valueFiles:
+          - $values/k8s/apps/api/kong-values.yaml
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+      targetRevision: main
+      ref: values
+    # The nginx Ingress for api.riotpiao.com. Kept in this Application rather
+    # than the central k8s/bootstrap/ingress/ingress.yaml because that one syncs
+    # in wave 1, before namespace `api` exists.
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+      targetRevision: main
+      path: k8s/apps/api
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: api
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      # The chart's CRDs exceed the annotation size limit that client-side
+      # apply relies on; server-side apply avoids the
+      # "metadata.annotations: Too long" failure CRDs commonly hit.
+      - ServerSideApply=true
+    retry:
+      limit: 3
+      backoff:
+        duration: 10s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/56-llm-serving.yaml

@@ -0,0 +1,58 @@
+# Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`).
+#
+# Syncs before wave 7 (Kong), so the predictor Services exist before the routes
+# that point at them. KServe itself is part of the substrate; this Application
+# owns only the InferenceServices.
+#
+# Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed
+# by hand — no ArgoCD ownership, present in no repo — so every change to them
+# was drift by definition. Each manifest was exported from the cluster and
+# verified with `kubectl diff` returning empty before this file existed; the
+# first sync therefore adopted them without restarting anything.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: llm-serving
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "6"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+    targetRevision: main
+    path: k8s/apps/llm-serving
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: llm-serving
+  syncPolicy:
+    automated:
+      # `prune: false` here, unlike every other Application in this repo, and it
+      # is not an oversight.
+      #
+      # ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label
+      # (argocd-cm `application.instanceLabelKey`). KServe copies an
+      # InferenceService's labels onto the Deployment and Service it generates —
+      # visible today as `app.kubernetes.io/name` and `part-of` on
+      # `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe
+      # propagates that tracking label to children that are not in git, ArgoCD
+      # reads them as extraneous, prunes them, and KServe recreates them. That
+      # loop churns GPU pods.
+      #
+      # Deleting an InferenceService therefore means deleting the file AND
+      # removing the object, rather than relying on prune.
+      prune: false
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      # KServe CRDs are large; server-side apply avoids the
+      # "metadata.annotations: Too long" failure client-side apply hits, and is
+      # the correct mode for adopting objects an operator also writes to.
+      - ServerSideApply=true
+    retry:
+      limit: 3
+      backoff:
+        duration: 10s
+        factor: 2
+        maxDuration: 3m

k8s/argocd/apps/60-applications.yaml

@@ -19,7 +19,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/temporal/temporal-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -48,7 +48,7 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/portainer/portainer-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
   destination:
@@ -71,7 +71,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/apps/cloudflared
   destination:
@@ -84,6 +84,60 @@ spec:
     syncOptions:
       - CreateNamespace=true
 ---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: agent-pod
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  source:
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+    targetRevision: main
+    path: k8s/apps/agent-pod
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: agent-pod
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+---
+# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
+# running the BlueBubbles server, plus its dedicated local StorageClass.
+#
+# Pinned to worker-2 via nodeSelector `workload: imessage` + a matching
+# toleration for that node's taint. Until worker-2 is provisioned this app
+# syncs everything except the pod, which stays Pending — that is expected.
+#
+# No CreateNamespace: namespace.yaml carries `pod-security: privileged`, which
+# the VM needs (/dev/kvm, privileged), and an ArgoCD-created namespace would
+# not have those labels.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: sms
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "8"
+spec:
+  project: homelab
+  source:
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+    targetRevision: main
+    path: k8s/apps/sms
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: sms
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+---
 # Consolidated: homarr + homarr-patches → homarr
 # Helm chart + values + PostSync hook patch (fix-probes-job.yaml)
 apiVersion: argoproj.io/v1alpha1
@@ -102,10 +156,10 @@ spec:
       helm:
         valueFiles:
           - $values/k8s/apps/homarr/homarr-values.yaml
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       ref: values
-    - repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       path: k8s/apps/homarr  # PostSync hook: fix-probes-job.yaml
   destination:

k8s/argocd/projects/homelab-project.yaml

@@ -11,7 +11,7 @@ metadata:
 spec:
   description: Homelab GitOps — single-repo, in-cluster destinations only
   sourceRepos:
-    - git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    - https://github.com/Riotpiaole/riotpiao.homelab.com.git
     # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
     - https://cloudnative-pg.github.io/charts
     - https://dl.gitea.com/charts/

k8s/argocd/root/homelab-root.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
     targetRevision: main
     path: k8s/argocd/apps
     directory:

k8s/argocd/secrets/agent-pod-models.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: ENC[AES256_GCM,data:zdc=,iv:VvjvrS5PVNAMIaOE0LaWU+tHcUIYVQDnCANQz6myktY=,tag:xGyWDhRCwwiNny7hPllf5g==,type:str]
+kind: ENC[AES256_GCM,data:2bf5Zfy8,iv:5Oz423GzUWmgdaaZHbrtedwRHIAPIuLh4iDMieLL05s=,tag:GNW1ROjlzGvO/4tIOSuH3Q==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:7RO0Qxkc+/sA,iv:wKe9A8d7QJSx/6rlEY5H6lU8V24TJqr5IpXQCBc8QgM=,tag:iF6z1MNEXG3pCz2cQ18gLg==,type:str]
+    namespace: ENC[AES256_GCM,data:RcduxHHLtjH9,iv:opOVx1lL2ltDqQgsleN7NdMAq0TyFr/YQO3FFsHh5AA=,tag:fIwO31apqIatRRzBvamw6g==,type:str]
+type: ENC[AES256_GCM,data:C+JjyZt5,iv:xs49Lz6zRzcf3spiPzdUKTm2HZ+VgFahN6wjIe81JI4=,tag:VllJ54MqU+klA8xAIebjrA==,type:str]
+stringData:
+    models.json: ENC[AES256_GCM,data:FvlbdMkcngJchi0GEjIEDjxXpQjRwh+3xnlZNpApgd3v1SpMd0qD2d3TbQr39+feSfsvpxhdmmyU/PmRHZYVdr/QpIKlZBWBD6qTCaOSq62NQbR4ck336muIaWD4AmahWCvRouYEJdEsnG7nedjAivw4cls18vMfnL5pNm2t96foScSyHosWoh3RdT6CWLmJE39+kS6/fwZ1D3Z/GEm/E2zHkVtEbEw/fk1PiqsnmZWt8QcSRAeX8aIDkphV7wOKlkri11r1NuFQKMVIau07VPce1YEi5rsJqsircvDxQelQiMGQt2y2M4GGlXx2NeJgKEtP4Hf9mVGhTeN3CYLcXi8Qyg7k1GIawGVW/Em0kWWfy0GMjvOMfYozNmIpi0YQdYBbr3j6pkixtVM2dWezvld5QIYLnHoGMxw4M6L4IqVIonC3j6tk9pV1DaL5IFskEXyb6ScTrmVY4mQ+VEXvfLYA38jRauWxx/4qf4I7B0RzfGzOx67UJh2jKj0bPWNFHKt5Dz4CK+ul39d1KUPHHMoyaH5gCpOAfLSkjMm44X1Cn+O6uY+QFCugh9MI7bJg3EbovTCZxz95GCupWAvDshv59f7gpGL0T8AuRZHlNp6yZMLdviR9yf79d2rW26PIM1QuNOUWELrOcTs5IAmyuFYr0PI5W5XSf4klOdz3UdhPfMDcyRqtSzVDsSo9oC/WeWOo1+yurtdqqW0wLiFpU/kTaZ4JN5Kl080twqFEqgyPnJrbmlxwWlt4XN2SV9D7pcfE61FzUTwoi5PD/8xt94Fp3XerSgJtwhQ6X4Neo47wCFoMR4Y69mlHrFgSJA4kLFezDc2ISnyAix0W5to0cjwJqQ2EgonIhCXD4xWTvYeApHJMOC2o4B74K5efOZ7klb6PwDMvzM3LyeGzjoJh47aJGvhWN/MpQZC2lSHT8PdMxdGDc+OkEVmsvMncddWMGjQhkTo+69sUexARxLY4H22TCt58azqQGcWa5e7s9NUHb6bRVKHzXh4HMiIhQ0jBev++Jz5gIOyYRnw5xoGbo3ROv8ndczVAqXyAla23EM/VBWT0hLMjr6xxonuLQacP9dPJE9QUofjRMeYblkVvQdIw8WuKGbAlbTR/bokQwGdVp2bywSFQZj7BLlPjzCr0/LJmxYMdq6CKB049PdTlS9/UKBHApddMQ/QRXfOYUCX+lm4k4J19jd7dWk34WNUrTEPSR2d1U/2wTnMvzlrnQHx/zk4/lmI1newfjIbybIbXypX8xVEqzFWXpxUe+iFLP19KmmmHG7gGsV32k1VSGvYFI0HOokZga98QsAbJGbi1gQq8gYpMR/mBdhjJVcnAjOYpg+YaR0vCyrhBN55aSCDoUbT1JOqxvFqmlBHDxTWorD/3KGFxEH/wvSsxussFmUCWdEgYmjGO1ktnQBw1YpOB7mhDUlzvYCSxfpexsYufDYqTbrb/R5gmATC0pEN6EgTth5YqFOL7AiIsogy14RGjRym1wvjVl5SUU9bMQlWGJbQuvNRRwL/TzR/QP210jMsnAh9t8sncDnFPIbhFgeBLsU/ctYoBs0nFVWGWzcuoP9HTnvdWTT2xKmWAeFz8I3DZ5sjm2B787Q4bCrAszTvVLeLiRw/pOE32BeNfG4nNiZDvMQhUomAKpaQypWTu2wLsv3ISm00gud5sRKB6ASJANuSu+EhG9k4OYcBEzAT9PKUxclupFbmsFnXx0u9CXDj3tErrPCUs7DuFSr+fz0ehCLMH+GrcqfytjNDVoTSBRZg+lJYKjAqVCGvq8uRf5V/rdulbt3ggx/uhGSiPzYrBQ0g9iMdjQqpNGBwjlxGkhRE9D2n/PoerAAnHTfwc9Hezffncostjp09xw2CB/xOx9HZFGalQ1R4F5+0lm0iR4lMzc+se+o38foNIiMeMIwAVBJu7IO++UMuKCxpxmdAGqEYj/Ddpzfi3ZcRwe/1HhanVvKRMJsPOhnv0SB2xBrdb7QHnEzLbRTriVRBBdOI77HXWgDt9uQru0tEXCEQ0n+f0BWqHVa/RLN4Yj0dQVWLXU75ddyHnRaCiVdLQnObMvG/QXqY1k/bJTzGNgVF8/KGY51NgdfqskOXC9Fv5IGiyWq+/DL7tQcrmThkzUhZlwDBmwUzPFmzuUC7Z5PkXv2EV+eUrYF16HYqp5KgqTkBHr/JXR8ljb/5gMN929Jk2fBVEuzaEG6W9Yek+Qbx1cnXYrMBI9GWJUJsMsxsAj+1bAjtQeQwt0nXgZesTaV90dfQYq+nTGXMNjqaMhsrJhR6lU1+CDpwY+IObHSttS8AKB+Czsl0nUB44o1a/zKRDNTCGGmPFuv1Q5Ub/1myxorLDDF1mYDpre5TedygtZ/1YSz/YOpKwmWdW32FsnY+m3qjEAcfV5f923N/z8ENVxVpeiWL6mqf7h6F3qTiaSsFq2O8m8tVm70X+oRdig3hPnKqlK/KoxVmzZErhEJ+KmP5I2uvIlb4PViE6imu8kw4VqMClb7jJ7Yk+2Zy84C8RGuDCN8GCDG7FvAahA9xVGxB6rmEpzvqFBxos2EeLOlyfewf7rCw1JAL3C9hZrciA9Jqb/t8B7KqCmCOsKK7xVt1yqyG1dsa364JG/1SKSMKebTheuymZ1+VnGjRZ+pgSAkQGjCtp7B3fTlhjt+WwxW4Ml27yfjokQQ==,iv:8Z39eWukGSMePh/3Dj35e6Zahejil+eeMSqwMYf3snI=,tag:FyBg1y0IQEg/m0mgpf1ESg==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZzB5REh6amhCNzRmY0ta
+            MkN3Y0ZlR29lQ3h2SWo3cW5CUThkL2RnU1NVCmt2ZkhIZTlHN1RQRkFrTjVvbjVw
+            RGRrTXRoYmdQcnlMSEo3ZWsrZUQ5cHMKLS0tIGU2TGJqZDRxUGJpZzRveEtZankx
+            MzhrT1R2akxxby9QVzd1RXB0RDY1LzQKOF+/e5z5lPX6Y1sMTAHuDj3YqW1m+sBd
+            u/0R0YnBonYM3wS5nJE3NZMkImaAdQlUjOzQepfBldG+lz++rlnAww==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-08-18T20:06:08Z"
+    mac: ENC[AES256_GCM,data:IM9HkpdwtQE2wCkjwDWOmHH4uP7TlIsrK4TVytiecvYz4SiLk6IRUSIu7I3a+F+dtltC2WtokoATaB69DTXPoI54amzzptirxiFD5FbaU+u2gLjo7KI7V0smYGuKqMYwnod2L/4GdlvP6xjVxFWuA01rQRaYBkFSumS4NABl/I4=,iv:al1MyBmFwni8gap7PPZxWaCwqFKCicfPq6nVmrSn9Xc=,tag:KJGTDQ8xRGF8qMOjoQ+dng==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/agent-pod-ssh-key.enc.yaml

@@ -0,0 +1,23 @@
+apiVersion: ENC[AES256_GCM,data:seY=,iv:1q2bg1QBNDDQPYNu1S08MZ+Ix8WLthxObsrqTMJEVhY=,tag:OMh/stn4vVScOEAyviuurw==,type:str]
+kind: ENC[AES256_GCM,data:QwqgPMrm,iv:tv6rUcezEgQigvHVw05Mevt6BHDD5AARuoUmQpry3AU=,tag:m9RLNezJsLYGSRWnozfkoQ==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:ZJtohnEDN2NBFrEhhvGuw4s=,iv:SU5LuA3Lqjh+q7nVxeVVtzeXiOxxVserNno5c7b47lM=,tag:YLprEk7oVjVsIDVANWDPrQ==,type:str]
+    namespace: ENC[AES256_GCM,data:LieEC+YWJzq+,iv:U00VXejzlpgOYS7Yxix/MnEW0PEvmwOuOXMUi2CWIps=,tag:QvFYZIexWhyvBYTruGMifQ==,type:str]
+type: ENC[AES256_GCM,data:U41EbP+H,iv:CKDqec0jfVtbOtEyqQpV9CHJNCwhneYi0wqK8QU2vnY=,tag:CSJ04pkPslevQrpSjhXNHw==,type:str]
+stringData:
+    id_forgejo: ENC[AES256_GCM,data:dgwezImUFb2r6MMZ2V1ZMuy32DAVZ9YMzlr9zYlXKEuzeCaWn29DuqgeP5PJQFFKwd37Y9jmzXwuKD5KrC9WCAzrS2VClxmJYHj2Bf842qtKa4x3SubHu4w6wp8v42wQYCU0Kygc5UgkCFYgOBgvpYCFKhiro6Mg/PBTWTIIgrLlSVwUWPIcaX6Sq4ZW4sy6CZ0H/pEMDAjU8drZ4KmGw7W4FshNynplMuP9ewbSs9yxtuAsKt4ezuUQBGHmAYdPHytC9HUzf67z/j6TF/zg6XgTe0AG116cNddAL8oNermCIkkKuTwb5xy1+3WRjjacC0K5Ire1fV5RyXtD/1JgEHJ8gbPK1w1V5fHZXX/ARvQFlP+uqxFYmPkCHBmK/R6/VvAghI/KYHlc76hdsAe0sd1UrPIHjVEtwgvDQl+4de9L9cHG3Qxj23ajlpISWA3tGADhwzXDxi0KYClckebrCKhizWFQLYXezALYklgY2yghG9jZHufV9jRjYSr9DLtwUmVf3oglAzKkWErNs8dWqEKYFx7HpFf0FiZh,iv:1aX4E+R1oFMm9eI1RENFIYCoatX8BdHo6PhrEGq3rtg=,tag:nwniLOOmpYd3Z6R92SvB3Q==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvRmtrdkpMOVhRdTVucVlr
+            QmlENGdDN2J1MUczSmRhNUpwYlBjWXhqbERzCnpLRFl1bVVtSFE3L1l4T3Rwd2ZQ
+            Z1MxTzJYZWovMmd2RTA3RFJVdVRRTEUKLS0tIDFQWHh2dmQxU1FUZnUxalhTdEd3
+            c3dUVzg2L2VOWFlmUG9XdFNWN2RvQkkKYqSmFMkDV/T7AOjKYQNJW85gUzFraRre
+            GhwAuPJ9oNQAhSRa5z2p6ghoUplSXtNZ6H2OzETRfOc4N2cHLljW/Q==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-08-18T04:18:28Z"
+    mac: ENC[AES256_GCM,data:ScGMBQqGm31fWDZurmlmk+rnlvyaQkalUcJ6ds/U5JJ2niTf7Llbruuqt2EzMQO09fT2kQqAIYr4cF61Czip6rDhpG8I3H3lBc2mOqUXbeCFyRL6mRpVYPOgkSzgyfXUfe8T++54UhQ0Clv/8Z4XB48oTs9detkUDzAfxP4azRE=,iv:edhsVviyjAm48Blm5CEQ0PBhcWVX97xi2hEbgDX9Dpg=,tag:Xnn85E99LibtB/LWwi5wcA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/model-invoke-apikey.enc.yaml

@@ -0,0 +1,25 @@
+apiVersion: ENC[AES256_GCM,data:894=,iv:Swg6ADUgmrqwz7wqAZHip9/qwFu0Rn8S2Lx4gBH8LJM=,tag:zbv0tFfRLtwFxBfpsuLt8A==,type:str]
+kind: ENC[AES256_GCM,data:jzVtJHYw,iv:ToZ0orfJqfGF/OnAPeYu/g2f4fXAMOZQDkA1+tmIccs=,tag:pi2skbwIU8qOQVEC86MdAA==,type:str]
+metadata:
+    name: ENC[AES256_GCM,data:t7zCeZLvAEbkJqUjWi26yD6UDA==,iv:MS1gq/bwKZdLA1itVDtsrdSOfI7e2CrhjvX5yhs0eQA=,tag:lC1gcoFMI5nfzC56U1WXrg==,type:str]
+    namespace: ENC[AES256_GCM,data:tp+d,iv:gnlet/60mgbSWLXEQpSlcWD98ky7NvlNCzGLTMys0JQ=,tag:PQYj9UeA50YenQESTCl7lg==,type:str]
+    labels:
+        konghq.com/credential: ENC[AES256_GCM,data:SOqQ9bLGLK0=,iv:a7En49UhRDwgHbv5NRB/XilEYIKQdaDqKH86WDrJB5I=,tag:JaDZ6Ko8ovY6AZ1hW9YMGQ==,type:str]
+type: ENC[AES256_GCM,data:r1K5gvop,iv:Gjv4oG2Unyql5rY9RTTljFqyd28xI81CDWtpavuuW5E=,tag:jF8lBs3AQwHPVEu9V+mONw==,type:str]
+stringData:
+    key: ENC[AES256_GCM,data:pm9GmSvX5MAsXO/e6ZcI4NF1Hwr3qjG6LaEEjvV0ihvSWhO23drlAEsrTtzppqgDZbMORf+5+P53mXU=,iv:5AbHNKeiMPoFQP/qTKdA0vEoYPzuaf4kIdGGcMSmfIQ=,tag:LD5w7XK+hiCS5D410+nCfQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUTmoxQkpuYUp5UlRQMkph
+            a0RaazdvaU5sWkNuL2gvVjlUYXVWV0dUWVVRClZhWDZCN2hpS2hnOG9Pck9zOTkx
+            RGNGMEI3RHpNbzVaaWNGcTNSSEdzZHMKLS0tIFZERnVJWUpreUh3TTlwbGw0dUx4
+            MGlCSkxuWWVEK2RaSDZPUzhNSUlCa28KHN0IsgQc/kBqmjQ6+4sgfb9PJy/45MwN
+            rXaLJ1htpqPZ9MJ8iOukRi0IKnKgQWXsoZengIxGmcOnEctpoH/kyQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-08-14T01:07:50Z"
+    mac: ENC[AES256_GCM,data:nY+YVwU1GuK8Yz+EZOQKkZKN28tm2L8afflc6hsgVFCFmsep5kVT+zId7AgemvQ+qnrho5N5nqxY2knB0gusFfWNKF3V5A5GBq40WtZCMaAtcwhJSex4kK7ZyaZD6oWDW/RTUumSrivSowkWlqt1XlDKyFLqSlpWQTd7JiGmUv8=,iv:zE8+B5UVYSuuAGYyvXsAwp1N/4vGduCoGyEMCNNEUnM=,tag:6RcrjneV2dOzhmoQi+5HsA==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -7,6 +7,8 @@ metadata:
       exec:
         path: ksops
 files:
+  - agent-pod-models.enc.yaml
+  - agent-pod-ssh-key.enc.yaml
   - authentik-secrets.enc.yaml
   - cloudflare-secrets.enc.yaml
   - forgejo-runner-token.enc.yaml
@@ -18,6 +20,7 @@ files:
   - homarr-secrets.enc.yaml
   - homelab-ca-secrets.enc.yaml
   - loki-secrets.enc.yaml
+  - model-invoke-apikey.enc.yaml
   - minio-secrets.enc.yaml
   - vault-secrets.enc.yaml
   - vault-unseal-keys.enc.yaml

k8s/argocd/seed-repo-secret.yaml

@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: github-repo-creds
-  namespace: argocd
-  labels:
-    argocd.argoproj.io/secret-type: repo-creds
-stringData:
-  type: git
-  url: [email protected]:[email protected]:Riotpiaole/riotpiao.homelab.com.git
-  sshPrivateKey: |
-    -----BEGIN OPENSSH PRIVATE KEY-----
-    b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
-    QyNTUxOQAAACAqZaKCvVj9z9JtQ8kyNpE42siEoEoUXTftc4zz+dAerQAAAKg5eEfbOXhH
-    2wAAAAtzc2gtZWQyNTUxOQAAACAqZaKCvVj9z9JtQ8kyNpE42siEoEoUXTftc4zz+dAerQ
-    AAAECTvujulFODUs/5miSpkwqqovKyuK3TSOSXFR8tNYOYKyplooK9WP3P0m1DyTI2kTja
-    yISgShRdN+1zjPP50B6tAAAAIXJvY2tsaWFuZ0BSb2NrZGVNYWNCb29rLVByby5sb2NhbA
-    ECAwQ=
-    -----END OPENSSH PRIVATE KEY-----

k8s/bootstrap/cilium/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: kube-system
-resources: []
-# Cilium deployed via Helm chart

k8s/bootstrap/cilium/l2-announcement-policy.yaml

@@ -1,39 +0,0 @@
-# k8s/cilium/l2-announcement-policy.yaml
-# CiliumL2AnnouncementPolicy — without this, LB-IPAM (lb-ipam-pool.yaml)
-# assigns IPs to LoadBalancer Services but nothing ARPs for them on the LAN,
-# so they're unreachable from outside the cluster even though `kubectl get
-# svc` shows a real EXTERNAL-IP. Confirmed both forgejo's .165 and
-# shadowsocks' .166 were 100% packet loss / incomplete ARP before this.
-#
-# loadBalancerIPs: true makes Cilium announce every Service's LB-IPAM IP via
-# ARP from whichever node currently holds the lease for it (one node per IP,
-# decided by leaderElection — not all nodes simultaneously, which would
-# otherwise cause ARP flapping/duplicate-IP confusion on the LAN).
-#
-# externalIPs/loadBalancerIPs split exists because Cilium also supports
-# announcing Service externalIPs (a different field, unused in this repo);
-# we only need loadBalancerIPs since every exposed Service here is type
-# LoadBalancer via lb-ipam-pool.yaml.
-#
-# requires kube-proxy replacement (already the case — see
-# k8s/talos-iam or helmfile.yaml.gotmpl kubeProxyReplacement=true) and a
-# Cilium build with L2 announcements enabled (default since v1.14).
-#
-# Apply once after cluster bootstrap, alongside lb-ipam-pool.yaml:
-#   kubectl apply -f k8s/cilium/l2-announcement-policy.yaml
-#
-# Verify:
-#   kubectl get ciliuml2announcementpolicy
-#   ping 192.168.1.165 && ping 192.168.1.166   # both should now respond
-#   arp -a | grep 192.168.1.16                 # should resolve to a real MAC
-apiVersion: cilium.io/v2alpha1
-kind: CiliumL2AnnouncementPolicy
-metadata:
-  name: homelab-l2-announce
-spec:
-  loadBalancerIPs: true
-  interfaces:
-    - eno1
-  # No nodeSelector restriction — all 3 nodes already run workloads
-  # (allowSchedulingOnControlPlanes: true in controlplane.yaml), and with
-  # 3 zone-labeled nodes, redundancy for per-IP leader election is maintained.

k8s/bootstrap/cilium/lb-ipam-pool.yaml

@@ -1,36 +0,0 @@
-# k8s/cilium/lb-ipam-pool.yaml
-# CiliumLoadBalancerIPPool — tells Cilium LB-IPAM which IPs it can assign
-# to LoadBalancer services in this cluster.
-#
-# CIDR 192.168.1.160/28 covers .160–.175 on the LAN:
-#   .160  talos-cp-1       (node — not assignable to services)
-#   .161  reserved
-#   .162  talos-worker-1   (node — not assignable to services)
-#   .163–.175  free for LoadBalancer services
-#
-# Current service IP assignments (via io.cilium/lb-ipam-ips annotation):
-#   192.168.1.165  forgejo-gitea-http (cicd)
-#   192.168.1.165  forgejo-gitea-ssh  (cicd)  — same IP, different ports
-#   192.168.1.166  shadowsocks        (vpn)
-#
-# Apply once after cluster bootstrap:
-#   kubectl apply -f k8s/cilium/lb-ipam-pool.yaml
-#
-# Verify assignment:
-#   kubectl get svc -n cicd forgejo-gitea-http forgejo-gitea-ssh
-#   # EXTERNAL-IP should change from <pending> to 192.168.1.165
-
-apiVersion: "cilium.io/v2alpha1"
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: homelab-pool
-spec:
-  blocks:
-    - cidr: "192.168.1.160/28"
-    # DO NOT add any 10.6.0.0/24 block here. That is the WireGuard subnet
-    # (10.6.0.1 = talos-cp-1 tunnel IP, 10.6.0.2 = DNS — see
-    # cluster-config/controlplane.yaml). A 10.6.0.x block let Cilium LB-IPAM
-    # auto-assign the CP's own tunnel IP to a Service, which broke the
-    # WireGuard tunnel and locked out the default kubectl context. It also
-    # can't work over WireGuard anyway — L2 announcements only ARP on eno1
-    # (the LAN interface), not wg0. Keep this pool LAN-only.

k8s/bootstrap/ingress/ingress-nginx-controller-alias.yaml

@@ -1,22 +0,0 @@
-# Service alias for CoreDNS compatibility
-# CoreDNS rewrites *.riotpiao.com → ingress-nginx-controller.ingress-nginx.svc
-# But bootstrap deployed as ingress-nginx-bootstrap-controller
-# This alias makes both names work
-apiVersion: v1
-kind: Service
-metadata:
-  name: ingress-nginx-controller
-  namespace: ingress-nginx
-spec:
-  type: ClusterIP
-  selector:
-    app.kubernetes.io/name: ingress-nginx
-    app.kubernetes.io/instance: ingress-nginx-bootstrap
-    app.kubernetes.io/component: controller
-  ports:
-  - name: http
-    port: 80
-    targetPort: http
-  - name: https
-    port: 443
-    targetPort: https

k8s/bootstrap/ingress/ingress.yaml

@@ -300,6 +300,10 @@ spec:
                 port:
                   number: 8080
 ---
+# NOTE: api.riotpiao.com (Kong) is deliberately NOT here. Its namespace `api` is
+# created in wave 7, and this Application syncs in wave 1 — an Ingress into a
+# namespace that doesn't exist yet would fail and mark this whole app
+# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced with Kong itself.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

k8s/bootstrap/ingress/kustomization.yaml

@@ -2,6 +2,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # No top-level namespace - resources declare their own namespaces
 resources:
-  - ingress-nginx-controller-alias.yaml  # Service alias for CoreDNS compatibility
+  # ingress-nginx-controller-alias.yaml REMOVED — it was a ClusterIP Service named
+  # ingress-nginx-controller with a stale selector (instance: ingress-nginx-bootstrap,
+  # a release that no longer exists). ingress-config's selfHeal kept re-applying it
+  # over the helm release's real LoadBalancer Service of the same name, reverting it
+  # to a ClusterIP with zero endpoints -> LB IP .160 unannounced -> cluster-wide
+  # outage. CoreDNS rewrites *.riotpiao.com to ingress-nginx-controller.ingress-nginx
+  # .svc, which is the helm Service directly — no alias needed.
   - riotpiao-com-cert.yaml                # Certificate for *.riotpiao.com (ingress-nginx namespace)
   - ingress.yaml                           # Ingress rules for all services (multiple namespaces)

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -1,6 +1,13 @@
 # Forgejo Helm Values — Single Source of Truth
 # Chart: https://codeberg.org/forgejo-contrib/forgejo-helm
 
+# Recreate (not RollingUpdate): the gitea data volume is a single RWO PVC. With
+# RollingUpdate the new pod tries to attach the PVC while the old pod still holds
+# it -> "Multi-Attach error", new pod stuck Init forever, rollout wedged. Recreate
+# terminates the old pod first so the PVC detaches before the new one starts.
+strategy:
+  type: Recreate
+
 # Disable bundled dependencies (use external CNPG + Redis instead)
 postgresql-ha:
   enabled: false
@@ -14,6 +21,18 @@ valkey-cluster:
 redis:
   enabled: false
 
+# External SSH access for git over the LAN. The chart's ssh Service becomes a
+# LoadBalancer with a stable IP from the Cilium homelab-pool (192.168.1.160/28,
+# L2-announced) so `git clone ssh://[email protected]:2222/...` works from the
+# LAN. gitea's sshd listens on 2222 in-pod; port 2222 is exposed directly to
+# avoid needing privileged :22.
+service:
+  ssh:
+    type: LoadBalancer
+    port: 2222
+    annotations:
+      lbipam.cilium.io/ips: "192.168.1.161"
+
 gitea:
   admin:
     existingSecret: forgejo-admin
@@ -22,8 +41,10 @@ gitea:
     server:
       DOMAIN: forgejo.riotpiao.com
       ROOT_URL: https://forgejo.riotpiao.com
-      SSH_DOMAIN: forgejo.riotpiao.com
-      SSH_PORT: 22
+      # SSH clone URLs advertise git.riotpiao.com:2222 (the LoadBalancer above).
+      SSH_DOMAIN: git.riotpiao.com
+      SSH_PORT: 2222
+      SSH_LISTEN_PORT: 2222
       
     database:
       DB_TYPE: postgres

k8s/bootstrap/phase4-argocd/argocd-values.yaml

@@ -131,12 +131,39 @@ configs:
     application.instanceLabelKey: argocd.argoproj.io/instance
     # Let every kustomize build run the ksops exec generator.
     kustomize.buildOptions: --enable-alpha-plugins --enable-exec
+    # External URL — required so OIDC redirect URIs are built correctly.
+    url: https://argocd.riotpiao.com
+    # Local accounts (in addition to Authentik SSO):
+    #   rock  — human admin; can log in with a password AND issue API tokens.
+    #   cicd  — automation-only; apiKey (token) for the CD pipeline, no UI login.
+    accounts.rock: apiKey,login
+    accounts.cicd: apiKey
+    # Authentik OIDC. clientSecret pulled from the argocd `oidc-secret` Secret
+    # (created by authentik-provision). The groups claim drives RBAC below.
+    oidc.config: |
+      name: Authentik
+      issuer: https://authentik.riotpiao.com/application/o/argocd/
+      clientID: argocd
+      clientSecret: $oidc-secret:client-secret
+      requestedScopes:
+        - openid
+        - profile
+        - email
+        - groups
+      requestedIDTokenClaims:
+        groups:
+          essential: true
 
   params:
     server.insecure: false
 
-  # RBAC (allow admin full access)
+  # RBAC. local `admin` + `rock` + the `cicd` pipeline account all get role:admin;
+  # the Authentik `homelab-admins` group (rock is a member) maps to admin so SSO
+  # logins are admin too.
   rbac:
     policy.default: role:readonly
     policy.csv: |
       g, admin, role:admin
+      g, rock, role:admin
+      g, cicd, role:admin
+      g, homelab-admins, role:admin

k8s/bootstrap/phase4-argocd/root-app-github.yaml

@@ -3,8 +3,8 @@
 # GitHub seed so ArgoCD can deploy everything after the control plane. After
 # Forgejo is healthy + mirroring GitHub, cut over with phase5-cutover/root-app-forgejo.yaml.
 #
-# repoURL is the SSH form — must match the `url` in the seed-repo deploy-key Secret
-# (see seed-repo-secret.example.yaml). Apply that Secret before this.
+# repoURL is anonymous HTTPS — the seed repo is public, so no deploy key and no
+# repository Secret are needed. Nothing to apply before this.
 ---
 apiVersion: v1
 kind: Namespace
@@ -44,7 +44,7 @@ metadata:
 spec:
   project: homelab
   source:
-    repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git   # GitHub seed (SSH)
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git   # GitHub seed (SSH)
     targetRevision: main
     path: k8s/argocd/apps
   destination:

k8s/bootstrap/phase4-argocd/seed-repo-secret.example.yaml

@@ -1,28 +0,0 @@
-# ArgoCD repo credential for the PRIVATE GitHub seed — deploy key (read-only).
-# Apply at Phase 0 BEFORE root-app-github.yaml. This is a TEMPLATE: never commit
-# the real private key.
-#
-#   ssh-keygen -t ed25519 -C "argocd@homelab" -f argocd_seed -N ""
-#   # add argocd_seed.pub → GitHub repo → Settings → Deploy keys (Read-only, no write)
-#   kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
-#   kubectl -n argocd create secret generic seed-github-repo \
-#     --from-literal=type=git \
-#     [email protected]:Riotpiaole/riotpiao.homelab.com.git \
-#     --from-file=sshPrivateKey=argocd_seed
-#   kubectl -n argocd label secret seed-github-repo argocd.argoproj.io/secret-type=repository
-#
-# url MUST match root-app-github.yaml's repoURL (SSH form).
-apiVersion: v1
-kind: Secret
-metadata:
-  name: seed-github-repo
-  namespace: argocd
-  labels:
-    argocd.argoproj.io/secret-type: repository
-stringData:
-  type: git
-  url: [email protected]:Riotpiaole/riotpiao.homelab.com.git
-  sshPrivateKey: |
-    -----BEGIN OPENSSH PRIVATE KEY-----
-    REPLACE-WITH-READ-ONLY-DEPLOY-KEY-PRIVATE-HALF — DO NOT COMMIT THE REAL KEY
-    -----END OPENSSH PRIVATE KEY-----

k8s/infra/iam/scripts/authentik-provision.py

@@ -88,7 +88,7 @@ def kubectl_get_secret_key(namespace, name, key):
     return base64.b64decode(p.stdout).decode()
 
 
-def kubectl_create_secret(namespace, name, literals: dict):
+def kubectl_create_secret(namespace, name, literals: dict, labels: dict = None):
     """Idempotent: create-or-update via dry-run|apply, same pattern used
     elsewhere in this repo (setup_vault.sh, apply-vault-secrets.sh)."""
     args = ["kubectl", "-n", namespace, "create", "secret", "generic", name]
@@ -103,6 +103,13 @@ def kubectl_create_secret(namespace, name, literals: dict):
     if apply.returncode != 0:
         die(f"applying secret {namespace}/{name}: {apply.stderr}")
     print(f"  secret {namespace}/{name}: {apply.stdout.strip()}")
+    if labels:
+        # argocd's `$secret:key` substitution only reads Secrets carrying
+        # app.kubernetes.io/part-of: argocd — without it OIDC login fails with
+        # oauth2 "invalid_client" (empty client_secret sent to the IdP).
+        label_args = ["kubectl", "-n", namespace, "label", "secret", name,
+                      "--overwrite"] + [f"{k}={v}" for k, v in labels.items()]
+        subprocess.run(label_args, capture_output=True, text=True)
 
 
 def get_or_create(list_path, create_path, query, payload, patch_existing=None):
@@ -269,6 +276,8 @@ SERVICES = {
         "client_secret_source": ("argocd", "oidc-secret", "client-secret"),
         "generate_if_missing": True,
         "extra_secret_literals": {"client-id": "argocd"},
+        # argocd only reads $secret refs from Secrets labelled part-of: argocd.
+        "secret_labels": {"app.kubernetes.io/part-of": "argocd"},
         "redirect_uris": ["https://argocd.riotpiao.com/auth/callback"],
         "launch_url": "https://argocd.riotpiao.com",
         "display_name": "Argo CD",
@@ -299,7 +308,8 @@ for name, cfg in SERVICES.items():
         client_secret = gen_secret(40)
         literals = {key: client_secret}
         literals.update(cfg.get("extra_secret_literals", {}))
-        kubectl_create_secret(ns, secret_name, literals)
+        kubectl_create_secret(ns, secret_name, literals,
+                              labels=cfg.get("secret_labels"))
         print(f"  {name}: generated new client secret -> {ns}/{secret_name}")
     else:
         print(f"  {name}: using existing client secret from {ns}/{secret_name}")

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -7,6 +7,22 @@ metadata:
   namespace: longhorn-system
   annotations:
     storageclass.kubernetes.io/is-default-class: "false"
+    # StorageClass.parameters is immutable. mkfsParams below was added after
+    # this class already existed, so every sync failed with
+    #   parameters: Invalid value: {...}: field is immutable
+    # and the Application sat OutOfSync permanently — ArgoCD retrying a change
+    # the API server can never accept.
+    #
+    # Replace=true makes ArgoCD delete and recreate instead of patching. Safe
+    # for a StorageClass: it is consulted only at provisioning time, so bound
+    # PVs and their data are untouched. New PVCs briefly fail if one is created
+    # during the window.
+    #
+    # Note what this does NOT do: the nine CNPG volumes already provisioned were
+    # formatted without mkfsParams and keep that format. Only volumes created
+    # after this recreate get it. Existing volumes that need to expand still
+    # need handling separately.
+    argocd.argoproj.io/sync-options: Replace=true,Force=true
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 parameters:

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,6 +1,16 @@
 # Longhorn Node CRDs for cp-2 and cp-3.
 # These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
 # Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
+#
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
+#
+# What these objects are actually for is `allowScheduling: true` on tainted
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
@@ -10,12 +20,6 @@ metadata:
 spec:
   name: talos-cp-2
   allowScheduling: true
-  disks:
-    default-disk:
-      allowScheduling: true
-      path: /var/lib/longhorn
-      storageReserved: 0
-      tags: []
   tags: []
 ---
 apiVersion: longhorn.io/v1beta2
@@ -26,10 +30,4 @@ metadata:
 spec:
   name: talos-cp-3
   allowScheduling: true
-  disks:
-    default-disk:
-      allowScheduling: true
-      path: /var/lib/longhorn
-      storageReserved: 0
-      tags: []
   tags: []

k8s/infra/monitoring/alerts/svc-forgejo-rules.yaml

@@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
   name: forgejo-rules
-  namespace: forgejo
+  namespace: cicd
 spec:
   groups:
     - name: forgejo.rules

k8s/infra/monitoring/dashboards/llm-frontend.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: llm-frontend-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "LLM"
+    # Request rate/error/latency/bandwidth now come from Kong's prometheus
+    # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
+    # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
+    # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
+    # embeddings/rerank uniformly without per-backend instrumentation.
+    # Token-count metrics are still not available: that needs response-body
+    # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
+    # Predictor-level metrics (native Ollama/vLLM stats) also still need a
+    # dedicated exporter -- not added here.
+data:
+  llm-frontend.json: |
+    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}

k8s/infra/monitoring/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
   - dashboards/control-plane-logs.yaml
   - dashboards/hardware-overview.yaml
   - dashboards/kube-controller-health.yaml
+  - dashboards/llm-frontend.yaml
   - dashboards/service-availability.yaml
   - dashboards/service-golden-signals.yaml
   - dashboards/service-internals.yaml

k8s/infra/monitoring/prometheus-values.yaml

@@ -20,8 +20,36 @@ grafana:
   enabled: false
 
 # ── Alertmanager ──────────────────────────────────────────────────────────────
+# Enabled with a default (null) receiver — every firing PrometheusRule lands in
+# the Alertmanager UI and Grafana's Alerting view; no external Slack/email/
+# PagerDuty notifier is wired yet (add a receiver + route later). Storage pinned
+# to az-a (sole Longhorn node) like Prometheus so the RWO PVC can attach.
 alertmanager:
-  enabled: false
+  enabled: true
+  alertmanagerSpec:
+    nodeSelector:
+      topology.kubernetes.io/zone: az-a
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: longhorn
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 2Gi
+  config:
+    route:
+      group_by: ["alertname", "namespace"]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+      receiver: "null"
+    receivers:
+      - name: "null"
 
 # ── Prometheus ────────────────────────────────────────────────────────────────
 prometheus:

terraform/files/cilium/l2-announcement.yaml

@@ -0,0 +1,20 @@
+# CiliumL2AnnouncementPolicy — ARPs each LoadBalancer IP (from lb-ippool) on the
+# LAN so the EXTERNAL-IP is actually reachable. Without it, LB-IPAM assigns IPs
+# but nothing answers ARP (100% packet loss / incomplete ARP). One node per IP
+# holds the lease (leaderElection) to avoid ARP flapping. Requires kube-proxy
+# replacement (enabled) and Cilium L2 announcements (default since v1.14).
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: homelab-l2-announce
+spec:
+  loadBalancerIPs: true
+  interfaces:
+    - eno1
+  # Only control-plane nodes may hold the L2 lease. They have the `eno1` LAN NIC;
+  # the GPU worker's NICs are enp28s0f*np* (Mellanox) with NO eno1 — if it won the
+  # lease it held the LB VIP but couldn't ARP it, turning .160 into a LAN black
+  # hole (cluster-wide outage that flapped as reboots reshuffled the lease).
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/control-plane: ""

terraform/files/cilium/lb-ippool.yaml

@@ -0,0 +1,15 @@
+# CiliumLoadBalancerIPPool — the IPs Cilium LB-IPAM may assign to LoadBalancer
+# Services. CIDR 192.168.1.160/28 covers .160–.175 on the LAN.
+#   .160  ingress-nginx (LoadBalancer)
+#   .161  forgejo-ssh   (LoadBalancer)
+#   .162–.175  free
+# Do NOT add a 10.6.0.0/24 block — that is the WireGuard subnet; letting LB-IPAM
+# hand out a CP tunnel IP breaks the tunnel, and L2 ARP only works on the LAN
+# interface (eno1) anyway. Keep this pool LAN-only.
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: homelab-pool
+spec:
+  blocks:
+    - cidr: "192.168.1.160/28"

terraform/files/coredns/Corefile

@@ -33,6 +33,11 @@
     rewrite name homarr.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name portainer.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    # Kong API gateway. Points at nginx, not kong-proxy, for the same reason as
+    # the rest: a direct rewrite would skip TLS termination. Pods that don't
+    # need TLS should call kong-proxy.api.svc.cluster.local instead of using
+    # this name at all.
+    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
     kubernetes cluster.local in-addr.arpa ip6.arpa {
         pods insecure

terraform/main.tf

@@ -42,6 +42,8 @@ resource "local_file" "controlplane_configs" {
     zone                   = each.value.zone
     allow_scheduling       = each.value.allow_scheduling
     coredns_corefile       = file("${path.module}/files/coredns/Corefile")
+    cilium_lb_ippool       = file("${path.module}/files/cilium/lb-ippool.yaml")
+    cilium_l2_announcement = file("${path.module}/files/cilium/l2-announcement.yaml")
 
     # Cloudflare Tunnel cert SANs (talos :50000 and kube-apiserver :6443)
     cloudflare_talos_sans     = each.value.cloudflare_talos_sans
@@ -101,6 +103,8 @@ resource "local_file" "worker_configs" {
     forgejo_hostname    = var.forgejo_hostname
     zone                = each.value.zone
     gpu_count           = each.value.gpu_count
+    node_labels         = each.value.node_labels
+    node_taints         = each.value.node_taints
     extra_disks         = each.value.extra_disks
     swap_size           = each.value.swap_size
     ephemeral_max_size  = each.value.ephemeral_max_size

terraform/templates/controlplane.tftpl

@@ -49,12 +49,15 @@ machine:
     image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version}
     wipe: true
     grubUseUKICmdline: true
+%{ if length(longhorn_disks) > 0 ~}
   disks:
-%{ for disk in longhorn_disks ~}
+%{ for idx, disk in longhorn_disks ~}
+    # ${disk.kind} — ${disk.device}
     - device: ${disk.device}
       partitions:
-        - mountpoint: ${disk.mountpoint}
+        - mountpoint: ${coalesce(disk.mountpoint, format("/var/lib/longhorn-disk%d", idx + 1))}
 %{ endfor ~}
+%{ endif ~}
   features:
     diskQuotaSupport: true
     kubePrism:
@@ -164,6 +167,18 @@ cluster:
         kind: Namespace
         metadata:
           name: kube-system
+    # Cilium LoadBalancer IPAM pool + L2 announcement policy. Substrate networking
+    # (owned here alongside the Cilium install), single source of truth in
+    # terraform/files/cilium/*.yaml. Provides LAN LoadBalancer IPs for ingress-nginx
+    # (.160) and forgejo-ssh (.161). Was previously an ArgoCD app whose empty
+    # kustomization never actually applied it (the live pool came from manual
+    # kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
+    - name: cilium-lb-ippool
+      contents: |
+        ${indent(8, cilium_lb_ippool)}
+    - name: cilium-l2-announcement
+      contents: |
+        ${indent(8, cilium_l2_announcement)}
     # CoreDNS Corefile with homelab hostname rewrites (single source of truth in
     # terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
     # the nginx ingress controller so OIDC auto-discovery against

terraform/templates/worker.tftpl

@@ -71,11 +71,20 @@ machine:
   nodeLabels:
     topology.kubernetes.io/region: homelab
     topology.kubernetes.io/zone: ${zone}
-    node-role.kubernetes.io/gpu-node: ""
 %{ if gpu_count > 0 ~}
+    node-role.kubernetes.io/gpu-node: ""
     nvidia.com/gpu: "true"
     gpu-count: "${gpu_count}"
 %{ endif ~}
+%{ for k, v in node_labels ~}
+    ${k}: "${v}"
+%{ endfor ~}
+%{ if length(node_taints) > 0 ~}
+  nodeTaints:
+%{ for t in node_taints ~}
+    ${t.key}: "${t.value}:${t.effect}"
+%{ endfor ~}
+%{ endif ~}
 
 cluster:
   id: ${cluster_id}

terraform/variables.tf

@@ -107,10 +107,24 @@ variable "controlplane_configs" {
     lan_subnet   = string
     lan_gateway  = string
     install_disk = string
-    longhorn_disks = list(object({
+    # Longhorn data disks, in mountpoint order. `device` should be a stable
+    # /dev/disk/by-id/wwn-* path: this hardware enumerates /dev/sdX by discovery
+    # order, and `install.wipe: true` means a renumber can aim the installer at a
+    # data disk.
+    #
+    # `mountpoint` defaults to /var/lib/longhorn-disk<N> by position. Never
+    # renumber or reorder existing entries — Longhorn keys its disks off the
+    # mountpoint, so a rename orphans the replicas already on that disk. Append
+    # new disks to the end.
+    #
+    # `kind` ("ssd"/"hdd") is a declared label only. PERC RAID controllers report
+    # every disk as rotational, so it can't be autodetected, and it deliberately
+    # does not affect ordering or mountpoints.
+    longhorn_disks = optional(list(object({
       device     = string
-      mountpoint = string
-    }))
+      mountpoint = optional(string)
+      kind       = optional(string, "hdd")
+    })), [])
     zone             = string
     allow_scheduling = bool
     # Extra cert SANs for this node — e.g. Cloudflare Tunnel public hostnames so
@@ -131,6 +145,15 @@ variable "worker_configs" {
     network_interface = optional(string, "eno1")
     zone              = string
     gpu_count         = optional(number, 0)
+    # Extra node labels beyond the topology/GPU defaults.
+    node_labels = optional(map(string), {})
+    # Taints make a node dedicated: only pods carrying a matching toleration
+    # schedule there. effect is NoSchedule | PreferNoSchedule | NoExecute.
+    node_taints = optional(list(object({
+      key    = string
+      value  = string
+      effect = string
+    })), [])
     factory_image      = optional(string)
     swap_size          = optional(string, "")
     ephemeral_max_size = optional(string, "700GiB")