fix(agent-pod): remote tui session for multi-agent

2026-08-18 14:23:55 -07:00
parent c52b4acc74
commit 225866945a
34 changed files with 1874 additions and 27 deletions
@@ -0,0 +1,18 @@
 apiVersion: v1
 data:
  settings.json: |
    {
      "defaultProvider": "homelab-ornith",
      "defaultModel": "ornith:35b",
      "defaultThinkingLevel": "medium",
      "theme": "light",
      "compaction": {
        "enabled": true,
        "reserveTokens": 8192,
        "keepRecentTokens": 12000
      }
    }
 kind: ConfigMap
 metadata:
  name: pi-config
  namespace: agent-pod
@@ -0,0 +1,36 @@
 # Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
 # (trigger a new session) -- both are routes on the same hub.js service.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: console
  namespace: agent-pod
  annotations:
    konghq.com/strip-path: "false"
 spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /console
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
          - path: /run
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
          - path: /sessions
            pathType: Prefix
            backend:
              service:
                name: agent-hub
                port:
                  number: 9090
@@ -0,0 +1,108 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: agent-pod
  namespace: agent-pod
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: agent-pod
  template:
    metadata:
      labels:
        app: agent-pod
    spec:
      # api.riotpiao.com has no in-cluster DNS record (only resolves from the
      # home network's own resolver) -- pin it to ingress-nginx-controller's
      # ClusterIP so pi's models.json baseUrl works unchanged. TLS still
      # terminates correctly since SNI/Host still say api.riotpiao.com.
      hostAliases:
        - ip: "10.101.128.185"
          hostnames:
            - "api.riotpiao.com"
      containers:
        # hub.js runs in the same container as pi (not a sidecar) so it can
        # spawn `pi -p --mode json` directly via child_process -- a separate
        # container can't exec into another container's filesystem/PATH.
        # It IS the container's long-running process now; no more `sleep
        # infinity` placeholder.
        - name: pi
          image: node:22-slim
          command:
            - sh
            - -c
            - |
              set -e
              apt-get update && apt-get install -y git curl jq openssh-client tmux
              ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub
              eval "$(ssh-agent -s)"
              ssh-add /root/.ssh/id_forgejo
              npm install -g @earendil-works/[email protected]
              npm install --prefix /root ws
              node /root/hub.js
          env:
            - name: PI_BIN
              value: pi
          ports:
            - containerPort: 9090
          resources:
            requests:
              cpu: "4"
              memory: 8Gi
            limits:
              cpu: "8"
              memory: 16Gi
          volumeMounts:
            - name: pi-config
              mountPath: /root/.pi/agent/settings.json
              subPath: settings.json
            - name: pi-models
              mountPath: /root/.pi/agent/models.json
              subPath: models.json
            - name: pi-skills
              mountPath: /root/.pi/agent/skills
            - name: hub-src
              mountPath: /root/hub.js
              subPath: hub.js
            - name: ssh-key
              mountPath: /root/.ssh/id_forgejo
              subPath: id_forgejo
            - name: ssh-config
              mountPath: /root/.ssh/config
              subPath: config
      volumes:
        - name: pi-config
          configMap:
            name: pi-config
        - name: pi-models
          secret:
            secretName: pi-models
        - name: pi-skills
          configMap:
            name: pi-skills
            items:
              - key: planner-SKILL.md
                path: planner/SKILL.md
              - key: investigator-SKILL.md
                path: investigator/SKILL.md
              - key: info-collector-SKILL.md
                path: info-collector/SKILL.md
              - key: implementer-SKILL.md
                path: implementer/SKILL.md
              - key: judge-SKILL.md
                path: judge/SKILL.md
              - key: resolver-SKILL.md
                path: resolver/SKILL.md
              - key: brave-search-SKILL.md
                path: brave-search/SKILL.md
        - name: hub-src
          configMap:
            name: hub-src
        - name: ssh-key
          secret:
            secretName: agent-pod-ssh-key
            defaultMode: 0600
        - name: ssh-config
          configMap:
            name: agent-pod-ssh-config
@@ -0,0 +1,700 @@
 apiVersion: v1
 data:
  hub.js: |
    #!/usr/bin/env node
    // agent-hub: lives inside the pi container (not a sidecar) so it can spawn
    // `pi` directly, and control the pod's own tmux server. One persistent
    // in-cluster service -- POST /run to trigger a single ad-hoc headless agent
    // run, POST /pipeline to run an ordered list of task phases against a
    // repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within
    // a phase run concurrently, each in its own git worktree). Within one task,
    // planner/investigator/implementer/judge are separate agents in separate
    // named tmux sessions (task-<id>-<role>, attachable via `kubectl exec -it --
    // tmux attach -t <name>` while running), coordinating only through what's on
    // disk in that task's worktree -- not one shared conversation. GET /console
    // (WebSocket) watches every concurrent headless run live, relaying pi's own
    // session protocol verbatim (same event shape Claude Code sessions use).
    const http = require("node:http");
    const crypto = require("node:crypto");
    const fs = require("node:fs");
    const path = require("node:path");
    const { spawn } = require("node:child_process");
    const readline = require("node:readline");
    const { WebSocketServer } = require("ws");
    const PORT = process.env.HUB_PORT || 9090;
    const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
    // Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide
    // with unrelated tools on this machine (a Rust CLI and a Datadog TUI,
    // respectively, discovered the hard way this session). Always invoke the
    // exact pinned @earendil-works/[email protected] installed locally under
    // .pi-cli/, by explicit path.
    const PI_BIN =
      process.env.PI_BIN ||
      path.join(
        __dirname,
        "..",
        ".pi-cli",
        "node_modules",
        "@earendil-works",
        "pi-coding-agent",
        "dist",
        "cli.js"
      );
    // Job-type skills under pi/skills/<name>/SKILL.md (mounted at
    // ~/.pi/agent/skills/<name>/ in agent-pod). When `agent` matches one of
    // these, the prompt is forced through pi's `/skill:<name> <args>` mechanism
    // instead of being sent bare -- see pi's skills.md docs on single-shot skill
    // loading. `resolver` is never dispatched directly by a caller; only the
    // pipeline driver invokes it, on stage crashes.
    const ROLE_SKILLS = new Set([
      "planner",
      "investigator",
      "info-collector",
      "implementer",
      "judge",
      "resolver",
    ]);
    const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?}
    const viewers = new Set(); // WebSocket connections watching /console
    function broadcast(type, session) {
      const msg = JSON.stringify({ type, session });
      for (const ws of viewers) {
        if (ws.readyState === ws.OPEN) ws.send(msg);
      }
    }
    function startSession(agent, extra = {}) {
      const id = extra.id || crypto.randomUUID();
      const session = {
        ...extra,
        id,
        agent,
        status: "running",
        events: [],
        startedAt: new Date().toISOString(),
      };
      sessions.set(id, session);
      broadcast("start", session);
      return session;
    }
    function addEvent(session, rawLine) {
      const event = JSON.parse(rawLine);
      session.events.push(event);
      broadcast("event", session);
      return event;
    }
    function endSession(session, status) {
      session.status = status;
      session.endedAt = new Date().toISOString();
      broadcast("end", session);
    }
    // Extracts the plain-text content of a message_end event, if any -- used to
    // find the VERDICT:/RESOLUTION: line judge/resolver skills are required to
    // end their final message with.
    function textOf(event) {
      if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) {
        return null;
      }
      return event.message.content
        .filter((c) => c.type === "text")
        .map((c) => c.text)
        .join("\n");
    }
    // Core spawn primitive. Spawns `pi -p --mode json <extraArgs> <prompt>`,
    // relays every line as a session event exactly like before. Returns
    // { session, done } -- `session` is available synchronously (so an HTTP
    // handler can respond with its id right away, same as the old runAgent),
    // `done` is a Promise resolving once the process exits, for callers that
    // need to wait on a stage (the pipeline driver) rather than fire-and-forget.
    function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) {
      const session = startSession(agent, sessionExtra);
      const args = ["-p", "--mode", "json"];
      if (provider) args.push("--provider", provider);
      if (model) args.push("--model", model);
      const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt;
      args.push(finalPrompt);
      const child = spawn(PI_BIN, args, {
        stdio: ["ignore", "pipe", "pipe"],
        cwd,
      });
      const rl = readline.createInterface({ input: child.stdout });
      let lastText = "";
      let stderrTail = "";
      rl.on("line", (line) => {
        if (!line.trim()) return;
        try {
          const event = addEvent(session, line);
          const text = textOf(event);
          if (text) lastText = text;
        } catch {
          // non-JSON stdout noise, ignore
        }
      });
      child.stderr.on("data", (chunk) => {
        process.stderr.write(chunk);
        stderrTail = (stderrTail + chunk.toString()).slice(-4000);
      });
      const done = new Promise((resolve) => {
        child.on("close", (code) => {
          endSession(session, code === 0 ? "done" : "error");
          resolve({ code, session, lastText, stderrTail });
        });
      });
      return { session, done };
    }
    function runAgent(agent, prompt, extraArgs = {}) {
      // Fire-and-forget: caller (the /run handler) doesn't await `done`.
      return spawnPi({ agent, prompt, ...extraArgs }).session;
    }
    function parseVerdictLine(text, label) {
      if (!text) return null;
      const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
      const m = text.match(re);
      return m ? m[1].toUpperCase() : null;
    }
    // Deterministic git operations, run directly by hub.js rather than left to
    // the model -- branch creation and pushing after each task are mechanical,
    // not judgment calls, and need to happen reliably every time regardless of
    // what a task's stages did or didn't remember to do.
    function runCmd(bin, args, cwd) {
      return new Promise((resolve) => {
        const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
        let out = "";
        child.stdout.on("data", (c) => (out += c));
        child.stderr.on("data", (c) => (out += c));
        child.on("close", (code) => resolve({ code, out: out.trim() }));
      });
    }
    function runGit(cwd, args) {
      return runCmd("git", args, cwd);
    }
    // A stage saying "commit" in its prompt is a request, not a guarantee -- seen
    // in practice: a stage writes a real file and simply never runs `git add`/
    // `git commit`, leaving it untracked and invisible to every later `git diff`.
    // Sweep and commit anything left dirty after every stage, deterministically.
    async function commitPending(cwd, message) {
      await runGit(cwd, ["add", "-A"]);
      const status = await runGit(cwd, ["status", "--porcelain"]);
      if (!status.out) return { committed: false };
      const commit = await runGit(cwd, ["commit", "-m", message]);
      return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
    }
    // Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide
    // RETRY vs ABORT. Shared by both crash-recovery paths below (headless
    // exit-code failures and interactive sentinel-file timeouts) -- the
    // diagnostic prompt differs per caller, but "ask resolver, parse the
    // RESOLUTION: line" is identical either way.
    async function askResolver(pipelineId, cwd, task, diagnosticPrompt) {
      const resolverResult = await spawnPi({
        agent: "resolver",
        prompt: diagnosticPrompt,
        cwd,
        sessionExtra: { pipelineId, stage: "resolver", task },
      }).done;
      return parseVerdictLine(resolverResult.lastText, "RESOLUTION");
    }
    // Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic
    // judge FAIL, which is handled separately), asks the resolver to diagnose
    // and decide RETRY vs ABORT. Retries the failed stage at most once,
    // regardless of what resolver recommends a second time -- a hard cap, not
    // indefinite trust in the model's judgment.
    async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) {
      let result = await spawnPi({
        agent: stage,
        prompt,
        cwd,
        sessionExtra: { pipelineId, stage, task },
      }).done;
      if (result.code === 0) return result;
      const resolution = await askResolver(
        pipelineId,
        cwd,
        task,
        `Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}`
      );
      if (resolution === "RETRY") {
        result = await spawnPi({
          agent: stage,
          prompt,
          cwd,
          sessionExtra: { pipelineId, stage, task },
        }).done;
      }
      return result;
    }
    // Deterministic tmux operations -- same rationale as runGit: mechanical,
    // not a judgment call, run directly rather than trusted to a prompt.
    function runTmux(args) {
      return runCmd("tmux", args);
    }
    function tmuxSessionName(task) {
      return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`;
    }
    // Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in
    // flight at once. No external dep; a plain in-order index cursor shared by
    // `limit` runner loops.
    async function runConcurrent(items, limit, worker) {
      const results = new Array(items.length);
      let i = 0;
      async function next() {
        while (i < items.length) {
          const idx = i++;
          results[idx] = await worker(items[idx], idx);
        }
      }
      await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
      return results;
    }
    const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
    const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
    const POLL_MS = 10 * 1000;
    async function waitForFile(filePath, limitMs) {
      const start = Date.now();
      while (!fs.existsSync(filePath)) {
        if (Date.now() - start > limitMs) return false;
        await new Promise((r) => setTimeout(r, POLL_MS));
      }
      return true;
    }
    const MAX_IMPLEMENT_ATTEMPTS = 5;
    const MAX_PLAN_REVISIONS = 3;
    // Runs one role as its own fresh interactive pi session in its own named
    // tmux session -- planner, investigator, implementer, and judge are
    // separate agents with separate context, not turns in one shared
    // conversation. They coordinate only through what's on disk in the task's
    // worktree: PLAN.md, committed code, judge's result file. Each session is
    // attachable while it runs (kubectl exec -it -- tmux attach -t <name>) and
    // killed once its sentinel file lands or it's abandoned after resolver
    // escalation.
    async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) {
      const sessionName = `${tmuxSessionName(task)}-${stageLabel}`;
      fs.rmSync(sentinelFile, { force: true });
      const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]);
      if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName };
      let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS);
      if (!ok) {
        const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]);
        const resolution = await askResolver(
          pipelineId,
          cwd,
          task,
          `Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` +
            `Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` +
            `real progress and worth nudging to wrap up, or stuck and worth abandoning?`
        );
        if (resolution === "RETRY") {
          await runTmux([
            "send-keys",
            "-t",
            sessionName,
            `Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`,
            "Enter",
          ]);
          ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS);
        }
      }
      await runTmux(["kill-session", "-t", sessionName]);
      return { ok, sessionName };
    }
    // Runs one task in its own git worktree (see runPhase): planner drafts
    // PLAN.md, investigator confirms it, then implementer and judge go back and
    // forth -- judge's FAIL rationale lands in .task-result-<task>, which the
    // next implementer attempt is told to read and address. After
    // MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to
    // judge whether the *plan* itself is wrong, not just the implementation; if
    // so it revises PLAN.md and the implementer gets a fresh attempt budget
    // against the new plan. MAX_PLAN_REVISIONS caps this from looping forever
    // on a task that's genuinely stuck.
    async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) {
      const resultFile = path.join(cwd, `.task-result-${task}`);
      fs.rmSync(resultFile, { force: true });
      const specHint = `the file under tasks/ starting with "${task}-"`;
      const runRole = async (stageLabel, prompt, sentinel) => {
        pipelineSession.activeTasks[task] = {
          stage: stageLabel,
          sessionName: `${tmuxSessionName(task)}-${stageLabel}`,
          startedAt: new Date().toISOString(),
        };
        broadcast("event", pipelineSession);
        const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel);
        await commitPending(cwd, `task: ${task} (${stageLabel})`);
        return result;
      };
      const abandon = (stageLabel, result, attempt) => {
        delete pipelineSession.activeTasks[task];
        broadcast("event", pipelineSession);
        return {
          task,
          status: result.crashed ? "spawn-crashed" : "timed-out",
          error: result.error,
          stoppedAt: stageLabel,
          ...(attempt !== undefined ? { attempt } : {}),
        };
      };
      // Task's implementation is inherited already-committed (e.g. from a base
      // branch of prior work) -- try one judge pass against the spec directly
      // (no PLAN.md exists yet) before paying for a full planner/investigator
      // redo. PASS ends the task here; FAIL falls through into the normal flow
      // below, so planner/implementer pick up with the judge's real feedback.
      if (judgeOnly) {
        const quick = await runRole(
          "judge",
          `Task ${task} may already be implemented on this branch -- check ` +
            `\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` +
            `(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` +
            `verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` +
            `one line of rationale, then run: touch .stage-done-${task}-judge-0`,
          path.join(cwd, `.stage-done-${task}-judge-0`)
        );
        if (!quick.ok) return abandon("judge", quick, 0);
        const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
        if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
          delete pipelineSession.activeTasks[task];
          broadcast("event", pipelineSession);
          return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
        }
      }
      let result = await runRole(
        "planner",
        `Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`,
        path.join(cwd, `.stage-done-${task}-planner`)
      );
      if (!result.ok) return abandon("planner", result);
      result = await runRole(
        "investigator",
        `Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`,
        path.join(cwd, `.stage-done-${task}-investigator`)
      );
      if (!result.ok) return abandon("investigator", result);
      let planRevisions = 0;
      let implementAttempt = 0;
      let verdict = null;
      let resultText = "";
      let justRevisedPlan = false;
      while (true) {
        implementAttempt++;
        const feedbackHint = fs.existsSync(resultFile)
          ? justRevisedPlan
            ? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
              `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
            : `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.`
          : "";
        justRevisedPlan = false;
        result = await runRole(
          "implementer",
          `Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`,
          path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
        );
        if (!result.ok) return abandon("implementer", result, implementAttempt);
        result = await runRole(
          "judge",
          `Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` +
            `.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` +
            `rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`,
          path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`)
        );
        if (!result.ok) return abandon("judge", result, implementAttempt);
        resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
        verdict = parseVerdictLine(resultText, "VERDICT");
        if (verdict === "PASS") break;
        if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
          if (planRevisions >= MAX_PLAN_REVISIONS) break;
          planRevisions++;
          result = await runRole(
            "planner-revise",
            `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
              `the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` +
              `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` +
              `commit. If you change the approach, also use the investigator skill to confirm the new approach against ` +
              `real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` +
              `touch .stage-done-${task}-planner-revise-${planRevisions}`,
            path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)
          );
          if (!result.ok) return abandon("planner-revise", result, planRevisions);
          implementAttempt = 0;
          justRevisedPlan = true;
        }
      }
      delete pipelineSession.activeTasks[task];
      broadcast("event", pipelineSession);
      if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
        return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
      }
      return {
        task,
        status: verdict === "PASS" ? "done" : "done-with-concerns",
        judgeRationale: resultText,
      };
    }
    const PHASE_CONCURRENCY = 3;
    // Runs one phase (a batch of tasks with no declared dependency on each
    // other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task
    // gets its own git worktree off workBranch -- concurrent pi sessions writing
    // into one shared working tree would corrupt the index; worktrees share the
    // same object database but give each task an isolated checkout. After a
    // task's session ends, its branch is merged back into workBranch and pushed,
    // one merge at a time (git ref updates aren't safe to run concurrently even
    // though the worktrees themselves are isolated).
    async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) {
      // Each entry is either a plain task id, or { id, judgeOnly: true } when
      // the task's implementation already exists (e.g. inherited from a base
      // branch) and just needs a real judge pass rather than a full
      // planner/investigator/implementer redo.
      const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
      const worktrees = {};
      for (const entry of entries) {
        const task = entry.id;
        const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`);
        const taskBranch = `task/${task}`;
        const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]);
        if (add.code !== 0) {
          pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out });
          continue;
        }
        worktrees[task] = { wtDir, taskBranch };
      }
      const runnable = entries.filter((e) => worktrees[e.id]);
      await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => {
        const { wtDir } = worktrees[entry.id];
        const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly);
        pipelineSession.taskResults.push(result);
        return result;
      });
      // Merge + push sequentially -- ref updates on the shared repo, one at a
      // time, in the declared task order for this phase.
      for (const entry of runnable) {
        const task = entry.id;
        const { wtDir, taskBranch } = worktrees[task];
        const result = pipelineSession.taskResults.find((r) => r.task === task);
        const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]);
        if (merge.code !== 0) {
          await runGit(cwd, ["merge", "--abort"]);
          if (result) {
            result.status = "merge-conflict";
            result.mergeError = merge.out;
          }
        } else {
          const push = await runGit(cwd, ["push", "-u", "origin", workBranch]);
          if (result) {
            result.pushed = push.code === 0;
            if (!result.pushed) result.pushError = push.out;
          }
        }
        await runGit(cwd, ["worktree", "remove", wtDir, "--force"]);
        await runGit(cwd, ["branch", "-D", taskBranch]);
        broadcast("event", pipelineSession);
      }
    }
    // tasks: array of phases, each phase an array of task ids with no declared
    // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) --
    // caller's responsibility to supply real phase grouping (see tasks/INDEX.md;
    // filename/numeric sort does NOT match execution order on boards like this).
    // A flat array of ids is also accepted and treated as one single phase.
    // Phases run strictly sequentially (a phase boundary is a real dependency
    // gate); tasks within a phase run concurrently, each in its own worktree --
    // see runPhase.
    function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) {
      const cwd = path.join(WORK_DIR, pipelineId);
      fs.mkdirSync(cwd, { recursive: true });
      const phases = Array.isArray(tasks[0]) ? tasks : [tasks];
      const pipelineSession = startSession("pipeline", {
        id: pipelineId,
        pipelineId,
        stage: "pipeline",
        taskResults: [],
        activeTasks: {},
        totalTasks: phases.flat().length,
      });
      (async () => {
        const clone = await runStageWithResolver(
          pipelineId,
          cwd,
          "planner",
          `Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`,
          "clone"
        );
        if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed");
        if (!fs.existsSync(path.join(cwd, ".git"))) {
          // The model deciding to `cd` elsewhere before cloning (instead of
          // cloning into the assigned cwd) is a real failure mode seen in
          // practice, not a hypothetical -- exit code 0 doesn't mean the clone
          // landed where every later stage's cwd assumes it did.
          return endSession(pipelineSession, "clone-missing");
        }
        // Dedicated branch, never main -- and pushed after every single task
        // (not just at the end) so a pod restart mid-run loses at most the
        // in-progress task's work, not everything since the start.
        const workBranch = branchName || `agent-run/${pipelineId}`;
        const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]);
        if (branchResult.code !== 0) {
          pipelineSession.gitError = branchResult.out;
          return endSession(pipelineSession, "branch-crashed");
        }
        // Seen in practice: a task manually downloads a dependency tarball
        // (crates.io registry access isn't guaranteed from every sandboxed
        // checkout) and it lands at repo root, outside whatever .gitignore
        // already covers -- then git add -A (ours or the model's own) commits
        // it. Append broad build-artifact/archive patterns before any task
        // runs, so it's excluded regardless of who stages files later.
        const gitignoreAdditions = [
          "",
          "# agent-harness: build artifacts and vendored archives never belong in source control",
          "*.tar.gz",
          "*.tgz",
          "*.crate",
          "*.zip",
          "*.bin",
          "*.whl",
          "vendor/",
          "node_modules/",
          "",
          "# agent-harness: task completion sentinel files, harness bookkeeping only",
          ".task-result-*",
          ".stage-done-*",
        ].join("\n");
        fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
        await runGit(cwd, ["add", ".gitignore"]);
        await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
        for (const phaseTasks of phases) {
          await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession);
        }
        const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed"));
        endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed");
      })();
      return pipelineSession;
    }
    const server = http.createServer((req, res) => {
      const url = new URL(req.url, "http://localhost");
      if (url.pathname === "/healthz") {
        res.writeHead(200).end();
        return;
      }
      if (url.pathname === "/sessions" && req.method === "GET") {
        res.writeHead(200, { "Content-Type": "application/json" });
        res.end(JSON.stringify([...sessions.values()]));
        return;
      }
      if (url.pathname === "/run" && req.method === "POST") {
        let body = "";
        req.on("data", (chunk) => (body += chunk));
        req.on("end", () => {
          try {
            const { agent, prompt, provider, model } = JSON.parse(body);
            if (!agent || !prompt) throw new Error("agent and prompt are required");
            const session = runAgent(agent, prompt, { provider, model });
            res.writeHead(200, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ id: session.id }));
          } catch (err) {
            res.writeHead(400, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ error: err.message }));
          }
        });
        return;
      }
      if (url.pathname === "/pipeline" && req.method === "POST") {
        let body = "";
        req.on("data", (chunk) => (body += chunk));
        req.on("end", () => {
          try {
            const { repo, baseBranch, tasks, branchName } = JSON.parse(body);
            if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) {
              throw new Error("repo, baseBranch, and a non-empty tasks array are required");
            }
            const pipelineId = crypto.randomUUID();
            runPipeline({ pipelineId, repo, baseBranch, tasks, branchName });
            res.writeHead(200, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ id: pipelineId }));
          } catch (err) {
            res.writeHead(400, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ error: err.message }));
          }
        });
        return;
      }
      res.writeHead(404).end();
    });
    const wss = new WebSocketServer({ server, path: "/console" });
    wss.on("connection", (ws) => {
      for (const session of sessions.values()) {
        ws.send(JSON.stringify({ type: "snapshot", session }));
      }
      viewers.add(ws);
      ws.on("close", () => viewers.delete(ws));
    });
    server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
 kind: ConfigMap
 metadata:
  name: hub-src
  namespace: agent-pod
@@ -0,0 +1,11 @@
 apiVersion: v1
 kind: Service
 metadata:
  name: agent-hub
  namespace: agent-pod
 spec:
  selector:
    app: agent-pod
  ports:
    - port: 9090
      targetPort: 9090
@@ -0,0 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: agent-pod
 resources:
  - deployment.yaml
  - configmap.yaml
  - hub-configmap.yaml
  - pi-skills-configmap.yaml
  - ssh-configmap.yaml
  - hub-service.yaml
  - console-ingress.yaml
@@ -0,0 +1,156 @@
 apiVersion: v1
 data:
  brave-search-SKILL.md: |
    ---
    name: brave-search
    description: Web search via the Brave Search API, called directly with curl. Use for searching documentation, facts, or any current web content.
    allowed-tools: Bash
    ---
    # Brave Search
    Direct HTTP call to the Brave Search API — no separate script or package, just `curl` (`BRAVE_API_KEY` is already set in the environment).
    ## Search
    ```bash
    curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
      --get --data-urlencode "q=<query>" --data-urlencode "count=5" \
      "https://api.search.brave.com/res/v1/web/search"
    ```
    Options (add as extra `--data-urlencode` pairs):
    - `count=<n>` — number of results (max 20, default 5)
    - `country=<code>` — two-letter country code (default US)
    - `freshness=pd|pw|pm|py` — past day/week/month/year, or `freshness=YYYY-MM-DDtoYYYY-MM-DD`
    Response is JSON; the results live at `.web.results[]`, each with `title`, `url`, `description`, `age`. Pipe through `jq` if you want a shorter view, e.g.:
    ```bash
    curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
      --get --data-urlencode "q=<query>" "https://api.search.brave.com/res/v1/web/search" \
      | jq -r '.web.results[] | "- \(.title)\n  \(.url)\n  \(.description)\n"'
    ```
    There's no page-content-extraction helper here — if a result needs reading in full, `curl` the URL directly and read the raw HTML/text; don't expect readability-cleaned markdown.
    ## When to Use
    - Searching for documentation or API references
    - Looking up facts or current information
    - Confirming a claim or approach against real sources
  implementer-SKILL.md: |
    ---
    name: implementer
    description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run.
    allowed-tools: Read Grep Find Ls Write Edit Bash
    ---
    Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions.
    - Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history.
    - Push only if the task explicitly asks for it.
    Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch.
    **Hard rules:**
    - Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing.
    - Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it.
    **Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it.
  info-collector-SKILL.md: |
    ---
    name: info-collector
    description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill).
    allowed-tools: Read Bash
    ---
    **Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there.
    **Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job).
    **Modes:**
    - **Collect mode** (default) — use the `brave-search` skill (`curl` against the Brave Search API) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill.
    - If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response.
  investigator-SKILL.md: |
    ---
    name: investigator
    description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct.
    allowed-tools: Read Bash
    ---
    Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one.
    - Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), use `brave-search` to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`.
    - No `PLAN.md`? Just answer the question asked, citing sources.
    Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next.
    **Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not.
  judge-SKILL.md: |
    ---
    name: judge
    description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria.
    allowed-tools: Read Bash
    ---
    Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp.
    - Run `git diff <base-branch>...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection?
    - FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale.
    - No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly.
    You MUST end your final message with a literal verdict line, exactly one of:
    ```
    VERDICT: PASS
    ```
    ```
    VERDICT: FAIL
    ```
    followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline.
  planner-SKILL.md: |
    ---
    name: planner
    description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it.
    allowed-tools: Read Grep Find Ls Write Bash
    ---
    Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms.
    - If cwd is empty: `git clone` the given repo/branch first (only at pipeline start).
    - Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity.
    - Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/<name>.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: <summary>"`.
    Don't touch any other file.
  resolver-SKILL.md: |
    ---
    name: resolver
    description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion.
    allowed-tools: Read Bash
    ---
    **Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again.
    **Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work.
    **Modes:**
    - **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix).
    You MUST end your final message with a literal resolution line, exactly one of:
    ```
    RESOLUTION: RETRY
    ```
    ```
    RESOLUTION: ABORT
    ```
    followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`.
 kind: ConfigMap
 metadata:
  name: pi-skills
  namespace: agent-pod
@@ -0,0 +1,12 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: agent-pod-ssh-config
  namespace: agent-pod
 data:
  config: |
    Host git.riotpiao.com
        IdentityFile /root/.ssh/id_forgejo
        Port 2222
        User git
        StrictHostKeyChecking accept-new
@@ -0,0 +1,83 @@
 # API Auth Layer — Authentik service account + Kong JWT (model invoke)
 Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik
 service account holding a valid **client_credentials** JWT can invoke the KServe
 models. "Invoke role" = **possession of a JWT from the dedicated model-invoke
 OAuth2 provider** (only the service account can obtain one).
 ## Flow
 ```
 service account ── client_credentials ──▶ Authentik token endpoint
                    (client_id + secret)   https://authentik.riotpiao.com/application/o/token/
        │
        ▼  RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/)
 client ── Authorization: Bearer <jwt> ──▶ Kong (api.riotpiao.com/*)
                                           jwt plugin: verify RS256 sig via Authentik JWKS,
                                           check iss/exp → map to KongConsumer → allow
                                           ▼
                                        KServe model (reasoning / ornith / ...)
 ```
 Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in
 **`jwt`** plugin: it validates an RS256 signature against a public key we pin on
 a KongConsumer, keyed by the token's `iss`.
 ## Changes
 ### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py)
 - New **service account** user `model-invoker` (type `service_account`, no
  password; Authentik issues an app-password/token for M2M).
 - New **OAuth2 provider + application** `model-invoke`:
  - `client_type: confidential`, `grant_types: ["client_credentials"]`
  - signing key = existing RS256 keypair (same as other providers)
  - mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes
    needed for M2M.
 - Client secret written to k8s Secret `api/model-invoke-oidc`
  (keys `client-id`, `client-secret`), labelled for whoever consumes it.
 - Bind the service account so it (and only it) can use the provider.
 ### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`)
 - **KongConsumer** `model-invoker` (ns api).
 - **`jwt` credential** on that consumer (a Secret of type
  `konghq.com/v1/credential`):
  - `algorithm: RS256`
  - `key`  = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/`
  - `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing
    cert (fetched from Authentik JWKS / cert, stored in git or ksops).
 - **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`).
 ### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml)
 - Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation
  (currently e.g. `llm-rewrite-reasoning`) → becomes
  `llm-rewrite-reasoning,jwt-auth`.
 - Leave `/models` list route open OR protect too (decision).
 ## Client usage (after build)
 ```bash
 TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \
  -d grant_type=client_credentials \
  -d client_id=model-invoke \
  -d client_secret=<secret> \
  -d scope=openid | jq -r .access_token)
 curl https://api.riotpiao.com/v1/chat/completions \
  -H "Authorization: Bearer $TOKEN" -d '{...}'
 ```
 ## Test plan
 1. No token → Kong returns 401.
 2. Valid client_credentials token → 200, model responds.
 3. Expired/garbage token → 401.
 4. Confirm the `/models` route behaviour matches the decision.
 ## Open items / risks
 - Authentik `client_credentials` for a *service account* may require an
  **app-password / JWT-assertion** flow rather than plain client_secret POST —
  verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs
  the SA token). Adjust step 1 accordingly before wiring Kong.
 - Pinning `rsa_public_key`: Authentik key rotation would break it — document a
  rotation runbook, or have the provision script re-export the cert PEM into the
  Kong credential on each run (keeps them in sync, same idea as ksops secrets).
 - Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's
  issuer is stable.
@@ -0,0 +1,19 @@
 # Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the
 # ingress controller apply it to every route on this Kong instance, so all
 # five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics
 # without touching llm-routes.yaml. Scraped via kong-values.yaml's
 # serviceMonitor (status listener, already on by chart default at :8100).
 apiVersion: configuration.konghq.com/v1
 kind: KongClusterPlugin
 metadata:
  name: prometheus
  annotations:
    kubernetes.io/ingress.class: kong
  labels:
    global: "true"
 plugin: prometheus
 config:
  status_code_metrics: true
  latency_metrics: true
  bandwidth_metrics: true
  upstream_health_metrics: true
@@ -57,6 +57,17 @@ env:
  # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the
  # buffered one dominates.
  nginx_proxy_proxy_buffering: "off"
  # Any plugin that rewrites the request body — request-transformer on the
  # llm-chat-* routes — reads it through `kong.request.get_body()`, and that
  # returns nothing once nginx has spilled the body past
  # client_body_buffer_size into a temp file. The plugin then re-serializes a
  # body with no `messages`, and the upstream answers
  #   HTTP 400 {"error":{"message":"[] is too short - 'messages'"}}
  # Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400.
  # An agent request carrying tool schemas clears that in one turn, so the
  # buffer has to hold a whole conversation, not a chat message.
  nginx_http_client_body_buffer_size: "16m"
  nginx_http_client_max_body_size: "16m"
 ingressController:
  enabled: true
@@ -107,6 +118,16 @@ podDisruptionBudget:
  enabled: true
  minAvailable: 1
 # Status listener (metrics/health) is on by default at :8100 (chart default,
 # verified via `helm show values`). This just wires the ServiceMonitor the
 # chart already knows how to generate for it, so kong_http_requests_total /
 # kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the
 # cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml.
 serviceMonitor:
  enabled: true
  labels:
    release: kube-prometheus-stack
 # Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
 # situation degrades to co-location instead of leaving a pod Pending.
 topologySpreadConstraints:
@@ -6,7 +6,9 @@ kind: Kustomization
 # or it is silently dropped with no error and no drift shown.
 resources:
  - ingress.yaml
  - kong-metrics.yaml
  - llm-routes.yaml
  - model-auth.yaml
 # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
 # namespace, and the transformer rewrites metadata.namespace on every resource
 # it builds, which is a trap for anything cross-namespace added later.
@@ -66,7 +66,7 @@ metadata:
  name: llm-models
  namespace: llm-serving
  annotations:
-    konghq.com/plugins: llm-models-list
+    konghq.com/plugins: llm-models-list,model-key-auth
    konghq.com/strip-path: "false"
    konghq.com/methods: "GET"
 spec:
@@ -110,7 +110,7 @@ metadata:
  name: llm-chat-reasoning
  namespace: llm-serving
  annotations:
-    konghq.com/plugins: llm-rewrite-reasoning
+    konghq.com/plugins: llm-rewrite-reasoning,model-key-auth
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
@@ -152,7 +152,7 @@ metadata:
  name: llm-chat-ornith
  namespace: llm-serving
  annotations:
-    konghq.com/plugins: llm-rewrite-ornith
+    konghq.com/plugins: llm-rewrite-ornith,model-key-auth
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
@@ -197,7 +197,7 @@ metadata:
  name: llm-chat-qwen
  namespace: llm-serving
  annotations:
-    konghq.com/plugins: llm-rewrite-qwen
+    konghq.com/plugins: llm-rewrite-qwen,model-key-auth
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
@@ -267,7 +267,7 @@ metadata:
  name: llm-rerank
  namespace: llm-serving
  annotations:
-    konghq.com/plugins: llm-rewrite-rerank
+    konghq.com/plugins: llm-rewrite-rerank,model-key-auth
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
@@ -0,0 +1,49 @@
 # API auth layer — Kong key-auth on the model routes.
 #
 # The model API (api.riotpiao.com/v1/...) requires a static API key, presented
 # OpenAI-style as `Authorization: Bearer <key>` (or `apikey: <key>`). The key
 # lives in the ksops-managed Secret model-invoke-apikey (labelled
 # konghq.com/credential: key-auth) and is bound to the KongConsumer below.
 #
 # Issue the key to rock; use it as the OpenAI SDK api_key. Rotate by updating the
 # ksops secret. This is self-contained in Kong — the invoke path does not depend
 # on an Authentik token (Authentik still fronts every *human* dashboard SSO).
 ---
 apiVersion: configuration.konghq.com/v1
 kind: KongConsumer
 metadata:
  name: model-invoker
  namespace: api
  annotations:
    kubernetes.io/ingress.class: kong
 username: model-invoker
 credentials:
  - model-invoke-apikey
 ---
 # key-auth: require the API key on the model routes. key_in_header accepts the
 # `apikey` header; key_in_bearer accepts `Authorization: Bearer <key>` so any
 # OpenAI-compatible SDK (api_key=..., base_url=https://api.riotpiao.com/v1) works
 # unchanged.
 #
 # Namespace `llm-serving`, not `api`: the ingress controller resolves a
 # `konghq.com/plugins` annotation against the annotated object's OWN namespace,
 # and all five model routes in llm-routes.yaml live in llm-serving. While this
 # sat in `api` the reference dangled, the plugin never bound, and every model
 # route served traffic with no key at all — verified: an unauthenticated
 # /v1/models and /v1/ornith/chat/completions both returned 200. A dangling
 # plugin reference is silent; it fails open, so re-test without a key after any
 # move rather than trusting that the object exists.
 apiVersion: configuration.konghq.com/v1
 kind: KongPlugin
 metadata:
  name: model-key-auth
  namespace: llm-serving
 plugin: key-auth
 config:
  key_names:
    - apikey
    - authorization
  key_in_header: true
  key_in_query: false
  key_in_body: false
  hide_credentials: true
@@ -0,0 +1,56 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-embeddings
    app.kubernetes.io/part-of: llm-serving
  name: embeddings
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model-id=nomic-ai/nomic-embed-text-v2-moe
      - --port=8080
      - --hostname=0.0.0.0
      - --auto-truncate
      env:
      - name: HUGGINGFACE_HUB_CACHE
        value: /mnt/models
      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 8Gi
        requests:
          cpu: '8'
          memory: 4Gi
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,19 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 # Explicit allowlist, matching k8s/apps/api. Anything added to this directory
 # and not listed here is silently dropped — no error, no drift shown.
 #
 # These five were adopted from live state on 2026-08-15; they had been applied
 # by hand and carried no ArgoCD ownership. Each was exported and verified with
 # `kubectl diff -f <file>` returning empty before the Application below was
 # created, so the first sync was a no-op rather than a redeploy. Re-verify that
 # way after any edit here: a GPU predictor restart is a weights reload measured
 # in tens of seconds, not a rolling update.
 resources:
  - embeddings.yaml
  - ornith.yaml
  - reasoning.yaml
  - reranker.yaml
  - verifier.yaml
 # No namespace transformer: every file sets its own, and the transformer would
 # rewrite metadata.namespace on anything cross-namespace added later.
@@ -0,0 +1,108 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # Kong reads its timeouts from the Kubernetes Service, not the Ingress —
    # Ingress annotations configure Route entities (strip-path, methods,
    # plugins), these configure the Service entity. They were on
    # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
    # default in force. KServe propagates InferenceService annotations to the
    # Service it generates, which is how they reach Kong from here.
    #
    # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
    # request ever waited on a cold load. A pod restart flushes VRAM, and
    # loading ornith:35b takes longer than 60s, so the first request after any
    # restart returned 504.
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "3600000"
    konghq.com/write-timeout: "3600000"
  labels:
    app.kubernetes.io/name: llm-ornith
    app.kubernetes.io/part-of: llm-serving
  name: ornith
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - command:
      - /bin/sh
      - -c
      - 'set -e
        ollama serve &
        SERVE_PID=$!
        until ollama list >/dev/null 2>&1; do sleep 2; done
        ollama pull ornith:35b
        ollama pull qwen2.5:3b-instruct
        ollama run ornith:35b "ok" >/dev/null 2>&1 || true
        ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
        wait $SERVE_PID
        '
      env:
      - name: OLLAMA_HOST
        value: 0.0.0.0:8080
      - name: OLLAMA_MODELS
        value: /mnt/models/ollama
      - name: OLLAMA_CONTEXT_LENGTH
        value: '32768'
      - name: OLLAMA_KEEP_ALIVE
        value: '-1'
      - name: OLLAMA_NUM_PARALLEL
        value: '1'
      - name: OLLAMA_MAX_LOADED_MODELS
        value: '2'
      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        exec:
          command:
          - /bin/sh
          - -c
          - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
            grep -q qwen2.5
        failureThreshold: 120
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,81 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reasoning
    app.kubernetes.io/part-of: llm-serving
  name: reasoning
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
      - --served-model-name=reasoning
      - --quantization=bitsandbytes
      - --dtype=float16
      - --kv-cache-dtype=auto
      - --tensor-parallel-size=1
      - --max-model-len=16384
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=4
      - --enable-chunked-prefill
      - --enable-prefix-caching
      - --reasoning-parser=deepseek_r1
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: TRITON_ATTN
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        failureThreshold: 80
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    deploymentStrategy:
      type: Recreate
    maxReplicas: 2
    minReplicas: 2
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 2Gi
      name: shm
@@ -0,0 +1,56 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-reranker
    app.kubernetes.io/part-of: llm-serving
  name: reranker
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model-id=BAAI/bge-reranker-base
      - --port=8080
      - --hostname=0.0.0.0
      - --auto-truncate
      env:
      - name: HUGGINGFACE_HUB_CACHE
        value: /mnt/models
      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 8Gi
        requests:
          cpu: '8'
          memory: 4Gi
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      volumeMounts:
      - mountPath: /mnt/models
        name: models
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
@@ -0,0 +1,76 @@
 apiVersion: serving.kserve.io/v1beta1
 kind: InferenceService
 metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
  labels:
    app.kubernetes.io/name: llm-verifier
    app.kubernetes.io/part-of: llm-serving
  name: verifier
  namespace: llm-serving
 spec:
  predictor:
    containers:
    - args:
      - --model=Qwen/Qwen2.5-Math-PRM-7B
      - --served-model-name=verifier
      - --runner=pooling
      - --dtype=float16
      - --tensor-parallel-size=1
      - --max-model-len=4096
      - --max-num-seqs=8
      - --host=0.0.0.0
      - --port=8080
      env:
      - name: VLLM_USE_FLASHINFER_SAMPLER
        value: '0'
      - name: VLLM_ATTENTION_BACKEND
        value: XFORMERS
      - name: HF_HOME
        value: /mnt/models
      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 16Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '4'
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        failureThreshold: 60
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    minReplicas: 1
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        sizeLimit: 1Gi
      name: shm
@@ -10,13 +10,22 @@ metadata:
  name: queue-operator
 rules:
  - apiGroups: ["kmsvc.io"]
-    resources: ["queues", "temporalworkers"]
+    resources: ["queues"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
-    resources: ["queues/status", "temporalworkers/status"]
+    resources: ["queues/status"]
    verbs: ["get", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
-    resources: ["queues/finalizers", "temporalworkers/finalizers"]
+    resources: ["queues/finalizers"]
    verbs: ["update"]
  - apiGroups: ["kmsvc.io"]
    resources: ["temporalworkers"]
    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
  - apiGroups: ["kmsvc.io"]
    resources: ["temporalworkers/status"]
    verbs: ["get", "update", "patch"]
  - apiGroups: ["kmsvc.io"]
    resources: ["temporalworkers/finalizers"]
    verbs: ["update"]
  - apiGroups: ["coordination.k8s.io"]
    resources: ["leases"]
@@ -72,6 +72,14 @@ spec:
  destination:
    server: https://kubernetes.default.svc
    namespace: longhorn-system
  # Longhorn writes disk state back into its own Node CRs — the disk key it
  # generates, storageReserved, diskType, evictionRequested. Git declares only
  # allowScheduling; without this the controller's writes read as drift forever.
  ignoreDifferences:
    - group: longhorn.io
      kind: Node
      jsonPointers:
        - /spec/disks
  syncPolicy:
    automated:
      prune: true
@@ -0,0 +1,58 @@
 # Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`).
 #
 # Syncs before wave 7 (Kong), so the predictor Services exist before the routes
 # that point at them. KServe itself is part of the substrate; this Application
 # owns only the InferenceServices.
 #
 # Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed
 # by hand — no ArgoCD ownership, present in no repo — so every change to them
 # was drift by definition. Each manifest was exported from the cluster and
 # verified with `kubectl diff` returning empty before this file existed; the
 # first sync therefore adopted them without restarting anything.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: llm-serving
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "6"
 spec:
  project: homelab
  revisionHistoryLimit: 3
  source:
    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
    targetRevision: main
    path: k8s/apps/llm-serving
  destination:
    server: https://kubernetes.default.svc
    namespace: llm-serving
  syncPolicy:
    automated:
      # `prune: false` here, unlike every other Application in this repo, and it
      # is not an oversight.
      #
      # ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label
      # (argocd-cm `application.instanceLabelKey`). KServe copies an
      # InferenceService's labels onto the Deployment and Service it generates —
      # visible today as `app.kubernetes.io/name` and `part-of` on
      # `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe
      # propagates that tracking label to children that are not in git, ArgoCD
      # reads them as extraneous, prunes them, and KServe recreates them. That
      # loop churns GPU pods.
      #
      # Deleting an InferenceService therefore means deleting the file AND
      # removing the object, rather than relying on prune.
      prune: false
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      # KServe CRDs are large; server-side apply avoids the
      # "metadata.annotations: Too long" failure client-side apply hits, and is
      # the correct mode for adopting objects an operator also writes to.
      - ServerSideApply=true
    retry:
      limit: 3
      backoff:
        duration: 10s
        factor: 2
        maxDuration: 3m
@@ -84,6 +84,29 @@ spec:
    syncOptions:
      - CreateNamespace=true
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: agent-pod
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "8"
 spec:
  project: homelab
  source:
    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
    targetRevision: main
    path: k8s/apps/agent-pod
  destination:
    server: https://kubernetes.default.svc
    namespace: agent-pod
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
 ---
 # iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
 # running the BlueBubbles server, plus its dedicated local StorageClass.
 #
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:zdc=,iv:VvjvrS5PVNAMIaOE0LaWU+tHcUIYVQDnCANQz6myktY=,tag:xGyWDhRCwwiNny7hPllf5g==,type:str]
 kind: ENC[AES256_GCM,data:2bf5Zfy8,iv:5Oz423GzUWmgdaaZHbrtedwRHIAPIuLh4iDMieLL05s=,tag:GNW1ROjlzGvO/4tIOSuH3Q==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:7RO0Qxkc+/sA,iv:wKe9A8d7QJSx/6rlEY5H6lU8V24TJqr5IpXQCBc8QgM=,tag:iF6z1MNEXG3pCz2cQ18gLg==,type:str]
    namespace: ENC[AES256_GCM,data:RcduxHHLtjH9,iv:opOVx1lL2ltDqQgsleN7NdMAq0TyFr/YQO3FFsHh5AA=,tag:fIwO31apqIatRRzBvamw6g==,type:str]
 type: ENC[AES256_GCM,data:C+JjyZt5,iv:xs49Lz6zRzcf3spiPzdUKTm2HZ+VgFahN6wjIe81JI4=,tag:VllJ54MqU+klA8xAIebjrA==,type:str]
 stringData:
    models.json: ENC[AES256_GCM,data:FvlbdMkcngJchi0GEjIEDjxXpQjRwh+3xnlZNpApgd3v1SpMd0qD2d3TbQr39+feSfsvpxhdmmyU/PmRHZYVdr/QpIKlZBWBD6qTCaOSq62NQbR4ck336muIaWD4AmahWCvRouYEJdEsnG7nedjAivw4cls18vMfnL5pNm2t96foScSyHosWoh3RdT6CWLmJE39+kS6/fwZ1D3Z/GEm/E2zHkVtEbEw/fk1PiqsnmZWt8QcSRAeX8aIDkphV7wOKlkri11r1NuFQKMVIau07VPce1YEi5rsJqsircvDxQelQiMGQt2y2M4GGlXx2NeJgKEtP4Hf9mVGhTeN3CYLcXi8Qyg7k1GIawGVW/Em0kWWfy0GMjvOMfYozNmIpi0YQdYBbr3j6pkixtVM2dWezvld5QIYLnHoGMxw4M6L4IqVIonC3j6tk9pV1DaL5IFskEXyb6ScTrmVY4mQ+VEXvfLYA38jRauWxx/4qf4I7B0RzfGzOx67UJh2jKj0bPWNFHKt5Dz4CK+ul39d1KUPHHMoyaH5gCpOAfLSkjMm44X1Cn+O6uY+QFCugh9MI7bJg3EbovTCZxz95GCupWAvDshv59f7gpGL0T8AuRZHlNp6yZMLdviR9yf79d2rW26PIM1QuNOUWELrOcTs5IAmyuFYr0PI5W5XSf4klOdz3UdhPfMDcyRqtSzVDsSo9oC/WeWOo1+yurtdqqW0wLiFpU/kTaZ4JN5Kl080twqFEqgyPnJrbmlxwWlt4XN2SV9D7pcfE61FzUTwoi5PD/8xt94Fp3XerSgJtwhQ6X4Neo47wCFoMR4Y69mlHrFgSJA4kLFezDc2ISnyAix0W5to0cjwJqQ2EgonIhCXD4xWTvYeApHJMOC2o4B74K5efOZ7klb6PwDMvzM3LyeGzjoJh47aJGvhWN/MpQZC2lSHT8PdMxdGDc+OkEVmsvMncddWMGjQhkTo+69sUexARxLY4H22TCt58azqQGcWa5e7s9NUHb6bRVKHzXh4HMiIhQ0jBev++Jz5gIOyYRnw5xoGbo3ROv8ndczVAqXyAla23EM/VBWT0hLMjr6xxonuLQacP9dPJE9QUofjRMeYblkVvQdIw8WuKGbAlbTR/bokQwGdVp2bywSFQZj7BLlPjzCr0/LJmxYMdq6CKB049PdTlS9/UKBHApddMQ/QRXfOYUCX+lm4k4J19jd7dWk34WNUrTEPSR2d1U/2wTnMvzlrnQHx/zk4/lmI1newfjIbybIbXypX8xVEqzFWXpxUe+iFLP19KmmmHG7gGsV32k1VSGvYFI0HOokZga98QsAbJGbi1gQq8gYpMR/mBdhjJVcnAjOYpg+YaR0vCyrhBN55aSCDoUbT1JOqxvFqmlBHDxTWorD/3KGFxEH/wvSsxussFmUCWdEgYmjGO1ktnQBw1YpOB7mhDUlzvYCSxfpexsYufDYqTbrb/R5gmATC0pEN6EgTth5YqFOL7AiIsogy14RGjRym1wvjVl5SUU9bMQlWGJbQuvNRRwL/TzR/QP210jMsnAh9t8sncDnFPIbhFgeBLsU/ctYoBs0nFVWGWzcuoP9HTnvdWTT2xKmWAeFz8I3DZ5sjm2B787Q4bCrAszTvVLeLiRw/pOE32BeNfG4nNiZDvMQhUomAKpaQypWTu2wLsv3ISm00gud5sRKB6ASJANuSu+EhG9k4OYcBEzAT9PKUxclupFbmsFnXx0u9CXDj3tErrPCUs7DuFSr+fz0ehCLMH+GrcqfytjNDVoTSBRZg+lJYKjAqVCGvq8uRf5V/rdulbt3ggx/uhGSiPzYrBQ0g9iMdjQqpNGBwjlxGkhRE9D2n/PoerAAnHTfwc9Hezffncostjp09xw2CB/xOx9HZFGalQ1R4F5+0lm0iR4lMzc+se+o38foNIiMeMIwAVBJu7IO++UMuKCxpxmdAGqEYj/Ddpzfi3ZcRwe/1HhanVvKRMJsPOhnv0SB2xBrdb7QHnEzLbRTriVRBBdOI77HXWgDt9uQru0tEXCEQ0n+f0BWqHVa/RLN4Yj0dQVWLXU75ddyHnRaCiVdLQnObMvG/QXqY1k/bJTzGNgVF8/KGY51NgdfqskOXC9Fv5IGiyWq+/DL7tQcrmThkzUhZlwDBmwUzPFmzuUC7Z5PkXv2EV+eUrYF16HYqp5KgqTkBHr/JXR8ljb/5gMN929Jk2fBVEuzaEG6W9Yek+Qbx1cnXYrMBI9GWJUJsMsxsAj+1bAjtQeQwt0nXgZesTaV90dfQYq+nTGXMNjqaMhsrJhR6lU1+CDpwY+IObHSttS8AKB+Czsl0nUB44o1a/zKRDNTCGGmPFuv1Q5Ub/1myxorLDDF1mYDpre5TedygtZ/1YSz/YOpKwmWdW32FsnY+m3qjEAcfV5f923N/z8ENVxVpeiWL6mqf7h6F3qTiaSsFq2O8m8tVm70X+oRdig3hPnKqlK/KoxVmzZErhEJ+KmP5I2uvIlb4PViE6imu8kw4VqMClb7jJ7Yk+2Zy84C8RGuDCN8GCDG7FvAahA9xVGxB6rmEpzvqFBxos2EeLOlyfewf7rCw1JAL3C9hZrciA9Jqb/t8B7KqCmCOsKK7xVt1yqyG1dsa364JG/1SKSMKebTheuymZ1+VnGjRZ+pgSAkQGjCtp7B3fTlhjt+WwxW4Ml27yfjokQQ==,iv:8Z39eWukGSMePh/3Dj35e6Zahejil+eeMSqwMYf3snI=,tag:FyBg1y0IQEg/m0mgpf1ESg==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZzB5REh6amhCNzRmY0ta
            MkN3Y0ZlR29lQ3h2SWo3cW5CUThkL2RnU1NVCmt2ZkhIZTlHN1RQRkFrTjVvbjVw
            RGRrTXRoYmdQcnlMSEo3ZWsrZUQ5cHMKLS0tIGU2TGJqZDRxUGJpZzRveEtZankx
            MzhrT1R2akxxby9QVzd1RXB0RDY1LzQKOF+/e5z5lPX6Y1sMTAHuDj3YqW1m+sBd
            u/0R0YnBonYM3wS5nJE3NZMkImaAdQlUjOzQepfBldG+lz++rlnAww==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-18T20:06:08Z"
    mac: ENC[AES256_GCM,data:IM9HkpdwtQE2wCkjwDWOmHH4uP7TlIsrK4TVytiecvYz4SiLk6IRUSIu7I3a+F+dtltC2WtokoATaB69DTXPoI54amzzptirxiFD5FbaU+u2gLjo7KI7V0smYGuKqMYwnod2L/4GdlvP6xjVxFWuA01rQRaYBkFSumS4NABl/I4=,iv:al1MyBmFwni8gap7PPZxWaCwqFKCicfPq6nVmrSn9Xc=,tag:KJGTDQ8xRGF8qMOjoQ+dng==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,23 @@
 apiVersion: ENC[AES256_GCM,data:seY=,iv:1q2bg1QBNDDQPYNu1S08MZ+Ix8WLthxObsrqTMJEVhY=,tag:OMh/stn4vVScOEAyviuurw==,type:str]
 kind: ENC[AES256_GCM,data:QwqgPMrm,iv:tv6rUcezEgQigvHVw05Mevt6BHDD5AARuoUmQpry3AU=,tag:m9RLNezJsLYGSRWnozfkoQ==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:ZJtohnEDN2NBFrEhhvGuw4s=,iv:SU5LuA3Lqjh+q7nVxeVVtzeXiOxxVserNno5c7b47lM=,tag:YLprEk7oVjVsIDVANWDPrQ==,type:str]
    namespace: ENC[AES256_GCM,data:LieEC+YWJzq+,iv:U00VXejzlpgOYS7Yxix/MnEW0PEvmwOuOXMUi2CWIps=,tag:QvFYZIexWhyvBYTruGMifQ==,type:str]
 type: ENC[AES256_GCM,data:U41EbP+H,iv:CKDqec0jfVtbOtEyqQpV9CHJNCwhneYi0wqK8QU2vnY=,tag:CSJ04pkPslevQrpSjhXNHw==,type:str]
 stringData:
    id_forgejo: ENC[AES256_GCM,data:dgwezImUFb2r6MMZ2V1ZMuy32DAVZ9YMzlr9zYlXKEuzeCaWn29DuqgeP5PJQFFKwd37Y9jmzXwuKD5KrC9WCAzrS2VClxmJYHj2Bf842qtKa4x3SubHu4w6wp8v42wQYCU0Kygc5UgkCFYgOBgvpYCFKhiro6Mg/PBTWTIIgrLlSVwUWPIcaX6Sq4ZW4sy6CZ0H/pEMDAjU8drZ4KmGw7W4FshNynplMuP9ewbSs9yxtuAsKt4ezuUQBGHmAYdPHytC9HUzf67z/j6TF/zg6XgTe0AG116cNddAL8oNermCIkkKuTwb5xy1+3WRjjacC0K5Ire1fV5RyXtD/1JgEHJ8gbPK1w1V5fHZXX/ARvQFlP+uqxFYmPkCHBmK/R6/VvAghI/KYHlc76hdsAe0sd1UrPIHjVEtwgvDQl+4de9L9cHG3Qxj23ajlpISWA3tGADhwzXDxi0KYClckebrCKhizWFQLYXezALYklgY2yghG9jZHufV9jRjYSr9DLtwUmVf3oglAzKkWErNs8dWqEKYFx7HpFf0FiZh,iv:1aX4E+R1oFMm9eI1RENFIYCoatX8BdHo6PhrEGq3rtg=,tag:nwniLOOmpYd3Z6R92SvB3Q==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvRmtrdkpMOVhRdTVucVlr
            QmlENGdDN2J1MUczSmRhNUpwYlBjWXhqbERzCnpLRFl1bVVtSFE3L1l4T3Rwd2ZQ
            Z1MxTzJYZWovMmd2RTA3RFJVdVRRTEUKLS0tIDFQWHh2dmQxU1FUZnUxalhTdEd3
            c3dUVzg2L2VOWFlmUG9XdFNWN2RvQkkKYqSmFMkDV/T7AOjKYQNJW85gUzFraRre
            GhwAuPJ9oNQAhSRa5z2p6ghoUplSXtNZ6H2OzETRfOc4N2cHLljW/Q==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-18T04:18:28Z"
    mac: ENC[AES256_GCM,data:ScGMBQqGm31fWDZurmlmk+rnlvyaQkalUcJ6ds/U5JJ2niTf7Llbruuqt2EzMQO09fT2kQqAIYr4cF61Czip6rDhpG8I3H3lBc2mOqUXbeCFyRL6mRpVYPOgkSzgyfXUfe8T++54UhQ0Clv/8Z4XB48oTs9detkUDzAfxP4azRE=,iv:edhsVviyjAm48Blm5CEQ0PBhcWVX97xi2hEbgDX9Dpg=,tag:Xnn85E99LibtB/LWwi5wcA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -0,0 +1,25 @@
 apiVersion: ENC[AES256_GCM,data:894=,iv:Swg6ADUgmrqwz7wqAZHip9/qwFu0Rn8S2Lx4gBH8LJM=,tag:zbv0tFfRLtwFxBfpsuLt8A==,type:str]
 kind: ENC[AES256_GCM,data:jzVtJHYw,iv:ToZ0orfJqfGF/OnAPeYu/g2f4fXAMOZQDkA1+tmIccs=,tag:pi2skbwIU8qOQVEC86MdAA==,type:str]
 metadata:
    name: ENC[AES256_GCM,data:t7zCeZLvAEbkJqUjWi26yD6UDA==,iv:MS1gq/bwKZdLA1itVDtsrdSOfI7e2CrhjvX5yhs0eQA=,tag:lC1gcoFMI5nfzC56U1WXrg==,type:str]
    namespace: ENC[AES256_GCM,data:tp+d,iv:gnlet/60mgbSWLXEQpSlcWD98ky7NvlNCzGLTMys0JQ=,tag:PQYj9UeA50YenQESTCl7lg==,type:str]
    labels:
        konghq.com/credential: ENC[AES256_GCM,data:SOqQ9bLGLK0=,iv:a7En49UhRDwgHbv5NRB/XilEYIKQdaDqKH86WDrJB5I=,tag:JaDZ6Ko8ovY6AZ1hW9YMGQ==,type:str]
 type: ENC[AES256_GCM,data:r1K5gvop,iv:Gjv4oG2Unyql5rY9RTTljFqyd28xI81CDWtpavuuW5E=,tag:jF8lBs3AQwHPVEu9V+mONw==,type:str]
 stringData:
    key: ENC[AES256_GCM,data:pm9GmSvX5MAsXO/e6ZcI4NF1Hwr3qjG6LaEEjvV0ihvSWhO23drlAEsrTtzppqgDZbMORf+5+P53mXU=,iv:5AbHNKeiMPoFQP/qTKdA0vEoYPzuaf4kIdGGcMSmfIQ=,tag:LD5w7XK+hiCS5D410+nCfQ==,type:str]
 sops:
    age:
        - enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUTmoxQkpuYUp5UlRQMkph
            a0RaazdvaU5sWkNuL2gvVjlUYXVWV0dUWVVRClZhWDZCN2hpS2hnOG9Pck9zOTkx
            RGNGMEI3RHpNbzVaaWNGcTNSSEdzZHMKLS0tIFZERnVJWUpreUh3TTlwbGw0dUx4
            MGlCSkxuWWVEK2RaSDZPUzhNSUlCa28KHN0IsgQc/kBqmjQ6+4sgfb9PJy/45MwN
            rXaLJ1htpqPZ9MJ8iOukRi0IKnKgQWXsoZengIxGmcOnEctpoH/kyQ==
            -----END AGE ENCRYPTED FILE-----
          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
    lastmodified: "2026-08-14T01:07:50Z"
    mac: ENC[AES256_GCM,data:nY+YVwU1GuK8Yz+EZOQKkZKN28tm2L8afflc6hsgVFCFmsep5kVT+zId7AgemvQ+qnrho5N5nqxY2knB0gusFfWNKF3V5A5GBq40WtZCMaAtcwhJSex4kK7ZyaZD6oWDW/RTUumSrivSowkWlqt1XlDKyFLqSlpWQTd7JiGmUv8=,iv:zE8+B5UVYSuuAGYyvXsAwp1N/4vGduCoGyEMCNNEUnM=,tag:6RcrjneV2dOzhmoQi+5HsA==,type:str]
    unencrypted_suffix: _unencrypted
    version: 3.13.2
@@ -7,6 +7,8 @@ metadata:
      exec:
        path: ksops
 files:
  - agent-pod-models.enc.yaml
  - agent-pod-ssh-key.enc.yaml
  - authentik-secrets.enc.yaml
  - cloudflare-secrets.enc.yaml
  - forgejo-runner-token.enc.yaml
@@ -18,6 +20,7 @@ files:
  - homarr-secrets.enc.yaml
  - homelab-ca-secrets.enc.yaml
  - loki-secrets.enc.yaml
  - model-invoke-apikey.enc.yaml
  - minio-secrets.enc.yaml
  - vault-secrets.enc.yaml
  - vault-unseal-keys.enc.yaml
@@ -7,6 +7,22 @@ metadata:
  namespace: longhorn-system
  annotations:
    storageclass.kubernetes.io/is-default-class: "false"
    # StorageClass.parameters is immutable. mkfsParams below was added after
    # this class already existed, so every sync failed with
    #   parameters: Invalid value: {...}: field is immutable
    # and the Application sat OutOfSync permanently — ArgoCD retrying a change
    # the API server can never accept.
    #
    # Replace=true makes ArgoCD delete and recreate instead of patching. Safe
    # for a StorageClass: it is consulted only at provisioning time, so bound
    # PVs and their data are untouched. New PVCs briefly fail if one is created
    # during the window.
    #
    # Note what this does NOT do: the nine CNPG volumes already provisioned were
    # formatted without mkfsParams and keep that format. Only volumes created
    # after this recreate get it. Existing volumes that need to expand still
    # need handling separately.
    argocd.argoproj.io/sync-options: Replace=true,Force=true
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 parameters:
@@ -1,6 +1,16 @@
 # Longhorn Node CRDs for cp-2 and cp-3.
 # These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
 # Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
 # `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
 # entry itself (`default-disk-080400000000`, not `default-disk`) and writes
 # `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
 # `default-disk` key here never matched the live one, so the Application sat
 # OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
 # same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
 # What these objects are actually for is `allowScheduling: true` on tainted
 # control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
@@ -10,12 +20,6 @@ metadata:
 spec:
  name: talos-cp-2
  allowScheduling: true
  disks:
    default-disk:
      allowScheduling: true
      path: /var/lib/longhorn
      storageReserved: 0
      tags: []
  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
@@ -26,10 +30,4 @@ metadata:
 spec:
  name: talos-cp-3
  allowScheduling: true
  disks:
    default-disk:
      allowScheduling: true
      path: /var/lib/longhorn
      storageReserved: 0
      tags: []
  tags: []
@@ -0,0 +1,21 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: llm-frontend-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "LLM"
    # Request rate/error/latency/bandwidth now come from Kong's prometheus
    # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
    # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
    # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
    # embeddings/rerank uniformly without per-backend instrumentation.
    # Token-count metrics are still not available: that needs response-body
    # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
    # Predictor-level metrics (native Ollama/vLLM stats) also still need a
    # dedicated exporter -- not added here.
 data:
  llm-frontend.json: |
    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
@@ -19,6 +19,7 @@ resources:
  - dashboards/control-plane-logs.yaml
  - dashboards/hardware-overview.yaml
  - dashboards/kube-controller-health.yaml
  - dashboards/llm-frontend.yaml
  - dashboards/service-availability.yaml
  - dashboards/service-golden-signals.yaml
  - dashboards/service-internals.yaml
@@ -49,12 +49,15 @@ machine:
    image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version}
    wipe: true
    grubUseUKICmdline: true
 %{ if length(longhorn_disks) > 0 ~}
  disks:
-%{ for disk in longhorn_disks ~}
+%{ for idx, disk in longhorn_disks ~}
    # ${disk.kind} — ${disk.device}
    - device: ${disk.device}
      partitions:
-        - mountpoint: ${disk.mountpoint}
+        - mountpoint: ${coalesce(disk.mountpoint, format("/var/lib/longhorn-disk%d", idx + 1))}
 %{ endfor ~}
 %{ endif ~}
  features:
    diskQuotaSupport: true
    kubePrism:
@@ -172,10 +175,10 @@ cluster:
    # kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
    - name: cilium-lb-ippool
      contents: |
-${indent(8, cilium_lb_ippool)}
+        ${indent(8, cilium_lb_ippool)}
    - name: cilium-l2-announcement
      contents: |
-${indent(8, cilium_l2_announcement)}
+        ${indent(8, cilium_l2_announcement)}
    # CoreDNS Corefile with homelab hostname rewrites (single source of truth in
    # terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
    # the nginx ingress controller so OIDC auto-discovery against
@@ -107,10 +107,24 @@ variable "controlplane_configs" {
    lan_subnet   = string
    lan_gateway  = string
    install_disk = string
-    longhorn_disks = list(object({
+    # Longhorn data disks, in mountpoint order. `device` should be a stable
    # /dev/disk/by-id/wwn-* path: this hardware enumerates /dev/sdX by discovery
    # order, and `install.wipe: true` means a renumber can aim the installer at a
    # data disk.
    #
    # `mountpoint` defaults to /var/lib/longhorn-disk<N> by position. Never
    # renumber or reorder existing entries — Longhorn keys its disks off the
    # mountpoint, so a rename orphans the replicas already on that disk. Append
    # new disks to the end.
    #
    # `kind` ("ssd"/"hdd") is a declared label only. PERC RAID controllers report
    # every disk as rotational, so it can't be autodetected, and it deliberately
    # does not affect ordering or mountpoints.
    longhorn_disks = optional(list(object({
      device     = string
-      mountpoint = string
+      mountpoint = optional(string)
-    }))
+      kind       = optional(string, "hdd")
    })), [])
    zone             = string
    allow_scheduling = bool
    # Extra cert SANs for this node — e.g. Cloudflare Tunnel public hostnames so