fix(agent-pod): remote tui session for multi-agent
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,18 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
settings.json: |
|
||||
{
|
||||
"defaultProvider": "homelab-ornith",
|
||||
"defaultModel": "ornith:35b",
|
||||
"defaultThinkingLevel": "medium",
|
||||
"theme": "light",
|
||||
"compaction": {
|
||||
"enabled": true,
|
||||
"reserveTokens": 8192,
|
||||
"keepRecentTokens": 12000
|
||||
}
|
||||
}
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: pi-config
|
||||
namespace: agent-pod
|
||||
@@ -0,0 +1,36 @@
|
||||
# Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
|
||||
# (trigger a new session) -- both are routes on the same hub.js service.
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: console
|
||||
namespace: agent-pod
|
||||
annotations:
|
||||
konghq.com/strip-path: "false"
|
||||
spec:
|
||||
ingressClassName: kong
|
||||
rules:
|
||||
- host: api.riotpiao.com
|
||||
http:
|
||||
paths:
|
||||
- path: /console
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: agent-hub
|
||||
port:
|
||||
number: 9090
|
||||
- path: /run
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: agent-hub
|
||||
port:
|
||||
number: 9090
|
||||
- path: /sessions
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: agent-hub
|
||||
port:
|
||||
number: 9090
|
||||
@@ -0,0 +1,108 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: agent-pod
|
||||
namespace: agent-pod
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: agent-pod
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: agent-pod
|
||||
spec:
|
||||
# api.riotpiao.com has no in-cluster DNS record (only resolves from the
|
||||
# home network's own resolver) -- pin it to ingress-nginx-controller's
|
||||
# ClusterIP so pi's models.json baseUrl works unchanged. TLS still
|
||||
# terminates correctly since SNI/Host still say api.riotpiao.com.
|
||||
hostAliases:
|
||||
- ip: "10.101.128.185"
|
||||
hostnames:
|
||||
- "api.riotpiao.com"
|
||||
containers:
|
||||
# hub.js runs in the same container as pi (not a sidecar) so it can
|
||||
# spawn `pi -p --mode json` directly via child_process -- a separate
|
||||
# container can't exec into another container's filesystem/PATH.
|
||||
# It IS the container's long-running process now; no more `sleep
|
||||
# infinity` placeholder.
|
||||
- name: pi
|
||||
image: node:22-slim
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
apt-get update && apt-get install -y git curl jq openssh-client tmux
|
||||
ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub
|
||||
eval "$(ssh-agent -s)"
|
||||
ssh-add /root/.ssh/id_forgejo
|
||||
npm install -g @earendil-works/[email protected]
|
||||
npm install --prefix /root ws
|
||||
node /root/hub.js
|
||||
env:
|
||||
- name: PI_BIN
|
||||
value: pi
|
||||
ports:
|
||||
- containerPort: 9090
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "8"
|
||||
memory: 16Gi
|
||||
volumeMounts:
|
||||
- name: pi-config
|
||||
mountPath: /root/.pi/agent/settings.json
|
||||
subPath: settings.json
|
||||
- name: pi-models
|
||||
mountPath: /root/.pi/agent/models.json
|
||||
subPath: models.json
|
||||
- name: pi-skills
|
||||
mountPath: /root/.pi/agent/skills
|
||||
- name: hub-src
|
||||
mountPath: /root/hub.js
|
||||
subPath: hub.js
|
||||
- name: ssh-key
|
||||
mountPath: /root/.ssh/id_forgejo
|
||||
subPath: id_forgejo
|
||||
- name: ssh-config
|
||||
mountPath: /root/.ssh/config
|
||||
subPath: config
|
||||
volumes:
|
||||
- name: pi-config
|
||||
configMap:
|
||||
name: pi-config
|
||||
- name: pi-models
|
||||
secret:
|
||||
secretName: pi-models
|
||||
- name: pi-skills
|
||||
configMap:
|
||||
name: pi-skills
|
||||
items:
|
||||
- key: planner-SKILL.md
|
||||
path: planner/SKILL.md
|
||||
- key: investigator-SKILL.md
|
||||
path: investigator/SKILL.md
|
||||
- key: info-collector-SKILL.md
|
||||
path: info-collector/SKILL.md
|
||||
- key: implementer-SKILL.md
|
||||
path: implementer/SKILL.md
|
||||
- key: judge-SKILL.md
|
||||
path: judge/SKILL.md
|
||||
- key: resolver-SKILL.md
|
||||
path: resolver/SKILL.md
|
||||
- key: brave-search-SKILL.md
|
||||
path: brave-search/SKILL.md
|
||||
- name: hub-src
|
||||
configMap:
|
||||
name: hub-src
|
||||
- name: ssh-key
|
||||
secret:
|
||||
secretName: agent-pod-ssh-key
|
||||
defaultMode: 0600
|
||||
- name: ssh-config
|
||||
configMap:
|
||||
name: agent-pod-ssh-config
|
||||
@@ -0,0 +1,700 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
hub.js: |
|
||||
#!/usr/bin/env node
|
||||
// agent-hub: lives inside the pi container (not a sidecar) so it can spawn
|
||||
// `pi` directly, and control the pod's own tmux server. One persistent
|
||||
// in-cluster service -- POST /run to trigger a single ad-hoc headless agent
|
||||
// run, POST /pipeline to run an ordered list of task phases against a
|
||||
// repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within
|
||||
// a phase run concurrently, each in its own git worktree). Within one task,
|
||||
// planner/investigator/implementer/judge are separate agents in separate
|
||||
// named tmux sessions (task-<id>-<role>, attachable via `kubectl exec -it --
|
||||
// tmux attach -t <name>` while running), coordinating only through what's on
|
||||
// disk in that task's worktree -- not one shared conversation. GET /console
|
||||
// (WebSocket) watches every concurrent headless run live, relaying pi's own
|
||||
// session protocol verbatim (same event shape Claude Code sessions use).
|
||||
const http = require("node:http");
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const { spawn } = require("node:child_process");
|
||||
const readline = require("node:readline");
|
||||
const { WebSocketServer } = require("ws");
|
||||
|
||||
const PORT = process.env.HUB_PORT || 9090;
|
||||
const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work");
|
||||
|
||||
// Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide
|
||||
// with unrelated tools on this machine (a Rust CLI and a Datadog TUI,
|
||||
// respectively, discovered the hard way this session). Always invoke the
|
||||
// exact pinned @earendil-works/[email protected] installed locally under
|
||||
// .pi-cli/, by explicit path.
|
||||
const PI_BIN =
|
||||
process.env.PI_BIN ||
|
||||
path.join(
|
||||
__dirname,
|
||||
"..",
|
||||
".pi-cli",
|
||||
"node_modules",
|
||||
"@earendil-works",
|
||||
"pi-coding-agent",
|
||||
"dist",
|
||||
"cli.js"
|
||||
);
|
||||
|
||||
// Job-type skills under pi/skills/<name>/SKILL.md (mounted at
|
||||
// ~/.pi/agent/skills/<name>/ in agent-pod). When `agent` matches one of
|
||||
// these, the prompt is forced through pi's `/skill:<name> <args>` mechanism
|
||||
// instead of being sent bare -- see pi's skills.md docs on single-shot skill
|
||||
// loading. `resolver` is never dispatched directly by a caller; only the
|
||||
// pipeline driver invokes it, on stage crashes.
|
||||
const ROLE_SKILLS = new Set([
|
||||
"planner",
|
||||
"investigator",
|
||||
"info-collector",
|
||||
"implementer",
|
||||
"judge",
|
||||
"resolver",
|
||||
]);
|
||||
|
||||
const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?}
|
||||
const viewers = new Set(); // WebSocket connections watching /console
|
||||
|
||||
function broadcast(type, session) {
|
||||
const msg = JSON.stringify({ type, session });
|
||||
for (const ws of viewers) {
|
||||
if (ws.readyState === ws.OPEN) ws.send(msg);
|
||||
}
|
||||
}
|
||||
|
||||
function startSession(agent, extra = {}) {
|
||||
const id = extra.id || crypto.randomUUID();
|
||||
const session = {
|
||||
...extra,
|
||||
id,
|
||||
agent,
|
||||
status: "running",
|
||||
events: [],
|
||||
startedAt: new Date().toISOString(),
|
||||
};
|
||||
sessions.set(id, session);
|
||||
broadcast("start", session);
|
||||
return session;
|
||||
}
|
||||
|
||||
function addEvent(session, rawLine) {
|
||||
const event = JSON.parse(rawLine);
|
||||
session.events.push(event);
|
||||
broadcast("event", session);
|
||||
return event;
|
||||
}
|
||||
|
||||
function endSession(session, status) {
|
||||
session.status = status;
|
||||
session.endedAt = new Date().toISOString();
|
||||
broadcast("end", session);
|
||||
}
|
||||
|
||||
// Extracts the plain-text content of a message_end event, if any -- used to
|
||||
// find the VERDICT:/RESOLUTION: line judge/resolver skills are required to
|
||||
// end their final message with.
|
||||
function textOf(event) {
|
||||
if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) {
|
||||
return null;
|
||||
}
|
||||
return event.message.content
|
||||
.filter((c) => c.type === "text")
|
||||
.map((c) => c.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
// Core spawn primitive. Spawns `pi -p --mode json <extraArgs> <prompt>`,
|
||||
// relays every line as a session event exactly like before. Returns
|
||||
// { session, done } -- `session` is available synchronously (so an HTTP
|
||||
// handler can respond with its id right away, same as the old runAgent),
|
||||
// `done` is a Promise resolving once the process exits, for callers that
|
||||
// need to wait on a stage (the pipeline driver) rather than fire-and-forget.
|
||||
function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) {
|
||||
const session = startSession(agent, sessionExtra);
|
||||
const args = ["-p", "--mode", "json"];
|
||||
if (provider) args.push("--provider", provider);
|
||||
if (model) args.push("--model", model);
|
||||
const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt;
|
||||
args.push(finalPrompt);
|
||||
|
||||
const child = spawn(PI_BIN, args, {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
cwd,
|
||||
});
|
||||
const rl = readline.createInterface({ input: child.stdout });
|
||||
let lastText = "";
|
||||
let stderrTail = "";
|
||||
|
||||
rl.on("line", (line) => {
|
||||
if (!line.trim()) return;
|
||||
try {
|
||||
const event = addEvent(session, line);
|
||||
const text = textOf(event);
|
||||
if (text) lastText = text;
|
||||
} catch {
|
||||
// non-JSON stdout noise, ignore
|
||||
}
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
process.stderr.write(chunk);
|
||||
stderrTail = (stderrTail + chunk.toString()).slice(-4000);
|
||||
});
|
||||
|
||||
const done = new Promise((resolve) => {
|
||||
child.on("close", (code) => {
|
||||
endSession(session, code === 0 ? "done" : "error");
|
||||
resolve({ code, session, lastText, stderrTail });
|
||||
});
|
||||
});
|
||||
|
||||
return { session, done };
|
||||
}
|
||||
|
||||
function runAgent(agent, prompt, extraArgs = {}) {
|
||||
// Fire-and-forget: caller (the /run handler) doesn't await `done`.
|
||||
return spawnPi({ agent, prompt, ...extraArgs }).session;
|
||||
}
|
||||
|
||||
function parseVerdictLine(text, label) {
|
||||
if (!text) return null;
|
||||
const re = new RegExp(`${label}:\\s*(\\w+)`, "i");
|
||||
const m = text.match(re);
|
||||
return m ? m[1].toUpperCase() : null;
|
||||
}
|
||||
|
||||
// Deterministic git operations, run directly by hub.js rather than left to
|
||||
// the model -- branch creation and pushing after each task are mechanical,
|
||||
// not judgment calls, and need to happen reliably every time regardless of
|
||||
// what a task's stages did or didn't remember to do.
|
||||
function runCmd(bin, args, cwd) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
child.stdout.on("data", (c) => (out += c));
|
||||
child.stderr.on("data", (c) => (out += c));
|
||||
child.on("close", (code) => resolve({ code, out: out.trim() }));
|
||||
});
|
||||
}
|
||||
|
||||
function runGit(cwd, args) {
|
||||
return runCmd("git", args, cwd);
|
||||
}
|
||||
|
||||
// A stage saying "commit" in its prompt is a request, not a guarantee -- seen
|
||||
// in practice: a stage writes a real file and simply never runs `git add`/
|
||||
// `git commit`, leaving it untracked and invisible to every later `git diff`.
|
||||
// Sweep and commit anything left dirty after every stage, deterministically.
|
||||
async function commitPending(cwd, message) {
|
||||
await runGit(cwd, ["add", "-A"]);
|
||||
const status = await runGit(cwd, ["status", "--porcelain"]);
|
||||
if (!status.out) return { committed: false };
|
||||
const commit = await runGit(cwd, ["commit", "-m", message]);
|
||||
return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined };
|
||||
}
|
||||
|
||||
// Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide
|
||||
// RETRY vs ABORT. Shared by both crash-recovery paths below (headless
|
||||
// exit-code failures and interactive sentinel-file timeouts) -- the
|
||||
// diagnostic prompt differs per caller, but "ask resolver, parse the
|
||||
// RESOLUTION: line" is identical either way.
|
||||
async function askResolver(pipelineId, cwd, task, diagnosticPrompt) {
|
||||
const resolverResult = await spawnPi({
|
||||
agent: "resolver",
|
||||
prompt: diagnosticPrompt,
|
||||
cwd,
|
||||
sessionExtra: { pipelineId, stage: "resolver", task },
|
||||
}).done;
|
||||
return parseVerdictLine(resolverResult.lastText, "RESOLUTION");
|
||||
}
|
||||
|
||||
// Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic
|
||||
// judge FAIL, which is handled separately), asks the resolver to diagnose
|
||||
// and decide RETRY vs ABORT. Retries the failed stage at most once,
|
||||
// regardless of what resolver recommends a second time -- a hard cap, not
|
||||
// indefinite trust in the model's judgment.
|
||||
async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) {
|
||||
let result = await spawnPi({
|
||||
agent: stage,
|
||||
prompt,
|
||||
cwd,
|
||||
sessionExtra: { pipelineId, stage, task },
|
||||
}).done;
|
||||
if (result.code === 0) return result;
|
||||
|
||||
const resolution = await askResolver(
|
||||
pipelineId,
|
||||
cwd,
|
||||
task,
|
||||
`Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}`
|
||||
);
|
||||
|
||||
if (resolution === "RETRY") {
|
||||
result = await spawnPi({
|
||||
agent: stage,
|
||||
prompt,
|
||||
cwd,
|
||||
sessionExtra: { pipelineId, stage, task },
|
||||
}).done;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Deterministic tmux operations -- same rationale as runGit: mechanical,
|
||||
// not a judgment call, run directly rather than trusted to a prompt.
|
||||
function runTmux(args) {
|
||||
return runCmd("tmux", args);
|
||||
}
|
||||
|
||||
function tmuxSessionName(task) {
|
||||
return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
||||
}
|
||||
|
||||
// Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in
|
||||
// flight at once. No external dep; a plain in-order index cursor shared by
|
||||
// `limit` runner loops.
|
||||
async function runConcurrent(items, limit, worker) {
|
||||
const results = new Array(items.length);
|
||||
let i = 0;
|
||||
async function next() {
|
||||
while (i < items.length) {
|
||||
const idx = i++;
|
||||
results[idx] = await worker(items[idx], idx);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next));
|
||||
return results;
|
||||
}
|
||||
|
||||
const STAGE_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
const NUDGE_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
const POLL_MS = 10 * 1000;
|
||||
|
||||
async function waitForFile(filePath, limitMs) {
|
||||
const start = Date.now();
|
||||
while (!fs.existsSync(filePath)) {
|
||||
if (Date.now() - start > limitMs) return false;
|
||||
await new Promise((r) => setTimeout(r, POLL_MS));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const MAX_IMPLEMENT_ATTEMPTS = 5;
|
||||
const MAX_PLAN_REVISIONS = 3;
|
||||
|
||||
// Runs one role as its own fresh interactive pi session in its own named
|
||||
// tmux session -- planner, investigator, implementer, and judge are
|
||||
// separate agents with separate context, not turns in one shared
|
||||
// conversation. They coordinate only through what's on disk in the task's
|
||||
// worktree: PLAN.md, committed code, judge's result file. Each session is
|
||||
// attachable while it runs (kubectl exec -it -- tmux attach -t <name>) and
|
||||
// killed once its sentinel file lands or it's abandoned after resolver
|
||||
// escalation.
|
||||
async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) {
|
||||
const sessionName = `${tmuxSessionName(task)}-${stageLabel}`;
|
||||
fs.rmSync(sentinelFile, { force: true });
|
||||
|
||||
const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]);
|
||||
if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName };
|
||||
|
||||
let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS);
|
||||
if (!ok) {
|
||||
const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]);
|
||||
const resolution = await askResolver(
|
||||
pipelineId,
|
||||
cwd,
|
||||
task,
|
||||
`Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` +
|
||||
`Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` +
|
||||
`real progress and worth nudging to wrap up, or stuck and worth abandoning?`
|
||||
);
|
||||
if (resolution === "RETRY") {
|
||||
await runTmux([
|
||||
"send-keys",
|
||||
"-t",
|
||||
sessionName,
|
||||
`Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`,
|
||||
"Enter",
|
||||
]);
|
||||
ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS);
|
||||
}
|
||||
}
|
||||
|
||||
await runTmux(["kill-session", "-t", sessionName]);
|
||||
return { ok, sessionName };
|
||||
}
|
||||
|
||||
// Runs one task in its own git worktree (see runPhase): planner drafts
|
||||
// PLAN.md, investigator confirms it, then implementer and judge go back and
|
||||
// forth -- judge's FAIL rationale lands in .task-result-<task>, which the
|
||||
// next implementer attempt is told to read and address. After
|
||||
// MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to
|
||||
// judge whether the *plan* itself is wrong, not just the implementation; if
|
||||
// so it revises PLAN.md and the implementer gets a fresh attempt budget
|
||||
// against the new plan. MAX_PLAN_REVISIONS caps this from looping forever
|
||||
// on a task that's genuinely stuck.
|
||||
async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) {
|
||||
const resultFile = path.join(cwd, `.task-result-${task}`);
|
||||
fs.rmSync(resultFile, { force: true });
|
||||
|
||||
const specHint = `the file under tasks/ starting with "${task}-"`;
|
||||
|
||||
const runRole = async (stageLabel, prompt, sentinel) => {
|
||||
pipelineSession.activeTasks[task] = {
|
||||
stage: stageLabel,
|
||||
sessionName: `${tmuxSessionName(task)}-${stageLabel}`,
|
||||
startedAt: new Date().toISOString(),
|
||||
};
|
||||
broadcast("event", pipelineSession);
|
||||
const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel);
|
||||
await commitPending(cwd, `task: ${task} (${stageLabel})`);
|
||||
return result;
|
||||
};
|
||||
|
||||
const abandon = (stageLabel, result, attempt) => {
|
||||
delete pipelineSession.activeTasks[task];
|
||||
broadcast("event", pipelineSession);
|
||||
return {
|
||||
task,
|
||||
status: result.crashed ? "spawn-crashed" : "timed-out",
|
||||
error: result.error,
|
||||
stoppedAt: stageLabel,
|
||||
...(attempt !== undefined ? { attempt } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
// Task's implementation is inherited already-committed (e.g. from a base
|
||||
// branch of prior work) -- try one judge pass against the spec directly
|
||||
// (no PLAN.md exists yet) before paying for a full planner/investigator
|
||||
// redo. PASS ends the task here; FAIL falls through into the normal flow
|
||||
// below, so planner/implementer pick up with the judge's real feedback.
|
||||
if (judgeOnly) {
|
||||
const quick = await runRole(
|
||||
"judge",
|
||||
`Task ${task} may already be implemented on this branch -- check ` +
|
||||
`\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` +
|
||||
`(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` +
|
||||
`verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` +
|
||||
`one line of rationale, then run: touch .stage-done-${task}-judge-0`,
|
||||
path.join(cwd, `.stage-done-${task}-judge-0`)
|
||||
);
|
||||
if (!quick.ok) return abandon("judge", quick, 0);
|
||||
|
||||
const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
|
||||
if (parseVerdictLine(quickText, "VERDICT") === "PASS") {
|
||||
delete pipelineSession.activeTasks[task];
|
||||
broadcast("event", pipelineSession);
|
||||
return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true };
|
||||
}
|
||||
}
|
||||
|
||||
let result = await runRole(
|
||||
"planner",
|
||||
`Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`,
|
||||
path.join(cwd, `.stage-done-${task}-planner`)
|
||||
);
|
||||
if (!result.ok) return abandon("planner", result);
|
||||
|
||||
result = await runRole(
|
||||
"investigator",
|
||||
`Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`,
|
||||
path.join(cwd, `.stage-done-${task}-investigator`)
|
||||
);
|
||||
if (!result.ok) return abandon("investigator", result);
|
||||
|
||||
let planRevisions = 0;
|
||||
let implementAttempt = 0;
|
||||
let verdict = null;
|
||||
let resultText = "";
|
||||
let justRevisedPlan = false;
|
||||
|
||||
while (true) {
|
||||
implementAttempt++;
|
||||
const feedbackHint = fs.existsSync(resultFile)
|
||||
? justRevisedPlan
|
||||
? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` +
|
||||
`PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.`
|
||||
: `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.`
|
||||
: "";
|
||||
justRevisedPlan = false;
|
||||
|
||||
result = await runRole(
|
||||
"implementer",
|
||||
`Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`,
|
||||
path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`)
|
||||
);
|
||||
if (!result.ok) return abandon("implementer", result, implementAttempt);
|
||||
|
||||
result = await runRole(
|
||||
"judge",
|
||||
`Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` +
|
||||
`.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` +
|
||||
`rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`,
|
||||
path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`)
|
||||
);
|
||||
if (!result.ok) return abandon("judge", result, implementAttempt);
|
||||
|
||||
resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : "";
|
||||
verdict = parseVerdictLine(resultText, "VERDICT");
|
||||
if (verdict === "PASS") break;
|
||||
|
||||
if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) {
|
||||
if (planRevisions >= MAX_PLAN_REVISIONS) break;
|
||||
planRevisions++;
|
||||
result = await runRole(
|
||||
"planner-revise",
|
||||
`Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` +
|
||||
`the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` +
|
||||
`whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` +
|
||||
`commit. If you change the approach, also use the investigator skill to confirm the new approach against ` +
|
||||
`real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` +
|
||||
`touch .stage-done-${task}-planner-revise-${planRevisions}`,
|
||||
path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`)
|
||||
);
|
||||
if (!result.ok) return abandon("planner-revise", result, planRevisions);
|
||||
implementAttempt = 0;
|
||||
justRevisedPlan = true;
|
||||
}
|
||||
}
|
||||
|
||||
delete pipelineSession.activeTasks[task];
|
||||
broadcast("event", pipelineSession);
|
||||
|
||||
if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) {
|
||||
return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions };
|
||||
}
|
||||
return {
|
||||
task,
|
||||
status: verdict === "PASS" ? "done" : "done-with-concerns",
|
||||
judgeRationale: resultText,
|
||||
};
|
||||
}
|
||||
|
||||
const PHASE_CONCURRENCY = 3;
|
||||
|
||||
// Runs one phase (a batch of tasks with no declared dependency on each
|
||||
// other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task
|
||||
// gets its own git worktree off workBranch -- concurrent pi sessions writing
|
||||
// into one shared working tree would corrupt the index; worktrees share the
|
||||
// same object database but give each task an isolated checkout. After a
|
||||
// task's session ends, its branch is merged back into workBranch and pushed,
|
||||
// one merge at a time (git ref updates aren't safe to run concurrently even
|
||||
// though the worktrees themselves are isolated).
|
||||
async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) {
|
||||
// Each entry is either a plain task id, or { id, judgeOnly: true } when
|
||||
// the task's implementation already exists (e.g. inherited from a base
|
||||
// branch) and just needs a real judge pass rather than a full
|
||||
// planner/investigator/implementer redo.
|
||||
const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t));
|
||||
|
||||
const worktrees = {};
|
||||
for (const entry of entries) {
|
||||
const task = entry.id;
|
||||
const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`);
|
||||
const taskBranch = `task/${task}`;
|
||||
const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]);
|
||||
if (add.code !== 0) {
|
||||
pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out });
|
||||
continue;
|
||||
}
|
||||
worktrees[task] = { wtDir, taskBranch };
|
||||
}
|
||||
|
||||
const runnable = entries.filter((e) => worktrees[e.id]);
|
||||
await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => {
|
||||
const { wtDir } = worktrees[entry.id];
|
||||
const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly);
|
||||
pipelineSession.taskResults.push(result);
|
||||
return result;
|
||||
});
|
||||
|
||||
// Merge + push sequentially -- ref updates on the shared repo, one at a
|
||||
// time, in the declared task order for this phase.
|
||||
for (const entry of runnable) {
|
||||
const task = entry.id;
|
||||
const { wtDir, taskBranch } = worktrees[task];
|
||||
const result = pipelineSession.taskResults.find((r) => r.task === task);
|
||||
|
||||
const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]);
|
||||
if (merge.code !== 0) {
|
||||
await runGit(cwd, ["merge", "--abort"]);
|
||||
if (result) {
|
||||
result.status = "merge-conflict";
|
||||
result.mergeError = merge.out;
|
||||
}
|
||||
} else {
|
||||
const push = await runGit(cwd, ["push", "-u", "origin", workBranch]);
|
||||
if (result) {
|
||||
result.pushed = push.code === 0;
|
||||
if (!result.pushed) result.pushError = push.out;
|
||||
}
|
||||
}
|
||||
|
||||
await runGit(cwd, ["worktree", "remove", wtDir, "--force"]);
|
||||
await runGit(cwd, ["branch", "-D", taskBranch]);
|
||||
broadcast("event", pipelineSession);
|
||||
}
|
||||
}
|
||||
|
||||
// tasks: array of phases, each phase an array of task ids with no declared
|
||||
// dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) --
|
||||
// caller's responsibility to supply real phase grouping (see tasks/INDEX.md;
|
||||
// filename/numeric sort does NOT match execution order on boards like this).
|
||||
// A flat array of ids is also accepted and treated as one single phase.
|
||||
// Phases run strictly sequentially (a phase boundary is a real dependency
|
||||
// gate); tasks within a phase run concurrently, each in its own worktree --
|
||||
// see runPhase.
|
||||
function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) {
|
||||
const cwd = path.join(WORK_DIR, pipelineId);
|
||||
fs.mkdirSync(cwd, { recursive: true });
|
||||
|
||||
const phases = Array.isArray(tasks[0]) ? tasks : [tasks];
|
||||
|
||||
const pipelineSession = startSession("pipeline", {
|
||||
id: pipelineId,
|
||||
pipelineId,
|
||||
stage: "pipeline",
|
||||
taskResults: [],
|
||||
activeTasks: {},
|
||||
totalTasks: phases.flat().length,
|
||||
});
|
||||
|
||||
(async () => {
|
||||
const clone = await runStageWithResolver(
|
||||
pipelineId,
|
||||
cwd,
|
||||
"planner",
|
||||
`Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`,
|
||||
"clone"
|
||||
);
|
||||
if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed");
|
||||
if (!fs.existsSync(path.join(cwd, ".git"))) {
|
||||
// The model deciding to `cd` elsewhere before cloning (instead of
|
||||
// cloning into the assigned cwd) is a real failure mode seen in
|
||||
// practice, not a hypothetical -- exit code 0 doesn't mean the clone
|
||||
// landed where every later stage's cwd assumes it did.
|
||||
return endSession(pipelineSession, "clone-missing");
|
||||
}
|
||||
|
||||
// Dedicated branch, never main -- and pushed after every single task
|
||||
// (not just at the end) so a pod restart mid-run loses at most the
|
||||
// in-progress task's work, not everything since the start.
|
||||
const workBranch = branchName || `agent-run/${pipelineId}`;
|
||||
const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]);
|
||||
if (branchResult.code !== 0) {
|
||||
pipelineSession.gitError = branchResult.out;
|
||||
return endSession(pipelineSession, "branch-crashed");
|
||||
}
|
||||
|
||||
// Seen in practice: a task manually downloads a dependency tarball
|
||||
// (crates.io registry access isn't guaranteed from every sandboxed
|
||||
// checkout) and it lands at repo root, outside whatever .gitignore
|
||||
// already covers -- then git add -A (ours or the model's own) commits
|
||||
// it. Append broad build-artifact/archive patterns before any task
|
||||
// runs, so it's excluded regardless of who stages files later.
|
||||
const gitignoreAdditions = [
|
||||
"",
|
||||
"# agent-harness: build artifacts and vendored archives never belong in source control",
|
||||
"*.tar.gz",
|
||||
"*.tgz",
|
||||
"*.crate",
|
||||
"*.zip",
|
||||
"*.bin",
|
||||
"*.whl",
|
||||
"vendor/",
|
||||
"node_modules/",
|
||||
"",
|
||||
"# agent-harness: task completion sentinel files, harness bookkeeping only",
|
||||
".task-result-*",
|
||||
".stage-done-*",
|
||||
].join("\n");
|
||||
fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
|
||||
await runGit(cwd, ["add", ".gitignore"]);
|
||||
await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
|
||||
|
||||
for (const phaseTasks of phases) {
|
||||
await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession);
|
||||
}
|
||||
|
||||
const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed"));
|
||||
endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed");
|
||||
})();
|
||||
|
||||
return pipelineSession;
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
const url = new URL(req.url, "http://localhost");
|
||||
|
||||
if (url.pathname === "/healthz") {
|
||||
res.writeHead(200).end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/sessions" && req.method === "GET") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify([...sessions.values()]));
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/run" && req.method === "POST") {
|
||||
let body = "";
|
||||
req.on("data", (chunk) => (body += chunk));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
const { agent, prompt, provider, model } = JSON.parse(body);
|
||||
if (!agent || !prompt) throw new Error("agent and prompt are required");
|
||||
const session = runAgent(agent, prompt, { provider, model });
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ id: session.id }));
|
||||
} catch (err) {
|
||||
res.writeHead(400, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ error: err.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (url.pathname === "/pipeline" && req.method === "POST") {
|
||||
let body = "";
|
||||
req.on("data", (chunk) => (body += chunk));
|
||||
req.on("end", () => {
|
||||
try {
|
||||
const { repo, baseBranch, tasks, branchName } = JSON.parse(body);
|
||||
if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) {
|
||||
throw new Error("repo, baseBranch, and a non-empty tasks array are required");
|
||||
}
|
||||
const pipelineId = crypto.randomUUID();
|
||||
runPipeline({ pipelineId, repo, baseBranch, tasks, branchName });
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ id: pipelineId }));
|
||||
} catch (err) {
|
||||
res.writeHead(400, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ error: err.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404).end();
|
||||
});
|
||||
|
||||
const wss = new WebSocketServer({ server, path: "/console" });
|
||||
wss.on("connection", (ws) => {
|
||||
for (const session of sessions.values()) {
|
||||
ws.send(JSON.stringify({ type: "snapshot", session }));
|
||||
}
|
||||
viewers.add(ws);
|
||||
ws.on("close", () => viewers.delete(ws));
|
||||
});
|
||||
|
||||
server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`));
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: hub-src
|
||||
namespace: agent-pod
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: agent-hub
|
||||
namespace: agent-pod
|
||||
spec:
|
||||
selector:
|
||||
app: agent-pod
|
||||
ports:
|
||||
- port: 9090
|
||||
targetPort: 9090
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: agent-pod
|
||||
resources:
|
||||
- deployment.yaml
|
||||
- configmap.yaml
|
||||
- hub-configmap.yaml
|
||||
- pi-skills-configmap.yaml
|
||||
- ssh-configmap.yaml
|
||||
- hub-service.yaml
|
||||
- console-ingress.yaml
|
||||
@@ -0,0 +1,156 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
brave-search-SKILL.md: |
|
||||
---
|
||||
name: brave-search
|
||||
description: Web search via the Brave Search API, called directly with curl. Use for searching documentation, facts, or any current web content.
|
||||
allowed-tools: Bash
|
||||
---
|
||||
|
||||
# Brave Search
|
||||
|
||||
Direct HTTP call to the Brave Search API — no separate script or package, just `curl` (`BRAVE_API_KEY` is already set in the environment).
|
||||
|
||||
## Search
|
||||
|
||||
```bash
|
||||
curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
|
||||
--get --data-urlencode "q=<query>" --data-urlencode "count=5" \
|
||||
"https://api.search.brave.com/res/v1/web/search"
|
||||
```
|
||||
|
||||
Options (add as extra `--data-urlencode` pairs):
|
||||
- `count=<n>` — number of results (max 20, default 5)
|
||||
- `country=<code>` — two-letter country code (default US)
|
||||
- `freshness=pd|pw|pm|py` — past day/week/month/year, or `freshness=YYYY-MM-DDtoYYYY-MM-DD`
|
||||
|
||||
Response is JSON; the results live at `.web.results[]`, each with `title`, `url`, `description`, `age`. Pipe through `jq` if you want a shorter view, e.g.:
|
||||
|
||||
```bash
|
||||
curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \
|
||||
--get --data-urlencode "q=<query>" "https://api.search.brave.com/res/v1/web/search" \
|
||||
| jq -r '.web.results[] | "- \(.title)\n \(.url)\n \(.description)\n"'
|
||||
```
|
||||
|
||||
There's no page-content-extraction helper here — if a result needs reading in full, `curl` the URL directly and read the raw HTML/text; don't expect readability-cleaned markdown.
|
||||
|
||||
## When to Use
|
||||
|
||||
- Searching for documentation or API references
|
||||
- Looking up facts or current information
|
||||
- Confirming a claim or approach against real sources
|
||||
implementer-SKILL.md: |
|
||||
---
|
||||
name: implementer
|
||||
description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run.
|
||||
allowed-tools: Read Grep Find Ls Write Edit Bash
|
||||
---
|
||||
|
||||
Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions.
|
||||
|
||||
- Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history.
|
||||
- Push only if the task explicitly asks for it.
|
||||
|
||||
Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch.
|
||||
|
||||
**Hard rules:**
|
||||
- Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing.
|
||||
- Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it.
|
||||
|
||||
**Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it.
|
||||
info-collector-SKILL.md: |
|
||||
---
|
||||
name: info-collector
|
||||
description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill).
|
||||
allowed-tools: Read Bash
|
||||
---
|
||||
|
||||
**Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there.
|
||||
|
||||
**Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job).
|
||||
|
||||
**Modes:**
|
||||
|
||||
- **Collect mode** (default) — use the `brave-search` skill (`curl` against the Brave Search API) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill.
|
||||
- If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response.
|
||||
investigator-SKILL.md: |
|
||||
---
|
||||
name: investigator
|
||||
description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct.
|
||||
allowed-tools: Read Bash
|
||||
---
|
||||
|
||||
Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one.
|
||||
|
||||
- Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), use `brave-search` to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`.
|
||||
- No `PLAN.md`? Just answer the question asked, citing sources.
|
||||
|
||||
Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next.
|
||||
|
||||
**Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not.
|
||||
judge-SKILL.md: |
|
||||
---
|
||||
name: judge
|
||||
description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria.
|
||||
allowed-tools: Read Bash
|
||||
---
|
||||
|
||||
Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp.
|
||||
|
||||
- Run `git diff <base-branch>...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection?
|
||||
- FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale.
|
||||
- No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly.
|
||||
|
||||
You MUST end your final message with a literal verdict line, exactly one of:
|
||||
|
||||
```
|
||||
VERDICT: PASS
|
||||
```
|
||||
```
|
||||
VERDICT: FAIL
|
||||
```
|
||||
|
||||
followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline.
|
||||
planner-SKILL.md: |
|
||||
---
|
||||
name: planner
|
||||
description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it.
|
||||
allowed-tools: Read Grep Find Ls Write Bash
|
||||
---
|
||||
|
||||
Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms.
|
||||
|
||||
- If cwd is empty: `git clone` the given repo/branch first (only at pipeline start).
|
||||
- Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity.
|
||||
- Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/<name>.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: <summary>"`.
|
||||
|
||||
Don't touch any other file.
|
||||
resolver-SKILL.md: |
|
||||
---
|
||||
name: resolver
|
||||
description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion.
|
||||
allowed-tools: Read Bash
|
||||
---
|
||||
|
||||
**Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again.
|
||||
|
||||
**Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work.
|
||||
|
||||
**Modes:**
|
||||
|
||||
- **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix).
|
||||
|
||||
You MUST end your final message with a literal resolution line, exactly one of:
|
||||
|
||||
```
|
||||
RESOLUTION: RETRY
|
||||
```
|
||||
```
|
||||
RESOLUTION: ABORT
|
||||
```
|
||||
|
||||
followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`.
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: pi-skills
|
||||
namespace: agent-pod
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: agent-pod-ssh-config
|
||||
namespace: agent-pod
|
||||
data:
|
||||
config: |
|
||||
Host git.riotpiao.com
|
||||
IdentityFile /root/.ssh/id_forgejo
|
||||
Port 2222
|
||||
User git
|
||||
StrictHostKeyChecking accept-new
|
||||
@@ -0,0 +1,83 @@
|
||||
# API Auth Layer — Authentik service account + Kong JWT (model invoke)
|
||||
|
||||
Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik
|
||||
service account holding a valid **client_credentials** JWT can invoke the KServe
|
||||
models. "Invoke role" = **possession of a JWT from the dedicated model-invoke
|
||||
OAuth2 provider** (only the service account can obtain one).
|
||||
|
||||
## Flow
|
||||
|
||||
```
|
||||
service account ── client_credentials ──▶ Authentik token endpoint
|
||||
(client_id + secret) https://authentik.riotpiao.com/application/o/token/
|
||||
│
|
||||
▼ RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/)
|
||||
client ── Authorization: Bearer <jwt> ──▶ Kong (api.riotpiao.com/*)
|
||||
jwt plugin: verify RS256 sig via Authentik JWKS,
|
||||
check iss/exp → map to KongConsumer → allow
|
||||
▼
|
||||
KServe model (reasoning / ornith / ...)
|
||||
```
|
||||
|
||||
Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in
|
||||
**`jwt`** plugin: it validates an RS256 signature against a public key we pin on
|
||||
a KongConsumer, keyed by the token's `iss`.
|
||||
|
||||
## Changes
|
||||
|
||||
### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py)
|
||||
- New **service account** user `model-invoker` (type `service_account`, no
|
||||
password; Authentik issues an app-password/token for M2M).
|
||||
- New **OAuth2 provider + application** `model-invoke`:
|
||||
- `client_type: confidential`, `grant_types: ["client_credentials"]`
|
||||
- signing key = existing RS256 keypair (same as other providers)
|
||||
- mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes
|
||||
needed for M2M.
|
||||
- Client secret written to k8s Secret `api/model-invoke-oidc`
|
||||
(keys `client-id`, `client-secret`), labelled for whoever consumes it.
|
||||
- Bind the service account so it (and only it) can use the provider.
|
||||
|
||||
### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`)
|
||||
- **KongConsumer** `model-invoker` (ns api).
|
||||
- **`jwt` credential** on that consumer (a Secret of type
|
||||
`konghq.com/v1/credential`):
|
||||
- `algorithm: RS256`
|
||||
- `key` = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/`
|
||||
- `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing
|
||||
cert (fetched from Authentik JWKS / cert, stored in git or ksops).
|
||||
- **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`).
|
||||
|
||||
### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml)
|
||||
- Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation
|
||||
(currently e.g. `llm-rewrite-reasoning`) → becomes
|
||||
`llm-rewrite-reasoning,jwt-auth`.
|
||||
- Leave `/models` list route open OR protect too (decision).
|
||||
|
||||
## Client usage (after build)
|
||||
```bash
|
||||
TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \
|
||||
-d grant_type=client_credentials \
|
||||
-d client_id=model-invoke \
|
||||
-d client_secret=<secret> \
|
||||
-d scope=openid | jq -r .access_token)
|
||||
|
||||
curl https://api.riotpiao.com/v1/chat/completions \
|
||||
-H "Authorization: Bearer $TOKEN" -d '{...}'
|
||||
```
|
||||
|
||||
## Test plan
|
||||
1. No token → Kong returns 401.
|
||||
2. Valid client_credentials token → 200, model responds.
|
||||
3. Expired/garbage token → 401.
|
||||
4. Confirm the `/models` route behaviour matches the decision.
|
||||
|
||||
## Open items / risks
|
||||
- Authentik `client_credentials` for a *service account* may require an
|
||||
**app-password / JWT-assertion** flow rather than plain client_secret POST —
|
||||
verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs
|
||||
the SA token). Adjust step 1 accordingly before wiring Kong.
|
||||
- Pinning `rsa_public_key`: Authentik key rotation would break it — document a
|
||||
rotation runbook, or have the provision script re-export the cert PEM into the
|
||||
Kong credential on each run (keeps them in sync, same idea as ksops secrets).
|
||||
- Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's
|
||||
issuer is stable.
|
||||
@@ -0,0 +1,19 @@
|
||||
# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the
|
||||
# ingress controller apply it to every route on this Kong instance, so all
|
||||
# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics
|
||||
# without touching llm-routes.yaml. Scraped via kong-values.yaml's
|
||||
# serviceMonitor (status listener, already on by chart default at :8100).
|
||||
apiVersion: configuration.konghq.com/v1
|
||||
kind: KongClusterPlugin
|
||||
metadata:
|
||||
name: prometheus
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: kong
|
||||
labels:
|
||||
global: "true"
|
||||
plugin: prometheus
|
||||
config:
|
||||
status_code_metrics: true
|
||||
latency_metrics: true
|
||||
bandwidth_metrics: true
|
||||
upstream_health_metrics: true
|
||||
@@ -57,6 +57,17 @@ env:
|
||||
# nginx Ingress in ingress.yaml; both hops have to be unbuffered or the
|
||||
# buffered one dominates.
|
||||
nginx_proxy_proxy_buffering: "off"
|
||||
# Any plugin that rewrites the request body — request-transformer on the
|
||||
# llm-chat-* routes — reads it through `kong.request.get_body()`, and that
|
||||
# returns nothing once nginx has spilled the body past
|
||||
# client_body_buffer_size into a temp file. The plugin then re-serializes a
|
||||
# body with no `messages`, and the upstream answers
|
||||
# HTTP 400 {"error":{"message":"[] is too short - 'messages'"}}
|
||||
# Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400.
|
||||
# An agent request carrying tool schemas clears that in one turn, so the
|
||||
# buffer has to hold a whole conversation, not a chat message.
|
||||
nginx_http_client_body_buffer_size: "16m"
|
||||
nginx_http_client_max_body_size: "16m"
|
||||
|
||||
ingressController:
|
||||
enabled: true
|
||||
@@ -107,6 +118,16 @@ podDisruptionBudget:
|
||||
enabled: true
|
||||
minAvailable: 1
|
||||
|
||||
# Status listener (metrics/health) is on by default at :8100 (chart default,
|
||||
# verified via `helm show values`). This just wires the ServiceMonitor the
|
||||
# chart already knows how to generate for it, so kong_http_requests_total /
|
||||
# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the
|
||||
# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml.
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
|
||||
# Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
|
||||
# situation degrades to co-location instead of leaving a pod Pending.
|
||||
topologySpreadConstraints:
|
||||
|
||||
@@ -6,7 +6,9 @@ kind: Kustomization
|
||||
# or it is silently dropped with no error and no drift shown.
|
||||
resources:
|
||||
- ingress.yaml
|
||||
- kong-metrics.yaml
|
||||
- llm-routes.yaml
|
||||
- model-auth.yaml
|
||||
# No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
|
||||
# namespace, and the transformer rewrites metadata.namespace on every resource
|
||||
# it builds, which is a trap for anything cross-namespace added later.
|
||||
|
||||
@@ -66,7 +66,7 @@ metadata:
|
||||
name: llm-models
|
||||
namespace: llm-serving
|
||||
annotations:
|
||||
konghq.com/plugins: llm-models-list
|
||||
konghq.com/plugins: llm-models-list,model-key-auth
|
||||
konghq.com/strip-path: "false"
|
||||
konghq.com/methods: "GET"
|
||||
spec:
|
||||
@@ -110,7 +110,7 @@ metadata:
|
||||
name: llm-chat-reasoning
|
||||
namespace: llm-serving
|
||||
annotations:
|
||||
konghq.com/plugins: llm-rewrite-reasoning
|
||||
konghq.com/plugins: llm-rewrite-reasoning,model-key-auth
|
||||
konghq.com/strip-path: "false"
|
||||
konghq.com/methods: "POST"
|
||||
konghq.com/connect-timeout: "10000"
|
||||
@@ -152,7 +152,7 @@ metadata:
|
||||
name: llm-chat-ornith
|
||||
namespace: llm-serving
|
||||
annotations:
|
||||
konghq.com/plugins: llm-rewrite-ornith
|
||||
konghq.com/plugins: llm-rewrite-ornith,model-key-auth
|
||||
konghq.com/strip-path: "false"
|
||||
konghq.com/methods: "POST"
|
||||
konghq.com/connect-timeout: "10000"
|
||||
@@ -197,7 +197,7 @@ metadata:
|
||||
name: llm-chat-qwen
|
||||
namespace: llm-serving
|
||||
annotations:
|
||||
konghq.com/plugins: llm-rewrite-qwen
|
||||
konghq.com/plugins: llm-rewrite-qwen,model-key-auth
|
||||
konghq.com/strip-path: "false"
|
||||
konghq.com/methods: "POST"
|
||||
konghq.com/connect-timeout: "10000"
|
||||
@@ -267,7 +267,7 @@ metadata:
|
||||
name: llm-rerank
|
||||
namespace: llm-serving
|
||||
annotations:
|
||||
konghq.com/plugins: llm-rewrite-rerank
|
||||
konghq.com/plugins: llm-rewrite-rerank,model-key-auth
|
||||
konghq.com/strip-path: "false"
|
||||
konghq.com/methods: "POST"
|
||||
konghq.com/connect-timeout: "10000"
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# API auth layer — Kong key-auth on the model routes.
|
||||
#
|
||||
# The model API (api.riotpiao.com/v1/...) requires a static API key, presented
|
||||
# OpenAI-style as `Authorization: Bearer <key>` (or `apikey: <key>`). The key
|
||||
# lives in the ksops-managed Secret model-invoke-apikey (labelled
|
||||
# konghq.com/credential: key-auth) and is bound to the KongConsumer below.
|
||||
#
|
||||
# Issue the key to rock; use it as the OpenAI SDK api_key. Rotate by updating the
|
||||
# ksops secret. This is self-contained in Kong — the invoke path does not depend
|
||||
# on an Authentik token (Authentik still fronts every *human* dashboard SSO).
|
||||
---
|
||||
apiVersion: configuration.konghq.com/v1
|
||||
kind: KongConsumer
|
||||
metadata:
|
||||
name: model-invoker
|
||||
namespace: api
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: kong
|
||||
username: model-invoker
|
||||
credentials:
|
||||
- model-invoke-apikey
|
||||
---
|
||||
# key-auth: require the API key on the model routes. key_in_header accepts the
|
||||
# `apikey` header; key_in_bearer accepts `Authorization: Bearer <key>` so any
|
||||
# OpenAI-compatible SDK (api_key=..., base_url=https://api.riotpiao.com/v1) works
|
||||
# unchanged.
|
||||
#
|
||||
# Namespace `llm-serving`, not `api`: the ingress controller resolves a
|
||||
# `konghq.com/plugins` annotation against the annotated object's OWN namespace,
|
||||
# and all five model routes in llm-routes.yaml live in llm-serving. While this
|
||||
# sat in `api` the reference dangled, the plugin never bound, and every model
|
||||
# route served traffic with no key at all — verified: an unauthenticated
|
||||
# /v1/models and /v1/ornith/chat/completions both returned 200. A dangling
|
||||
# plugin reference is silent; it fails open, so re-test without a key after any
|
||||
# move rather than trusting that the object exists.
|
||||
apiVersion: configuration.konghq.com/v1
|
||||
kind: KongPlugin
|
||||
metadata:
|
||||
name: model-key-auth
|
||||
namespace: llm-serving
|
||||
plugin: key-auth
|
||||
config:
|
||||
key_names:
|
||||
- apikey
|
||||
- authorization
|
||||
key_in_header: true
|
||||
key_in_query: false
|
||||
key_in_body: false
|
||||
hide_credentials: true
|
||||
@@ -0,0 +1,56 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-embeddings
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
name: embeddings
|
||||
namespace: llm-serving
|
||||
spec:
|
||||
predictor:
|
||||
containers:
|
||||
- args:
|
||||
- --model-id=nomic-ai/nomic-embed-text-v2-moe
|
||||
- --port=8080
|
||||
- --hostname=0.0.0.0
|
||||
- --auto-truncate
|
||||
env:
|
||||
- name: HUGGINGFACE_HUB_CACHE
|
||||
value: /mnt/models
|
||||
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
|
||||
name: kserve-container
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 8Gi
|
||||
requests:
|
||||
cpu: '8'
|
||||
memory: 4Gi
|
||||
startupProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
# Explicit allowlist, matching k8s/apps/api. Anything added to this directory
|
||||
# and not listed here is silently dropped — no error, no drift shown.
|
||||
#
|
||||
# These five were adopted from live state on 2026-08-15; they had been applied
|
||||
# by hand and carried no ArgoCD ownership. Each was exported and verified with
|
||||
# `kubectl diff -f <file>` returning empty before the Application below was
|
||||
# created, so the first sync was a no-op rather than a redeploy. Re-verify that
|
||||
# way after any edit here: a GPU predictor restart is a weights reload measured
|
||||
# in tens of seconds, not a rolling update.
|
||||
resources:
|
||||
- embeddings.yaml
|
||||
- ornith.yaml
|
||||
- reasoning.yaml
|
||||
- reranker.yaml
|
||||
- verifier.yaml
|
||||
# No namespace transformer: every file sets its own, and the transformer would
|
||||
# rewrite metadata.namespace on anything cross-namespace added later.
|
||||
@@ -0,0 +1,108 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
# Kong reads its timeouts from the Kubernetes Service, not the Ingress —
|
||||
# Ingress annotations configure Route entities (strip-path, methods,
|
||||
# plugins), these configure the Service entity. They were on
|
||||
# llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
|
||||
# default in force. KServe propagates InferenceService annotations to the
|
||||
# Service it generates, which is how they reach Kong from here.
|
||||
#
|
||||
# This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
|
||||
# request ever waited on a cold load. A pod restart flushes VRAM, and
|
||||
# loading ornith:35b takes longer than 60s, so the first request after any
|
||||
# restart returned 504.
|
||||
konghq.com/connect-timeout: "10000"
|
||||
konghq.com/read-timeout: "3600000"
|
||||
konghq.com/write-timeout: "3600000"
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-ornith
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
name: ornith
|
||||
namespace: llm-serving
|
||||
spec:
|
||||
predictor:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- 'set -e
|
||||
|
||||
ollama serve &
|
||||
|
||||
SERVE_PID=$!
|
||||
|
||||
until ollama list >/dev/null 2>&1; do sleep 2; done
|
||||
|
||||
ollama pull ornith:35b
|
||||
|
||||
ollama pull qwen2.5:3b-instruct
|
||||
|
||||
ollama run ornith:35b "ok" >/dev/null 2>&1 || true
|
||||
|
||||
ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true
|
||||
|
||||
wait $SERVE_PID
|
||||
|
||||
'
|
||||
env:
|
||||
- name: OLLAMA_HOST
|
||||
value: 0.0.0.0:8080
|
||||
- name: OLLAMA_MODELS
|
||||
value: /mnt/models/ollama
|
||||
- name: OLLAMA_CONTEXT_LENGTH
|
||||
value: '32768'
|
||||
- name: OLLAMA_KEEP_ALIVE
|
||||
value: '-1'
|
||||
- name: OLLAMA_NUM_PARALLEL
|
||||
value: '1'
|
||||
- name: OLLAMA_MAX_LOADED_MODELS
|
||||
value: '2'
|
||||
image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
|
||||
name: kserve-container
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
||||
grep -q qwen2.5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: '1'
|
||||
requests:
|
||||
cpu: '8'
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: '1'
|
||||
startupProbe:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null |
|
||||
grep -q qwen2.5
|
||||
failureThreshold: 120
|
||||
periodSeconds: 15
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-reasoning
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
name: reasoning
|
||||
namespace: llm-serving
|
||||
spec:
|
||||
predictor:
|
||||
containers:
|
||||
- args:
|
||||
- --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
|
||||
- --served-model-name=reasoning
|
||||
- --quantization=bitsandbytes
|
||||
- --dtype=float16
|
||||
- --kv-cache-dtype=auto
|
||||
- --tensor-parallel-size=1
|
||||
- --max-model-len=16384
|
||||
- --gpu-memory-utilization=0.90
|
||||
- --max-num-seqs=4
|
||||
- --enable-chunked-prefill
|
||||
- --enable-prefix-caching
|
||||
- --reasoning-parser=deepseek_r1
|
||||
- --host=0.0.0.0
|
||||
- --port=8080
|
||||
env:
|
||||
- name: VLLM_USE_FLASHINFER_SAMPLER
|
||||
value: '0'
|
||||
- name: VLLM_ATTENTION_BACKEND
|
||||
value: TRITON_ATTN
|
||||
- name: HF_HOME
|
||||
value: /mnt/models
|
||||
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
||||
name: kserve-container
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: '1'
|
||||
requests:
|
||||
cpu: '8'
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: '1'
|
||||
startupProbe:
|
||||
failureThreshold: 80
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 15
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
- mountPath: /dev/shm
|
||||
name: shm
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
maxReplicas: 2
|
||||
minReplicas: 2
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 2Gi
|
||||
name: shm
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-reranker
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
name: reranker
|
||||
namespace: llm-serving
|
||||
spec:
|
||||
predictor:
|
||||
containers:
|
||||
- args:
|
||||
- --model-id=BAAI/bge-reranker-base
|
||||
- --port=8080
|
||||
- --hostname=0.0.0.0
|
||||
- --auto-truncate
|
||||
env:
|
||||
- name: HUGGINGFACE_HUB_CACHE
|
||||
value: /mnt/models
|
||||
image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
|
||||
name: kserve-container
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 8Gi
|
||||
requests:
|
||||
cpu: '8'
|
||||
memory: 4Gi
|
||||
startupProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
apiVersion: serving.kserve.io/v1beta1
|
||||
kind: InferenceService
|
||||
metadata:
|
||||
annotations:
|
||||
serving.kserve.io/deploymentMode: RawDeployment
|
||||
labels:
|
||||
app.kubernetes.io/name: llm-verifier
|
||||
app.kubernetes.io/part-of: llm-serving
|
||||
name: verifier
|
||||
namespace: llm-serving
|
||||
spec:
|
||||
predictor:
|
||||
containers:
|
||||
- args:
|
||||
- --model=Qwen/Qwen2.5-Math-PRM-7B
|
||||
- --served-model-name=verifier
|
||||
- --runner=pooling
|
||||
- --dtype=float16
|
||||
- --tensor-parallel-size=1
|
||||
- --max-model-len=4096
|
||||
- --max-num-seqs=8
|
||||
- --host=0.0.0.0
|
||||
- --port=8080
|
||||
env:
|
||||
- name: VLLM_USE_FLASHINFER_SAMPLER
|
||||
value: '0'
|
||||
- name: VLLM_ATTENTION_BACKEND
|
||||
value: XFORMERS
|
||||
- name: HF_HOME
|
||||
value: /mnt/models
|
||||
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
||||
name: kserve-container
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 16Gi
|
||||
nvidia.com/gpu: '1'
|
||||
requests:
|
||||
cpu: '4'
|
||||
memory: 8Gi
|
||||
nvidia.com/gpu: '1'
|
||||
startupProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
periodSeconds: 15
|
||||
volumeMounts:
|
||||
- mountPath: /mnt/models
|
||||
name: models
|
||||
- mountPath: /dev/shm
|
||||
name: shm
|
||||
deploymentStrategy:
|
||||
type: Recreate
|
||||
maxReplicas: 1
|
||||
minReplicas: 1
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: worker-1
|
||||
runtimeClassName: nvidia
|
||||
volumes:
|
||||
- name: models
|
||||
persistentVolumeClaim:
|
||||
claimName: llm-models
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 1Gi
|
||||
name: shm
|
||||
|
||||
@@ -10,13 +10,22 @@ metadata:
|
||||
name: queue-operator
|
||||
rules:
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["queues", "temporalworkers"]
|
||||
resources: ["queues"]
|
||||
verbs: ["get", "list", "watch", "update", "patch"]
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["queues/status", "temporalworkers/status"]
|
||||
resources: ["queues/status"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["queues/finalizers", "temporalworkers/finalizers"]
|
||||
resources: ["queues/finalizers"]
|
||||
verbs: ["update"]
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["temporalworkers"]
|
||||
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["temporalworkers/status"]
|
||||
verbs: ["get", "update", "patch"]
|
||||
- apiGroups: ["kmsvc.io"]
|
||||
resources: ["temporalworkers/finalizers"]
|
||||
verbs: ["update"]
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
|
||||
@@ -72,6 +72,14 @@ spec:
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: longhorn-system
|
||||
# Longhorn writes disk state back into its own Node CRs — the disk key it
|
||||
# generates, storageReserved, diskType, evictionRequested. Git declares only
|
||||
# allowScheduling; without this the controller's writes read as drift forever.
|
||||
ignoreDifferences:
|
||||
- group: longhorn.io
|
||||
kind: Node
|
||||
jsonPointers:
|
||||
- /spec/disks
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`).
|
||||
#
|
||||
# Syncs before wave 7 (Kong), so the predictor Services exist before the routes
|
||||
# that point at them. KServe itself is part of the substrate; this Application
|
||||
# owns only the InferenceServices.
|
||||
#
|
||||
# Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed
|
||||
# by hand — no ArgoCD ownership, present in no repo — so every change to them
|
||||
# was drift by definition. Each manifest was exported from the cluster and
|
||||
# verified with `kubectl diff` returning empty before this file existed; the
|
||||
# first sync therefore adopted them without restarting anything.
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: llm-serving
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "6"
|
||||
spec:
|
||||
project: homelab
|
||||
revisionHistoryLimit: 3
|
||||
source:
|
||||
repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
|
||||
targetRevision: main
|
||||
path: k8s/apps/llm-serving
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: llm-serving
|
||||
syncPolicy:
|
||||
automated:
|
||||
# `prune: false` here, unlike every other Application in this repo, and it
|
||||
# is not an oversight.
|
||||
#
|
||||
# ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label
|
||||
# (argocd-cm `application.instanceLabelKey`). KServe copies an
|
||||
# InferenceService's labels onto the Deployment and Service it generates —
|
||||
# visible today as `app.kubernetes.io/name` and `part-of` on
|
||||
# `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe
|
||||
# propagates that tracking label to children that are not in git, ArgoCD
|
||||
# reads them as extraneous, prunes them, and KServe recreates them. That
|
||||
# loop churns GPU pods.
|
||||
#
|
||||
# Deleting an InferenceService therefore means deleting the file AND
|
||||
# removing the object, rather than relying on prune.
|
||||
prune: false
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
# KServe CRDs are large; server-side apply avoids the
|
||||
# "metadata.annotations: Too long" failure client-side apply hits, and is
|
||||
# the correct mode for adopting objects an operator also writes to.
|
||||
- ServerSideApply=true
|
||||
retry:
|
||||
limit: 3
|
||||
backoff:
|
||||
duration: 10s
|
||||
factor: 2
|
||||
maxDuration: 3m
|
||||
@@ -84,6 +84,29 @@ spec:
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: agent-pod
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "8"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
|
||||
targetRevision: main
|
||||
path: k8s/apps/agent-pod
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: agent-pod
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
---
|
||||
# iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX)
|
||||
# running the BlueBubbles server, plus its dedicated local StorageClass.
|
||||
#
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: ENC[AES256_GCM,data:zdc=,iv:VvjvrS5PVNAMIaOE0LaWU+tHcUIYVQDnCANQz6myktY=,tag:xGyWDhRCwwiNny7hPllf5g==,type:str]
|
||||
kind: ENC[AES256_GCM,data:2bf5Zfy8,iv:5Oz423GzUWmgdaaZHbrtedwRHIAPIuLh4iDMieLL05s=,tag:GNW1ROjlzGvO/4tIOSuH3Q==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:7RO0Qxkc+/sA,iv:wKe9A8d7QJSx/6rlEY5H6lU8V24TJqr5IpXQCBc8QgM=,tag:iF6z1MNEXG3pCz2cQ18gLg==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:RcduxHHLtjH9,iv:opOVx1lL2ltDqQgsleN7NdMAq0TyFr/YQO3FFsHh5AA=,tag:fIwO31apqIatRRzBvamw6g==,type:str]
|
||||
type: ENC[AES256_GCM,data:C+JjyZt5,iv:xs49Lz6zRzcf3spiPzdUKTm2HZ+VgFahN6wjIe81JI4=,tag:VllJ54MqU+klA8xAIebjrA==,type:str]
|
||||
stringData:
|
||||
models.json: ENC[AES256_GCM,data:FvlbdMkcngJchi0GEjIEDjxXpQjRwh+3xnlZNpApgd3v1SpMd0qD2d3TbQr39+feSfsvpxhdmmyU/PmRHZYVdr/QpIKlZBWBD6qTCaOSq62NQbR4ck336muIaWD4AmahWCvRouYEJdEsnG7nedjAivw4cls18vMfnL5pNm2t96foScSyHosWoh3RdT6CWLmJE39+kS6/fwZ1D3Z/GEm/E2zHkVtEbEw/fk1PiqsnmZWt8QcSRAeX8aIDkphV7wOKlkri11r1NuFQKMVIau07VPce1YEi5rsJqsircvDxQelQiMGQt2y2M4GGlXx2NeJgKEtP4Hf9mVGhTeN3CYLcXi8Qyg7k1GIawGVW/Em0kWWfy0GMjvOMfYozNmIpi0YQdYBbr3j6pkixtVM2dWezvld5QIYLnHoGMxw4M6L4IqVIonC3j6tk9pV1DaL5IFskEXyb6ScTrmVY4mQ+VEXvfLYA38jRauWxx/4qf4I7B0RzfGzOx67UJh2jKj0bPWNFHKt5Dz4CK+ul39d1KUPHHMoyaH5gCpOAfLSkjMm44X1Cn+O6uY+QFCugh9MI7bJg3EbovTCZxz95GCupWAvDshv59f7gpGL0T8AuRZHlNp6yZMLdviR9yf79d2rW26PIM1QuNOUWELrOcTs5IAmyuFYr0PI5W5XSf4klOdz3UdhPfMDcyRqtSzVDsSo9oC/WeWOo1+yurtdqqW0wLiFpU/kTaZ4JN5Kl080twqFEqgyPnJrbmlxwWlt4XN2SV9D7pcfE61FzUTwoi5PD/8xt94Fp3XerSgJtwhQ6X4Neo47wCFoMR4Y69mlHrFgSJA4kLFezDc2ISnyAix0W5to0cjwJqQ2EgonIhCXD4xWTvYeApHJMOC2o4B74K5efOZ7klb6PwDMvzM3LyeGzjoJh47aJGvhWN/MpQZC2lSHT8PdMxdGDc+OkEVmsvMncddWMGjQhkTo+69sUexARxLY4H22TCt58azqQGcWa5e7s9NUHb6bRVKHzXh4HMiIhQ0jBev++Jz5gIOyYRnw5xoGbo3ROv8ndczVAqXyAla23EM/VBWT0hLMjr6xxonuLQacP9dPJE9QUofjRMeYblkVvQdIw8WuKGbAlbTR/bokQwGdVp2bywSFQZj7BLlPjzCr0/LJmxYMdq6CKB049PdTlS9/UKBHApddMQ/QRXfOYUCX+lm4k4J19jd7dWk34WNUrTEPSR2d1U/2wTnMvzlrnQHx/zk4/lmI1newfjIbybIbXypX8xVEqzFWXpxUe+iFLP19KmmmHG7gGsV32k1VSGvYFI0HOokZga98QsAbJGbi1gQq8gYpMR/mBdhjJVcnAjOYpg+YaR0vCyrhBN55aSCDoUbT1JOqxvFqmlBHDxTWorD/3KGFxEH/wvSsxussFmUCWdEgYmjGO1ktnQBw1YpOB7mhDUlzvYCSxfpexsYufDYqTbrb/R5gmATC0pEN6EgTth5YqFOL7AiIsogy14RGjRym1wvjVl5SUU9bMQlWGJbQuvNRRwL/TzR/QP210jMsnAh9t8sncDnFPIbhFgeBLsU/ctYoBs0nFVWGWzcuoP9HTnvdWTT2xKmWAeFz8I3DZ5sjm2B787Q4bCrAszTvVLeLiRw/pOE32BeNfG4nNiZDvMQhUomAKpaQypWTu2wLsv3ISm00gud5sRKB6ASJANuSu+EhG9k4OYcBEzAT9PKUxclupFbmsFnXx0u9CXDj3tErrPCUs7DuFSr+fz0ehCLMH+GrcqfytjNDVoTSBRZg+lJYKjAqVCGvq8uRf5V/rdulbt3ggx/uhGSiPzYrBQ0g9iMdjQqpNGBwjlxGkhRE9D2n/PoerAAnHTfwc9Hezffncostjp09xw2CB/xOx9HZFGalQ1R4F5+0lm0iR4lMzc+se+o38foNIiMeMIwAVBJu7IO++UMuKCxpxmdAGqEYj/Ddpzfi3ZcRwe/1HhanVvKRMJsPOhnv0SB2xBrdb7QHnEzLbRTriVRBBdOI77HXWgDt9uQru0tEXCEQ0n+f0BWqHVa/RLN4Yj0dQVWLXU75ddyHnRaCiVdLQnObMvG/QXqY1k/bJTzGNgVF8/KGY51NgdfqskOXC9Fv5IGiyWq+/DL7tQcrmThkzUhZlwDBmwUzPFmzuUC7Z5PkXv2EV+eUrYF16HYqp5KgqTkBHr/JXR8ljb/5gMN929Jk2fBVEuzaEG6W9Yek+Qbx1cnXYrMBI9GWJUJsMsxsAj+1bAjtQeQwt0nXgZesTaV90dfQYq+nTGXMNjqaMhsrJhR6lU1+CDpwY+IObHSttS8AKB+Czsl0nUB44o1a/zKRDNTCGGmPFuv1Q5Ub/1myxorLDDF1mYDpre5TedygtZ/1YSz/YOpKwmWdW32FsnY+m3qjEAcfV5f923N/z8ENVxVpeiWL6mqf7h6F3qTiaSsFq2O8m8tVm70X+oRdig3hPnKqlK/KoxVmzZErhEJ+KmP5I2uvIlb4PViE6imu8kw4VqMClb7jJ7Yk+2Zy84C8RGuDCN8GCDG7FvAahA9xVGxB6rmEpzvqFBxos2EeLOlyfewf7rCw1JAL3C9hZrciA9Jqb/t8B7KqCmCOsKK7xVt1yqyG1dsa364JG/1SKSMKebTheuymZ1+VnGjRZ+pgSAkQGjCtp7B3fTlhjt+WwxW4Ml27yfjokQQ==,iv:8Z39eWukGSMePh/3Dj35e6Zahejil+eeMSqwMYf3snI=,tag:FyBg1y0IQEg/m0mgpf1ESg==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZzB5REh6amhCNzRmY0ta
|
||||
MkN3Y0ZlR29lQ3h2SWo3cW5CUThkL2RnU1NVCmt2ZkhIZTlHN1RQRkFrTjVvbjVw
|
||||
RGRrTXRoYmdQcnlMSEo3ZWsrZUQ5cHMKLS0tIGU2TGJqZDRxUGJpZzRveEtZankx
|
||||
MzhrT1R2akxxby9QVzd1RXB0RDY1LzQKOF+/e5z5lPX6Y1sMTAHuDj3YqW1m+sBd
|
||||
u/0R0YnBonYM3wS5nJE3NZMkImaAdQlUjOzQepfBldG+lz++rlnAww==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
|
||||
lastmodified: "2026-08-18T20:06:08Z"
|
||||
mac: ENC[AES256_GCM,data:IM9HkpdwtQE2wCkjwDWOmHH4uP7TlIsrK4TVytiecvYz4SiLk6IRUSIu7I3a+F+dtltC2WtokoATaB69DTXPoI54amzzptirxiFD5FbaU+u2gLjo7KI7V0smYGuKqMYwnod2L/4GdlvP6xjVxFWuA01rQRaYBkFSumS4NABl/I4=,iv:al1MyBmFwni8gap7PPZxWaCwqFKCicfPq6nVmrSn9Xc=,tag:KJGTDQ8xRGF8qMOjoQ+dng==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: ENC[AES256_GCM,data:seY=,iv:1q2bg1QBNDDQPYNu1S08MZ+Ix8WLthxObsrqTMJEVhY=,tag:OMh/stn4vVScOEAyviuurw==,type:str]
|
||||
kind: ENC[AES256_GCM,data:QwqgPMrm,iv:tv6rUcezEgQigvHVw05Mevt6BHDD5AARuoUmQpry3AU=,tag:m9RLNezJsLYGSRWnozfkoQ==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:ZJtohnEDN2NBFrEhhvGuw4s=,iv:SU5LuA3Lqjh+q7nVxeVVtzeXiOxxVserNno5c7b47lM=,tag:YLprEk7oVjVsIDVANWDPrQ==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:LieEC+YWJzq+,iv:U00VXejzlpgOYS7Yxix/MnEW0PEvmwOuOXMUi2CWIps=,tag:QvFYZIexWhyvBYTruGMifQ==,type:str]
|
||||
type: ENC[AES256_GCM,data:U41EbP+H,iv:CKDqec0jfVtbOtEyqQpV9CHJNCwhneYi0wqK8QU2vnY=,tag:CSJ04pkPslevQrpSjhXNHw==,type:str]
|
||||
stringData:
|
||||
id_forgejo: ENC[AES256_GCM,data:dgwezImUFb2r6MMZ2V1ZMuy32DAVZ9YMzlr9zYlXKEuzeCaWn29DuqgeP5PJQFFKwd37Y9jmzXwuKD5KrC9WCAzrS2VClxmJYHj2Bf842qtKa4x3SubHu4w6wp8v42wQYCU0Kygc5UgkCFYgOBgvpYCFKhiro6Mg/PBTWTIIgrLlSVwUWPIcaX6Sq4ZW4sy6CZ0H/pEMDAjU8drZ4KmGw7W4FshNynplMuP9ewbSs9yxtuAsKt4ezuUQBGHmAYdPHytC9HUzf67z/j6TF/zg6XgTe0AG116cNddAL8oNermCIkkKuTwb5xy1+3WRjjacC0K5Ire1fV5RyXtD/1JgEHJ8gbPK1w1V5fHZXX/ARvQFlP+uqxFYmPkCHBmK/R6/VvAghI/KYHlc76hdsAe0sd1UrPIHjVEtwgvDQl+4de9L9cHG3Qxj23ajlpISWA3tGADhwzXDxi0KYClckebrCKhizWFQLYXezALYklgY2yghG9jZHufV9jRjYSr9DLtwUmVf3oglAzKkWErNs8dWqEKYFx7HpFf0FiZh,iv:1aX4E+R1oFMm9eI1RENFIYCoatX8BdHo6PhrEGq3rtg=,tag:nwniLOOmpYd3Z6R92SvB3Q==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvRmtrdkpMOVhRdTVucVlr
|
||||
QmlENGdDN2J1MUczSmRhNUpwYlBjWXhqbERzCnpLRFl1bVVtSFE3L1l4T3Rwd2ZQ
|
||||
Z1MxTzJYZWovMmd2RTA3RFJVdVRRTEUKLS0tIDFQWHh2dmQxU1FUZnUxalhTdEd3
|
||||
c3dUVzg2L2VOWFlmUG9XdFNWN2RvQkkKYqSmFMkDV/T7AOjKYQNJW85gUzFraRre
|
||||
GhwAuPJ9oNQAhSRa5z2p6ghoUplSXtNZ6H2OzETRfOc4N2cHLljW/Q==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
|
||||
lastmodified: "2026-08-18T04:18:28Z"
|
||||
mac: ENC[AES256_GCM,data:ScGMBQqGm31fWDZurmlmk+rnlvyaQkalUcJ6ds/U5JJ2niTf7Llbruuqt2EzMQO09fT2kQqAIYr4cF61Czip6rDhpG8I3H3lBc2mOqUXbeCFyRL6mRpVYPOgkSzgyfXUfe8T++54UhQ0Clv/8Z4XB48oTs9detkUDzAfxP4azRE=,iv:edhsVviyjAm48Blm5CEQ0PBhcWVX97xi2hEbgDX9Dpg=,tag:Xnn85E99LibtB/LWwi5wcA==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -0,0 +1,25 @@
|
||||
apiVersion: ENC[AES256_GCM,data:894=,iv:Swg6ADUgmrqwz7wqAZHip9/qwFu0Rn8S2Lx4gBH8LJM=,tag:zbv0tFfRLtwFxBfpsuLt8A==,type:str]
|
||||
kind: ENC[AES256_GCM,data:jzVtJHYw,iv:ToZ0orfJqfGF/OnAPeYu/g2f4fXAMOZQDkA1+tmIccs=,tag:pi2skbwIU8qOQVEC86MdAA==,type:str]
|
||||
metadata:
|
||||
name: ENC[AES256_GCM,data:t7zCeZLvAEbkJqUjWi26yD6UDA==,iv:MS1gq/bwKZdLA1itVDtsrdSOfI7e2CrhjvX5yhs0eQA=,tag:lC1gcoFMI5nfzC56U1WXrg==,type:str]
|
||||
namespace: ENC[AES256_GCM,data:tp+d,iv:gnlet/60mgbSWLXEQpSlcWD98ky7NvlNCzGLTMys0JQ=,tag:PQYj9UeA50YenQESTCl7lg==,type:str]
|
||||
labels:
|
||||
konghq.com/credential: ENC[AES256_GCM,data:SOqQ9bLGLK0=,iv:a7En49UhRDwgHbv5NRB/XilEYIKQdaDqKH86WDrJB5I=,tag:JaDZ6Ko8ovY6AZ1hW9YMGQ==,type:str]
|
||||
type: ENC[AES256_GCM,data:r1K5gvop,iv:Gjv4oG2Unyql5rY9RTTljFqyd28xI81CDWtpavuuW5E=,tag:jF8lBs3AQwHPVEu9V+mONw==,type:str]
|
||||
stringData:
|
||||
key: ENC[AES256_GCM,data:pm9GmSvX5MAsXO/e6ZcI4NF1Hwr3qjG6LaEEjvV0ihvSWhO23drlAEsrTtzppqgDZbMORf+5+P53mXU=,iv:5AbHNKeiMPoFQP/qTKdA0vEoYPzuaf4kIdGGcMSmfIQ=,tag:LD5w7XK+hiCS5D410+nCfQ==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUTmoxQkpuYUp5UlRQMkph
|
||||
a0RaazdvaU5sWkNuL2gvVjlUYXVWV0dUWVVRClZhWDZCN2hpS2hnOG9Pck9zOTkx
|
||||
RGNGMEI3RHpNbzVaaWNGcTNSSEdzZHMKLS0tIFZERnVJWUpreUh3TTlwbGw0dUx4
|
||||
MGlCSkxuWWVEK2RaSDZPUzhNSUlCa28KHN0IsgQc/kBqmjQ6+4sgfb9PJy/45MwN
|
||||
rXaLJ1htpqPZ9MJ8iOukRi0IKnKgQWXsoZengIxGmcOnEctpoH/kyQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
|
||||
lastmodified: "2026-08-14T01:07:50Z"
|
||||
mac: ENC[AES256_GCM,data:nY+YVwU1GuK8Yz+EZOQKkZKN28tm2L8afflc6hsgVFCFmsep5kVT+zId7AgemvQ+qnrho5N5nqxY2knB0gusFfWNKF3V5A5GBq40WtZCMaAtcwhJSex4kK7ZyaZD6oWDW/RTUumSrivSowkWlqt1XlDKyFLqSlpWQTd7JiGmUv8=,iv:zE8+B5UVYSuuAGYyvXsAwp1N/4vGduCoGyEMCNNEUnM=,tag:6RcrjneV2dOzhmoQi+5HsA==,type:str]
|
||||
unencrypted_suffix: _unencrypted
|
||||
version: 3.13.2
|
||||
@@ -7,6 +7,8 @@ metadata:
|
||||
exec:
|
||||
path: ksops
|
||||
files:
|
||||
- agent-pod-models.enc.yaml
|
||||
- agent-pod-ssh-key.enc.yaml
|
||||
- authentik-secrets.enc.yaml
|
||||
- cloudflare-secrets.enc.yaml
|
||||
- forgejo-runner-token.enc.yaml
|
||||
@@ -18,6 +20,7 @@ files:
|
||||
- homarr-secrets.enc.yaml
|
||||
- homelab-ca-secrets.enc.yaml
|
||||
- loki-secrets.enc.yaml
|
||||
- model-invoke-apikey.enc.yaml
|
||||
- minio-secrets.enc.yaml
|
||||
- vault-secrets.enc.yaml
|
||||
- vault-unseal-keys.enc.yaml
|
||||
|
||||
@@ -7,6 +7,22 @@ metadata:
|
||||
namespace: longhorn-system
|
||||
annotations:
|
||||
storageclass.kubernetes.io/is-default-class: "false"
|
||||
# StorageClass.parameters is immutable. mkfsParams below was added after
|
||||
# this class already existed, so every sync failed with
|
||||
# parameters: Invalid value: {...}: field is immutable
|
||||
# and the Application sat OutOfSync permanently — ArgoCD retrying a change
|
||||
# the API server can never accept.
|
||||
#
|
||||
# Replace=true makes ArgoCD delete and recreate instead of patching. Safe
|
||||
# for a StorageClass: it is consulted only at provisioning time, so bound
|
||||
# PVs and their data are untouched. New PVCs briefly fail if one is created
|
||||
# during the window.
|
||||
#
|
||||
# Note what this does NOT do: the nine CNPG volumes already provisioned were
|
||||
# formatted without mkfsParams and keep that format. Only volumes created
|
||||
# after this recreate get it. Existing volumes that need to expand still
|
||||
# need handling separately.
|
||||
argocd.argoproj.io/sync-options: Replace=true,Force=true
|
||||
provisioner: driver.longhorn.io
|
||||
allowVolumeExpansion: true
|
||||
parameters:
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
# Longhorn Node CRDs for cp-2 and cp-3.
|
||||
# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
|
||||
# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
|
||||
#
|
||||
# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
|
||||
# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
|
||||
# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
|
||||
# `default-disk` key here never matched the live one, so the Application sat
|
||||
# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
|
||||
# same /var/lib/longhorn path — which is worse than the drift it was fixing.
|
||||
#
|
||||
# What these objects are actually for is `allowScheduling: true` on tainted
|
||||
# control-plane nodes. That is all they need to declare.
|
||||
---
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
kind: Node
|
||||
@@ -10,12 +20,6 @@ metadata:
|
||||
spec:
|
||||
name: talos-cp-2
|
||||
allowScheduling: true
|
||||
disks:
|
||||
default-disk:
|
||||
allowScheduling: true
|
||||
path: /var/lib/longhorn
|
||||
storageReserved: 0
|
||||
tags: []
|
||||
tags: []
|
||||
---
|
||||
apiVersion: longhorn.io/v1beta2
|
||||
@@ -26,10 +30,4 @@ metadata:
|
||||
spec:
|
||||
name: talos-cp-3
|
||||
allowScheduling: true
|
||||
disks:
|
||||
default-disk:
|
||||
allowScheduling: true
|
||||
path: /var/lib/longhorn
|
||||
storageReserved: 0
|
||||
tags: []
|
||||
tags: []
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: llm-frontend-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "LLM"
|
||||
# Request rate/error/latency/bandwidth now come from Kong's prometheus
|
||||
# plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
|
||||
# chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
|
||||
# every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
|
||||
# embeddings/rerank uniformly without per-backend instrumentation.
|
||||
# Token-count metrics are still not available: that needs response-body
|
||||
# parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
|
||||
# Predictor-level metrics (native Ollama/vLLM stats) also still need a
|
||||
# dedicated exporter -- not added here.
|
||||
data:
|
||||
llm-frontend.json: |
|
||||
{"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
|
||||
@@ -19,6 +19,7 @@ resources:
|
||||
- dashboards/control-plane-logs.yaml
|
||||
- dashboards/hardware-overview.yaml
|
||||
- dashboards/kube-controller-health.yaml
|
||||
- dashboards/llm-frontend.yaml
|
||||
- dashboards/service-availability.yaml
|
||||
- dashboards/service-golden-signals.yaml
|
||||
- dashboards/service-internals.yaml
|
||||
|
||||
@@ -49,12 +49,15 @@ machine:
|
||||
image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version}
|
||||
wipe: true
|
||||
grubUseUKICmdline: true
|
||||
%{ if length(longhorn_disks) > 0 ~}
|
||||
disks:
|
||||
%{ for disk in longhorn_disks ~}
|
||||
%{ for idx, disk in longhorn_disks ~}
|
||||
# ${disk.kind} — ${disk.device}
|
||||
- device: ${disk.device}
|
||||
partitions:
|
||||
- mountpoint: ${disk.mountpoint}
|
||||
- mountpoint: ${coalesce(disk.mountpoint, format("/var/lib/longhorn-disk%d", idx + 1))}
|
||||
%{ endfor ~}
|
||||
%{ endif ~}
|
||||
features:
|
||||
diskQuotaSupport: true
|
||||
kubePrism:
|
||||
@@ -172,10 +175,10 @@ cluster:
|
||||
# kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
|
||||
- name: cilium-lb-ippool
|
||||
contents: |
|
||||
${indent(8, cilium_lb_ippool)}
|
||||
${indent(8, cilium_lb_ippool)}
|
||||
- name: cilium-l2-announcement
|
||||
contents: |
|
||||
${indent(8, cilium_l2_announcement)}
|
||||
${indent(8, cilium_l2_announcement)}
|
||||
# CoreDNS Corefile with homelab hostname rewrites (single source of truth in
|
||||
# terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
|
||||
# the nginx ingress controller so OIDC auto-discovery against
|
||||
|
||||
+17
-3
@@ -107,10 +107,24 @@ variable "controlplane_configs" {
|
||||
lan_subnet = string
|
||||
lan_gateway = string
|
||||
install_disk = string
|
||||
longhorn_disks = list(object({
|
||||
# Longhorn data disks, in mountpoint order. `device` should be a stable
|
||||
# /dev/disk/by-id/wwn-* path: this hardware enumerates /dev/sdX by discovery
|
||||
# order, and `install.wipe: true` means a renumber can aim the installer at a
|
||||
# data disk.
|
||||
#
|
||||
# `mountpoint` defaults to /var/lib/longhorn-disk<N> by position. Never
|
||||
# renumber or reorder existing entries — Longhorn keys its disks off the
|
||||
# mountpoint, so a rename orphans the replicas already on that disk. Append
|
||||
# new disks to the end.
|
||||
#
|
||||
# `kind` ("ssd"/"hdd") is a declared label only. PERC RAID controllers report
|
||||
# every disk as rotational, so it can't be autodetected, and it deliberately
|
||||
# does not affect ordering or mountpoints.
|
||||
longhorn_disks = optional(list(object({
|
||||
device = string
|
||||
mountpoint = string
|
||||
}))
|
||||
mountpoint = optional(string)
|
||||
kind = optional(string, "hdd")
|
||||
})), [])
|
||||
zone = string
|
||||
allow_scheduling = bool
|
||||
# Extra cert SANs for this node — e.g. Cloudflare Tunnel public hostnames so
|
||||
|
||||
Reference in New Issue
Block a user