diff --git a/k8s/apps/agent-pod/configmap.yaml b/k8s/apps/agent-pod/configmap.yaml new file mode 100644 index 0000000..174a1ae --- /dev/null +++ b/k8s/apps/agent-pod/configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +data: + settings.json: | + { + "defaultProvider": "homelab-ornith", + "defaultModel": "ornith:35b", + "defaultThinkingLevel": "medium", + "theme": "light", + "compaction": { + "enabled": true, + "reserveTokens": 8192, + "keepRecentTokens": 12000 + } + } +kind: ConfigMap +metadata: + name: pi-config + namespace: agent-pod diff --git a/k8s/apps/agent-pod/console-ingress.yaml b/k8s/apps/agent-pod/console-ingress.yaml new file mode 100644 index 0000000..afc8229 --- /dev/null +++ b/k8s/apps/agent-pod/console-ingress.yaml @@ -0,0 +1,36 @@ +# Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run +# (trigger a new session) -- both are routes on the same hub.js service. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: console + namespace: agent-pod + annotations: + konghq.com/strip-path: "false" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /console + pathType: Prefix + backend: + service: + name: agent-hub + port: + number: 9090 + - path: /run + pathType: Prefix + backend: + service: + name: agent-hub + port: + number: 9090 + - path: /sessions + pathType: Prefix + backend: + service: + name: agent-hub + port: + number: 9090 diff --git a/k8s/apps/agent-pod/deployment.yaml b/k8s/apps/agent-pod/deployment.yaml new file mode 100644 index 0000000..f9807b3 --- /dev/null +++ b/k8s/apps/agent-pod/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-pod + namespace: agent-pod +spec: + replicas: 1 + selector: + matchLabels: + app: agent-pod + template: + metadata: + labels: + app: agent-pod + spec: + # api.riotpiao.com has no in-cluster DNS record (only resolves from the + # home network's own resolver) -- pin it to ingress-nginx-controller's + # ClusterIP so pi's models.json baseUrl works unchanged. TLS still + # terminates correctly since SNI/Host still say api.riotpiao.com. + hostAliases: + - ip: "10.101.128.185" + hostnames: + - "api.riotpiao.com" + containers: + # hub.js runs in the same container as pi (not a sidecar) so it can + # spawn `pi -p --mode json` directly via child_process -- a separate + # container can't exec into another container's filesystem/PATH. + # It IS the container's long-running process now; no more `sleep + # infinity` placeholder. + - name: pi + image: node:22-slim + command: + - sh + - -c + - | + set -e + apt-get update && apt-get install -y git curl jq openssh-client tmux + ssh-keygen -y -f /root/.ssh/id_forgejo > /root/.ssh/id_forgejo.pub + eval "$(ssh-agent -s)" + ssh-add /root/.ssh/id_forgejo + npm install -g @earendil-works/pi-coding-agent@0.84.2 + npm install --prefix /root ws + node /root/hub.js + env: + - name: PI_BIN + value: pi + ports: + - containerPort: 9090 + resources: + requests: + cpu: "4" + memory: 8Gi + limits: + cpu: "8" + memory: 16Gi + volumeMounts: + - name: pi-config + mountPath: /root/.pi/agent/settings.json + subPath: settings.json + - name: pi-models + mountPath: /root/.pi/agent/models.json + subPath: models.json + - name: pi-skills + mountPath: /root/.pi/agent/skills + - name: hub-src + mountPath: /root/hub.js + subPath: hub.js + - name: ssh-key + mountPath: /root/.ssh/id_forgejo + subPath: id_forgejo + - name: ssh-config + mountPath: /root/.ssh/config + subPath: config + volumes: + - name: pi-config + configMap: + name: pi-config + - name: pi-models + secret: + secretName: pi-models + - name: pi-skills + configMap: + name: pi-skills + items: + - key: planner-SKILL.md + path: planner/SKILL.md + - key: investigator-SKILL.md + path: investigator/SKILL.md + - key: info-collector-SKILL.md + path: info-collector/SKILL.md + - key: implementer-SKILL.md + path: implementer/SKILL.md + - key: judge-SKILL.md + path: judge/SKILL.md + - key: resolver-SKILL.md + path: resolver/SKILL.md + - key: brave-search-SKILL.md + path: brave-search/SKILL.md + - name: hub-src + configMap: + name: hub-src + - name: ssh-key + secret: + secretName: agent-pod-ssh-key + defaultMode: 0600 + - name: ssh-config + configMap: + name: agent-pod-ssh-config diff --git a/k8s/apps/agent-pod/hub-configmap.yaml b/k8s/apps/agent-pod/hub-configmap.yaml new file mode 100644 index 0000000..4bdcc42 --- /dev/null +++ b/k8s/apps/agent-pod/hub-configmap.yaml @@ -0,0 +1,700 @@ +apiVersion: v1 +data: + hub.js: | + #!/usr/bin/env node + // agent-hub: lives inside the pi container (not a sidecar) so it can spawn + // `pi` directly, and control the pod's own tmux server. One persistent + // in-cluster service -- POST /run to trigger a single ad-hoc headless agent + // run, POST /pipeline to run an ordered list of task phases against a + // repo/branch (phases run sequentially, up to PHASE_CONCURRENCY tasks within + // a phase run concurrently, each in its own git worktree). Within one task, + // planner/investigator/implementer/judge are separate agents in separate + // named tmux sessions (task--, attachable via `kubectl exec -it -- + // tmux attach -t ` while running), coordinating only through what's on + // disk in that task's worktree -- not one shared conversation. GET /console + // (WebSocket) watches every concurrent headless run live, relaying pi's own + // session protocol verbatim (same event shape Claude Code sessions use). + const http = require("node:http"); + const crypto = require("node:crypto"); + const fs = require("node:fs"); + const path = require("node:path"); + const { spawn } = require("node:child_process"); + const readline = require("node:readline"); + const { WebSocketServer } = require("ws"); + + const PORT = process.env.HUB_PORT || 9090; + const WORK_DIR = process.env.HUB_WORK_DIR || path.join(require("node:os").tmpdir(), "agent-harness-work"); + + // Never rely on a bare `pi` on $PATH -- both `pi` and `agent-console` collide + // with unrelated tools on this machine (a Rust CLI and a Datadog TUI, + // respectively, discovered the hard way this session). Always invoke the + // exact pinned @earendil-works/pi-coding-agent@0.84.2 installed locally under + // .pi-cli/, by explicit path. + const PI_BIN = + process.env.PI_BIN || + path.join( + __dirname, + "..", + ".pi-cli", + "node_modules", + "@earendil-works", + "pi-coding-agent", + "dist", + "cli.js" + ); + + // Job-type skills under pi/skills//SKILL.md (mounted at + // ~/.pi/agent/skills// in agent-pod). When `agent` matches one of + // these, the prompt is forced through pi's `/skill: ` mechanism + // instead of being sent bare -- see pi's skills.md docs on single-shot skill + // loading. `resolver` is never dispatched directly by a caller; only the + // pipeline driver invokes it, on stage crashes. + const ROLE_SKILLS = new Set([ + "planner", + "investigator", + "info-collector", + "implementer", + "judge", + "resolver", + ]); + + const sessions = new Map(); // id -> {id, agent, status, events, startedAt, endedAt, pipelineId?, stage?} + const viewers = new Set(); // WebSocket connections watching /console + + function broadcast(type, session) { + const msg = JSON.stringify({ type, session }); + for (const ws of viewers) { + if (ws.readyState === ws.OPEN) ws.send(msg); + } + } + + function startSession(agent, extra = {}) { + const id = extra.id || crypto.randomUUID(); + const session = { + ...extra, + id, + agent, + status: "running", + events: [], + startedAt: new Date().toISOString(), + }; + sessions.set(id, session); + broadcast("start", session); + return session; + } + + function addEvent(session, rawLine) { + const event = JSON.parse(rawLine); + session.events.push(event); + broadcast("event", session); + return event; + } + + function endSession(session, status) { + session.status = status; + session.endedAt = new Date().toISOString(); + broadcast("end", session); + } + + // Extracts the plain-text content of a message_end event, if any -- used to + // find the VERDICT:/RESOLUTION: line judge/resolver skills are required to + // end their final message with. + function textOf(event) { + if (event.type !== "message_end" || !event.message || !Array.isArray(event.message.content)) { + return null; + } + return event.message.content + .filter((c) => c.type === "text") + .map((c) => c.text) + .join("\n"); + } + + // Core spawn primitive. Spawns `pi -p --mode json `, + // relays every line as a session event exactly like before. Returns + // { session, done } -- `session` is available synchronously (so an HTTP + // handler can respond with its id right away, same as the old runAgent), + // `done` is a Promise resolving once the process exits, for callers that + // need to wait on a stage (the pipeline driver) rather than fire-and-forget. + function spawnPi({ agent, prompt, provider, model, cwd, sessionExtra = {} }) { + const session = startSession(agent, sessionExtra); + const args = ["-p", "--mode", "json"]; + if (provider) args.push("--provider", provider); + if (model) args.push("--model", model); + const finalPrompt = ROLE_SKILLS.has(agent) ? `/skill:${agent} ${prompt}` : prompt; + args.push(finalPrompt); + + const child = spawn(PI_BIN, args, { + stdio: ["ignore", "pipe", "pipe"], + cwd, + }); + const rl = readline.createInterface({ input: child.stdout }); + let lastText = ""; + let stderrTail = ""; + + rl.on("line", (line) => { + if (!line.trim()) return; + try { + const event = addEvent(session, line); + const text = textOf(event); + if (text) lastText = text; + } catch { + // non-JSON stdout noise, ignore + } + }); + child.stderr.on("data", (chunk) => { + process.stderr.write(chunk); + stderrTail = (stderrTail + chunk.toString()).slice(-4000); + }); + + const done = new Promise((resolve) => { + child.on("close", (code) => { + endSession(session, code === 0 ? "done" : "error"); + resolve({ code, session, lastText, stderrTail }); + }); + }); + + return { session, done }; + } + + function runAgent(agent, prompt, extraArgs = {}) { + // Fire-and-forget: caller (the /run handler) doesn't await `done`. + return spawnPi({ agent, prompt, ...extraArgs }).session; + } + + function parseVerdictLine(text, label) { + if (!text) return null; + const re = new RegExp(`${label}:\\s*(\\w+)`, "i"); + const m = text.match(re); + return m ? m[1].toUpperCase() : null; + } + + // Deterministic git operations, run directly by hub.js rather than left to + // the model -- branch creation and pushing after each task are mechanical, + // not judgment calls, and need to happen reliably every time regardless of + // what a task's stages did or didn't remember to do. + function runCmd(bin, args, cwd) { + return new Promise((resolve) => { + const child = spawn(bin, args, { cwd, stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + child.stdout.on("data", (c) => (out += c)); + child.stderr.on("data", (c) => (out += c)); + child.on("close", (code) => resolve({ code, out: out.trim() })); + }); + } + + function runGit(cwd, args) { + return runCmd("git", args, cwd); + } + + // A stage saying "commit" in its prompt is a request, not a guarantee -- seen + // in practice: a stage writes a real file and simply never runs `git add`/ + // `git commit`, leaving it untracked and invisible to every later `git diff`. + // Sweep and commit anything left dirty after every stage, deterministically. + async function commitPending(cwd, message) { + await runGit(cwd, ["add", "-A"]); + const status = await runGit(cwd, ["status", "--porcelain"]); + if (!status.out) return { committed: false }; + const commit = await runGit(cwd, ["commit", "-m", message]); + return { committed: commit.code === 0, error: commit.code !== 0 ? commit.out : undefined }; + } + + // Invokes the `resolver` skill to diagnose a stuck/crashed stage and decide + // RETRY vs ABORT. Shared by both crash-recovery paths below (headless + // exit-code failures and interactive sentinel-file timeouts) -- the + // diagnostic prompt differs per caller, but "ask resolver, parse the + // RESOLUTION: line" is identical either way. + async function askResolver(pipelineId, cwd, task, diagnosticPrompt) { + const resolverResult = await spawnPi({ + agent: "resolver", + prompt: diagnosticPrompt, + cwd, + sessionExtra: { pipelineId, stage: "resolver", task }, + }).done; + return parseVerdictLine(resolverResult.lastText, "RESOLUTION"); + } + + // Runs one pipeline stage, and if it crashes (nonzero exit -- not a semantic + // judge FAIL, which is handled separately), asks the resolver to diagnose + // and decide RETRY vs ABORT. Retries the failed stage at most once, + // regardless of what resolver recommends a second time -- a hard cap, not + // indefinite trust in the model's judgment. + async function runStageWithResolver(pipelineId, cwd, stage, prompt, task) { + let result = await spawnPi({ + agent: stage, + prompt, + cwd, + sessionExtra: { pipelineId, stage, task }, + }).done; + if (result.code === 0) return result; + + const resolution = await askResolver( + pipelineId, + cwd, + task, + `Stage "${stage}" exited with code ${result.code}. Its stderr tail:\n${result.stderrTail}` + ); + + if (resolution === "RETRY") { + result = await spawnPi({ + agent: stage, + prompt, + cwd, + sessionExtra: { pipelineId, stage, task }, + }).done; + } + return result; + } + + // Deterministic tmux operations -- same rationale as runGit: mechanical, + // not a judgment call, run directly rather than trusted to a prompt. + function runTmux(args) { + return runCmd("tmux", args); + } + + function tmuxSessionName(task) { + return `task-${task.replace(/[^a-zA-Z0-9]/g, "-")}`; + } + + // Bounded-concurrency pool -- runs `worker` over `items`, at most `limit` in + // flight at once. No external dep; a plain in-order index cursor shared by + // `limit` runner loops. + async function runConcurrent(items, limit, worker) { + const results = new Array(items.length); + let i = 0; + async function next() { + while (i < items.length) { + const idx = i++; + results[idx] = await worker(items[idx], idx); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, next)); + return results; + } + + const STAGE_TIMEOUT_MS = 10 * 60 * 1000; + const NUDGE_TIMEOUT_MS = 5 * 60 * 1000; + const POLL_MS = 10 * 1000; + + async function waitForFile(filePath, limitMs) { + const start = Date.now(); + while (!fs.existsSync(filePath)) { + if (Date.now() - start > limitMs) return false; + await new Promise((r) => setTimeout(r, POLL_MS)); + } + return true; + } + + const MAX_IMPLEMENT_ATTEMPTS = 5; + const MAX_PLAN_REVISIONS = 3; + + // Runs one role as its own fresh interactive pi session in its own named + // tmux session -- planner, investigator, implementer, and judge are + // separate agents with separate context, not turns in one shared + // conversation. They coordinate only through what's on disk in the task's + // worktree: PLAN.md, committed code, judge's result file. Each session is + // attachable while it runs (kubectl exec -it -- tmux attach -t ) and + // killed once its sentinel file lands or it's abandoned after resolver + // escalation. + async function runStage(pipelineId, cwd, task, stageLabel, stagePrompt, sentinelFile) { + const sessionName = `${tmuxSessionName(task)}-${stageLabel}`; + fs.rmSync(sentinelFile, { force: true }); + + const spawned = await runTmux(["new-session", "-d", "-s", sessionName, "-c", cwd, PI_BIN, stagePrompt]); + if (spawned.code !== 0) return { ok: false, crashed: true, error: spawned.out, sessionName }; + + let ok = await waitForFile(sentinelFile, STAGE_TIMEOUT_MS); + if (!ok) { + const pane = await runTmux(["capture-pane", "-t", sessionName, "-p", "-S", "-200"]); + const resolution = await askResolver( + pipelineId, + cwd, + task, + `Task ${task}'s "${stageLabel}" stage hasn't finished after 10 minutes. ` + + `Its pane tail:\n${pane.out.slice(-3000)}\n\nDecide: is it still making ` + + `real progress and worth nudging to wrap up, or stuck and worth abandoning?` + ); + if (resolution === "RETRY") { + await runTmux([ + "send-keys", + "-t", + sessionName, + `Please wrap up the "${stageLabel}" stage now and touch ${path.basename(sentinelFile)} when done.`, + "Enter", + ]); + ok = await waitForFile(sentinelFile, NUDGE_TIMEOUT_MS); + } + } + + await runTmux(["kill-session", "-t", sessionName]); + return { ok, sessionName }; + } + + // Runs one task in its own git worktree (see runPhase): planner drafts + // PLAN.md, investigator confirms it, then implementer and judge go back and + // forth -- judge's FAIL rationale lands in .task-result-, which the + // next implementer attempt is told to read and address. After + // MAX_IMPLEMENT_ATTEMPTS straight fails, planner is brought back in to + // judge whether the *plan* itself is wrong, not just the implementation; if + // so it revises PLAN.md and the implementer gets a fresh attempt budget + // against the new plan. MAX_PLAN_REVISIONS caps this from looping forever + // on a task that's genuinely stuck. + async function runTaskInteractive(pipelineId, cwd, baseBranch, task, pipelineSession, judgeOnly) { + const resultFile = path.join(cwd, `.task-result-${task}`); + fs.rmSync(resultFile, { force: true }); + + const specHint = `the file under tasks/ starting with "${task}-"`; + + const runRole = async (stageLabel, prompt, sentinel) => { + pipelineSession.activeTasks[task] = { + stage: stageLabel, + sessionName: `${tmuxSessionName(task)}-${stageLabel}`, + startedAt: new Date().toISOString(), + }; + broadcast("event", pipelineSession); + const result = await runStage(pipelineId, cwd, task, stageLabel, prompt, sentinel); + await commitPending(cwd, `task: ${task} (${stageLabel})`); + return result; + }; + + const abandon = (stageLabel, result, attempt) => { + delete pipelineSession.activeTasks[task]; + broadcast("event", pipelineSession); + return { + task, + status: result.crashed ? "spawn-crashed" : "timed-out", + error: result.error, + stoppedAt: stageLabel, + ...(attempt !== undefined ? { attempt } : {}), + }; + }; + + // Task's implementation is inherited already-committed (e.g. from a base + // branch of prior work) -- try one judge pass against the spec directly + // (no PLAN.md exists yet) before paying for a full planner/investigator + // redo. PASS ends the task here; FAIL falls through into the normal flow + // below, so planner/implementer pick up with the judge's real feedback. + if (judgeOnly) { + const quick = await runRole( + "judge", + `Task ${task} may already be implemented on this branch -- check ` + + `\`git log --oneline --grep '${task}'\` and the current code directly against its spec ` + + `(${specHint})'s acceptance criteria (no PLAN.md exists for this task yet). Write your ` + + `verdict to .task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus ` + + `one line of rationale, then run: touch .stage-done-${task}-judge-0`, + path.join(cwd, `.stage-done-${task}-judge-0`) + ); + if (!quick.ok) return abandon("judge", quick, 0); + + const quickText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : ""; + if (parseVerdictLine(quickText, "VERDICT") === "PASS") { + delete pipelineSession.activeTasks[task]; + broadcast("event", pipelineSession); + return { task, status: "done", judgeRationale: quickText, judgeOnlyPass: true }; + } + } + + let result = await runRole( + "planner", + `Use the planner skill to draft PLAN.md for task ${task}, reading its spec (${specHint}). Commit PLAN.md, then run: touch .stage-done-${task}-planner`, + path.join(cwd, `.stage-done-${task}-planner`) + ); + if (!result.ok) return abandon("planner", result); + + result = await runRole( + "investigator", + `Use the investigator skill to confirm PLAN.md against real sources, append findings, commit. Then run: touch .stage-done-${task}-investigator`, + path.join(cwd, `.stage-done-${task}-investigator`) + ); + if (!result.ok) return abandon("investigator", result); + + let planRevisions = 0; + let implementAttempt = 0; + let verdict = null; + let resultText = ""; + let justRevisedPlan = false; + + while (true) { + implementAttempt++; + const feedbackHint = fs.existsSync(resultFile) + ? justRevisedPlan + ? `.task-result-${task} holds the judge's feedback against the OLD plan, which prompted a plan revision -- ` + + `PLAN.md has since changed. Read the current PLAN.md as the source of truth, not the old feedback verbatim.` + : `A previous judge review exists at .task-result-${task} -- read it and address every issue it raises.` + : ""; + justRevisedPlan = false; + + result = await runRole( + "implementer", + `Use the implementer skill to implement what the current PLAN.md specifies (commit as you go). ${feedbackHint} Then run: touch .stage-done-${task}-implementer-${implementAttempt}`, + path.join(cwd, `.stage-done-${task}-implementer-${implementAttempt}`) + ); + if (!result.ok) return abandon("implementer", result, implementAttempt); + + result = await runRole( + "judge", + `Use the judge skill to review the diff against ${baseBranch}...HEAD. Write your verdict to ` + + `.task-result-${task} as a single "VERDICT: PASS" or "VERDICT: FAIL" line plus one line of ` + + `rationale, then run: touch .stage-done-${task}-judge-${implementAttempt}`, + path.join(cwd, `.stage-done-${task}-judge-${implementAttempt}`) + ); + if (!result.ok) return abandon("judge", result, implementAttempt); + + resultText = fs.existsSync(resultFile) ? fs.readFileSync(resultFile, "utf8") : ""; + verdict = parseVerdictLine(resultText, "VERDICT"); + if (verdict === "PASS") break; + + if (implementAttempt >= MAX_IMPLEMENT_ATTEMPTS) { + if (planRevisions >= MAX_PLAN_REVISIONS) break; + planRevisions++; + result = await runRole( + "planner-revise", + `Implementer failed judge review ${MAX_IMPLEMENT_ATTEMPTS} times in a row for task ${task}. Read PLAN.md, ` + + `the judge's feedback in .task-result-${task}, and the current diff against ${baseBranch}...HEAD. Decide ` + + `whether the plan's approach itself is wrong, not just the implementation -- if so, revise PLAN.md and ` + + `commit. If you change the approach, also use the investigator skill to confirm the new approach against ` + + `real sources before committing. If the plan is sound, note why in PLAN.md and leave it as-is. Then run: ` + + `touch .stage-done-${task}-planner-revise-${planRevisions}`, + path.join(cwd, `.stage-done-${task}-planner-revise-${planRevisions}`) + ); + if (!result.ok) return abandon("planner-revise", result, planRevisions); + implementAttempt = 0; + justRevisedPlan = true; + } + } + + delete pipelineSession.activeTasks[task]; + broadcast("event", pipelineSession); + + if (verdict !== "PASS" && planRevisions >= MAX_PLAN_REVISIONS) { + return { task, status: "unresolved", judgeRationale: resultText, implementAttempts: implementAttempt, planRevisions }; + } + return { + task, + status: verdict === "PASS" ? "done" : "done-with-concerns", + judgeRationale: resultText, + }; + } + + const PHASE_CONCURRENCY = 3; + + // Runs one phase (a batch of tasks with no declared dependency on each + // other) with up to PHASE_CONCURRENCY tasks in flight at once. Each task + // gets its own git worktree off workBranch -- concurrent pi sessions writing + // into one shared working tree would corrupt the index; worktrees share the + // same object database but give each task an isolated checkout. After a + // task's session ends, its branch is merged back into workBranch and pushed, + // one merge at a time (git ref updates aren't safe to run concurrently even + // though the worktrees themselves are isolated). + async function runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession) { + // Each entry is either a plain task id, or { id, judgeOnly: true } when + // the task's implementation already exists (e.g. inherited from a base + // branch) and just needs a real judge pass rather than a full + // planner/investigator/implementer redo. + const entries = phaseTasks.map((t) => (typeof t === "string" ? { id: t, judgeOnly: false } : t)); + + const worktrees = {}; + for (const entry of entries) { + const task = entry.id; + const wtDir = path.join(WORK_DIR, pipelineId, `wt-${task.replace(/[^a-zA-Z0-9]/g, "-")}`); + const taskBranch = `task/${task}`; + const add = await runGit(cwd, ["worktree", "add", "-b", taskBranch, wtDir, workBranch]); + if (add.code !== 0) { + pipelineSession.taskResults.push({ task, status: "worktree-crashed", error: add.out }); + continue; + } + worktrees[task] = { wtDir, taskBranch }; + } + + const runnable = entries.filter((e) => worktrees[e.id]); + await runConcurrent(runnable, PHASE_CONCURRENCY, async (entry) => { + const { wtDir } = worktrees[entry.id]; + const result = await runTaskInteractive(pipelineId, wtDir, workBranch, entry.id, pipelineSession, entry.judgeOnly); + pipelineSession.taskResults.push(result); + return result; + }); + + // Merge + push sequentially -- ref updates on the shared repo, one at a + // time, in the declared task order for this phase. + for (const entry of runnable) { + const task = entry.id; + const { wtDir, taskBranch } = worktrees[task]; + const result = pipelineSession.taskResults.find((r) => r.task === task); + + const merge = await runGit(cwd, ["merge", "--no-ff", taskBranch, "-m", `merge: ${task}`]); + if (merge.code !== 0) { + await runGit(cwd, ["merge", "--abort"]); + if (result) { + result.status = "merge-conflict"; + result.mergeError = merge.out; + } + } else { + const push = await runGit(cwd, ["push", "-u", "origin", workBranch]); + if (result) { + result.pushed = push.code === 0; + if (!result.pushed) result.pushError = push.out; + } + } + + await runGit(cwd, ["worktree", "remove", wtDir, "--force"]); + await runGit(cwd, ["branch", "-D", taskBranch]); + broadcast("event", pipelineSession); + } + } + + // tasks: array of phases, each phase an array of task ids with no declared + // dependency on each other (e.g. [["T0.1","T0.2"], ["T1.1","T1.2","T1.3"]]) -- + // caller's responsibility to supply real phase grouping (see tasks/INDEX.md; + // filename/numeric sort does NOT match execution order on boards like this). + // A flat array of ids is also accepted and treated as one single phase. + // Phases run strictly sequentially (a phase boundary is a real dependency + // gate); tasks within a phase run concurrently, each in its own worktree -- + // see runPhase. + function runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }) { + const cwd = path.join(WORK_DIR, pipelineId); + fs.mkdirSync(cwd, { recursive: true }); + + const phases = Array.isArray(tasks[0]) ? tasks : [tasks]; + + const pipelineSession = startSession("pipeline", { + id: pipelineId, + pipelineId, + stage: "pipeline", + taskResults: [], + activeTasks: {}, + totalTasks: phases.flat().length, + }); + + (async () => { + const clone = await runStageWithResolver( + pipelineId, + cwd, + "planner", + `Run exactly this command, verbatim, no variation: git clone --branch ${baseBranch} ${repo} . -- the trailing dot is required, it clones directly into the current directory instead of creating a subdirectory. Do not cd anywhere first or after. Do nothing else.`, + "clone" + ); + if (clone.code !== 0) return endSession(pipelineSession, "clone-crashed"); + if (!fs.existsSync(path.join(cwd, ".git"))) { + // The model deciding to `cd` elsewhere before cloning (instead of + // cloning into the assigned cwd) is a real failure mode seen in + // practice, not a hypothetical -- exit code 0 doesn't mean the clone + // landed where every later stage's cwd assumes it did. + return endSession(pipelineSession, "clone-missing"); + } + + // Dedicated branch, never main -- and pushed after every single task + // (not just at the end) so a pod restart mid-run loses at most the + // in-progress task's work, not everything since the start. + const workBranch = branchName || `agent-run/${pipelineId}`; + const branchResult = await runGit(cwd, ["checkout", "-b", workBranch]); + if (branchResult.code !== 0) { + pipelineSession.gitError = branchResult.out; + return endSession(pipelineSession, "branch-crashed"); + } + + // Seen in practice: a task manually downloads a dependency tarball + // (crates.io registry access isn't guaranteed from every sandboxed + // checkout) and it lands at repo root, outside whatever .gitignore + // already covers -- then git add -A (ours or the model's own) commits + // it. Append broad build-artifact/archive patterns before any task + // runs, so it's excluded regardless of who stages files later. + const gitignoreAdditions = [ + "", + "# agent-harness: build artifacts and vendored archives never belong in source control", + "*.tar.gz", + "*.tgz", + "*.crate", + "*.zip", + "*.bin", + "*.whl", + "vendor/", + "node_modules/", + "", + "# agent-harness: task completion sentinel files, harness bookkeeping only", + ".task-result-*", + ".stage-done-*", + ].join("\n"); + fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n"); + await runGit(cwd, ["add", ".gitignore"]); + await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]); + + for (const phaseTasks of phases) { + await runPhase(pipelineId, cwd, workBranch, phaseTasks, pipelineSession); + } + + const crashed = pipelineSession.taskResults.filter((r) => r.status.endsWith("-crashed")); + endSession(pipelineSession, crashed.length > 0 ? "completed-with-crashes" : "completed"); + })(); + + return pipelineSession; + } + + const server = http.createServer((req, res) => { + const url = new URL(req.url, "http://localhost"); + + if (url.pathname === "/healthz") { + res.writeHead(200).end(); + return; + } + + if (url.pathname === "/sessions" && req.method === "GET") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify([...sessions.values()])); + return; + } + + if (url.pathname === "/run" && req.method === "POST") { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + try { + const { agent, prompt, provider, model } = JSON.parse(body); + if (!agent || !prompt) throw new Error("agent and prompt are required"); + const session = runAgent(agent, prompt, { provider, model }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: session.id })); + } catch (err) { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: err.message })); + } + }); + return; + } + + if (url.pathname === "/pipeline" && req.method === "POST") { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + try { + const { repo, baseBranch, tasks, branchName } = JSON.parse(body); + if (!repo || !baseBranch || !Array.isArray(tasks) || tasks.length === 0) { + throw new Error("repo, baseBranch, and a non-empty tasks array are required"); + } + const pipelineId = crypto.randomUUID(); + runPipeline({ pipelineId, repo, baseBranch, tasks, branchName }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: pipelineId })); + } catch (err) { + res.writeHead(400, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: err.message })); + } + }); + return; + } + + res.writeHead(404).end(); + }); + + const wss = new WebSocketServer({ server, path: "/console" }); + wss.on("connection", (ws) => { + for (const session of sessions.values()) { + ws.send(JSON.stringify({ type: "snapshot", session })); + } + viewers.add(ws); + ws.on("close", () => viewers.delete(ws)); + }); + + server.listen(PORT, () => console.log(`agent-hub listening on :${PORT}`)); +kind: ConfigMap +metadata: + name: hub-src + namespace: agent-pod diff --git a/k8s/apps/agent-pod/hub-service.yaml b/k8s/apps/agent-pod/hub-service.yaml new file mode 100644 index 0000000..c1527e3 --- /dev/null +++ b/k8s/apps/agent-pod/hub-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: agent-hub + namespace: agent-pod +spec: + selector: + app: agent-pod + ports: + - port: 9090 + targetPort: 9090 diff --git a/k8s/apps/agent-pod/kustomization.yaml b/k8s/apps/agent-pod/kustomization.yaml new file mode 100644 index 0000000..e18d614 --- /dev/null +++ b/k8s/apps/agent-pod/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-pod +resources: + - deployment.yaml + - configmap.yaml + - hub-configmap.yaml + - pi-skills-configmap.yaml + - ssh-configmap.yaml + - hub-service.yaml + - console-ingress.yaml diff --git a/k8s/apps/agent-pod/pi-skills-configmap.yaml b/k8s/apps/agent-pod/pi-skills-configmap.yaml new file mode 100644 index 0000000..09598d2 --- /dev/null +++ b/k8s/apps/agent-pod/pi-skills-configmap.yaml @@ -0,0 +1,156 @@ +apiVersion: v1 +data: + brave-search-SKILL.md: | + --- + name: brave-search + description: Web search via the Brave Search API, called directly with curl. Use for searching documentation, facts, or any current web content. + allowed-tools: Bash + --- + + # Brave Search + + Direct HTTP call to the Brave Search API — no separate script or package, just `curl` (`BRAVE_API_KEY` is already set in the environment). + + ## Search + + ```bash + curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \ + --get --data-urlencode "q=" --data-urlencode "count=5" \ + "https://api.search.brave.com/res/v1/web/search" + ``` + + Options (add as extra `--data-urlencode` pairs): + - `count=` — number of results (max 20, default 5) + - `country=` — two-letter country code (default US) + - `freshness=pd|pw|pm|py` — past day/week/month/year, or `freshness=YYYY-MM-DDtoYYYY-MM-DD` + + Response is JSON; the results live at `.web.results[]`, each with `title`, `url`, `description`, `age`. Pipe through `jq` if you want a shorter view, e.g.: + + ```bash + curl -s -H "Accept: application/json" -H "X-Subscription-Token: $BRAVE_API_KEY" \ + --get --data-urlencode "q=" "https://api.search.brave.com/res/v1/web/search" \ + | jq -r '.web.results[] | "- \(.title)\n \(.url)\n \(.description)\n"' + ``` + + There's no page-content-extraction helper here — if a result needs reading in full, `curl` the URL directly and read the raw HTML/text; don't expect readability-cleaned markdown. + + ## When to Use + + - Searching for documentation or API references + - Looking up facts or current information + - Confirming a claim or approach against real sources + implementer-SKILL.md: | + --- + name: implementer + description: Turns a confirmed PLAN.md into real code changes in the current checkout, committing incrementally. Use as the implementation stage of a spec-to-push pipeline, after planner and investigator have run. + allowed-tools: Read Grep Find Ls Write Edit Bash + --- + + Execute the already-agreed plan; don't re-litigate it. `PLAN.md` (plus any `## Investigation` flags) is the source of truth for *what*; use judgment only for *how*, within the codebase's existing conventions. + + - Read `PLAN.md` top to bottom. Treat flagged/unconfirmed steps conservatively (safer, more literal reading; note it in the commit). Work steps in order. Commit after each meaningful step (`git add -A && git commit -m "..."`), not one giant commit — the judge stage needs real diff history. + - Push only if the task explicitly asks for it. + + Match existing style. Don't refactor or "improve" code the plan didn't ask you to touch. + + **Hard rules:** + - Follow DRY and SOLID. Don't duplicate logic that already exists elsewhere in the codebase you're touching — reuse or extract instead. Keep each unit responsible for one thing. + - Never commit anything that doesn't belong in source control: build artifacts, downloaded/vendored dependencies, secrets, scratch/debug files. `.gitignore` already blocks common patterns; if you create something outside those patterns, delete it before committing rather than relying on `.gitignore` to catch it. + + **Never vendor a dependency by downloading/extracting it into the repo.** Use the language's real package manager (`cargo add`, `npm install`, etc.) so the dependency is declared in the manifest and lockfile, not a tarball or extracted source tree sitting in the checkout. If the package manager can't reach its registry from here, say so in your commit message rather than working around it — a later commit sweep (`git add -A`) commits whatever's in the checkout, including anything downloaded for a workaround, even if you never intended to keep it. + info-collector-SKILL.md: | + --- + name: info-collector + description: Gathers and summarizes information on a topic from the web without judging or confirming any particular approach. Use standalone when you need raw research/context on a subject, not a verdict on a specific plan (that's the investigator skill). + allowed-tools: Read Bash + --- + + **Persona:** You are a research assistant. Your job is to gather relevant information on a topic and summarize it neutrally — you are not asked to approve, reject, or recommend anything, just to collect and organize what's out there. + + **Thinking mode:** Medium — breadth of coverage matters more than deep verification here (that's the investigator skill's job). + + **Modes:** + + - **Collect mode** (default) — use the `brave-search` skill (`curl` against the Brave Search API) with varied queries to cover the topic from multiple angles, then produce a structured summary: topic areas found, key facts, and links to sources for each. Do not editorialize about which approach is "right" — that's out of scope for this skill. + - If asked to write the summary to a file, write it and report the path; otherwise return it directly in your response. + investigator-SKILL.md: | + --- + name: investigator + description: Reads an existing PLAN.md and confirms its approach against real, current sources via web search, appending findings and flags. Use to sanity-check a plan before implementation, or standalone to verify a claimed approach is actually correct. + allowed-tools: Read Bash + --- + + Check whether the plan's claims about the real world are actually true right now. Cite sources; don't assert without one. + + - Read `PLAN.md` and the spec docs on disk. For each claim depending on external facts (a library's current API, a service's behavior), use `brave-search` to confirm or refute it. Append a `## Investigation` section to `PLAN.md`: each claim, its source(s), PASS/FLAG. Commit: `git add PLAN.md && git commit -m "investigate: confirm plan against sources"`. + - No `PLAN.md`? Just answer the question asked, citing sources. + + Flag unconfirmed/contradicted claims rather than silently fixing them — that decision belongs to whoever reads the flag next. + + **Never download anything into the repo checkout.** Need to inspect a dependency's real source/docs? Fetch into `/tmp/`, not the repo tree — a later `git add -A` sweep commits whatever's sitting in the checkout, staged or not. + judge-SKILL.md: | + --- + name: judge + description: LLM-as-judge. Reviews a git diff against PLAN.md and the original spec, and returns a PASS/FAIL verdict with rationale. Use as the final review/report stage of a spec-to-push pipeline (the implementer stage already pushed; this reports on what shipped), or standalone to review any diff against stated criteria. + allowed-tools: Read Bash + --- + + Independent reviewer. Judge whether the implementation satisfies the plan and spec, on the evidence in front of you — not on how confident the commit messages sound. Don't rubber-stamp. + + - Run `git diff ...HEAD` to see exactly what changed. Compare against `PLAN.md`'s steps and the spec docs. Does every step have a corresponding change? Does the diff contradict any investigator flag? Anything obviously broken on inspection? + - FAIL on DRY/SOLID violations (duplicated logic that should reuse existing code, mixed-responsibility units) and on anything committed that doesn't belong in source control (build artifacts, vendored dependencies, secrets, scratch files) — name the specific file/lines in your rationale. + - No `PLAN.md`/base given? Review whatever diff/criteria are in the task directly. + + You MUST end your final message with a literal verdict line, exactly one of: + + ``` + VERDICT: PASS + ``` + ``` + VERDICT: FAIL + ``` + + followed by your rationale. The pipeline driver parses this exact line mechanically to record the outcome — omitting it or rephrasing it breaks the pipeline. + planner-SKILL.md: | + --- + name: planner + description: Clones a target repo/branch, reads its markdown specs, and writes a verifiable step-by-step implementation plan (PLAN.md). Use as the first stage of a spec-to-PR pipeline, or standalone when asked to plan out a task before implementing it. + allowed-tools: Read Grep Find Ls Write Bash + --- + + Plan a concrete, verifiable implementation. Don't implement — that's `implementer`'s job, after `investigator` confirms. + + - If cwd is empty: `git clone` the given repo/branch first (only at pipeline start). + - Read every markdown spec file for the task. Decompose into a numbered list of concrete steps, each naming the files/areas it touches and how to verify it's done. State assumptions explicitly; if ambiguous, pick the literal reading and note the ambiguity. + - Write the plan to exactly `./PLAN.md` in the repo root — not `tasks/PLAN.md`, not `plans/.md`, not any other name or location. Every later stage looks for the plan at that exact path. Commit: `git add PLAN.md && git commit -m "plan: "`. + + Don't touch any other file. + resolver-SKILL.md: | + --- + name: resolver + description: Diagnoses why a pipeline stage crashed (nonzero exit, not a semantic pass/fail) and decides whether it's safe to retry. Invoked by the pipeline driver when a planner/investigator/implementer/judge stage process fails to run to completion. + allowed-tools: Read Bash + --- + + **Persona:** You are an incident triager, not a fixer. A pipeline stage stopped running — your job is to look at what's on disk and what the failed stage's own output said, figure out why, and decide whether re-running that stage is likely to succeed or would just fail the same way again. + + **Thinking mode:** Medium — this is triage (root cause + retry/no-retry judgment), not deep design work. + + **Modes:** + + - **Triage mode** (default) — read the failed stage's name and its last stdout/stderr tail (given in the task). Check the working directory's current state (`git status`, `git log -1`) to see what, if anything, that stage managed to do before stopping. Distinguish transient causes (network blip, a flaky command, an interrupted git operation left in a bad-but-fixable state) from structural ones (the plan itself is broken, a required tool/credential is missing, the repo is in a state no retry will fix). + + You MUST end your final message with a literal resolution line, exactly one of: + + ``` + RESOLUTION: RETRY + ``` + ``` + RESOLUTION: ABORT + ``` + + followed by your rationale. The pipeline driver parses this exact line mechanically and retries the failed stage **at most once** regardless of what you recommend a second time — don't assume unlimited retries. If the working directory is left in a broken state that a retry needs cleaned up first (e.g. a half-finished `git` operation), say so and do that cleanup yourself (via `bash`) before recommending `RETRY`. +kind: ConfigMap +metadata: + name: pi-skills + namespace: agent-pod diff --git a/k8s/apps/agent-pod/ssh-configmap.yaml b/k8s/apps/agent-pod/ssh-configmap.yaml new file mode 100644 index 0000000..c150794 --- /dev/null +++ b/k8s/apps/agent-pod/ssh-configmap.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-pod-ssh-config + namespace: agent-pod +data: + config: | + Host git.riotpiao.com + IdentityFile /root/.ssh/id_forgejo + Port 2222 + User git + StrictHostKeyChecking accept-new diff --git a/k8s/apps/api/AUTH-PLAN.md b/k8s/apps/api/AUTH-PLAN.md new file mode 100644 index 0000000..d3feb1c --- /dev/null +++ b/k8s/apps/api/AUTH-PLAN.md @@ -0,0 +1,83 @@ +# API Auth Layer — Authentik service account + Kong JWT (model invoke) + +Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik +service account holding a valid **client_credentials** JWT can invoke the KServe +models. "Invoke role" = **possession of a JWT from the dedicated model-invoke +OAuth2 provider** (only the service account can obtain one). + +## Flow + +``` +service account ── client_credentials ──▶ Authentik token endpoint + (client_id + secret) https://authentik.riotpiao.com/application/o/token/ + │ + ▼ RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/) +client ── Authorization: Bearer ──▶ Kong (api.riotpiao.com/*) + jwt plugin: verify RS256 sig via Authentik JWKS, + check iss/exp → map to KongConsumer → allow + ▼ + KServe model (reasoning / ornith / ...) +``` + +Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in +**`jwt`** plugin: it validates an RS256 signature against a public key we pin on +a KongConsumer, keyed by the token's `iss`. + +## Changes + +### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py) +- New **service account** user `model-invoker` (type `service_account`, no + password; Authentik issues an app-password/token for M2M). +- New **OAuth2 provider + application** `model-invoke`: + - `client_type: confidential`, `grant_types: ["client_credentials"]` + - signing key = existing RS256 keypair (same as other providers) + - mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes + needed for M2M. +- Client secret written to k8s Secret `api/model-invoke-oidc` + (keys `client-id`, `client-secret`), labelled for whoever consumes it. +- Bind the service account so it (and only it) can use the provider. + +### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`) +- **KongConsumer** `model-invoker` (ns api). +- **`jwt` credential** on that consumer (a Secret of type + `konghq.com/v1/credential`): + - `algorithm: RS256` + - `key` = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/` + - `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing + cert (fetched from Authentik JWKS / cert, stored in git or ksops). +- **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`). + +### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml) +- Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation + (currently e.g. `llm-rewrite-reasoning`) → becomes + `llm-rewrite-reasoning,jwt-auth`. +- Leave `/models` list route open OR protect too (decision). + +## Client usage (after build) +```bash +TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \ + -d grant_type=client_credentials \ + -d client_id=model-invoke \ + -d client_secret= \ + -d scope=openid | jq -r .access_token) + +curl https://api.riotpiao.com/v1/chat/completions \ + -H "Authorization: Bearer $TOKEN" -d '{...}' +``` + +## Test plan +1. No token → Kong returns 401. +2. Valid client_credentials token → 200, model responds. +3. Expired/garbage token → 401. +4. Confirm the `/models` route behaviour matches the decision. + +## Open items / risks +- Authentik `client_credentials` for a *service account* may require an + **app-password / JWT-assertion** flow rather than plain client_secret POST — + verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs + the SA token). Adjust step 1 accordingly before wiring Kong. +- Pinning `rsa_public_key`: Authentik key rotation would break it — document a + rotation runbook, or have the provision script re-export the cert PEM into the + Kong credential on each run (keeps them in sync, same idea as ksops secrets). +- Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's + issuer is stable. diff --git a/k8s/apps/api/kong-metrics.yaml b/k8s/apps/api/kong-metrics.yaml new file mode 100644 index 0000000..d490e06 --- /dev/null +++ b/k8s/apps/api/kong-metrics.yaml @@ -0,0 +1,19 @@ +# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the +# ingress controller apply it to every route on this Kong instance, so all +# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics +# without touching llm-routes.yaml. Scraped via kong-values.yaml's +# serviceMonitor (status listener, already on by chart default at :8100). +apiVersion: configuration.konghq.com/v1 +kind: KongClusterPlugin +metadata: + name: prometheus + annotations: + kubernetes.io/ingress.class: kong + labels: + global: "true" +plugin: prometheus +config: + status_code_metrics: true + latency_metrics: true + bandwidth_metrics: true + upstream_health_metrics: true diff --git a/k8s/apps/api/kong-values.yaml b/k8s/apps/api/kong-values.yaml index ac69f34..8f284e5 100644 --- a/k8s/apps/api/kong-values.yaml +++ b/k8s/apps/api/kong-values.yaml @@ -57,6 +57,17 @@ env: # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the # buffered one dominates. nginx_proxy_proxy_buffering: "off" + # Any plugin that rewrites the request body — request-transformer on the + # llm-chat-* routes — reads it through `kong.request.get_body()`, and that + # returns nothing once nginx has spilled the body past + # client_body_buffer_size into a temp file. The plugin then re-serializes a + # body with no `messages`, and the upstream answers + # HTTP 400 {"error":{"message":"[] is too short - 'messages'"}} + # Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400. + # An agent request carrying tool schemas clears that in one turn, so the + # buffer has to hold a whole conversation, not a chat message. + nginx_http_client_body_buffer_size: "16m" + nginx_http_client_max_body_size: "16m" ingressController: enabled: true @@ -107,6 +118,16 @@ podDisruptionBudget: enabled: true minAvailable: 1 +# Status listener (metrics/health) is on by default at :8100 (chart default, +# verified via `helm show values`). This just wires the ServiceMonitor the +# chart already knows how to generate for it, so kong_http_requests_total / +# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the +# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml. +serviceMonitor: + enabled: true + labels: + release: kube-prometheus-stack + # Spread the two replicas across nodes; `ScheduleAnyway` so a single-node # situation degrades to co-location instead of leaving a pod Pending. topologySpreadConstraints: diff --git a/k8s/apps/api/kustomization.yaml b/k8s/apps/api/kustomization.yaml index 14753b8..5a9ea06 100644 --- a/k8s/apps/api/kustomization.yaml +++ b/k8s/apps/api/kustomization.yaml @@ -6,7 +6,9 @@ kind: Kustomization # or it is silently dropped with no error and no drift shown. resources: - ingress.yaml + - kong-metrics.yaml - llm-routes.yaml + - model-auth.yaml # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own # namespace, and the transformer rewrites metadata.namespace on every resource # it builds, which is a trap for anything cross-namespace added later. diff --git a/k8s/apps/api/llm-routes.yaml b/k8s/apps/api/llm-routes.yaml index 5fe1602..c58bf96 100644 --- a/k8s/apps/api/llm-routes.yaml +++ b/k8s/apps/api/llm-routes.yaml @@ -66,7 +66,7 @@ metadata: name: llm-models namespace: llm-serving annotations: - konghq.com/plugins: llm-models-list + konghq.com/plugins: llm-models-list,model-key-auth konghq.com/strip-path: "false" konghq.com/methods: "GET" spec: @@ -110,7 +110,7 @@ metadata: name: llm-chat-reasoning namespace: llm-serving annotations: - konghq.com/plugins: llm-rewrite-reasoning + konghq.com/plugins: llm-rewrite-reasoning,model-key-auth konghq.com/strip-path: "false" konghq.com/methods: "POST" konghq.com/connect-timeout: "10000" @@ -152,7 +152,7 @@ metadata: name: llm-chat-ornith namespace: llm-serving annotations: - konghq.com/plugins: llm-rewrite-ornith + konghq.com/plugins: llm-rewrite-ornith,model-key-auth konghq.com/strip-path: "false" konghq.com/methods: "POST" konghq.com/connect-timeout: "10000" @@ -197,7 +197,7 @@ metadata: name: llm-chat-qwen namespace: llm-serving annotations: - konghq.com/plugins: llm-rewrite-qwen + konghq.com/plugins: llm-rewrite-qwen,model-key-auth konghq.com/strip-path: "false" konghq.com/methods: "POST" konghq.com/connect-timeout: "10000" @@ -267,7 +267,7 @@ metadata: name: llm-rerank namespace: llm-serving annotations: - konghq.com/plugins: llm-rewrite-rerank + konghq.com/plugins: llm-rewrite-rerank,model-key-auth konghq.com/strip-path: "false" konghq.com/methods: "POST" konghq.com/connect-timeout: "10000" diff --git a/k8s/apps/api/model-auth.yaml b/k8s/apps/api/model-auth.yaml new file mode 100644 index 0000000..c796cf7 --- /dev/null +++ b/k8s/apps/api/model-auth.yaml @@ -0,0 +1,49 @@ +# API auth layer — Kong key-auth on the model routes. +# +# The model API (api.riotpiao.com/v1/...) requires a static API key, presented +# OpenAI-style as `Authorization: Bearer ` (or `apikey: `). The key +# lives in the ksops-managed Secret model-invoke-apikey (labelled +# konghq.com/credential: key-auth) and is bound to the KongConsumer below. +# +# Issue the key to rock; use it as the OpenAI SDK api_key. Rotate by updating the +# ksops secret. This is self-contained in Kong — the invoke path does not depend +# on an Authentik token (Authentik still fronts every *human* dashboard SSO). +--- +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: model-invoker + namespace: api + annotations: + kubernetes.io/ingress.class: kong +username: model-invoker +credentials: + - model-invoke-apikey +--- +# key-auth: require the API key on the model routes. key_in_header accepts the +# `apikey` header; key_in_bearer accepts `Authorization: Bearer ` so any +# OpenAI-compatible SDK (api_key=..., base_url=https://api.riotpiao.com/v1) works +# unchanged. +# +# Namespace `llm-serving`, not `api`: the ingress controller resolves a +# `konghq.com/plugins` annotation against the annotated object's OWN namespace, +# and all five model routes in llm-routes.yaml live in llm-serving. While this +# sat in `api` the reference dangled, the plugin never bound, and every model +# route served traffic with no key at all — verified: an unauthenticated +# /v1/models and /v1/ornith/chat/completions both returned 200. A dangling +# plugin reference is silent; it fails open, so re-test without a key after any +# move rather than trusting that the object exists. +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: model-key-auth + namespace: llm-serving +plugin: key-auth +config: + key_names: + - apikey + - authorization + key_in_header: true + key_in_query: false + key_in_body: false + hide_credentials: true diff --git a/k8s/apps/llm-serving/embeddings.yaml b/k8s/apps/llm-serving/embeddings.yaml new file mode 100644 index 0000000..5ceb6b3 --- /dev/null +++ b/k8s/apps/llm-serving/embeddings.yaml @@ -0,0 +1,56 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-embeddings + app.kubernetes.io/part-of: llm-serving + name: embeddings + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model-id=nomic-ai/nomic-embed-text-v2-moe + - --port=8080 + - --hostname=0.0.0.0 + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 8Gi + requests: + cpu: '8' + memory: 4Gi + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumeMounts: + - mountPath: /mnt/models + name: models + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml new file mode 100644 index 0000000..ea788ac --- /dev/null +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Explicit allowlist, matching k8s/apps/api. Anything added to this directory +# and not listed here is silently dropped — no error, no drift shown. +# +# These five were adopted from live state on 2026-08-15; they had been applied +# by hand and carried no ArgoCD ownership. Each was exported and verified with +# `kubectl diff -f ` returning empty before the Application below was +# created, so the first sync was a no-op rather than a redeploy. Re-verify that +# way after any edit here: a GPU predictor restart is a weights reload measured +# in tens of seconds, not a rolling update. +resources: + - embeddings.yaml + - ornith.yaml + - reasoning.yaml + - reranker.yaml + - verifier.yaml +# No namespace transformer: every file sets its own, and the transformer would +# rewrite metadata.namespace on anything cross-namespace added later. diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml new file mode 100644 index 0000000..fb0c4bc --- /dev/null +++ b/k8s/apps/llm-serving/ornith.yaml @@ -0,0 +1,108 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + # Kong reads its timeouts from the Kubernetes Service, not the Ingress — + # Ingress annotations configure Route entities (strip-path, methods, + # plugins), these configure the Service entity. They were on + # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s + # default in force. KServe propagates InferenceService annotations to the + # Service it generates, which is how they reach Kong from here. + # + # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no + # request ever waited on a cold load. A pod restart flushes VRAM, and + # loading ornith:35b takes longer than 60s, so the first request after any + # restart returned 504. + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" + labels: + app.kubernetes.io/name: llm-ornith + app.kubernetes.io/part-of: llm-serving + name: ornith + namespace: llm-serving +spec: + predictor: + containers: + - command: + - /bin/sh + - -c + - 'set -e + + ollama serve & + + SERVE_PID=$! + + until ollama list >/dev/null 2>&1; do sleep 2; done + + ollama pull ornith:35b + + ollama pull qwen2.5:3b-instruct + + ollama run ornith:35b "ok" >/dev/null 2>&1 || true + + ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true + + wait $SERVE_PID + + ' + env: + - name: OLLAMA_HOST + value: 0.0.0.0:8080 + - name: OLLAMA_MODELS + value: /mnt/models/ollama + - name: OLLAMA_CONTEXT_LENGTH + value: '32768' + - name: OLLAMA_KEEP_ALIVE + value: '-1' + - name: OLLAMA_NUM_PARALLEL + value: '1' + - name: OLLAMA_MAX_LOADED_MODELS + value: '2' + image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -c + - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | + grep -q qwen2.5 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + exec: + command: + - /bin/sh + - -c + - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | + grep -q qwen2.5 + failureThreshold: 120 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml new file mode 100644 index 0000000..44f091c --- /dev/null +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -0,0 +1,81 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-reasoning + app.kubernetes.io/part-of: llm-serving + name: reasoning + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit + - --served-model-name=reasoning + - --quantization=bitsandbytes + - --dtype=float16 + - --kv-cache-dtype=auto + - --tensor-parallel-size=1 + - --max-model-len=16384 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=4 + - --enable-chunked-prefill + - --enable-prefix-caching + - --reasoning-parser=deepseek_r1 + - --host=0.0.0.0 + - --port=8080 + env: + - name: VLLM_USE_FLASHINFER_SAMPLER + value: '0' + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 80 + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + deploymentStrategy: + type: Recreate + maxReplicas: 2 + minReplicas: 2 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - emptyDir: + medium: Memory + sizeLimit: 2Gi + name: shm + diff --git a/k8s/apps/llm-serving/reranker.yaml b/k8s/apps/llm-serving/reranker.yaml new file mode 100644 index 0000000..eded1cd --- /dev/null +++ b/k8s/apps/llm-serving/reranker.yaml @@ -0,0 +1,56 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-reranker + app.kubernetes.io/part-of: llm-serving + name: reranker + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model-id=BAAI/bge-reranker-base + - --port=8080 + - --hostname=0.0.0.0 + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 8Gi + requests: + cpu: '8' + memory: 4Gi + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumeMounts: + - mountPath: /mnt/models + name: models + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/verifier.yaml b/k8s/apps/llm-serving/verifier.yaml new file mode 100644 index 0000000..8e20847 --- /dev/null +++ b/k8s/apps/llm-serving/verifier.yaml @@ -0,0 +1,76 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-verifier + app.kubernetes.io/part-of: llm-serving + name: verifier + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model=Qwen/Qwen2.5-Math-PRM-7B + - --served-model-name=verifier + - --runner=pooling + - --dtype=float16 + - --tensor-parallel-size=1 + - --max-model-len=4096 + - --max-num-seqs=8 + - --host=0.0.0.0 + - --port=8080 + env: + - name: VLLM_USE_FLASHINFER_SAMPLER + value: '0' + - name: VLLM_ATTENTION_BACKEND + value: XFORMERS + - name: HF_HOME + value: /mnt/models + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '4' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - emptyDir: + medium: Memory + sizeLimit: 1Gi + name: shm + diff --git a/k8s/apps/messaging/queue-crd/templates/rbac.yaml b/k8s/apps/messaging/queue-crd/templates/rbac.yaml index f85a429..20da39b 100644 --- a/k8s/apps/messaging/queue-crd/templates/rbac.yaml +++ b/k8s/apps/messaging/queue-crd/templates/rbac.yaml @@ -10,13 +10,22 @@ metadata: name: queue-operator rules: - apiGroups: ["kmsvc.io"] - resources: ["queues", "temporalworkers"] + resources: ["queues"] verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["kmsvc.io"] - resources: ["queues/status", "temporalworkers/status"] + resources: ["queues/status"] verbs: ["get", "update", "patch"] - apiGroups: ["kmsvc.io"] - resources: ["queues/finalizers", "temporalworkers/finalizers"] + resources: ["queues/finalizers"] + verbs: ["update"] + - apiGroups: ["kmsvc.io"] + resources: ["temporalworkers"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["kmsvc.io"] + resources: ["temporalworkers/status"] + verbs: ["get", "update", "patch"] + - apiGroups: ["kmsvc.io"] + resources: ["temporalworkers/finalizers"] verbs: ["update"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] diff --git a/k8s/argocd/apps/10-storage-observability.yaml b/k8s/argocd/apps/10-storage-observability.yaml index 8e26140..ef5b794 100644 --- a/k8s/argocd/apps/10-storage-observability.yaml +++ b/k8s/argocd/apps/10-storage-observability.yaml @@ -72,6 +72,14 @@ spec: destination: server: https://kubernetes.default.svc namespace: longhorn-system + # Longhorn writes disk state back into its own Node CRs — the disk key it + # generates, storageReserved, diskType, evictionRequested. Git declares only + # allowScheduling; without this the controller's writes read as drift forever. + ignoreDifferences: + - group: longhorn.io + kind: Node + jsonPointers: + - /spec/disks syncPolicy: automated: prune: true diff --git a/k8s/argocd/apps/56-llm-serving.yaml b/k8s/argocd/apps/56-llm-serving.yaml new file mode 100644 index 0000000..5544104 --- /dev/null +++ b/k8s/argocd/apps/56-llm-serving.yaml @@ -0,0 +1,58 @@ +# Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`). +# +# Syncs before wave 7 (Kong), so the predictor Services exist before the routes +# that point at them. KServe itself is part of the substrate; this Application +# owns only the InferenceServices. +# +# Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed +# by hand — no ArgoCD ownership, present in no repo — so every change to them +# was drift by definition. Each manifest was exported from the cluster and +# verified with `kubectl diff` returning empty before this file existed; the +# first sync therefore adopted them without restarting anything. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: llm-serving + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + project: homelab + revisionHistoryLimit: 3 + source: + repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/llm-serving + destination: + server: https://kubernetes.default.svc + namespace: llm-serving + syncPolicy: + automated: + # `prune: false` here, unlike every other Application in this repo, and it + # is not an oversight. + # + # ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label + # (argocd-cm `application.instanceLabelKey`). KServe copies an + # InferenceService's labels onto the Deployment and Service it generates — + # visible today as `app.kubernetes.io/name` and `part-of` on + # `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe + # propagates that tracking label to children that are not in git, ArgoCD + # reads them as extraneous, prunes them, and KServe recreates them. That + # loop churns GPU pods. + # + # Deleting an InferenceService therefore means deleting the file AND + # removing the object, rather than relying on prune. + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=true + # KServe CRDs are large; server-side apply avoids the + # "metadata.annotations: Too long" failure client-side apply hits, and is + # the correct mode for adopting objects an operator also writes to. + - ServerSideApply=true + retry: + limit: 3 + backoff: + duration: 10s + factor: 2 + maxDuration: 3m diff --git a/k8s/argocd/apps/60-applications.yaml b/k8s/argocd/apps/60-applications.yaml index 2122e9b..aa996f0 100644 --- a/k8s/argocd/apps/60-applications.yaml +++ b/k8s/argocd/apps/60-applications.yaml @@ -84,6 +84,29 @@ spec: syncOptions: - CreateNamespace=true --- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: agent-pod + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "8" +spec: + project: homelab + source: + repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/agent-pod + destination: + server: https://kubernetes.default.svc + namespace: agent-pod + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +--- # iMessage/SMS delivery. Raw manifests: a privileged macOS VM (Docker-OSX) # running the BlueBubbles server, plus its dedicated local StorageClass. # diff --git a/k8s/argocd/secrets/agent-pod-models.enc.yaml b/k8s/argocd/secrets/agent-pod-models.enc.yaml new file mode 100644 index 0000000..5d29449 --- /dev/null +++ b/k8s/argocd/secrets/agent-pod-models.enc.yaml @@ -0,0 +1,23 @@ +apiVersion: ENC[AES256_GCM,data:zdc=,iv:VvjvrS5PVNAMIaOE0LaWU+tHcUIYVQDnCANQz6myktY=,tag:xGyWDhRCwwiNny7hPllf5g==,type:str] +kind: ENC[AES256_GCM,data:2bf5Zfy8,iv:5Oz423GzUWmgdaaZHbrtedwRHIAPIuLh4iDMieLL05s=,tag:GNW1ROjlzGvO/4tIOSuH3Q==,type:str] +metadata: + name: ENC[AES256_GCM,data:7RO0Qxkc+/sA,iv:wKe9A8d7QJSx/6rlEY5H6lU8V24TJqr5IpXQCBc8QgM=,tag:iF6z1MNEXG3pCz2cQ18gLg==,type:str] + namespace: ENC[AES256_GCM,data:RcduxHHLtjH9,iv:opOVx1lL2ltDqQgsleN7NdMAq0TyFr/YQO3FFsHh5AA=,tag:fIwO31apqIatRRzBvamw6g==,type:str] +type: ENC[AES256_GCM,data:C+JjyZt5,iv:xs49Lz6zRzcf3spiPzdUKTm2HZ+VgFahN6wjIe81JI4=,tag:VllJ54MqU+klA8xAIebjrA==,type:str] +stringData: + models.json: ENC[AES256_GCM,data:FvlbdMkcngJchi0GEjIEDjxXpQjRwh+3xnlZNpApgd3v1SpMd0qD2d3TbQr39+feSfsvpxhdmmyU/PmRHZYVdr/QpIKlZBWBD6qTCaOSq62NQbR4ck336muIaWD4AmahWCvRouYEJdEsnG7nedjAivw4cls18vMfnL5pNm2t96foScSyHosWoh3RdT6CWLmJE39+kS6/fwZ1D3Z/GEm/E2zHkVtEbEw/fk1PiqsnmZWt8QcSRAeX8aIDkphV7wOKlkri11r1NuFQKMVIau07VPce1YEi5rsJqsircvDxQelQiMGQt2y2M4GGlXx2NeJgKEtP4Hf9mVGhTeN3CYLcXi8Qyg7k1GIawGVW/Em0kWWfy0GMjvOMfYozNmIpi0YQdYBbr3j6pkixtVM2dWezvld5QIYLnHoGMxw4M6L4IqVIonC3j6tk9pV1DaL5IFskEXyb6ScTrmVY4mQ+VEXvfLYA38jRauWxx/4qf4I7B0RzfGzOx67UJh2jKj0bPWNFHKt5Dz4CK+ul39d1KUPHHMoyaH5gCpOAfLSkjMm44X1Cn+O6uY+QFCugh9MI7bJg3EbovTCZxz95GCupWAvDshv59f7gpGL0T8AuRZHlNp6yZMLdviR9yf79d2rW26PIM1QuNOUWELrOcTs5IAmyuFYr0PI5W5XSf4klOdz3UdhPfMDcyRqtSzVDsSo9oC/WeWOo1+yurtdqqW0wLiFpU/kTaZ4JN5Kl080twqFEqgyPnJrbmlxwWlt4XN2SV9D7pcfE61FzUTwoi5PD/8xt94Fp3XerSgJtwhQ6X4Neo47wCFoMR4Y69mlHrFgSJA4kLFezDc2ISnyAix0W5to0cjwJqQ2EgonIhCXD4xWTvYeApHJMOC2o4B74K5efOZ7klb6PwDMvzM3LyeGzjoJh47aJGvhWN/MpQZC2lSHT8PdMxdGDc+OkEVmsvMncddWMGjQhkTo+69sUexARxLY4H22TCt58azqQGcWa5e7s9NUHb6bRVKHzXh4HMiIhQ0jBev++Jz5gIOyYRnw5xoGbo3ROv8ndczVAqXyAla23EM/VBWT0hLMjr6xxonuLQacP9dPJE9QUofjRMeYblkVvQdIw8WuKGbAlbTR/bokQwGdVp2bywSFQZj7BLlPjzCr0/LJmxYMdq6CKB049PdTlS9/UKBHApddMQ/QRXfOYUCX+lm4k4J19jd7dWk34WNUrTEPSR2d1U/2wTnMvzlrnQHx/zk4/lmI1newfjIbybIbXypX8xVEqzFWXpxUe+iFLP19KmmmHG7gGsV32k1VSGvYFI0HOokZga98QsAbJGbi1gQq8gYpMR/mBdhjJVcnAjOYpg+YaR0vCyrhBN55aSCDoUbT1JOqxvFqmlBHDxTWorD/3KGFxEH/wvSsxussFmUCWdEgYmjGO1ktnQBw1YpOB7mhDUlzvYCSxfpexsYufDYqTbrb/R5gmATC0pEN6EgTth5YqFOL7AiIsogy14RGjRym1wvjVl5SUU9bMQlWGJbQuvNRRwL/TzR/QP210jMsnAh9t8sncDnFPIbhFgeBLsU/ctYoBs0nFVWGWzcuoP9HTnvdWTT2xKmWAeFz8I3DZ5sjm2B787Q4bCrAszTvVLeLiRw/pOE32BeNfG4nNiZDvMQhUomAKpaQypWTu2wLsv3ISm00gud5sRKB6ASJANuSu+EhG9k4OYcBEzAT9PKUxclupFbmsFnXx0u9CXDj3tErrPCUs7DuFSr+fz0ehCLMH+GrcqfytjNDVoTSBRZg+lJYKjAqVCGvq8uRf5V/rdulbt3ggx/uhGSiPzYrBQ0g9iMdjQqpNGBwjlxGkhRE9D2n/PoerAAnHTfwc9Hezffncostjp09xw2CB/xOx9HZFGalQ1R4F5+0lm0iR4lMzc+se+o38foNIiMeMIwAVBJu7IO++UMuKCxpxmdAGqEYj/Ddpzfi3ZcRwe/1HhanVvKRMJsPOhnv0SB2xBrdb7QHnEzLbRTriVRBBdOI77HXWgDt9uQru0tEXCEQ0n+f0BWqHVa/RLN4Yj0dQVWLXU75ddyHnRaCiVdLQnObMvG/QXqY1k/bJTzGNgVF8/KGY51NgdfqskOXC9Fv5IGiyWq+/DL7tQcrmThkzUhZlwDBmwUzPFmzuUC7Z5PkXv2EV+eUrYF16HYqp5KgqTkBHr/JXR8ljb/5gMN929Jk2fBVEuzaEG6W9Yek+Qbx1cnXYrMBI9GWJUJsMsxsAj+1bAjtQeQwt0nXgZesTaV90dfQYq+nTGXMNjqaMhsrJhR6lU1+CDpwY+IObHSttS8AKB+Czsl0nUB44o1a/zKRDNTCGGmPFuv1Q5Ub/1myxorLDDF1mYDpre5TedygtZ/1YSz/YOpKwmWdW32FsnY+m3qjEAcfV5f923N/z8ENVxVpeiWL6mqf7h6F3qTiaSsFq2O8m8tVm70X+oRdig3hPnKqlK/KoxVmzZErhEJ+KmP5I2uvIlb4PViE6imu8kw4VqMClb7jJ7Yk+2Zy84C8RGuDCN8GCDG7FvAahA9xVGxB6rmEpzvqFBxos2EeLOlyfewf7rCw1JAL3C9hZrciA9Jqb/t8B7KqCmCOsKK7xVt1yqyG1dsa364JG/1SKSMKebTheuymZ1+VnGjRZ+pgSAkQGjCtp7B3fTlhjt+WwxW4Ml27yfjokQQ==,iv:8Z39eWukGSMePh/3Dj35e6Zahejil+eeMSqwMYf3snI=,tag:FyBg1y0IQEg/m0mgpf1ESg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZzB5REh6amhCNzRmY0ta + MkN3Y0ZlR29lQ3h2SWo3cW5CUThkL2RnU1NVCmt2ZkhIZTlHN1RQRkFrTjVvbjVw + RGRrTXRoYmdQcnlMSEo3ZWsrZUQ5cHMKLS0tIGU2TGJqZDRxUGJpZzRveEtZankx + MzhrT1R2akxxby9QVzd1RXB0RDY1LzQKOF+/e5z5lPX6Y1sMTAHuDj3YqW1m+sBd + u/0R0YnBonYM3wS5nJE3NZMkImaAdQlUjOzQepfBldG+lz++rlnAww== + -----END AGE ENCRYPTED FILE----- + recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + lastmodified: "2026-08-18T20:06:08Z" + mac: ENC[AES256_GCM,data:IM9HkpdwtQE2wCkjwDWOmHH4uP7TlIsrK4TVytiecvYz4SiLk6IRUSIu7I3a+F+dtltC2WtokoATaB69DTXPoI54amzzptirxiFD5FbaU+u2gLjo7KI7V0smYGuKqMYwnod2L/4GdlvP6xjVxFWuA01rQRaYBkFSumS4NABl/I4=,iv:al1MyBmFwni8gap7PPZxWaCwqFKCicfPq6nVmrSn9Xc=,tag:KJGTDQ8xRGF8qMOjoQ+dng==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 diff --git a/k8s/argocd/secrets/agent-pod-ssh-key.enc.yaml b/k8s/argocd/secrets/agent-pod-ssh-key.enc.yaml new file mode 100644 index 0000000..cf42beb --- /dev/null +++ b/k8s/argocd/secrets/agent-pod-ssh-key.enc.yaml @@ -0,0 +1,23 @@ +apiVersion: ENC[AES256_GCM,data:seY=,iv:1q2bg1QBNDDQPYNu1S08MZ+Ix8WLthxObsrqTMJEVhY=,tag:OMh/stn4vVScOEAyviuurw==,type:str] +kind: ENC[AES256_GCM,data:QwqgPMrm,iv:tv6rUcezEgQigvHVw05Mevt6BHDD5AARuoUmQpry3AU=,tag:m9RLNezJsLYGSRWnozfkoQ==,type:str] +metadata: + name: ENC[AES256_GCM,data:ZJtohnEDN2NBFrEhhvGuw4s=,iv:SU5LuA3Lqjh+q7nVxeVVtzeXiOxxVserNno5c7b47lM=,tag:YLprEk7oVjVsIDVANWDPrQ==,type:str] + namespace: ENC[AES256_GCM,data:LieEC+YWJzq+,iv:U00VXejzlpgOYS7Yxix/MnEW0PEvmwOuOXMUi2CWIps=,tag:QvFYZIexWhyvBYTruGMifQ==,type:str] +type: ENC[AES256_GCM,data:U41EbP+H,iv:CKDqec0jfVtbOtEyqQpV9CHJNCwhneYi0wqK8QU2vnY=,tag:CSJ04pkPslevQrpSjhXNHw==,type:str] +stringData: + id_forgejo: ENC[AES256_GCM,data:dgwezImUFb2r6MMZ2V1ZMuy32DAVZ9YMzlr9zYlXKEuzeCaWn29DuqgeP5PJQFFKwd37Y9jmzXwuKD5KrC9WCAzrS2VClxmJYHj2Bf842qtKa4x3SubHu4w6wp8v42wQYCU0Kygc5UgkCFYgOBgvpYCFKhiro6Mg/PBTWTIIgrLlSVwUWPIcaX6Sq4ZW4sy6CZ0H/pEMDAjU8drZ4KmGw7W4FshNynplMuP9ewbSs9yxtuAsKt4ezuUQBGHmAYdPHytC9HUzf67z/j6TF/zg6XgTe0AG116cNddAL8oNermCIkkKuTwb5xy1+3WRjjacC0K5Ire1fV5RyXtD/1JgEHJ8gbPK1w1V5fHZXX/ARvQFlP+uqxFYmPkCHBmK/R6/VvAghI/KYHlc76hdsAe0sd1UrPIHjVEtwgvDQl+4de9L9cHG3Qxj23ajlpISWA3tGADhwzXDxi0KYClckebrCKhizWFQLYXezALYklgY2yghG9jZHufV9jRjYSr9DLtwUmVf3oglAzKkWErNs8dWqEKYFx7HpFf0FiZh,iv:1aX4E+R1oFMm9eI1RENFIYCoatX8BdHo6PhrEGq3rtg=,tag:nwniLOOmpYd3Z6R92SvB3Q==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvRmtrdkpMOVhRdTVucVlr + QmlENGdDN2J1MUczSmRhNUpwYlBjWXhqbERzCnpLRFl1bVVtSFE3L1l4T3Rwd2ZQ + Z1MxTzJYZWovMmd2RTA3RFJVdVRRTEUKLS0tIDFQWHh2dmQxU1FUZnUxalhTdEd3 + c3dUVzg2L2VOWFlmUG9XdFNWN2RvQkkKYqSmFMkDV/T7AOjKYQNJW85gUzFraRre + GhwAuPJ9oNQAhSRa5z2p6ghoUplSXtNZ6H2OzETRfOc4N2cHLljW/Q== + -----END AGE ENCRYPTED FILE----- + recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + lastmodified: "2026-08-18T04:18:28Z" + mac: ENC[AES256_GCM,data:ScGMBQqGm31fWDZurmlmk+rnlvyaQkalUcJ6ds/U5JJ2niTf7Llbruuqt2EzMQO09fT2kQqAIYr4cF61Czip6rDhpG8I3H3lBc2mOqUXbeCFyRL6mRpVYPOgkSzgyfXUfe8T++54UhQ0Clv/8Z4XB48oTs9detkUDzAfxP4azRE=,iv:edhsVviyjAm48Blm5CEQ0PBhcWVX97xi2hEbgDX9Dpg=,tag:Xnn85E99LibtB/LWwi5wcA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 diff --git a/k8s/argocd/secrets/model-invoke-apikey.enc.yaml b/k8s/argocd/secrets/model-invoke-apikey.enc.yaml new file mode 100644 index 0000000..e5f4dba --- /dev/null +++ b/k8s/argocd/secrets/model-invoke-apikey.enc.yaml @@ -0,0 +1,25 @@ +apiVersion: ENC[AES256_GCM,data:894=,iv:Swg6ADUgmrqwz7wqAZHip9/qwFu0Rn8S2Lx4gBH8LJM=,tag:zbv0tFfRLtwFxBfpsuLt8A==,type:str] +kind: ENC[AES256_GCM,data:jzVtJHYw,iv:ToZ0orfJqfGF/OnAPeYu/g2f4fXAMOZQDkA1+tmIccs=,tag:pi2skbwIU8qOQVEC86MdAA==,type:str] +metadata: + name: ENC[AES256_GCM,data:t7zCeZLvAEbkJqUjWi26yD6UDA==,iv:MS1gq/bwKZdLA1itVDtsrdSOfI7e2CrhjvX5yhs0eQA=,tag:lC1gcoFMI5nfzC56U1WXrg==,type:str] + namespace: ENC[AES256_GCM,data:tp+d,iv:gnlet/60mgbSWLXEQpSlcWD98ky7NvlNCzGLTMys0JQ=,tag:PQYj9UeA50YenQESTCl7lg==,type:str] + labels: + konghq.com/credential: ENC[AES256_GCM,data:SOqQ9bLGLK0=,iv:a7En49UhRDwgHbv5NRB/XilEYIKQdaDqKH86WDrJB5I=,tag:JaDZ6Ko8ovY6AZ1hW9YMGQ==,type:str] +type: ENC[AES256_GCM,data:r1K5gvop,iv:Gjv4oG2Unyql5rY9RTTljFqyd28xI81CDWtpavuuW5E=,tag:jF8lBs3AQwHPVEu9V+mONw==,type:str] +stringData: + key: ENC[AES256_GCM,data:pm9GmSvX5MAsXO/e6ZcI4NF1Hwr3qjG6LaEEjvV0ihvSWhO23drlAEsrTtzppqgDZbMORf+5+P53mXU=,iv:5AbHNKeiMPoFQP/qTKdA0vEoYPzuaf4kIdGGcMSmfIQ=,tag:LD5w7XK+hiCS5D410+nCfQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUTmoxQkpuYUp5UlRQMkph + a0RaazdvaU5sWkNuL2gvVjlUYXVWV0dUWVVRClZhWDZCN2hpS2hnOG9Pck9zOTkx + RGNGMEI3RHpNbzVaaWNGcTNSSEdzZHMKLS0tIFZERnVJWUpreUh3TTlwbGw0dUx4 + MGlCSkxuWWVEK2RaSDZPUzhNSUlCa28KHN0IsgQc/kBqmjQ6+4sgfb9PJy/45MwN + rXaLJ1htpqPZ9MJ8iOukRi0IKnKgQWXsoZengIxGmcOnEctpoH/kyQ== + -----END AGE ENCRYPTED FILE----- + recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + lastmodified: "2026-08-14T01:07:50Z" + mac: ENC[AES256_GCM,data:nY+YVwU1GuK8Yz+EZOQKkZKN28tm2L8afflc6hsgVFCFmsep5kVT+zId7AgemvQ+qnrho5N5nqxY2knB0gusFfWNKF3V5A5GBq40WtZCMaAtcwhJSex4kK7ZyaZD6oWDW/RTUumSrivSowkWlqt1XlDKyFLqSlpWQTd7JiGmUv8=,iv:zE8+B5UVYSuuAGYyvXsAwp1N/4vGduCoGyEMCNNEUnM=,tag:6RcrjneV2dOzhmoQi+5HsA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.2 diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml index 33dc375..96c131a 100644 --- a/k8s/argocd/secrets/secret-generator.yaml +++ b/k8s/argocd/secrets/secret-generator.yaml @@ -7,6 +7,8 @@ metadata: exec: path: ksops files: + - agent-pod-models.enc.yaml + - agent-pod-ssh-key.enc.yaml - authentik-secrets.enc.yaml - cloudflare-secrets.enc.yaml - forgejo-runner-token.enc.yaml @@ -18,6 +20,7 @@ files: - homarr-secrets.enc.yaml - homelab-ca-secrets.enc.yaml - loki-secrets.enc.yaml + - model-invoke-apikey.enc.yaml - minio-secrets.enc.yaml - vault-secrets.enc.yaml - vault-unseal-keys.enc.yaml diff --git a/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml b/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml index 580d5ab..cca83f3 100644 --- a/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml +++ b/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml @@ -7,6 +7,22 @@ metadata: namespace: longhorn-system annotations: storageclass.kubernetes.io/is-default-class: "false" + # StorageClass.parameters is immutable. mkfsParams below was added after + # this class already existed, so every sync failed with + # parameters: Invalid value: {...}: field is immutable + # and the Application sat OutOfSync permanently — ArgoCD retrying a change + # the API server can never accept. + # + # Replace=true makes ArgoCD delete and recreate instead of patching. Safe + # for a StorageClass: it is consulted only at provisioning time, so bound + # PVs and their data are untouched. New PVCs briefly fail if one is created + # during the window. + # + # Note what this does NOT do: the nine CNPG volumes already provisioned were + # formatted without mkfsParams and keep that format. Only volumes created + # after this recreate get it. Existing volumes that need to expand still + # need handling separately. + argocd.argoproj.io/sync-options: Replace=true,Force=true provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: diff --git a/k8s/infra/longhorn/longhorn-nodes.yaml b/k8s/infra/longhorn/longhorn-nodes.yaml index 5d6e5b2..52e4b3d 100644 --- a/k8s/infra/longhorn/longhorn-nodes.yaml +++ b/k8s/infra/longhorn/longhorn-nodes.yaml @@ -1,6 +1,16 @@ # Longhorn Node CRDs for cp-2 and cp-3. # These nodes have the control-plane taint, so Longhorn doesn't auto-discover them. # Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes. +# +# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the +# entry itself (`default-disk-080400000000`, not `default-disk`) and writes +# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a +# `default-disk` key here never matched the live one, so the Application sat +# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the +# same /var/lib/longhorn path — which is worse than the drift it was fixing. +# +# What these objects are actually for is `allowScheduling: true` on tainted +# control-plane nodes. That is all they need to declare. --- apiVersion: longhorn.io/v1beta2 kind: Node @@ -10,12 +20,6 @@ metadata: spec: name: talos-cp-2 allowScheduling: true - disks: - default-disk: - allowScheduling: true - path: /var/lib/longhorn - storageReserved: 0 - tags: [] tags: [] --- apiVersion: longhorn.io/v1beta2 @@ -26,10 +30,4 @@ metadata: spec: name: talos-cp-3 allowScheduling: true - disks: - default-disk: - allowScheduling: true - path: /var/lib/longhorn - storageReserved: 0 - tags: [] tags: [] diff --git a/k8s/infra/monitoring/dashboards/llm-frontend.yaml b/k8s/infra/monitoring/dashboards/llm-frontend.yaml new file mode 100644 index 0000000..8537562 --- /dev/null +++ b/k8s/infra/monitoring/dashboards/llm-frontend.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: llm-frontend-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "LLM" + # Request rate/error/latency/bandwidth now come from Kong's prometheus + # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the + # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) -- + # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/ + # embeddings/rerank uniformly without per-backend instrumentation. + # Token-count metrics are still not available: that needs response-body + # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only). + # Predictor-level metrics (native Ollama/vLLM stats) also still need a + # dedicated exporter -- not added here. +data: + llm-frontend.json: | + {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]} diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index eb14f2c..b328de0 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -19,6 +19,7 @@ resources: - dashboards/control-plane-logs.yaml - dashboards/hardware-overview.yaml - dashboards/kube-controller-health.yaml + - dashboards/llm-frontend.yaml - dashboards/service-availability.yaml - dashboards/service-golden-signals.yaml - dashboards/service-internals.yaml diff --git a/terraform/templates/controlplane.tftpl b/terraform/templates/controlplane.tftpl index 0d998fd..8669820 100644 --- a/terraform/templates/controlplane.tftpl +++ b/terraform/templates/controlplane.tftpl @@ -49,12 +49,15 @@ machine: image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version} wipe: true grubUseUKICmdline: true +%{ if length(longhorn_disks) > 0 ~} disks: -%{ for disk in longhorn_disks ~} +%{ for idx, disk in longhorn_disks ~} + # ${disk.kind} — ${disk.device} - device: ${disk.device} partitions: - - mountpoint: ${disk.mountpoint} + - mountpoint: ${coalesce(disk.mountpoint, format("/var/lib/longhorn-disk%d", idx + 1))} %{ endfor ~} +%{ endif ~} features: diskQuotaSupport: true kubePrism: @@ -172,10 +175,10 @@ cluster: # kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs. - name: cilium-lb-ippool contents: | -${indent(8, cilium_lb_ippool)} + ${indent(8, cilium_lb_ippool)} - name: cilium-l2-announcement contents: | -${indent(8, cilium_l2_announcement)} + ${indent(8, cilium_l2_announcement)} # CoreDNS Corefile with homelab hostname rewrites (single source of truth in # terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to # the nginx ingress controller so OIDC auto-discovery against diff --git a/terraform/variables.tf b/terraform/variables.tf index fc40506..8693c03 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -107,10 +107,24 @@ variable "controlplane_configs" { lan_subnet = string lan_gateway = string install_disk = string - longhorn_disks = list(object({ + # Longhorn data disks, in mountpoint order. `device` should be a stable + # /dev/disk/by-id/wwn-* path: this hardware enumerates /dev/sdX by discovery + # order, and `install.wipe: true` means a renumber can aim the installer at a + # data disk. + # + # `mountpoint` defaults to /var/lib/longhorn-disk by position. Never + # renumber or reorder existing entries — Longhorn keys its disks off the + # mountpoint, so a rename orphans the replicas already on that disk. Append + # new disks to the end. + # + # `kind` ("ssd"/"hdd") is a declared label only. PERC RAID controllers report + # every disk as rotational, so it can't be autodetected, and it deliberately + # does not affect ordering or mountpoints. + longhorn_disks = optional(list(object({ device = string - mountpoint = string - })) + mountpoint = optional(string) + kind = optional(string, "hdd") + })), []) zone = string allow_scheduling = bool # Extra cert SANs for this node — e.g. Cloudflare Tunnel public hostnames so