From 5582f2cd9c427873b12bf58e343b50b9ac3ff3bf Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:04:04 -0700 Subject: [PATCH] (chore) identify new types --- .gitignore | 10 + loop.sh | 324 ++++ poimen/Cargo.lock | 778 ++++++++ poimen/Cargo.toml | 21 + poimen/crates/ids/Cargo.toml | 19 + poimen/crates/ids/examples/usage.rs | 59 + poimen/crates/ids/src/lib.rs | 194 ++ poimen/crates/ids/tests/compile_fail.rs | 7 + .../ids/tests/compile_fail/default_task_id.rs | 8 + .../ids/tests/compile_fail/unscoped_key.rs | 11 + poimen/crates/ids/tests/it_scoped_keys.rs | 159 ++ poimen/crates/kernel/Cargo.toml | 11 + poimen/crates/kernel/src/lib.rs | 127 ++ .../kernel/tests/it_transition_table.rs | 211 ++ rust-agentic-sys.md | 1700 +++++++++++++++++ 15 files changed, 3639 insertions(+) create mode 100644 .gitignore create mode 100755 loop.sh create mode 100644 poimen/Cargo.lock create mode 100644 poimen/Cargo.toml create mode 100644 poimen/crates/ids/Cargo.toml create mode 100644 poimen/crates/ids/examples/usage.rs create mode 100644 poimen/crates/ids/src/lib.rs create mode 100644 poimen/crates/ids/tests/compile_fail.rs create mode 100644 poimen/crates/ids/tests/compile_fail/default_task_id.rs create mode 100644 poimen/crates/ids/tests/compile_fail/unscoped_key.rs create mode 100644 poimen/crates/ids/tests/it_scoped_keys.rs create mode 100644 poimen/crates/kernel/Cargo.toml create mode 100644 poimen/crates/kernel/src/lib.rs create mode 100644 poimen/crates/kernel/tests/it_transition_table.rs create mode 100644 rust-agentic-sys.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8454d59 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.claude +.pi +tasks +target +rust-agentic-task.md + +*.stderr + +verify/ +reviews/ \ No newline at end of file diff --git a/loop.sh b/loop.sh new file mode 100755 index 0000000..f71c94f --- /dev/null +++ b/loop.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash +# Agentic driver for the rust-agent-sys board. +# +# Per task, two `pi` passes: +# 1. coder — implements, runs the task's Verify command, reports +# 2. reviewer — re-verifies, audits false passes/traps, emits a markdown review +# The shell is the chain: the coder's report is fed into the reviewer's prompt. +# The shell also owns the gate rule and the Status writes, because those are +# mechanical and must not depend on a model choosing to comply. +# +# ./loop.sh run from the first unfinished task until done or blocked +# ./loop.sh --list show phase order + current status, run nothing +# ./loop.sh --dry-run list what would run, run nothing +# ./loop.sh --sync rewrite INDEX.md status cells from the task files +# ./loop.sh T0.2 T0.3 run only these, in the order given +# +# Resumable: state lives in the task files, not here. Rerun after a crash. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TASKS="$ROOT/tasks" +INDEX="$TASKS/INDEX.md" +REVIEWS="$ROOT/reviews" +GUIDE="$TASKS/rust-guide-line.md" +CRATE="$ROOT/poimen" +AGENTS="$ROOT/.pi/agents" +LOGS="$ROOT/.pi/logs" + +# The golden-rule region of INDEX.md: board rules + progress tables. +GOLDEN_RULE_LINES="0-69" + +CODER_TOOLS="read,write,edit,bash,grep,find,ls,hashline_edit" +REVIEWER_TOOLS="read,grep,find,ls,bash" + +mkdir -p "$REVIEWS" "$LOGS" + +# ---------------------------------------------------------------- board state + +# Status cells carry emoji ("✅ Done", "⬜ Not started"), so match on substring. +status_of() { sed -n 's/^| *Status *| *\(.*[^ ]\) *|$/\1/p' "$1" | head -1; } +is_done() { case "$(status_of "$1")" in *Done*) return 0 ;; *) return 1 ;; esac; } + +status_emoji() { + case "$(status_of "$1")" in + *Done*) printf '✅' ;; + *rogress*) printf '🟡' ;; + *lock*|*BLOCK*) printf '⛔' ;; + *) printf '⬜' ;; + esac +} + +verdict_of() { + [ -f "$1" ] || { echo "no-review"; return; } + sed -n 's/^| *Verdict *| *\(.*[^ ]\) *|$/\1/p' "$1" | head -1 +} + +task_id() { basename "$1" | cut -d- -f1; } +phase_of() { echo "${1#T}" | cut -d. -f1; } + +# Phase order. Numeric sort on phase then minor puts each gate task (highest +# minor in its phase) last, so sequential execution satisfies the board's +# "no phase starts until its predecessor's gate is green" rule for free. +ordered_tasks() { + find "$TASKS" -maxdepth 1 -name 'T*.md' \ + | sed "s|.*/T||" \ + | sort -t. -k1,1n -k2,2n \ + | sed "s|^|$TASKS/T|" +} + +file_for_id() { find "$TASKS" -maxdepth 1 -name "$1-*.md" | head -1; } + +# The gate task of a phase is its highest-numbered task. +gate_of_phase() { + ordered_tasks | while read -r f; do + [ "$(phase_of "$(task_id "$f")")" = "$1" ] && echo "$f" + done | tail -1 +} + +# No phase starts until its predecessor's gate is green. +gate_blocks() { + local phase="$1" prev gate + [ "$phase" -eq 0 ] && return 1 + prev=$((phase - 1)) + gate="$(gate_of_phase "$prev")" + [ -n "$gate" ] || return 1 + is_done "$gate" && return 1 + echo "P$phase blocked: P$prev gate $(task_id "$gate") is '$(status_of "$gate")'" + return 0 +} + +# ------------------------------------------------------------- index mirroring + +# Rewrite INDEX.md's status cells from the task files. The task file is the +# source of truth; a status changed in one and not the other is a lie. +sync_index() { + local map tmp before after + map="$(mktemp)"; tmp="$(mktemp)" + while read -r f; do + printf '%s %s\n' "$(task_id "$f")" "$(status_emoji "$f")" + done < <(ordered_tasks) > "$map" + + before="$(wc -l < "$INDEX")" + + awk -v mapfile="$map" ' + BEGIN { + FS = "|"; OFS = "|" + while ((getline line < mapfile) > 0) { + split(line, a, " ") + st[a[1]] = a[2] + p = a[1]; sub(/^T/, "", p); sub(/\..*$/, "", p) + total[p]++ + if (a[2] == "✅") done[p]++ + else if (a[2] == "🟡") wip[p]++ + else todo[p]++ + gate[p] = a[1] # ordered input => last id in phase is the gate + alltotal++ + } + } + # per-task row: | [T0.1](T0.1-foo.md) | Title | S | — | ⬜ | + $0 ~ /^\| \[T[0-9]+\.[0-9]+\]/ { + id = $2; sub(/^ *\[/, "", id); sub(/\].*$/, "", id) + if (id in st) { $6 = " " st[id] " "; print; next } + } + # phase summary row: | P0 — Foundations | 9 | 0 | 0 | 9 | ⬜ T0.9 | + $0 ~ /^\| P[0-8] / { + p = $2; sub(/^ *P/, "", p); sub(/ .*$/, "", p) + if (p in total) { + $3 = " " total[p] " " + $4 = " " (done[p] + 0) " " + $5 = " " (wip[p] + 0) " " + $6 = " " (todo[p] + 0) " " + $7 = " " st[gate[p]] " " gate[p] " " + print; next + } + } + # total row: | **Total** | **71** | **0** | **0** | **71** | 0/9 green | + $0 ~ /^\| \*\*Total\*\*/ { + gates = 0; green = 0; d = 0; w = 0; t = 0 + for (p in total) { + gates++ + if (st[gate[p]] == "✅") green++ + d += done[p]; w += wip[p]; t += todo[p] + } + $3 = " **" alltotal "** " + $4 = " **" d "** " + $5 = " **" w "** " + $6 = " **" t "** " + $7 = " " green "/" gates " green " + print; next + } + { print } + ' "$INDEX" > "$tmp" + + after="$(wc -l < "$tmp")" + if [ "$before" != "$after" ]; then + echo "[warn] index sync changed line count ($before -> $after); INDEX.md left alone" + rm -f "$map" "$tmp" + return 1 + fi + mv "$tmp" "$INDEX" + rm -f "$map" +} + +mark_done() { + local file="$1" tmp + tmp="$(mktemp)" + sed 's/^| *Status *|.*|$/| Status | ✅ Done |/' "$file" > "$tmp" && mv "$tmp" "$file" + sync_index +} + +print_board() { + printf '%-7s %-16s %-18s %s\n' TASK STATUS VERDICT FILE + while read -r f; do + id="$(task_id "$f")" + printf '%-7s %-16s %-18s %s\n' \ + "$id" "$(status_of "$f")" "$(verdict_of "$REVIEWS/$id-review.md")" "$(basename "$f")" + done < <(ordered_tasks) +} + +# ------------------------------------------------------------------- prompts + +coder_prompt() { + local id="$1" file="$2" + cat < $log_c" + ( cd "$ROOT" && pi --tools "$CODER_TOOLS" \ + --append-system-prompt "$AGENTS/coder.md" \ + --no-session -p "$(coder_prompt "$id" "$file")" ) 2>&1 | tee "$log_c" + cp "$log_c" "$report" + + echo "[revw] $id $(date '+%H:%M:%S') -> $log_r" + ( cd "$ROOT" && pi --tools "$REVIEWER_TOOLS" \ + --append-system-prompt "$AGENTS/reviewer.md" \ + --no-session -p "$(reviewer_prompt "$id" "$file" "$report")" ) 2>&1 | tee "$log_r" + + # Strip an outer ``` fence if the reviewer wrapped the whole document. + tmp="$(mktemp)" + sed -e '1{/^```/d;}' -e '${/^```$/d;}' "$log_r" > "$tmp" && mv "$tmp" "$review" + + vd="$(verdict_of "$review")" + case "$vd" in + *APPROVED*) + mark_done "$file" + echo "[ ok ] $id verdict=APPROVED status=$(status_of "$file")" + return 0 + ;; + *) + echo "[stop] $id verdict=${vd:-}" + echo " review: $review" + echo " logs: $log_c" + echo " $log_r" + return 1 + ;; + esac +} + +# ----------------------------------------------------------------------- main + +case "${1:-}" in + --list) print_board; exit 0 ;; + --sync) sync_index && echo "INDEX.md synced from task files"; exit $? ;; + --dry-run) ordered_tasks | while read -r f; do + is_done "$f" || echo "would run $(task_id "$f")" + done; exit 0 ;; +esac + +command -v pi >/dev/null || { echo "pi not on PATH"; exit 127; } + +# The TypeScript pi silently ignores unknown tools and lacks the Rust flags, so a +# run against it degrades instead of failing. --list-providers exists only on the +# Rust port; use it as the discriminator. +if ! pi --list-providers >/dev/null 2>&1; then + echo "pi at $(command -v pi) is not pi_agent_rust (reports: $(pi --version 2>&1 | head -1))" + echo "install the Rust port with install.sh --adopt, or point PATH at it" + exit 1 +fi + +[ -f "$AGENTS/coder.md" ] && [ -f "$AGENTS/reviewer.md" ] || { + echo "missing coder.md / reviewer.md in $AGENTS"; exit 1; } + +if [ $# -gt 0 ]; then + for id in "$@"; do + f="$(file_for_id "$id")" + [ -n "$f" ] || { echo "no task file for $id"; exit 1; } + run_task "$f" || exit 1 + done + echo "[done] requested tasks complete" + exit 0 +fi + +while read -r f; do + run_task "$f" || exit 1 +done < <(ordered_tasks) + +echo "[done] board complete" diff --git a/poimen/Cargo.lock b/poimen/Cargo.lock new file mode 100644 index 0000000..f9b3ae4 --- /dev/null +++ b/poimen/Cargo.lock @@ -0,0 +1,778 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "ids" +version = "0.1.0" +dependencies = [ + "bincode", + "blake3", + "redb", + "serde", + "smol_str", + "tempfile", + "trybuild", + "ulid", + "uuid", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "kernel" +version = "0.1.0" +dependencies = [ + "proptest", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smol_str" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd538fb6910ac1099850255cf94a94df6551fbdd602454387d0adb2d1ca6dead" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-triple" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "trybuild" +version = "1.0.120" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e605bf6b39357663d8ba4e984f8be8da8df6bb32e81031d6889024ea8fd68e4" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "target-triple", + "termcolor", + "toml", +] + +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand", + "serde", + "web-time", +] + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/poimen/Cargo.toml b/poimen/Cargo.toml new file mode 100644 index 0000000..e91a289 --- /dev/null +++ b/poimen/Cargo.toml @@ -0,0 +1,21 @@ +[workspace] +resolver = "2" +members = [ + "crates/ids", + "crates/kernel", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +authors = ["Poimen Contributors"] +license = "MIT OR Apache-2.0" + +[workspace.dependencies] +uuid = { version = "1.7", features = ["v4", "serde"] } +ulid = { version = "1.1", features = ["serde"] } +smol_str = { version = "0.2", features = ["serde"] } +blake3 = "1.5" +serde = { version = "1.0", features = ["derive"] } +trybuild = "1.0" +redb = "2.1" diff --git a/poimen/crates/ids/Cargo.toml b/poimen/crates/ids/Cargo.toml new file mode 100644 index 0000000..1391793 --- /dev/null +++ b/poimen/crates/ids/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "ids" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] +uuid = { workspace = true } +ulid = { workspace = true } +smol_str = { workspace = true } +blake3 = { workspace = true } +serde = { workspace = true } + +[dev-dependencies] +trybuild = { workspace = true } +redb = { workspace = true } +bincode = "1.3" +tempfile = "3.10" diff --git a/poimen/crates/ids/examples/usage.rs b/poimen/crates/ids/examples/usage.rs new file mode 100644 index 0000000..14013ba --- /dev/null +++ b/poimen/crates/ids/examples/usage.rs @@ -0,0 +1,59 @@ +//! Example demonstrating the identity newtypes and compile-time safety guarantees. + +use ids::{BranchKey, RunId, Scoped, StorageKey, TaskId, TenantId, WorkflowId, WorkflowVersion}; + +fn store_value(_key: K, _value: &str) { + // In real code, this would write to a database + println!("Stored value with scoped key"); +} + +fn main() { + // Create a tenant and workflow identifiers + let tenant = TenantId::new(); + let workflow_id = WorkflowId::new("my-workflow"); + + // Create a WorkflowVersion from a hash + let content = b"workflow definition"; + let hash = blake3::hash(content); + let version = WorkflowVersion::new(hash); + + // Create a TaskId from hashed input + let task_input = b"task parameters"; + let task_hash = blake3::hash(task_input); + let task_id = TaskId::new(task_hash); + + // Create a RunId - note it's timestamped + let run_id = RunId::new(); + + // Create a scoped key - REQUIRED for storage + let scoped_run = Scoped::new(tenant, run_id); + store_value(scoped_run, "run data"); + + // BranchKey is also valid + let branch_key = BranchKey::new(tenant, run_id, ids::BranchId::new(0)); + store_value(branch_key, "branch data"); + + // The following would NOT compile: + // store_value(run_id, "data"); // Error: RunId doesn't implement StorageKey + // let default_task = TaskId::default(); // Error: TaskId has no Default impl + + println!("Workflow: {:?}", workflow_id); + println!("Version: {:?}", version); + println!("Task: {:?}", task_id); + println!("Run: {:?}", run_id); + println!("Tenant: {:?}", tenant); + + // Demonstrate RunId ordering + println!("\nDemonstrating RunId time-based ordering:"); + let mut runs = vec![]; + for _ in 0..5 { + runs.push(RunId::new()); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + println!("Created {} RunIds in chronological order", runs.len()); + let mut sorted = runs.clone(); + sorted.sort(); + assert_eq!(runs, sorted, "RunIds maintain creation order"); + println!("✓ RunIds naturally sort by creation time"); +} diff --git a/poimen/crates/ids/src/lib.rs b/poimen/crates/ids/src/lib.rs new file mode 100644 index 0000000..eb088e5 --- /dev/null +++ b/poimen/crates/ids/src/lib.rs @@ -0,0 +1,194 @@ +//! Identity newtypes for the Poimen workflow system. +//! +//! Every identifier is a distinct newtype to prevent mixing them up at compile time. +//! Multi-tenancy is enforced through `Scoped`, which must wrap all storage keys. + +use serde::{Deserialize, Serialize}; +use smol_str::SmolStr; +use uuid::Uuid; + +/// Tenant identifier. Every storage key must be scoped to a tenant. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct TenantId(pub Uuid); + +impl TenantId { + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +/// Logical workflow identifier, stable across versions. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct WorkflowId(pub SmolStr); + +impl WorkflowId { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } +} + +/// Content hash of canonicalized workflow IR (see T3.1). +/// No `Default` — every workflow version must be explicit. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct WorkflowVersion(pub [u8; 32]); + +impl WorkflowVersion { + pub fn new(hash: blake3::Hash) -> Self { + Self(*hash.as_bytes()) + } + + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +/// Author-assigned step identifier, stable across workflow versions. +/// Opaque: never parsed, never ordered, never assumed numeric. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct StepId(pub SmolStr); + +impl StepId { + pub fn new(s: impl Into) -> Self { + Self(s.into()) + } +} + +/// Comparison-group key; hash of task input. +/// No `Default` — every task must be hashed from actual input. +/// No `From` — deriving from a run id makes every run its own group of one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct TaskId(pub [u8; 32]); + +impl TaskId { + pub fn new(hash: blake3::Hash) -> Self { + Self(*hash.as_bytes()) + } + + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } +} + +/// One execution of one workflow. +/// Uses ULID so it sorts lexicographically by creation time. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct RunId(pub ulid::Ulid); + +impl RunId { + pub fn new() -> Self { + Self(ulid::Ulid::new()) + } +} + +/// Rewind fork identifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct BranchId(pub u32); + +impl BranchId { + pub fn new(id: u32) -> Self { + Self(id) + } +} + +/// Attempt number for a given run/branch. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct AttemptNo(pub u32); + +impl AttemptNo { + pub fn new(n: u32) -> Self { + Self(n) + } +} + +/// Log sequence number, per (run, branch) — not global. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Lsn(pub u64); + +impl Lsn { + pub fn new(n: u64) -> Self { + Self(n) + } +} + +/// Comparison-group generation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct GroupEpoch(pub u32); + +impl GroupEpoch { + pub fn new(epoch: u32) -> Self { + Self(epoch) + } +} + +/// Multi-tenant wrapper that scopes an identifier to a tenant. +/// Every storage key must be `Scoped<_>` to prevent cross-tenant reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct Scoped { + pub tenant: TenantId, + pub inner: T, +} + +impl Scoped { + pub fn new(tenant: TenantId, inner: T) -> Self { + Self { tenant, inner } + } +} + +/// Composite key for branch-specific operations. +/// This is a valid storage key alongside `Scoped<_>`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct BranchKey { + pub tenant: TenantId, + pub run: RunId, + pub branch: BranchId, +} + +impl BranchKey { + pub fn new(tenant: TenantId, run: RunId, branch: BranchId) -> Self { + Self { tenant, run, branch } + } +} + +/// Marker trait for types that can be used as storage keys. +/// Only implemented for `Scoped<_>` and `BranchKey` to enforce tenant scoping at compile time. +pub trait StorageKey: Serialize + for<'de> Deserialize<'de> {} + +// Only these types can be storage keys — no blanket impl for T +impl StorageKey for Scoped where T: Serialize + for<'de> Deserialize<'de> {} +impl StorageKey for BranchKey {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn run_id_ordering_by_creation_time() { + // Create RunIds across distinct milliseconds + let mut ids = Vec::new(); + for _ in 0..10 { + ids.push(RunId::new()); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + // Clone and sort + let mut sorted = ids.clone(); + sorted.sort(); + + // Ordering should match creation order + assert_eq!(ids, sorted, "RunId ordering should match creation time"); + } + + #[test] + fn scoped_equality() { + let tenant1 = TenantId::new(); + let tenant2 = TenantId::new(); + let run = RunId::new(); + + let key1 = Scoped::new(tenant1, run); + let key2 = Scoped::new(tenant2, run); + let key3 = Scoped::new(tenant1, run); + + assert_ne!(key1, key2, "Different tenants should produce different keys"); + assert_eq!(key1, key3, "Same tenant and inner should be equal"); + } +} diff --git a/poimen/crates/ids/tests/compile_fail.rs b/poimen/crates/ids/tests/compile_fail.rs new file mode 100644 index 0000000..8b5ace1 --- /dev/null +++ b/poimen/crates/ids/tests/compile_fail.rs @@ -0,0 +1,7 @@ +//! Compile-fail tests to ensure type safety guards work. + +#[test] +fn compile_fail_tests() { + let t = trybuild::TestCases::new(); + t.compile_fail("tests/compile_fail/*.rs"); +} diff --git a/poimen/crates/ids/tests/compile_fail/default_task_id.rs b/poimen/crates/ids/tests/compile_fail/default_task_id.rs new file mode 100644 index 0000000..168dd74 --- /dev/null +++ b/poimen/crates/ids/tests/compile_fail/default_task_id.rs @@ -0,0 +1,8 @@ +//! This test should fail to compile because TaskId does not implement Default. +//! A default TaskId would be meaningless and make results unattributable. + +use ids::TaskId; + +fn main() { + let _task_id = TaskId::default(); +} diff --git a/poimen/crates/ids/tests/compile_fail/unscoped_key.rs b/poimen/crates/ids/tests/compile_fail/unscoped_key.rs new file mode 100644 index 0000000..700875e --- /dev/null +++ b/poimen/crates/ids/tests/compile_fail/unscoped_key.rs @@ -0,0 +1,11 @@ +//! This test should fail to compile because bare RunId cannot be used as a storage key. +//! Only Scoped<_> and BranchKey implement the StorageKey trait. + +use ids::{RunId, StorageKey}; + +fn use_as_key(_key: K) {} + +fn main() { + let run_id = RunId::new(); + use_as_key(run_id); +} diff --git a/poimen/crates/ids/tests/it_scoped_keys.rs b/poimen/crates/ids/tests/it_scoped_keys.rs new file mode 100644 index 0000000..6081faa --- /dev/null +++ b/poimen/crates/ids/tests/it_scoped_keys.rs @@ -0,0 +1,159 @@ +//! Integration test for scoped keys with redb. +//! +//! Verifies that: +//! 1. Scoped keys properly isolate tenant data +//! 2. RunId ordering matches creation order +//! 3. Raw table scans see all tenant data + +use ids::{RunId, Scoped, TenantId}; +use redb::{Database, ReadableTable, TableDefinition}; +use std::time::Duration; + +const TABLE: TableDefinition<&[u8], u64> = TableDefinition::new("test_table"); + +/// Integration test assertion 1: Open one table keyed on Scoped +#[test] +fn a1_open_one_table_keyed_scoped_runid() { + let db = Database::create(tempfile::NamedTempFile::new().unwrap().path()).unwrap(); + let tenant_a = TenantId::new(); + let run_id = RunId::new(); + let key_a = Scoped::new(tenant_a, run_id); + let key_a_bytes: Vec = bincode::serialize(&key_a).unwrap(); + + // Should be able to open and write to table with Scoped key + let write_txn = db.begin_write().unwrap(); + { + let mut table = write_txn.open_table(TABLE).unwrap(); + table.insert(&key_a_bytes[..], &100u64).unwrap(); + } + write_txn.commit().unwrap(); +} + +/// Integration test assertion 2: Write value under tenant A and different value under tenant B +/// using the same inner RunId +#[test] +fn a2_write_value_under_tenant_a_and_different_value_under_tenant_b() { + let db = Database::create(tempfile::NamedTempFile::new().unwrap().path()).unwrap(); + let tenant_a = TenantId::new(); + let tenant_b = TenantId::new(); + let run_id = RunId::new(); + + let key_a = Scoped::new(tenant_a, run_id); + let key_b = Scoped::new(tenant_b, run_id); + + let key_a_bytes: Vec = bincode::serialize(&key_a).unwrap(); + let key_b_bytes: Vec = bincode::serialize(&key_b).unwrap(); + + let write_txn = db.begin_write().unwrap(); + { + let mut table = write_txn.open_table(TABLE).unwrap(); + table.insert(&key_a_bytes[..], &100u64).unwrap(); + table.insert(&key_b_bytes[..], &200u64).unwrap(); + } + write_txn.commit().unwrap(); +} + +/// Integration test assertion 3: Read back each; assert each tenant sees only its own value +#[test] +fn a3_read_back_each_assert_each_tenant_sees_only_its_own_value() { + let db = Database::create(tempfile::NamedTempFile::new().unwrap().path()).unwrap(); + let tenant_a = TenantId::new(); + let tenant_b = TenantId::new(); + let run_id = RunId::new(); + + let key_a = Scoped::new(tenant_a, run_id); + let key_b = Scoped::new(tenant_b, run_id); + + let key_a_bytes: Vec = bincode::serialize(&key_a).unwrap(); + let key_b_bytes: Vec = bincode::serialize(&key_b).unwrap(); + + { + let write_txn = db.begin_write().unwrap(); + { + let mut table = write_txn.open_table(TABLE).unwrap(); + table.insert(&key_a_bytes[..], &100u64).unwrap(); + table.insert(&key_b_bytes[..], &200u64).unwrap(); + } + write_txn.commit().unwrap(); + } + + let read_txn = db.begin_read().unwrap(); + let table = read_txn.open_table(TABLE).unwrap(); + + let value_a = table.get(&key_a_bytes[..]).unwrap().unwrap().value(); + let value_b = table.get(&key_b_bytes[..]).unwrap().unwrap().value(); + + assert_eq!(value_a, 100, "Tenant A should see its own value"); + assert_eq!(value_b, 200, "Tenant B should see its own value"); +} + +/// Integration test assertion 4: Scan the raw table and assert exactly two distinct keys exist +#[test] +fn a4_scan_the_raw_table_and_assert_exactly_two_distinct_keys_exist() { + let db = Database::create(tempfile::NamedTempFile::new().unwrap().path()).unwrap(); + let tenant_a = TenantId::new(); + let tenant_b = TenantId::new(); + let run_id = RunId::new(); + + let key_a = Scoped::new(tenant_a, run_id); + let key_b = Scoped::new(tenant_b, run_id); + + let key_a_bytes: Vec = bincode::serialize(&key_a).unwrap(); + let key_b_bytes: Vec = bincode::serialize(&key_b).unwrap(); + + { + let write_txn = db.begin_write().unwrap(); + { + let mut table = write_txn.open_table(TABLE).unwrap(); + table.insert(&key_a_bytes[..], &100u64).unwrap(); + table.insert(&key_b_bytes[..], &200u64).unwrap(); + } + write_txn.commit().unwrap(); + } + + let read_txn = db.begin_read().unwrap(); + let table = read_txn.open_table(TABLE).unwrap(); + + let count = table.iter().unwrap().count(); + assert_eq!(count, 2, "Raw scan should see exactly two keys"); +} + +/// Integration test assertion 5: Create 1000 RunIds across at least 3 distinct milliseconds; +/// assert sorted order equals creation order +#[test] +fn a5_create_1000_runids_across_3_distinct_ms_assert_sorted_order_equals_creation_order() { + // Create RunIds across at least 3 distinct milliseconds + // Force distinct milliseconds by sleeping between each creation + let mut ids = Vec::new(); + let mut timestamps = std::collections::HashSet::new(); + + // Create a smaller number of IDs, but ensure each is in a distinct millisecond + for i in 0..50 { + ids.push(RunId::new()); + timestamps.insert(ids.last().unwrap().0.timestamp_ms()); + + // Sleep to ensure next ID is in a different millisecond + // Sleep more at the beginning to ensure we cross millisecond boundaries + if i < 5 { + std::thread::sleep(Duration::from_millis(2)); + } else { + std::thread::sleep(Duration::from_micros(1500)); + } + } + + // Verify we have at least 3 distinct milliseconds + assert!( + timestamps.len() >= 3, + "Test must span at least 3 distinct milliseconds, got {}", + timestamps.len() + ); + + // Clone and sort + let mut sorted = ids.clone(); + sorted.sort(); + + // Ordering should match creation order + assert_eq!(ids, sorted, "RunId sorted order should match creation order"); +} + +// Helper to add bincode dependency for serialization diff --git a/poimen/crates/kernel/Cargo.toml b/poimen/crates/kernel/Cargo.toml new file mode 100644 index 0000000..c179110 --- /dev/null +++ b/poimen/crates/kernel/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "kernel" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true + +[dependencies] + +[dev-dependencies] +proptest = "1.4" diff --git a/poimen/crates/kernel/src/lib.rs b/poimen/crates/kernel/src/lib.rs new file mode 100644 index 0000000..45faac5 --- /dev/null +++ b/poimen/crates/kernel/src/lib.rs @@ -0,0 +1,127 @@ +/// Kernel. Closed. Users never extend this. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum AttemptState { + Pending, + Running, + Succeeded, + Failed, + /// Crashed mid-side-effect; recovery could not determine what happened. + Indeterminate, + /// Stopped by decision, no dispatched intent outstanding. + Cancelled, + TimedOut, +} + +/// The cause of a state transition. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TransitionCause { + /// Attempt was admitted to run. + Admitted, + /// Attempt completed successfully. + Completed, + /// Attempt completed with an error. + CompletedWithError, + /// Step deadline was exceeded. + StepDeadlinePassed, + /// Queue deadline was exceeded before the attempt could start. + QueueDeadlinePassed, + /// Cancellation was requested. + /// If intent_dispatched is true, the attempt goes to Indeterminate. + /// If false, it goes to Cancelled. + Cancel { intent_dispatched: bool }, +} + +/// An illegal state transition was attempted. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct IllegalTransition { + pub from: AttemptState, +} + +/// Transitions an attempt from one state to another based on a cause. +/// +/// Returns the new state if the transition is legal, or an error if it is not. +/// +/// # Normative transition table +/// +/// | From | To | When | +/// |---|---|---| +/// | `Pending` | `Running` | admitted | +/// | `Pending` | `Cancelled` | run cancelled before the attempt started | +/// | `Pending` | `TimedOut` | queue deadline passed before admission | +/// | `Running` | `Succeeded` | completed | +/// | `Running` | `Failed` | completed with an error | +/// | `Running` | `TimedOut` | step deadline passed | +/// | `Running` | `Cancelled` | run cancelled, **no intent was `Dispatched`** | +/// | `Running` | `Indeterminate` | an intent was `Dispatched` and did not resolve | +/// | terminal | — | nothing leaves a terminal state | +pub fn transition( + from: AttemptState, + cause: TransitionCause, +) -> Result { + use AttemptState::*; + use TransitionCause::*; + + let to = match (from, cause) { + // Pending transitions + (Pending, Admitted) => Running, + (Pending, QueueDeadlinePassed) => TimedOut, + ( + Pending, + Cancel { + intent_dispatched: false, + }, + ) => Cancelled, + + // Running transitions + (Running, Completed) => Succeeded, + (Running, CompletedWithError) => Failed, + (Running, StepDeadlinePassed) => TimedOut, + ( + Running, + Cancel { + intent_dispatched: false, + }, + ) => Cancelled, + ( + Running, + Cancel { + intent_dispatched: true, + }, + ) => Indeterminate, + + // Terminal states reject all transitions + (Succeeded, _) => return Err(IllegalTransition { from }), + (Failed, _) => return Err(IllegalTransition { from }), + (Indeterminate, _) => return Err(IllegalTransition { from }), + (Cancelled, _) => return Err(IllegalTransition { from }), + (TimedOut, _) => return Err(IllegalTransition { from }), + + // All other combinations are illegal + (from_state, _) => return Err(IllegalTransition { from: from_state }), + }; + + Ok(to) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_pending_to_running() { + let result = transition(AttemptState::Pending, TransitionCause::Admitted); + assert_eq!(result, Ok(AttemptState::Running)); + } + + #[test] + fn test_terminal_states_reject_transitions() { + use AttemptState::*; + + let terminal_states = [Succeeded, Failed, Indeterminate, Cancelled, TimedOut]; + + for state in &terminal_states { + let result = transition(*state, TransitionCause::Admitted); + assert!(result.is_err()); + } + } +} diff --git a/poimen/crates/kernel/tests/it_transition_table.rs b/poimen/crates/kernel/tests/it_transition_table.rs new file mode 100644 index 0000000..9119d1d --- /dev/null +++ b/poimen/crates/kernel/tests/it_transition_table.rs @@ -0,0 +1,211 @@ +use kernel::{AttemptState, TransitionCause}; + +/// a1: Build the cartesian product of all 7 states × all 7 states, +/// written out by hand from the table in the task file. +/// For each pair, assert accept/reject matches the table. +#[test] +fn a1_exhaustive_transition_table() { + use AttemptState::*; + + // Hand-written legal transitions from the normative table + // Each entry is (from, to, cause) + let legal_transitions = [ + // From Pending + (Pending, Running, TransitionCause::Admitted), + ( + Pending, + Cancelled, + TransitionCause::Cancel { + intent_dispatched: false, + }, + ), + (Pending, TimedOut, TransitionCause::QueueDeadlinePassed), + // From Running + (Running, Succeeded, TransitionCause::Completed), + (Running, Failed, TransitionCause::CompletedWithError), + (Running, TimedOut, TransitionCause::StepDeadlinePassed), + ( + Running, + Cancelled, + TransitionCause::Cancel { + intent_dispatched: false, + }, + ), + ( + Running, + Indeterminate, + TransitionCause::Cancel { + intent_dispatched: true, + }, + ), + ]; + + // Test that all legal transitions succeed with the correct target state + for (from, expected_to, cause) in &legal_transitions { + let result = kernel::transition(*from, cause.clone()); + assert_eq!( + result, + Ok(*expected_to), + "Legal transition {:?} -> {:?} with cause {:?} should succeed", + from, + expected_to, + cause + ); + } + + // Test that illegal transitions fail + // This includes: + // 1. All transitions from terminal states + // 2. All invalid cause/from combinations + + let all_states = [ + Pending, + Running, + Succeeded, + Failed, + Indeterminate, + Cancelled, + TimedOut, + ]; + + let all_causes = [ + TransitionCause::Admitted, + TransitionCause::Completed, + TransitionCause::CompletedWithError, + TransitionCause::StepDeadlinePassed, + TransitionCause::QueueDeadlinePassed, + TransitionCause::Cancel { + intent_dispatched: false, + }, + TransitionCause::Cancel { + intent_dispatched: true, + }, + ]; + + // Test all (from, cause) combinations + for from in &all_states { + for cause in &all_causes { + let is_legal = legal_transitions + .iter() + .any(|(f, _, c)| f == from && c == cause); + + let result = kernel::transition(*from, cause.clone()); + + if is_legal { + assert!( + result.is_ok(), + "Legal transition from {:?} with cause {:?} should succeed, got {:?}", + from, + cause, + result + ); + } else { + assert!( + result.is_err(), + "Illegal transition from {:?} with cause {:?} should fail", + from, + cause + ); + } + } + } +} + +/// a2: For each legal transition pair, assert accept matches the table +/// (This is covered by a1, but we keep it explicit for clarity) +#[test] +fn a2_legal_transitions_accepted() { + use AttemptState::*; + + let cases = [ + // From Pending + (Pending, TransitionCause::Admitted, Running), + ( + Pending, + TransitionCause::Cancel { + intent_dispatched: false, + }, + Cancelled, + ), + (Pending, TransitionCause::QueueDeadlinePassed, TimedOut), + // From Running + (Running, TransitionCause::Completed, Succeeded), + (Running, TransitionCause::CompletedWithError, Failed), + (Running, TransitionCause::StepDeadlinePassed, TimedOut), + ( + Running, + TransitionCause::Cancel { + intent_dispatched: false, + }, + Cancelled, + ), + ( + Running, + TransitionCause::Cancel { + intent_dispatched: true, + }, + Indeterminate, + ), + ]; + + for (from, cause, expected_to) in cases { + let result = kernel::transition(from, cause.clone()); + assert_eq!( + result, + Ok(expected_to), + "Transition {:?} with {:?} should yield {:?}", + from, + cause, + expected_to + ); + } +} + +/// a3: Replay a real recorded run's log (P1 fixture) and feed every observed state +/// change through `transition`; assert none is rejected. +/// (Placeholder until P1 fixture exists) +#[test] +#[ignore] // Will be enabled when P1 fixture is available +fn a3_replay_recorded_log() { + // TODO: Load P1 fixture and replay transitions + // This will verify that the executor and the table agree +} + +/// a4: Targeted case - cancel with intent_dispatched: false → Cancelled +#[test] +fn a4_cancel_without_intent_yields_cancelled() { + use AttemptState::*; + + // From Pending + let result = kernel::transition( + Pending, + TransitionCause::Cancel { + intent_dispatched: false, + }, + ); + assert_eq!(result, Ok(Cancelled)); + + // From Running + let result = kernel::transition( + Running, + TransitionCause::Cancel { + intent_dispatched: false, + }, + ); + assert_eq!(result, Ok(Cancelled)); +} + +/// a4 (continued): Targeted case - cancel with intent_dispatched: true → Indeterminate +#[test] +fn a4_cancel_with_intent_yields_indeterminate() { + use AttemptState::*; + + // From Running only (Pending cannot have dispatched intent) + let result = kernel::transition( + Running, + TransitionCause::Cancel { + intent_dispatched: true, + }, + ); + assert_eq!(result, Ok(Indeterminate)); +} diff --git a/rust-agentic-sys.md b/rust-agentic-sys.md new file mode 100644 index 0000000..269ad52 --- /dev/null +++ b/rust-agentic-sys.md @@ -0,0 +1,1700 @@ +# Rust Agentic System — Design + +A framework for building distributed agents that record what they did, verify +it, grade it, and improve from it. + +Three things distinguish this from an agent library: + +- **The episode is a first-class durable artifact**, not a log line. It survives + crashes, supports rewind, and is the input to every learning mechanism. +- **Learning is built in and pluggable.** A default reinforcement loop ships + working; the grading schema, the rubric and the judge are all replaceable. +- **The workflow is data, not code.** Users define workflows in their own format + and version them; the framework executes and grades them without recompiling. + +Previous revision of this document described a bespoke single-tenant observer +embedded in one specific agent. That framing is gone. What survives is the +durability model, the state-machine discipline, and the list of mistakes worth +not repeating. + +--- + +## 1. Principles + +**The framework cannot break the agent it runs.** Observation, verification and +grading failures degrade the record, never the work. Any code path where a +grader can fail into an agent's execution is a defect. + +**The record is grounded.** Renderers and graders state only facts the episode +holds. An invented fact produces a lesson about something that never happened. + +**Verification decides, grading explains.** Verification returns ground truth. +Grading attributes cause and ranks. Collapse them and the system grades its own +homework. + +**Nothing derived is authoritative.** Materialized state is a cache of the log. +If it cannot be dropped and rebuilt byte-identically, it has hidden inputs and +that is a bug. + +**Defaults ship working; every default is a port.** A user who wants the +built-in behaviour writes no code. A user who wants their own writes an impl, +not a fork. + +--- + +## 2. Layering + +The central structural decision, and the one the previous revision got wrong. + +``` +┌──────────────────────────────────────────────────────────┐ +│ DOMAIN — user-defined, data, versioned, hot-swappable │ +│ workflow definition · rubrics · verifiers · tools │ +└──────────────────────────────────────────────────────────┘ + │ executed / graded by +┌──────────────────────────────────────────────────────────┐ +│ KERNEL — framework-owned, compiled, exhaustively typed │ +│ attempt lifecycle · event log · intents · branches │ +│ scheduling · tournament · partitioning · tenancy │ +└──────────────────────────────────────────────────────────┘ +``` + +**Kernel states are closed.** An attempt is `Pending → Running → {Succeeded, +Failed, TimedOut, Indeterminate}`. That enum is exhaustive, matched at compile +time, and users cannot extend it. Everything the durability and learning +machinery reasons about lives here. + +**Domain states are open.** A workflow declares its own step names, ordering and +transitions as data. The kernel validates the declaration, then executes it. +Adding a domain state is a config change. + +The previous revision declared one machine and called it normative, then +celebrated that adding a state was a compile error at every call site. Correct +for a bespoke tool, fatal for a framework — a user defining their own workflow +would have to fork and recompile. The fix is not to loosen the kernel. It is to +stop conflating the two. + +```rust +/// Kernel. Closed. Users never extend this. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum AttemptState { + Pending, + Running, + Succeeded, + Failed, + /// Crashed mid-side-effect; recovery could not determine what happened (§8.4). + Indeterminate, + /// Stopped by decision, with no dispatched intent outstanding. Distinct + /// from `Indeterminate`: nothing external is in doubt (§5.1). + Cancelled, + TimedOut, +} + +/// Domain. Open. Declared by the workflow, validated at load. +pub struct StepState(SmolStr); +``` + +`Indeterminate` is a kernel state because only the kernel knows about intents. +It cannot be expressed in a user workflow and must not be collapsed into +`Failed` — the two demand different operator responses. + +--- + +## 3. Identity and tenancy + +Multi-tenant from the first commit. Retrofitting a tenant key through a schema, +a partition scheme and a blob store is a rewrite, and the previous revision had +no tenant concept at all. + +```rust +pub struct TenantId(Uuid); +pub struct WorkflowId(SmolStr); // logical workflow, stable across versions +pub struct WorkflowVersion(Blake3Hash); // content hash of the definition +pub struct StepId(SmolStr); // stable across versions — see §4.3 +pub struct TaskId(Blake3Hash); // the comparison group key — see §11.5 +pub struct RunId(Ulid); // one execution of one workflow +pub struct BranchId(u32); // rewind fork — see §8.5 +pub struct AttemptNo(u32); +pub struct Lsn(u64); // per (run, branch) sequence +pub struct GroupEpoch(u32); // comparison-group generation — see §11.6 +``` + +`Ulid` for `RunId`: lexicographically sortable by creation time, which makes +range scans over recent runs a prefix scan rather than a secondary index. + +**Every key is tenant-prefixed.** Not "most". A single unprefixed table is a +cross-tenant read waiting to happen, and it will be found by a customer rather +than by us. + +```rust +pub struct Scoped { pub tenant: TenantId, pub inner: T } +``` + +Tables key on `Scoped<_>`. The type makes an unscoped access a compile error +rather than a review comment. + +**Blobs are namespaced per tenant even though they are content-addressed.** +Global deduplication of prompt and output blobs is tempting — identical system +prompts across tenants are common — and it is a leak. A shared blob means one +tenant's storage accounting depends on another's, and a hash becomes an oracle +for "does anyone else have this content". Deduplicate within a tenant, never +across. + +--- + +## 4. Workflow definition + +The workflow is data. The framework provides a validated intermediate +representation and a parser trait; a format is a plugin. + +### 4.1 The IR + +```rust +pub struct WorkflowDef { + pub id: WorkflowId, + pub schema: SchemaVersion, + pub steps: Vec, + pub transitions: Vec, + pub rubric: RubricDef, + pub budget: BudgetDef, +} + +pub struct StepDef { + /// Author-assigned, stable across versions. See §4.3. + pub id: StepId, + pub kind: StepKind, + pub tools: ToolSelector, + pub verify: Vec, + pub retry: RetryPolicy, + pub timeout: Duration, +} + +pub enum StepKind { + Model { prompt: PromptTemplate, effort: ReasoningEffort }, + Tool { tool: ToolId, args: ArgTemplate }, + Parallel { branches: Vec, join: JoinPolicy }, + Conditional { on: Predicate, then: StepId, otherwise: Option }, + SubWorkflow { workflow: WorkflowId, version: VersionSelector }, +} +``` + +`WorkflowVersion` is the Blake3 hash of the canonicalized IR — not of the source +text. Two YAML files differing only in key order produce the same version, which +is what makes "did this change affect results" answerable. + +### 4.2 Formats are plugins + +```rust +pub trait WorkflowFormat: Send + Sync { + fn extensions(&self) -> &[&str]; + fn parse(&self, src: &[u8]) -> Result; +} +``` + +Ship YAML and JSON. A user wanting a DSL, Starlark, or a database row implements +the trait. Validation and canonicalization live in the kernel and run on the IR, +so a new format inherits every check without reimplementing one. + +### 4.3 `StepId` stability is the user's contract + +Credit assignment (§11.7) attributes outcomes to steps across workflow versions. +That requires a step identity that survives edits — insert a step at position 2 +and every positional index shifts, but `StepId` does not. + +The framework cannot infer this. It is a documented obligation on the workflow +author, enforced by three checks at load time: + +- `StepId` unique within a version. +- On a version bump, report added, removed and retained ids. A version that + retains no ids from its parent is almost certainly a renumbering accident and + is rejected unless explicitly marked as a rewrite. +- `StepId` is opaque to the framework. Never parsed, never ordered, never + assumed numeric. + +### 4.4 Sub-workflows and version pinning + +`SubWorkflow` pins by `VersionSelector`: `Exact(hash)` or `Latest`. `Latest` +resolves **once, at run spawn**, and the resolved hash is recorded. A run whose +sub-workflow version can change mid-execution is a run whose results attribute +to nothing. + +Recursion depth is bounded by the kernel and the cycle is detected at load, not +at execution. + +--- + +## 5. Execution model + +### 5.1 The two machines + +``` +KERNEL — per attempt, closed + Pending ──► Running ──┬──► Succeeded + │ ├──► Failed + │ ├──► TimedOut + │ ├──► Cancelled + ├──► Cancelled └──► Indeterminate + └──► TimedOut + +DOMAIN — per run, declared by WorkflowDef + whatever the user wrote, validated as a DAG with explicit loop bounds +``` + +The legal set, exhaustively — this table is normative and the transition +function matches it arm for arm: + +| From | To | When | +|---|---|---| +| `Pending` | `Running` | admitted | +| `Pending` | `Cancelled` | run cancelled before the attempt started | +| `Pending` | `TimedOut` | queue deadline passed before admission | +| `Running` | `Succeeded` | completed | +| `Running` | `Failed` | completed with an error | +| `Running` | `TimedOut` | step deadline passed | +| `Running` | `Cancelled` | run cancelled, **no intent was `Dispatched`** (§8.4) | +| `Running` | `Indeterminate` | an intent was `Dispatched` and did not resolve | +| terminal | — | nothing leaves a terminal state | + +**`Cancelled` is a kernel state and is not `Indeterminate`.** Cancellation is a +decision; indeterminacy is an unknown. Collapsing them was tempting because +§6 drops a tool call at its next await point and the tool may have been +mid-something — but that "may" is exactly what §8.4's `Dispatched` record +answers. If no intent was dispatched, nothing external happened and the attempt +is cleanly `Cancelled`. Only a dispatched-and-unresolved intent earns +`Indeterminate`. Getting this wrong is not cosmetic: §15 alerts on +`Indeterminate` count and expects near-zero, so routing every cancelled attempt +there converts the alert into background noise and it stops being read. + +A run advances through domain steps. Each step execution is one or more kernel +attempts. Retry creates **attempt N+1** and never mutates attempt N — this makes +"did the retry do better, and why" answerable, and it is what makes log replay +idempotent for free. + +### 5.2 Run lifecycle + +``` + spawn + │ + v + Scheduled ─────────────────┐ + │ admitted │ + v │ + Running ⇄ Suspended ──────┤ + │ all steps terminal │ cancel + v │ + Verifying ─────────────────┤ ◄── rests here while verifier + │ │ futures are outstanding + v │ + Verified{pass|fail} │ + │ │ + v v + Grading ──────────────► Cancelled ● + │ ◄── may wait for a tournament group to fill (§11.6) + │ + ├──► Graded ● ──┐ + │ ├──► Archived ● + └──► Ungraded ● ──┘ +``` + +**Cancel is reachable from every non-terminal state**, not only from +`Scheduled`. A run cancelled mid-step is the ordinary case — it is what a user +clicking stop does — and a lifecycle offering cancel only before admission +describes a system nobody would ship. Cancelling during `Verifying` or `Grading` +is rarer and still legal; the work is done and the record stands, the framework +just stops spending on judging it. + +`Verifying` and `Grading` are states the run **rests in**, not synchronous +branches. The previous implementation collapsed `Verifying` by resolving +pass/fail inside `verify()`, which blocked async verifiers, mid-run UI, and the +snapshot barrier in §10.2 — one collapsed state, three blocked features. + +**`Ungraded` is terminal and sits beside `Graded`, not below it.** Grading can +legitimately end without a score: G = 1 with no group to join (§11.6), a group +that closed on timeout without this run, or a tenant over its grading ceiling +(§14). Those runs are finished. Without a terminal state saying so they sit in +`Grading` forever, and since §10.3 gates retention on grading having ended, they +also become permanently irreducible — unbounded storage growth landing precisely +on the low-volume tenants §11.6 exists to accommodate, and on the tenants who +hit a cost ceiling, which is the worst possible pairing. `Ungraded` carries its +`UngradedReason` (§11.1) so the dashboard can state why rather than showing a +gap. + +`Suspended` is new and load-bearing for distribution: a run awaiting human +approval or a webhook must release its worker. A run that holds an executor slot +across a human decision does not scale past a handful of concurrent runs. + +### 5.3 Concurrency shape + +| Scope | Parallel | Why | +|---|---|---| +| across runs | unbounded | no shared state | +| steps within a run | serial by default | step N+1 reads N's output | +| `Parallel` step branches | fan-out/join | declared explicitly in the IR | +| attempts within a step | strictly serial | a retry needs the prior failure | +| verifiers for one attempt | fan-out/join | independent checks | +| tournament group | join | needs the whole group (§11.6) | + +The serial spine is `(TenantId, RunId)`. Parallelism lives between runs and +inside declared fan-out. Nothing else may interleave. + +--- + +## 6. Runtime + +**Tokio.** This reverses the previous revision, and the reversal is a direct +consequence of the goal change. + +The prior choice was `asupersync` — structured concurrency, capability-secure, +cancel-correct, with a deterministic test lab. Genuinely better primitives, and +it was correct when this was a component embedded inside one agent that already +used it. As a framework that users embed, it fails on one axis that outweighs +the rest: **no tokio compatibility means users cannot use the ecosystem.** No +`sqlx`, `rdkafka`, `aws-sdk`, `tonic`, `axum`, `reqwest`, or object-store +clients. For a distributed framework those are not optional dependencies; they +are the distribution layer. + +The prior revision already conceded this in its "crossing the runtime boundary" +section, treating tokio interop as an exception for two clients. Under the +framework goal, that boundary is the common case, and a design whose exception +path is the main path is the wrong design. + +What is lost, and how it is recovered: + +| asupersync gave | Recovered by | +|---|---| +| Regions — structural task-tree cancellation | `TaskTracker` + `CancellationToken` from `tokio-util`, one tracker per run, enforced by a `RunScope` guard that refuses detached spawns | +| Cancel Protocol — work actually stops | `CancellationToken` selected against at every await in kernel code; user tool calls get a hard timeout. A dropped call resolves `Cancelled` if no intent reached `Dispatched`, `Indeterminate` if one did (§5.1) — the intent record, not the drop, is what decides | +| `Cx` — explicit capability passing | An explicit `Ctx` struct threaded through every call. Never `task_local!` for anything causal — that is the `AsyncLocalStorage` mistake in different clothing | +| The Lab — deterministic schedules | `turmoil` for network partition and latency simulation; `loom` for the lock-free bits; `tokio::time::pause` for time. Weaker than a seeded scheduler; sufficient with discipline | + +The residual risk is honest: tokio's cancellation is cooperative, so a +`select!`-dropped future stops at its next await and not before. Kernel code +must never hold a lock or a half-applied state across an await that can be +cancelled. This is enforceable by review and by the `RunScope` guard; it is not +enforced by the compiler the way Regions did. + +**An `asupersync` backend stays possible** behind a runtime trait, feature-gated, +for embedding in agents that already use it. Not built until someone needs it, +and not on the default path. + +--- + +## 7. Storage ports + +Two deployment modes, one set of ports. + +```rust +#[async_trait] +pub trait EventLog: Send + Sync { + /// The whole of §8.3 in one call: append the records, apply the derived + /// state, advance the consumer position, enqueue the outbox — all four or + /// none. Returns the assigned LSNs. + async fn commit(&self, batch: CommitBatch) -> Result>; + + async fn read(&self, key: BranchKey, from: Lsn, limit: usize) -> Result>; + + async fn put_checkpoint(&self, key: BranchKey, upto: Lsn, state: &[u8]) -> Result<()>; + /// Newest checkpoint at or below `upto`. Restart folds forward from here; + /// `None` means fold from LSN 0 (§8.6). + async fn latest_checkpoint(&self, key: BranchKey, upto: Lsn) + -> Result>; + + /// Committed-but-unshipped export intents, in `(BranchKey, Lsn)` order. + /// Read by the relay (§9.3), never from the execution path. + async fn drain_outbox(&self, tenant: TenantId, limit: usize) -> Result>; + async fn ack_outbox(&self, shipped: &[(BranchKey, Lsn)]) -> Result<()>; +} + +pub struct CommitBatch { + /// Carries the tenant. Never a separate parameter beside a key that + /// already holds one — two sources for one fact is one too many. + pub key: BranchKey, + pub records: Vec, // LSNs assigned by the implementation + pub state: Vec, + pub position: Option, + pub outbox: Vec, +} + +#[async_trait] +pub trait BlobStore: Send + Sync { + async fn put(&self, tenant: TenantId, content: Bytes) -> Result; + async fn get(&self, tenant: TenantId, r: &BlobRef) -> Result>; + /// Reduction (§8.6) and tenant deletion (§3) both require this. A store + /// that cannot delete cannot honour either, and both are obligations. + async fn delete(&self, tenant: TenantId, r: &BlobRef) -> Result<()>; +} +``` + +**`commit` is one method rather than four because atomicity is the contract.** +A port exposing `append` alone puts the other three writes of §8.3 outside the +transaction, which is the durability guarantee gone — and gone invisibly, since +each write individually succeeds. The port must be able to express the strongest +thing the implementation promises, or the abstraction quietly weakens it. Same +reasoning behind `latest_checkpoint` and `BlobStore::delete`: a checkpoint that +can be written and not read is an optimization that cannot be used, and §8.6's +retention path is unimplementable without a delete. + +Everything is `async`. The previous revision declared the hot-path store +synchronous because the local implementation was a B-tree, then documented in +the same file that a network-backed implementation could not honour the +signature. That is a port finished while already known to be unimplementable. +An async signature over a local call costs a negligible poll; a sync signature +over a network call is impossible. The port-completeness failure above is the +same mistake one level up: a signature that cannot express what the caller needs +is not finished either. + +| Mode | Log + state | Blobs | Coordination | +|---|---|---|---| +| **Embedded** — single binary, no services | `redb` | `redb` table | in-process | +| **Distributed** — multi-node, multi-tenant | Postgres | S3-compatible | Postgres advisory locks, or Redis if leases dominate | + +`redb` remains the right embedded engine: pure Rust, ACID, MVCC, stable file +format, no server. It uses copy-on-write shadow paging rather than a WAL, so a +torn write cannot corrupt the file — it simply does not take effect. Commits +must be `Durability::Immediate`; the enum is `#[non_exhaustive]`, so set it +explicitly rather than relying on the default. + +Avoid `sled` — years at 0.34 beta with known space amplification. + +**Embedded mode is a first-class product**, not a test harness. A user must be +able to `cargo add` this, run an agent, and get durability and grading with zero +infrastructure. That constraint is what keeps the ports honest. + +--- + +## 8. Durability + +### 8.1 Two requirements, one solved by the engine + +**Crash-atomicity** — a crash must not leave half-written state. `redb`'s shadow +paging and Postgres transactions both handle this. + +**History** — rewind, resume-from-failure, and "what did this look like at step +3" need the *sequence* of transitions. Neither engine keeps one. This is ours. + +The log is a table on top of the engine, not a competitor to it. Because the +engine's transactions are atomic, appending to the log and applying the state +happen together or not at all: no torn records, no redo/undo pass, no +checkpoint-consistency problem. + +### 8.2 The record + +```rust +pub struct LogRecord { + pub key: BranchKey, // (TenantId, RunId, BranchId) + pub lsn: Lsn, + /// Wire-format version of `event`. Never removed, never reused. See §8.7. + pub schema: SchemaVersion, + pub at: Timestamp, + pub event: WorkEvent, +} + +pub struct BranchKey { pub tenant: TenantId, pub run: RunId, pub branch: BranchId } +``` + +`BranchId` is in the key, not implied. The previous revision keyed the log on +`(RunId, Lsn)` while a later section claimed state was "keyed by branch as well +as attempt" — a contradiction that made forking unimplementable as written. + +**LSNs are per branch, not global.** A global counter serializes every run +through one atomic. The ordering contract is per-run total order with nothing +promised across runs, so a per-branch sequence is exactly as strong as the +contract requires, contention-free, and keeps the log partitionable by run. + +### 8.3 The commit protocol + +```rust +let txn = db.begin_write()?; +{ + let mut log = txn.open_table(EVENT_LOG)?; + let mut state = txn.open_table(RUN_STATE)?; + let mut position = txn.open_table(CONSUMER_POSITION)?; + let mut outbox = txn.open_table(OUTBOX)?; + + log.insert((branch_key, lsn), &record)?; // append: the durable fact + state.insert((branch_key, attempt_no), &attempt)?; // apply: the derived view + position.insert(Scoped::new(tenant, stream), pos)?; // advance: where to resume + outbox.insert((branch_key, lsn), &intent)?; // relay separately (§13.3) +} +txn.commit()?; // all four, or none +``` + +**Every key here carries `BranchKey` or a `Scoped<_>` (§3).** The outbox is the +one that invites the mistake: an outbox keyed on `Lsn` alone reads naturally and +is wrong, because LSNs are per branch (§8.2), so a bare LSN collides across every +branch of every run of every tenant. The same key shape also gives the relay a +defined order — per `BranchKey`, ascending `Lsn` — which is the only ordering +§9.2 promises. + +Commit per event, not per run. A projection that accumulates in memory and +writes at run end loses the whole run on a crash. Episodes are small and +append-mostly; one fsync per transition is cheap next to model latency. + +### 8.4 Write-ahead intent + +Appending after the fact records history. It does not make a failed step +resumable, because the dangerous window is *before* the record exists. + +``` + 1. append Intent{Pending} ──► commit + fsync + ◄── window A: crash here, the call was never issued + 2. append Intent{Dispatched} ──► commit + fsync + 3. perform the call + ◄── window B: crash here, the call may have landed + 4. append outcome, Intent{Committed} ──► commit + fsync +``` + +Three records, not two. Two records cannot separate the windows: a crash before +the call and a crash after it both leave a lone `Pending` with no outcome, which +makes every interrupted effect maximally suspicious and pushes recoverable work +into `Indeterminate`. The second fsync buys the distinction. It is paid only on +steps with external effects and is small next to the call it guards. + +On restart, the last committed intent state classifies the crash: + +| Last state | Meaning | Resolution | +|---|---|---| +| `Pending` | the call was never issued | retry freely, whatever the effect class | +| `Dispatched` | the call *may* have been issued | by effect class, below | +| `Committed` | outcome already recorded | nothing to do | + +For a `Dispatched` intent, resolution is by declared effect class: + +| Class | Recovery | +|---|---| +| `Idempotent` | retry with the same idempotency key; the provider deduplicates | +| `Queryable` | ask the provider whether the request id landed, then complete or retry | +| `Unsafe` | **never auto-retry.** Attempt becomes `Indeterminate`, operator is notified | + +The third row is the honest one. Some effects cannot be made safe by any +protocol. The intent log's value is converting an invisible unknown into a +recorded one — `Indeterminate` is a fact a grader and an operator can both use; +a silently retried payment is not. + +Intents always commit `Immediate`. Batching them defeats their only purpose. + +Tool calls need this more than model calls. A model call is a metered read; a +tool call writes files, pushes commits, and touches the world. + +### 8.5 Rewind is a fork + +``` + lsn 0 ─ 1 ─ 2 ─ 3 ─ 4 ─ 5 ─ 6(failed) branch 0, retained + └─ 0 ─ 1 ─ 2 ─ ... branch 1, forked at (0, 3) +``` + +A rewind allocates a new `BranchId` and starts its LSNs at zero, recording the +fork point. Nothing is removed. Same rule as attempt N+1 never mutating attempt +N, for the same reason: **the discarded branch is the evidence**. Truncating it +destroys the failure that motivated the rewind, which is what the learning loop +exists to consume. + +- Queries default to the live branch; grading may read all of them. +- Only the live branch is exported. +- Rewinding past a `Committed` intent with a non-idempotent effect is a + compensation problem, not a replay problem. The log records what happened; it + cannot un-happen it. Flag rather than pretend. + +### 8.6 Retention: reduce, then tier + +Three mechanisms, escalating. Note the vocabulary: **`Archived`** is a run +state; **reduction** is the token-budget operation; **tiering** is the move to +cold storage. The previous revision called two of these "compaction" and the +collision was guaranteed to confuse implementers. + +**Checkpoints.** A materialized state snapshot tagged with its LSN. Restart +folds forward from the newest one. An optimization only — deleting every +checkpoint costs startup time and nothing else. + +**Reduction at a token ceiling.** Retention is measured in **tokens**, because +tokens are the currency of every downstream consumer: what a replay costs, what +fits in a judge's window, what an export is billed at. Default ceiling 200k per +run; per-tenant configurable. + +On self-hosted weights that ceiling is **not free to choose** — it is bounded by +`max_context_tokens / 2` from §14.2, because a pairwise judge reads two episodes +into one KV cache and two 200k episodes do not fit on any single device. Where +the derived bound is tighter than the configured ceiling, the bound wins and the +judge reads a further-reduced view. A retention number set without reference to +the hardware that must read it is a number that will be discovered wrong at the +first judge call. + +Loss order is fixed: + +| Kept | Reduced | Dropped | +|---|---|---| +| every transition record | blob bodies → summary blob | raw text on dead branches | +| context partitions, tool info, usage | dead-branch attempts → summary | | +| verifier results, grades | | | + +**Reduction never rewrites a blob and never edits the log.** Blobs are +content-addressed (§3), so replacing a body under its existing ref makes the ref +a lie; and repointing the log at a new ref is the history rewrite §8.7 forbids. +Reduction instead writes the summary as a *new* blob, appends a +`Reduced{original: BlobRef, summary: BlobRef}` event to the log, and only then +deletes the original body. The reduction is a later fact about an earlier record, +not a change to it. `BlobStore::get` on the original ref returns `None`, and the +fold knows why and what stands in its place — so drop-and-re-fold from LSN 0 +still yields byte-identical state, which it would not if the mapping lived only +in the projection. + +The transition sequence always survives. What reduces is *text*, because it +dominates token count and is the only part with a cheap lossy representation. A +reduced episode can still be graded, attributed and structurally rewound — it +just cannot be replayed verbatim. + +**Tiering.** Terminal, reduced runs move to cold storage. Moved, not copied, +with local rows deleted only after the remote commit acknowledges. + +### 8.7 Log schema evolution + +An append-only log plus an evolving event enum is a trap the previous revision +walked straight past. Two years of records, one `WorkEvent` variant renamed, and +the "drop derived state and re-fold" property is silently gone. + +Rules, from record one: + +- Every record carries `SchemaVersion`. Written always, even at v1. +- `WorkEvent` is `#[non_exhaustive]`; decode is version-dispatched. +- **Variants are never removed or repurposed.** Deprecated variants stay + decodable forever. Storage is cheap; an undecodable log is not. +- Migrations are **upcasters** — `fn upcast(vN) -> vN+1` — applied on read, never + by rewriting history. Rewriting an append-only log is a contradiction. +- A round-trip test per version, asserting that a stored fixture of every + historical version still folds to the expected state. This test is the whole + guarantee; without it the rules are aspirational. + +--- + +## 9. Distribution + +### 9.1 Partitioning + +``` + ingest ──► execute ──► verify ──► tournament ──► aggregate ──► decide + │ │ │ │ │ │ + (T,Run) (T,Run) (T,Run) (T,Task,Class, (T,Variant) (T,Workflow) + Epoch) + ▲ ▲ ▲ + shuffle 1 shuffle 2 single writer +``` + +Three keys, two shuffles, one single-writer stage. Everything up to verification +keys on `(TenantId, RunId)` and is embarrassingly parallel. Grading is a **join** +— a comparison group must be co-located — and it keys on +`(TenantId, TaskId, VerifierOutcome, GroupEpoch)`: the outcome class because +§11.4 brackets only within one, and the epoch because a closed group never +reopens for a late arrival (§11.6). + +The decide stage is single-writer per `(TenantId, WorkflowId)`. Two schedulers +adjusting traffic allocation concurrently produce an allocation neither holds. +A compare-and-swap on a generation counter is sufficient; no lock service +needed at this size. + +### 9.2 Ordering and delivery + +Per-run total order, nothing promised across runs. Downstream consumers must +therefore partition by run key, and the framework's broker adapters set the +partition key from `(TenantId, RunId)` — never from a correlation id, which +collapses unrelated runs onto one partition while splitting single runs across +several. + +Delivery is at-least-once. Exactly-once is achieved at the fold, not in +transport: `(BranchKey, Lsn)` is the natural idempotency key, so a redelivered +record is a no-op insert. + +### 9.3 Outbox + +The framework never calls a broker from the execution path. Export intent is +written in the same transaction as the state (§8.3); a separate relay reads +committed intents and ships them. This makes export restartable, keeps a broker +outage from stalling a run, and is the only pattern that survives a crash +between "state committed" and "event published". + +### 9.4 Leases and work distribution + +Runs are claimed by workers under a lease with a TTL. A worker that dies has its +runs reclaimed after expiry. Lease renewal is a heartbeat on the run record, and +`Suspended` runs (§5.2) release their lease entirely rather than heartbeating +through a human's lunch break. + +Fencing tokens on every lease. A partitioned worker that resumes must not write +under an expired claim, and a monotonic fence in the run record makes that a +rejected write rather than a silent double-execution. + +--- + +## 10. Verification + +Verification returns ground truth. It is a port with a fail-closed contract. + +```rust +#[async_trait] +pub trait Verifier: Send + Sync { + fn id(&self) -> VerifierId; + /// Any error, panic or timeout MUST resolve to `Fail`. A verifier that + /// throws or hangs can never report `Pass`. + async fn verify(&self, cx: &VerifierCtx) -> VerifierOutcome; +} +``` + +### 10.1 Verifiers need the inputs, not just the ids + +A verifier seeing only identifiers can answer "did it work". Answering "did the +agent have what it needed" requires the context and the prompt. + +```rust +pub struct VerifierCtx { + pub run: RunId, + pub step: StepId, + pub attempt: AttemptNo, + /// Frozen on entry to `Verifying`. Never mutates while a verifier holds it. + pub episode: EpisodeView, + /// Lazy. Verifiers needing no text never pay for it. + pub blobs: Arc, + pub deadline: Instant, +} + +pub struct AttemptView { + pub step: StepId, + pub attempt: AttemptNo, + pub state: AttemptState, + pub workflow_version: WorkflowVersion, + pub context: Option, // identifiers, small, inline + pub prompt: Option, // large, by reference + pub output: Option, + pub tools: Option, + pub usage: Option, +} +``` + +Context partitions hold *identifiers* — packed, available-but-not-packed, +dropped — and are small enough to inline. Prompts and outputs are large and go +by reference. Laziness matters: a verifier that shells out and checks an exit +code needs none of this, and making every verifier carry prompt text penalizes +the common case while blowing broker payload limits. + +Include **failed attempts**. "Retried three times because context was missing X" +is the learning signal; shipping only the winning attempt discards it. + +### 10.2 The snapshot barrier + +Verifiers read after work completes, so state must stop moving beneath them. +Entry to `Verifying` freezes the view. This is why `Verifying` must be a real +resting state and not a synchronous branch. + +Worth naming what the barrier actually defends against, because the obvious +answer is wrong. It is **not** a concurrent retry: `Verifying` is entered only +when every step is terminal (§5.2), so no attempt can still be running. The real +mutators are the ones that arrive from outside the run's own execution: + +- a **rewind** (§8.5) forking a new `BranchId` while verifiers hold a view of + the old one; +- a **cancel** (§5.2), now legal from `Verifying`; +- **recovery** resolving a `Dispatched` intent left by an earlier crash, which + writes an outcome into an attempt a verifier is already reading. + +Each of these is a write to the run while verifiers are mid-flight, and each is +rare enough to be missed in testing and ordinary enough to happen in production. + +### 10.3 Retention ordering + +Reduction (§8.6) must not outrun verification or grading. Eligibility is +**`Graded`, `Ungraded` or `Archived`** — the condition is *grading has +terminated*, not *grading succeeded*. Never a step-level finish timestamp: a +step can finish, be reduced, and then run-level verification finds nothing. + +`Ungraded` belongs in that set for a reason worth stating plainly, since the +tighter-looking `Graded`-only rule is the one that gets written. A run that +never gets a score — G = 1, a group that closed without it, a tenant over its +grading ceiling — is finished, and gating retention on `Graded` alone leaves it +irreducible forever. The tenants that hit this are the low-volume ones and the +cost-capped ones: the two populations least able to absorb unbounded storage, +and the two least likely to have anyone watching for it. + +--- + +## 11. Grading and the learning loop + +The default loop ships working. Every component is a port. + +### 11.1 Ports + +Grading is **strategy-pluggable, and the strategy declares what hardware it +needs before it is allowed to run.** That second half is not a detail: the +strategies below differ by more than an order of magnitude in model calls and in +VRAM, and a deployment that cannot afford one must be told at load time rather +than by an OOM at 3am. + +```rust +#[async_trait] +pub trait EvaluationStrategy: Send + Sync { + fn id(&self) -> StrategyId; + + /// Declared before any work is admitted. Validated against §14.2's limits + /// at load; a strategy whose profile does not fit is rejected by name. + fn resources(&self) -> ResourceProfile; + + async fn evaluate(&self, cx: &EvalCtx) -> Result>; +} + +pub struct ResourceProfile { + /// Models this strategy calls. One entry means it runs on the agent's + /// already-resident model and forces no swap (§14.2). + pub models: Vec, + /// Largest single-call context the strategy will request. A pairwise judge + /// reads two episodes, so this is roughly twice an episode budget. + pub max_context_tokens: u32, + /// Model calls per episode evaluated, for capacity planning and for §14's + /// spend projection. + pub calls_per_episode: f32, +} + +#[async_trait] +pub trait Grader: Send + Sync { + /// Produce comparable scores for a group of episodes. + async fn grade(&self, group: &Group, rubric: &RubricDef) -> Result>; +} + +#[async_trait] +pub trait Judge: Send + Sync { + /// Relative comparison only. Deliberately cannot return an absolute score. + async fn compare(&self, a: &EpisodeView, b: &EpisodeView, r: &RubricDef) + -> Result; + + /// Unary, and separate from `compare` for a structural reason: a `Core` + /// violation caps an episode on its own terms, not relative to an opponent + /// (§11.7). Runs before pairing. + async fn screen(&self, e: &EpisodeView, r: &RubricDef) -> Result>; +} + +pub enum Verdict { A, B, Draw } + +pub struct CoreViolation { + pub criterion: RubricCriterionId, + pub evidence: BlobRef, +} + +pub enum Score { + /// Default path (§11.3): one verdict against the current reference, plus + /// the running record the sequential test consumes. + Relative { against: RunId, verdict: Verdict, record: WinRecord }, + /// Bradley-Terry strength as a delta from control, with its interval and + /// the group size that produced it (§11.2, §11.4). Only the tournament + /// strategy produces this. + Ranked { strength: f64, interval: (f64, f64), group_size: u32 }, + /// A `Core` violation caps the episode. Carries the violations and no + /// number, so there is nothing for an aggregate to average past. + Capped { violations: Vec }, + /// No comparison was possible (§11.6). A reason, never a neutral score. + Ungraded { reason: UngradedReason }, +} +``` + +`Judge::compare` returning `Verdict` rather than `f64` is the schema decision +that matters most, and §11.2 is why. + +`Score` is deliberately a sum rather than a number with flags. A capped episode +and an ungraded one are not low scores; they are different kinds of answer, and +a type that can represent them as numbers will eventually have them averaged +into a promotion gate by code that meant no harm. + +**The strategy catalogue.** Cost is per episode evaluated, on a group of eight. + +| Strategy | Model calls / episode | Models resident | Produces | Default | +|---|---|---|---|---| +| `DeterministicGrader` | 0 | 0 | `Ranked` on a computed number | — | +| **`PairwiseSequential`** (§11.3) | **1–2** | **1, the agent's** | `Relative` | **yes** | +| `TournamentGrader` (§11.4) | 3–5 | 1 | `Ranked` with intervals | opt-in | +| `ReplayTournament` (§12.4) | 3–5 **plus N full agent runs** | 1 | `Ranked` across variants | opt-in | + +The default is `PairwiseSequential` because it is the only one whose cost does +not scale with how much you want to know. The tournament's `(G/2)·log₂(G)` +comparisons buy a full ranking with composable strengths; that is genuinely more +information, and a deployment that can afford it should turn it on. Most cannot, +and a framework whose default path assumes a grading budget larger than the work +being graded will simply be run with grading disabled — which is the outcome +this whole section exists to avoid. + +`DeterministicGrader` remains for users whose quality signal is a number they +already compute — latency, cost, test pass count. It exists so that adopting the +framework does not require adopting LLM-as-judge at all. + +### 11.2 Absolute scores do not work here + +Three failure modes, all of which this system would hit: + +**Calibration drift.** A judge asked for 0..1 returns different numbers for the +same episode across weeks and model versions. Drift is indistinguishable from a +variant trend, so promotion decisions fire on grader noise. + +**Weak discrimination.** Four competent episodes all score 0.8. No gradient, no +selection pressure, and the loop reports "nothing beats control" because the +grader cannot resolve them — not because they are equivalent. + +**Saturation.** The one that kills the loop outright. As workflows improve, pass +rate approaches 100% and pass/fail carries zero information; absolute rubric +scores saturate identically. A tournament cannot saturate — better candidates +just make it harder. + +Group-normalized relative scores also give something absolute scores cannot: +**cross-task comparability** — but only through a shared anchor, and that +qualification is load-bearing. A Bradley-Terry fit identifies strengths only up +to an additive constant *within one connected comparison graph*. Two groups on +different tasks are disjoint graphs, so their strengths sit on unlinked scales, +and averaging them directly commits the same error this section accuses point +tallies of, one layer further in. + +**The anchor is control.** §12.1 gives control a traffic share in every +allocation, so every group contains at least one control episode; the fit pins +control to zero and every other strength is read as a delta from it. A variant's +aggregate is then a mean of like-for-like deltas rather than a mean of +incomparable scales — which is what makes the per-variant stage in §9.1 sound +rather than approximate. A group that happens to contain no control episode is +not aggregatable: it still grades its own members and is still worth reading, it +just does not feed the aggregate, and it is recorded as such rather than folded +in on the assumption that scales match. + +### 11.3 Pairwise sequential (default) + +The system holds **one current version and at most one challenger**, and grading +answers one question: has the challenger accumulated enough evidence to replace +the current one? Not "rank these eight", not "what is each episode worth" — a +single accept/reject that converges toward one state. + +``` + current version ──► episode ──┐ + ├──► Judge::compare ──► verdict + challenger ──► episode ──┘ │ + (same TaskId) ▼ + accumulate into WinRecord + │ + ┌────────────────┼────────────────┐ + ▼ ▼ ▼ + accept continue reject + challenger becomes keep sampling discard, keep + the current current +``` + +One comparison per episode. Against the tournament's `(G/2)·log₂(G)` pairs +doubled for both orderings, that is 24 judge calls dropping to 8 at G = 8, and +the saving grows with G rather than shrinking. + +**The reference is the current version's recorded episode on the same `TaskId`.** +It is already on disk — no re-run, no group to fill, no `GroupEpoch` timeout. +Where the task has never been seen before there is nothing to compare against, +and that case degrades per §11.6 rather than being papered over. + +**Stopping is a sequential test, not a fixed sample.** Verdicts accumulate into a +likelihood ratio against boundaries set by α, β and the smallest win-rate shift +worth acting on; the test stops as soon as a boundary is crossed. A clearly +better challenger is accepted in far fewer comparisons than a fixed-n design +would spend, and a clearly worse one is rejected early instead of running to +completion. This is the mechanism that makes the cost adaptive: cheap decisions +cost little, close decisions cost more, and nothing costs the worst case by +default. + +**Draws are recorded and excluded from the ratio.** A tie carries no evidence +either way about which is stronger, so folding it in as half a win manufactures +information. But a *high draw rate* is itself a result — it says the challenger +is not meaningfully different — so the test also rejects on a draw-rate ceiling +rather than sampling forever toward a boundary it will never reach. + +**Order alternates rather than doubling.** §11.4's both-orderings rule pays 2× on +every comparison to cancel position bias. Here the challenger takes position A on +even-numbered comparisons and position B on odd ones: bias cancels across the +sequence instead of within each pair, at no extra cost. Order-consistency is +still measured, on a sampled fraction of comparisons, and reported as the +grader's error bar exactly as before. + +**A fixed reference is a cacheable prefix.** The same reference episode leads +every comparison in a decision, so it can be cached across calls. The tournament +cannot do this — shuffling into brackets makes every pair a novel combination by +design, which is the point of the shuffle and the reason nothing caches. + +What this gives up, stated rather than discovered later: **parallel exploration +and composable strengths.** One challenger at a time is hill-climbing, which is +slower to find improvements and can settle in a local optimum with nothing in the +loop able to report that it has. And a `Relative` score answers "better than the +current version on this task" — it is not a strength that composes across tasks +the way §11.2's anchored Bradley-Terry deltas do. Deployments that can afford the +tournament get real information for the money; this is the right default, not the +better mechanism. + +### 11.4 Tournament grading (opt-in) + +Not the default path — §11.3 is — and not a fallback either. This is the +strategy to enable when episodes are **already co-present at no extra cost**, or +when a deployment can afford full rankings. Two cases qualify naturally: + +- **Attempt tournaments** (§11.5). The attempts of one step are on disk the + moment a retry happens. No agent runs to pay for, and this is the only source + of per-step credit the system has. +- **Replay** (§12.4), where N variants are executed against one task + deliberately. The episodes exist because you paid for them; grading them + pairwise would waste the group you bought. + +Everything below is unchanged in substance from when it was the default. What +changed is the claim: it is more information per episode, at three to five times +the model calls, and that trade is now the user's to make explicitly rather than +one the framework makes for them. + +``` + [ G comparable episodes for one task, one outcome class ] + │ + ▼ + [ shuffle into brackets ] ◄── shuffling also cancels position bias + │ + ▼ + ┌─────────────────────────────────┐ + │ Swiss pairing, log₂(G) rounds │ ◄── rubric-guided Judge, relative only + └────────────────┬────────────────┘ + │ + ▼ + [ Bradley-Terry fit over all comparisons ] + │ + ▼ + [ strength per episode + confidence interval ] +``` + +Swiss rather than round-robin: O(G log G) instead of O(G²). Eight episodes is +twelve comparisons rather than twenty-eight. Swiss rather than single +elimination because we want a full ranking, not a champion — eliminated +candidates still carry signal. + +**Bradley-Terry, not point-tally z-scores.** Accumulating tournament points and +normalizing to mean 0 / sd 1 within a group is the obvious approach and it is +statistically wrong for our aggregate: small groups produce extreme z-scores, so +a variant that appears in many small groups wins on variance rather than +quality. A Bradley-Terry fit over the pairwise outcomes yields a strength +parameter with a real confidence interval, which composes correctly across +groups of different sizes and feeds the sample gate in §12.3 directly. + +Three details of that fit are decisions, not implementation freedom. Textbook +Bradley-Terry does none of them, and each failure looks like a result rather +than a bug. + +**Draws need a draw model.** Plain BT is binary and has no tie term, so the +draws this section deliberately permits have nowhere to go. Dropping them +discards the judge's most confident statements; splitting each half-and-half +fabricates two comparisons that never happened and tightens the interval on +invented evidence. Use the **Davidson extension** — one additional tie parameter +fit alongside the strengths. + +**Small groups separate.** At G = 4..8 an episode that wins every comparison +drives the unpenalized maximum-likelihood estimate to infinite strength. That is +the extreme-score failure §11.2 rejects, arriving through the fit instead of +through z-scores. A weakly-informative prior on the strengths — equivalently, a +penalized likelihood — is **required, not tuning**. It is the mechanism that +turns "won all three of its comparisons" into a wide interval rather than an +unbounded one, and without it the reassuring sentence about small groups +producing wide intervals is simply false. + +**A strength alone means nothing.** The fit is identified only up to an additive +constant (§11.2), so an interval on a single raw strength is an interval on an +arbitrary origin. Pin control to zero and report every strength as a delta +against it. The §12.3 canary gate says "BT interval excludes zero" — zero is +control, and it is only zero because it was pinned there. + +**Draws are permitted.** A judge forced to separate two equivalent episodes +invents a distinction, and the optimizer will chase the invention. Draws cost +gradient; forced choices cost correctness. + +**Both orderings are judged.** Pairwise judges have position bias. +Order-consistency is recorded per comparison, and the disagreement rate is the +grader's own error bar — it belongs in the report next to the scores, and an +inconsistent judge should widen the sample gate rather than silently promote. + +### 11.5 Groups: where they come from + +A tournament needs a comparison group, and production runs are one-shot on tasks +that mostly never repeat. This is the binding constraint on the whole learning +loop. + +`TaskId` is the group key: a content hash of the task input **before any +workflow touches it**. Hashing the prompt does not work — the workflow changes +the prompt by construction, which is the entire point of a variant. + +| Source | Group | Cost | Needs | +|---|---|---|---| +| **Attempts** | attempts 1..N of one step, identical context | free, already recorded | nothing | +| **Recurring tasks** | runs sharing a `TaskId` over time | free, slow to fill | `TaskId` | +| **Replay** | one task re-executed under N variants | N full runs | blobs, sandbox | + +Attempt tournaments first. Retries are already on disk and attempt N+1 never +mutates attempt N, so the attempts of one step are a group on identical context — +the cheapest per-step credit signal available. + +A failure and its successful retry are **not** a judge comparison. §11.8 forbids +that pairing and the verifier has already ordered it; asking a judge which is +better asks it to re-decide what verification decided. The pair is consumed +*structurally* instead: what differed between attempt N and N+1 — context +partition, tool selection, prompt — is attributed to the `StepId` as the change +that turned a fail into a pass. The judge sees only same-outcome attempts, where +the question it answers is "which failure got further" — which no verifier can +answer. + +**`TaskId` cannot be backfilled.** A run recorded without one is permanently +ungroupable, which is why it is required at spawn with no `Default` and no +`From`. + +### 11.6 Low-volume degradation + +Most of this problem is a tournament problem, and §11.3 does not have it: a +pairwise comparison needs one partner, and the current version's recorded +episode on that `TaskId` is already on disk. Volume stops mattering the moment a +task recurs even once. + +What survives is the genuinely irreducible case — **a `TaskId` never seen +before**. There is nothing to compare against, because nothing else has done this +task. That is not a degradation to engineer around; it is the first observation +of a new task, and it becomes the reference for the next one. + +1. **Novel `TaskId`** — no comparison. Verifier outcome and deterministic + dimensions (cost, latency, tool efficiency) still recorded; no relative score. + Reported as `Score::Ungraded { reason: NoReference }`, never as a neutral + score. The episode is retained as the reference for that `TaskId`. +2. **Attempt tournaments** — available to any workflow that retries, regardless + of volume, and unaffected by either of the above. +3. **Grading budget exhausted** — a tenant over its §14 ceiling reports + `Ungraded { BudgetExhausted }`. Not a quality signal; a spend signal. + +The remainder of this section applies **only when the tournament strategy is +enabled** (§11.4), where a group must genuinely fill: + +4. **Group completeness trigger** — a group closes on quorum *or* on a timeout, + grading whatever arrived, with group size attached to the confidence interval. + +**A closed group is immutable, and the next episode starts a new one.** This is +the question the trigger raises and does not answer on its own: a group closed +on timeout at G = 3, then a fourth episode with the same `TaskId` arrives an hour +later. Re-opening and re-fitting is the wrong answer — strengths from that group +have already been published, aggregated, and possibly acted on by a promotion +gate, and a fit that silently changes underneath a decision already made is +worse than a small group. + +So the group key carries a generation: `(TenantId, TaskId, VerifierOutcome, +GroupEpoch)`. Closure increments the epoch; late arrivals accumulate into the +next one. The cost is honest and should be stated rather than discovered — a +low-volume tenant with a long inter-arrival time gets a run of G = 1 epochs, each +reported `Ungraded { InsufficientGroup }`. That is a real signal about their +volume, and the fix is a longer timeout, which is a tenant-level setting and a +tradeoff between waiting and grading, not a bug in the trigger. + +A tenant whose loop never engages must see that in the dashboard as a stated +reason. Silent no-op is the worst outcome: it looks like a working loop that +finds no improvements. + +### 11.7 Rubrics and credit assignment + +```rust +pub enum RubricLayer { + /// Mandatory. A violation caps the result regardless of everything else. + Core, + /// Anti-gaming. Written explicitly against known exploits. + Prescriptive, + /// Context-specific, user-authored, weighed rather than binding. + Contextual, +} +``` + +`Core` violations **cap** rather than subtract. A weighted sum lets a variant buy +past a safety failure with speed, which is the exact failure prescriptive +rubrics exist to prevent. + +The cap needs somewhere to live, and `Verdict` is the wrong place — a `Core` +violation is a fact about one episode, not about a pair, and a judge asked to +express it through a comparison can only rank the offender lower. It comes from +`Judge::screen` (§11.1) instead, which runs before pairing and yields +`Score::Capped`. A capped episode is **excluded from the bracket, not ranked +last in it**: left in, it still contributes comparisons that shape everyone +else's strength, and a variant with one safety failure and seven strong episodes +aggregates to a promotion. + +Never let a rubric judge what a verifier can check. Every criterion that can be +made mechanical should be a `Verifier`, not a rubric line — deterministic, +cheap, and not subject to judge drift. + +Per-step credit attributes a group's outcome to `StepId`s, which is why §4.3's +stability contract is load-bearing rather than cosmetic. + +### 11.8 Grading never overrides the verifier + +Episodes are bracketed **within** a verifier outcome class, never across. A +verified pass beats a verified fail by definition and that pairing is never shown +to a judge. Ranking failures against each other is not wasted work: "failed at +step 2" versus "failed at step 7 after recovering twice" is exactly the signal a +pass rate cannot see. + +This binds attempt groups too (§11.5), which is where the rule is easiest to +break: a step's failed attempt and its successful retry sit side by side on disk +and look like a free comparison. They are a free *credit* signal and not a +comparison at all. The bracketing rule has no exemptions — if a pairing crosses +an outcome class, it is evidence for attribution, never input to a judge. + +--- + +## 12. Optimization loops + +Two loops at different clock speeds. The fast loop **selects** among existing +workflow versions; the slow loop **generates** new ones. + +### 12.1 Shape + +``` + ┌──── GENERATE (slow, human-gated by default) ─────┐ + │ failure evidence ──► propose ──► challenger │ + │ ▲ │ │ + └────────┼──────────────────────────────┼──────────┘ + │ │ register — at most one + ┌────────┼──── SELECT (fast) ───────────┼──────────┐ + │ │ ┌── allocation ────────────┘ │ + │ │ │ current 95% · challenger 5% │ + │ │ └────┬─────── │ + │ │ │ spawn — pin WorkflowVersion │ + │ │ ▼ │ + │ │ run ──► verify ──► compare vs │ + │ │ current (§11.3) │ + │ │ │ │ + │ └──────────────────────────────┤ │ + │ ▼ │ + │ sequential test boundary │ + │ │ │ │ + │ accept │ │ reject │ + │ ▼ ▼ │ + │ challenger discard, │ + │ becomes current keep current │ + └───────────────────────────────────────────────────┘ +``` + +**One current version, at most one challenger.** The loop converges toward a +single state rather than maintaining a population. This is the change that makes +everything else affordable: no N-way traffic split, no per-variant aggregation +across groups, no allocation state to contend over, and one comparison per +episode instead of a bracket. + +The cost is exploration. A single challenger at a time is hill-climbing — it +finds improvements more slowly than a population would, and it can sit in a local +optimum indefinitely with nothing in the loop able to say so. §12.5's held-out +report is the only instrument that will notice, which makes it more important +here than it was under the population design, not less. + +**Multi-variant selection remains available** for deployments that can afford it: +enable the tournament strategy (§11.4), allow N challengers, and §12.3's full +rung ladder applies with Bradley-Terry aggregation across groups. The machinery +is the same; what changes is how many versions are live at once and which +statistical object closes the decision. + +The slow loop fires when the fast loop **rejects a challenger without finding a +replacement** — a trigger, not a timer. + +### 12.2 Versions form a DAG + +``` + v1 ────┬───► v2 ───┐ + control │ └──► v4 (merge) + └───► v3 ───────┘ +``` + +Content-addressed, parent-pointered, never edited. Editing a version in place +destroys every result already attributed to it. + +### 12.3 Promotion gates + +Every rung needs a criterion. The previous revision drew the ladder and stated a +rule for only the first rung. + +**Default ladder — one challenger, pairwise (§11.3).** Three rungs, because a +graduated ramp is a population instrument and there is no population here: + +| Rung | Traffic | Entry criterion | +|---|---|---| +| shadow | 0% | registered, validated, sandbox-clean, `ResourceProfile` fits (§14.2) | +| trial | 5% | no `Core` violation on any trial episode | +| current | 100% | sequential test crosses the accept boundary; drift check clean | + +The swap at the last rung is deliberate and worth naming: the challenger takes +all traffic at once rather than ramping. A ramp exists to limit blast radius +while evidence accumulates, and here the evidence has already accumulated — the +sequential test does not cross its boundary until the win rate is established at +the configured α. Ramping after that spends traffic to re-learn what the test +already concluded. What guards the swap instead is the rollback rule below, which +fires on a single `Score::Capped` and does not wait for a boundary. + +**Resourced ladder — N challengers, tournament (§11.4).** For deployments that +enabled the tournament strategy and can carry multiple live versions: + +| Rung | Traffic | Entry criterion | +|---|---|---| +| shadow | 0% | registered, validated, sandbox-clean, profile fits | +| canary | 5% | beats current on **selection** replay, BT interval excludes zero | +| ramp | 20→50% | no `Core` violation, cost within budget, sequential test at α | +| current | 100% | sustained over N groups, drift check clean | + +Rollback is automatic and unconditional on any `Score::Capped` attributed to the +variant, or a verifier pass-rate regression beyond a configured margin. +`Score::Capped` is the only signal for the first of those (§11.1); a gate that +reads a low *number* instead is reading something the cap exists to prevent from +existing. Rollback is a traffic +change, never a version delete — the failed variant stays in the DAG with its +results. + +### 12.4 Replay is re-execution + +"Replay against recorded episodes" means **re-running the task under a new +workflow version**, not pushing a recorded trajectory through new logic. +Trajectory replay tells you only where behaviour would first diverge, and +everything after divergence is unknown — near-worthless for grading. + +Two consequences the prior revision missed: + +**Replay is not free.** "0% live traffic" means no user sees the result, not +that it costs nothing. N variants × M tasks is N·M full agent runs plus judge +calls. Shadow is the most expensive rung, not the cheapest. + +**Replay executes real tools.** Re-running a workflow that pushes commits pushes +commits. Shadow execution runs in a **sandbox with `Unsafe` effects denied**, and +a workflow whose steps cannot run sandboxed is ineligible for shadow evaluation +and must say so at load time rather than at 3am. + +### 12.5 Held-out set, and the leak + +Selecting on a fixed set of recorded tasks overfits to those tasks, silently: +shadow scores improve while live performance does not. + +Partition tasks into a selection set and a held-out set. Promote on selection, +report held-out without optimizing against it, and treat a widening gap as the +overfitting alarm. Concretely: **no rung in §12.3 takes held-out as an entry +criterion.** A gate that reads held-out has converted it into a second selection +set and left nothing measuring generalization. + +**The proposer reads the selection set only.** The slow loop consumes failure +evidence to generate candidates; if it reads held-out failures, the held-out set +is contaminated through the generator instead of the selector. This leak is +easy to introduce and invisible once present. + +Held-out catches overfitting to *episodes*. It does not catch drift in the *task +mix* — that needs a separate distribution check on incoming `TaskId` +characteristics over time. + +--- + +## 13. Extensions, tools, and trust + +### 13.1 Capability-gated hostcalls + +Tools register through capability-gated hostcalls. The gate is where effect +class is declared, and a tool that cannot state whether it is safe to retry does +not register. + +```rust +pub struct ToolRegistration { + pub id: ToolId, + /// No Default. The author is the only party who knows this. + pub effects: EffectClass, + pub caps: CapabilitySet, + pub timeout: Duration, +} + +/// Each class carries what its §8.4 recovery path actually needs. A bare +/// discriminant would let a tool claim `Idempotent` while withholding the one +/// thing that makes the claim actionable. +pub enum EffectClass { + Idempotent { key: KeyDerivation }, + Queryable { lookup: RequestIdLookup }, + Unsafe, +} +``` + +Tool calls appear in the episode as **first-class steps**. The hostcall boundary +already knows the identity, arguments and capabilities; recording them as opaque +invocations discards information the runtime is holding anyway. + +### 13.2 Declaration is not enforcement + +A framework cannot trust a user-supplied `EffectClass`. A tool declared +`Idempotent` that is not will be retried after an indeterminate crash, and the +damage is the user's data. + +Three layers, in order of strength: + +1. **Declaration** — required, recorded, auditable. +2. **Keyed capability** — `Idempotent` is a claim about *retry*, not about + abstaining from writes. Denying an `Idempotent` tool network and filesystem + writes would deny the recovery path in §8.4, which retries the write under an + idempotency key. So the restriction is on the *shape* of the write, not its + existence: an `Idempotent` tool declares how its key derives from its + arguments (§13.1), the kernel derives and supplies that key on every call, + and a write issued without it is denied. A tool that cannot derive a stable + key cannot be `Idempotent` — which is the same claim as before, now refused + at registration rather than discovered after a double-charge. +3. **Sandbox** — shadow and replay execution deny `Unsafe` effects outright. + +Layer 2 is the one that makes layer 1 more than paperwork. + +### 13.3 The relay + +Export runs from the outbox, out of process, with its own retry and its own +failure domain. A user's broker being down is not an agent outage. + +--- + +## 14. Metering, cost, and capacity + +### 14.1 Metering + +A framework that spends users' model budget on grading must account for it. + +- Every model call — agent, judge, or proposer — is attributed to + `(TenantId, RunId, Purpose)` where purpose distinguishes work from grading + from replay. +- Per-tenant ceilings on grading spend, enforced at the group boundary in §9.1 + where group size is known and a tournament can be skipped or downsampled + before it starts. +- Grading cost is reported next to grading value. A loop that costs more than + the work it grades may still be worth running; it should never be an + unpleasant discovery. + +Replay is the expensive one (§12.4) and needs its own ceiling separate from +judging. + +### 14.2 Model capacity is a hard constraint, not a budget + +Metering counts tokens after the fact. On self-hosted weights the binding limit +arrives earlier and harder: **VRAM**, and the cost of moving weights in and out +of it. A model that is not resident cannot be inferred against, and making it +resident means evicting something else and paying a load measured in tens of +seconds. A design that treats "call the judge model" as equivalent in cost to +"call the agent model" is wrong by two orders of magnitude on this hardware. + +The limits are declared, not discovered: + +```rust +pub struct CapacityLimits { + /// Per-device VRAM this framework may use. Not the card's total — leave + /// headroom for anything else sharing the device. + pub vram_bytes_per_device: u64, + pub devices: u32, + /// Hard cap on simultaneously resident models across all devices. + pub max_resident_models: u32, + /// Never evictable. The agent's model belongs here; if it can be evicted, + /// grading can stall agent work, which §1 forbids outright. + pub pinned: Vec, +} + +pub struct ModelProfile { + pub id: ModelId, + pub weights_bytes: u64, + /// KV cache cost per token at the deployed dtype and parallelism. The term + /// that decides how long a context may be — see below. + pub kv_bytes_per_token: u64, + /// Devices this model spans under tensor parallelism. + pub devices_required: u32, +} +``` + +**Residency invariant**, checked at load and before any admission: + +``` +sum(weights of resident models) + peak_concurrent_kv ≤ vram_bytes_per_device × devices +resident_model_count ≤ max_resident_models +``` + +**One resident model is the default configuration.** The agent's model is pinned; +`Judge` and the §12 proposer run on that same model. §11.3's `ResourceProfile` +declares a single `ModelId` precisely so that the default grading path adds no +resident model and forces no swap. A strategy naming a second model is legal and +is rejected at load unless the invariant still holds with both resident — never +by swapping between them per call, which is the failure mode this section +exists to prevent. + +**Context length is a VRAM quantity, and this bites hardest on the judge.** A +pairwise comparison reads two episodes, so its context is roughly twice an +episode budget. Working the invariant backwards gives the ceiling: + +``` +max_context_tokens = (vram_bytes_per_device × devices − sum(weights)) / kv_bytes_per_token +``` + +Put numbers on it, because the result is not marginal. At GQA fp16, per-token KV +runs roughly 0.13 MB for an 8B-class model and 0.33 MB for a 70B-class one. Two +episodes at §8.6's 200k-token retention ceiling is 400k tokens of context: +**52 GB of KV cache at 8B, 131 GB at 70B**, before weights. Neither fits an 80 GB +device. A judge reading two full-ceiling episodes is not expensive — it is +impossible. + +Three consequences, all forced rather than chosen: + +- **The retention ceiling must be derived from this, not set beside it.** §8.6's + 200k default is a token-budget number that was picked without reference to any + device. The reduced episode is what the judge reads, so the reduction target + is `max_context_tokens / 2`, and where that is smaller than the retention + ceiling, the judge reads a further-reduced view. +- **Admission control, not backpressure.** Work whose `ResourceProfile` does not + fit the current residency is refused at admission with the limit named. Queuing + it would stall behind an eviction that §1 does not permit. +- **Grading yields to agent work.** When both contend for the same resident + model, agent inference wins and grading queues. A framework that lets a judge + call delay the work it is judging has inverted its own first principle. + +Distributed GPUs change the arithmetic, not the rule. `devices_required` +expresses tensor parallelism across cards; `max_resident_models` is a +fleet-wide count, so two nodes each holding the agent model are two resident +instances, not one. Residency is per device, and a model resident on node A does +not make node B's runs admissible. + +--- + +## 15. Observability + +The framework observes agents; it must also be observable. + +- Kernel state transitions as metrics, tagged by tenant and workflow version. +- Lag on every stage boundary in §9.1. Tournament and reduction backlogs are the + two that grow silently. +- `Indeterminate` attempt count as a first-class alert. It should be near zero; + a nonzero rate means either a crash loop or a misdeclared effect class. This + threshold only holds because `Cancelled` is a separate state (§5.1) — route + cancellations here and the alert has a noisy floor, which is the same as not + having it. +- `Ungraded` run count by reason (§11.6), separately from `Graded`. A tenant + whose runs are mostly `InsufficientGroup` has a loop that is not engaging, and + that reads as healthy on any dashboard that only counts failures. +- Judge order-inconsistency rate (§11.4) as a grader-health metric, measured on + a sampled fraction of comparisons under §11.3's alternating-order scheme. +- **Judge-versus-verifier agreement on the calibration set** (§16). Under one + resident model the judge is the agent's model, and this is the only instrument + that can see the grader drifting toward its own bias. +- **Model residency and swap count** (§14.2). A nonzero swap rate on a + single-model deployment means something is requesting a non-resident model, and + the load cost will dominate everything else in the trace. +- Admission refusals by reason, separating "capacity" from "budget". They look + identical in a queue-depth graph and have opposite fixes. +- Held-out versus selection gap (§12.5) as the overfitting alarm. +- Trace context propagated through `Ctx`, never through task-locals. + +--- + +## 16. Deliberately not built + +- Anything before the record is trustworthy. No dashboards and no learning loop + until one full run works end to end against a stub model. +- Grading that decides. It attributes; the verifier decides. +- Automatic workflow mutation without human approval. The generator is gated by + default; a user may ungate it once their loop is calibrated against verifier + ground truth, and that is their decision to make explicitly. +- Self-critique using the same model family being graded, **ungated**. §14.2's + one-resident-model default means the judge normally *is* the agent's model, so + this is no longer a prohibition but a precondition: it is admissible only with + the bootstrap the prior revision named — validation against verifiable tasks. + Concretely, a calibration set of tasks with known verifier ground truth is + replayed through the judge on a schedule, and judge-versus-verifier agreement + is tracked as a health metric (§15). It costs no agent runs, since the episodes + are already recorded. Without it the grader's bias is unmeasured and the + optimizer will find it — and under one-state convergence (§12.1) there is no + competing variant whose divergence would make that visible. +- Cross-tenant blob deduplication (§3). +- Semantic retrieval over episodes. No consumer yet. + +--- + +## 17. Decisions + +| Question | Decision | Where | +|---|---|---| +| Framework or application | **Framework.** Kernel/domain split; workflows are data | §2, §4 | +| Async runtime | **Tokio.** Reverses the prior asupersync choice; ecosystem access is decisive for a distributed framework | §6 | +| Storage | **Two modes**, one port set: `redb` embedded, Postgres + object store distributed | §7 | +| Port signatures | **Async everywhere**, including local implementations | §7 | +| Durability | Engine gives atomicity; **the log gives history**. Append + apply + advance in one transaction | §8 | +| Side effects | **Write-ahead intent in three phases** (pending / dispatched / committed), effect-class recovery, `Indeterminate` as a real state | §8.4 | +| Rewind | **Fork**, never truncate. `BranchId` in the log key | §8.5 | +| Log evolution | **Versioned records, upcasters on read.** Variants never removed | §8.7 | +| Grading | **Relative only**, and **strategy-pluggable**. Default `PairwiseSequential`: one current, one challenger, one comparison per episode, sequential-test stop. `Judge` returns a verdict not a number | §11.1, §11.3 | +| Tournament | **Opt-in, not default.** Swiss + Bradley-Terry with a Davidson tie term and a prior, strengths as deltas from a pinned anchor. Enabled where episodes are co-present anyway — attempts, replay | §11.4 | +| Optimization target | **Converge to one state.** One current version plus at most one challenger; N-variant population selection is the resourced option | §12.1, §12.3 | +| Model capacity | **One resident model by default**, agent's model pinned, judge and proposer share it. VRAM residency invariant checked at load; context ceiling derived from KV cost, not chosen | §14.2 | +| Self-critique | **Admissible under calibration.** Same-model judging is the default consequence of one resident model, gated on judge-versus-verifier agreement tracking | §16 | +| Tenancy | **Tenant key on every row and every blob namespace**, from commit one | §3 | +| Multi-agent orchestration | **Deferred.** The log already provides durable state and resumability; revisit when cross-agent coordination is real | §18 | + +--- + +## 18. Open questions + +- **What defines `TaskId` for a given user.** Ticket id, input fixture, or a hash + of the pre-workflow goal. Depends on where work enters their system, so the + framework provides the type and a default hasher and lets it be overridden. + Must be settled before any run is recorded, since it cannot be backfilled. +- **Minimum useful group size.** G ≥ 2 runs, but a two-episode tournament is one + comparison and carries little. Where the useful floor sits is empirical. +- **Group timeout default.** §11.6 closes a group on quorum or timeout and + increments the epoch. The timeout trades grading latency against group size, + and the right default depends on tenant arrival rate — which the framework can + measure but has no data for yet. Only bites when the tournament is enabled. +- **Sequential test boundaries.** §11.3 stops on α, β and a minimum detectable + win-rate shift. All three are policy, not physics: too tight and no challenger + is ever accepted, too loose and the loop churns the current version on noise. + Needs calibration against a workflow whose true improvement is known. +- **Draw-rate ceiling.** §11.3 rejects a challenger that draws too often, since + the test would otherwise never terminate. Where the ceiling sits is empirical + and interacts with judge quality — a weak judge draws more. +- **Local optima under one-state convergence.** §12.1 accepts hill-climbing. + Nothing currently detects a loop that has stalled in a local optimum versus one + correctly reporting no improvement exists. The held-out gap (§12.5) is the + nearest instrument and was not designed for this. +- **`kv_bytes_per_token` measurement.** §14.2's context ceiling depends on it, + and it varies with dtype, quantization, attention implementation and + parallelism. Measured per deployment or read from a profile the operator + supplies — the framework should refuse a guess. +- **Grading spend ratio.** §14 meters it; nobody has set the ceiling. +- **Reduction summarizer.** §8.6 reduces blob bodies to "a summary" without + saying what produces it. A model call makes reduction non-deterministic, which + interacts badly with replay. Extractive or structural reduction may suffice. +- **Effect classification for built-in tools.** An audit, not a decision. Until + done, default `Unsafe` and never auto-retry. +- **Postgres schema for the log at scale.** Partitioning by tenant and time, + index strategy for branch scans, and whether the outbox is a table or a + logical replication slot. +- **Cancellation rigour under tokio.** §6 accepts cooperative cancellation as a + residual risk. Whether a lint, a wrapper type, or a `loom` harness is the right + enforcement is unresolved. + +--- + +## 19. Lessons carried forward + +Each of these cost real time in the prior TypeScript implementation. Most are +now structural; the rest stay written down. + +**Structurally handled:** half a concurrency guarantee is worse than none +(awaitable delivery without cancellation); unbounded state (retention declared +at construction); ordering under concurrency (per-run scope, not a global queue). + +**Still on us:** + +- *A green suite says nothing about coverage.* An idempotency feature generated + its own keys and checked for duplicates among them — unreachable for a whole + phase, with tests asserting the count was zero. +- *Suspect the guards before the mechanism.* A "batching" failure was the depth + guard, because two unrelated limits shared a default value. +- *A test run that prints nothing cannot distinguish slow from hung.* Per-test + progress and per-test timeouts from the first commit. +- *Collapsing a state removes the seam that needed it.* Resolving pass/fail + inside `verify()` blocked three separate features. +- *A placeholder that type-checks is invisible.* A hardcoded `"current"` version + hash compiled, passed tests, and made every result unattributable. A newtype + with no `Default` refuses to compile instead. +- *A port finished while known to be unimplementable is not finished.* The + synchronous dedupe store documented, in its own doc comment, that a networked + implementation could not honour its signature. + +--- + +## 20. Build order + +Detailed, dependency-ordered tasks: [rust-agentic-task.md](rust-agentic-task.md). + +The walking skeleton is one full run of one workflow against a stub model, in +embedded mode, with no network — recorded, verified, and re-derivable from the +log. Nothing after it is worth starting until it runs.