Story Crater BotandClaude Opus 5
a1a3737f8f
Adds the log crate: SchemaVersion, LogRecord, BranchKey, and WorkEvent with all eight variants, plus tests/it_fixture_roundtrip.rs walking tests/fixtures/*/ and decoding every file. Fixtures are local-only, not committed. The .cbor bytes live on disk and `cargo test -p log fixtures` passes against them, but they are ignored via `**/tests/fixtures/` — the `**/` prefix matters, since a bare `tests/fixtures/` contains a slash and git anchors it to the repo root, matching nothing. Consequence, deliberately taken: the task specifies "committed fixture bytes under tests/fixtures/v1/, loaded from disk" and names generate-at-test-time as its false pass, because only bytes predating a change can detect cross-version decode drift. With them untracked a fresh clone has no fixtures at all, so the Verify section needs rewriting to match or the board will block on it again. Co-Authored-By: Claude Opus 5 <[email protected]>
452 lines
16 KiB
Bash
Executable File
452 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Agentic driver for the rust-agent-sys board.
|
|
#
|
|
# Per task, two `pi` passes:
|
|
# 1. coder — implements, runs the task's Verify command, reports
|
|
# 2. reviewer — re-verifies, audits false passes/traps, emits a markdown review
|
|
# The shell is the chain: the coder's report is fed into the reviewer's prompt.
|
|
# The shell also owns the gate rule and the Status writes, because those are
|
|
# mechanical and must not depend on a model choosing to comply.
|
|
#
|
|
# ./loop.sh run from the first unfinished task until done or blocked
|
|
# ./loop.sh --list show phase order + current status, run nothing
|
|
# ./loop.sh --dry-run list what would run, run nothing
|
|
# ./loop.sh --sync rewrite INDEX.md status cells from the task files
|
|
# ./loop.sh --cost print the per-task token/cost ledger
|
|
# ./loop.sh T0.2 T0.3 run only these, in the order given
|
|
#
|
|
# Every generated file lands in tasks/artifacts/<TaskId>/:
|
|
# review.md coder-report.md cost.json coder.jsonl reviewer.jsonl *.stderr
|
|
# plus a board-wide tasks/artifacts/cost-ledger.tsv.
|
|
#
|
|
# Resumable: state lives in the task files, not here. Rerun after a crash.
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
TASKS="$ROOT/tasks"
|
|
INDEX="$TASKS/INDEX.md"
|
|
GUIDE="$TASKS/rust-guide-line.md"
|
|
CRATE="$ROOT/poimen"
|
|
AGENTS="$ROOT/.pi/agents"
|
|
|
|
# Every generated artifact lands under tasks/artifacts/<TaskId>/ — reviews,
|
|
# coder reports, run logs, and anything an agent is told to write. Nothing
|
|
# generated belongs anywhere else in the tree.
|
|
ARTIFACTS="$TASKS/artifacts"
|
|
|
|
# The golden-rule region of INDEX.md: board rules + progress tables.
|
|
GOLDEN_RULE_LINES="1-76"
|
|
|
|
CODER_TOOLS="read,write,edit,bash,grep,find,ls,hashline_edit"
|
|
REVIEWER_TOOLS="read,grep,find,ls,bash"
|
|
|
|
# The task files are fully specified, so the coder is mostly transcription —
|
|
# a cheap model is enough. The reviewer is the only thing between a false pass
|
|
# and a Done, so it gets the stronger model. Override per run:
|
|
# CODER_MODEL=... REVIEWER_MODEL=... ./loop.sh T0.3
|
|
CODER_MODEL="${CODER_MODEL:-claude-haiku-4-5}"
|
|
REVIEWER_MODEL="${REVIEWER_MODEL:-claude-sonnet-4-5}"
|
|
|
|
task_dir() { echo "$ARTIFACTS/$1"; }
|
|
review_path() { echo "$ARTIFACTS/$1/review.md"; }
|
|
|
|
# ---------------------------------------------------------------- board state
|
|
|
|
# Status cells carry emoji ("✅ Done", "⬜ Not started"), so match on substring.
|
|
status_of() { sed -n 's/^| *Status *| *\(.*[^ ]\) *|$/\1/p' "$1" | head -1; }
|
|
is_done() { case "$(status_of "$1")" in *Done*) return 0 ;; *) return 1 ;; esac; }
|
|
|
|
status_emoji() {
|
|
case "$(status_of "$1")" in
|
|
*Done*) printf '✅' ;;
|
|
*rogress*) printf '🟡' ;;
|
|
*lock*|*BLOCK*) printf '⛔' ;;
|
|
*) printf '⬜' ;;
|
|
esac
|
|
}
|
|
|
|
verdict_of() {
|
|
[ -f "$1" ] || { echo "no-review"; return; }
|
|
sed -n 's/^| *Verdict *| *\(.*[^ ]\) *|$/\1/p' "$1" | head -1
|
|
}
|
|
|
|
task_id() { basename "$1" | cut -d- -f1; }
|
|
phase_of() { echo "${1#T}" | cut -d. -f1; }
|
|
|
|
# Phase order. Numeric sort on phase then minor puts each gate task (highest
|
|
# minor in its phase) last, so sequential execution satisfies the board's
|
|
# "no phase starts until its predecessor's gate is green" rule for free.
|
|
ordered_tasks() {
|
|
find "$TASKS" -maxdepth 1 -name 'T*.md' \
|
|
| sed "s|.*/T||" \
|
|
| sort -t. -k1,1n -k2,2n \
|
|
| sed "s|^|$TASKS/T|"
|
|
}
|
|
|
|
file_for_id() { find "$TASKS" -maxdepth 1 -name "$1-*.md" | head -1; }
|
|
|
|
# The gate task of a phase is its highest-numbered task.
|
|
gate_of_phase() {
|
|
ordered_tasks | while read -r f; do
|
|
[ "$(phase_of "$(task_id "$f")")" = "$1" ] && echo "$f"
|
|
done | tail -1
|
|
}
|
|
|
|
# No phase starts until its predecessor's gate is green.
|
|
gate_blocks() {
|
|
local phase="$1" prev gate
|
|
[ "$phase" -eq 0 ] && return 1
|
|
prev=$((phase - 1))
|
|
gate="$(gate_of_phase "$prev")"
|
|
[ -n "$gate" ] || return 1
|
|
is_done "$gate" && return 1
|
|
echo "P$phase blocked: P$prev gate $(task_id "$gate") is '$(status_of "$gate")'"
|
|
return 0
|
|
}
|
|
|
|
# ------------------------------------------------------------- index mirroring
|
|
|
|
# Rewrite INDEX.md's status cells from the task files. The task file is the
|
|
# source of truth; a status changed in one and not the other is a lie.
|
|
sync_index() {
|
|
local map tmp before after ledger
|
|
map="$(mktemp)"; tmp="$(mktemp)"
|
|
ledger="$ARTIFACTS/cost-ledger.tsv"
|
|
while read -r f; do
|
|
id="$(task_id "$f")"
|
|
printf '%s %s %s %s\n' "$id" "$(status_emoji "$f")" \
|
|
"$(awk -F'\t' -v t="$id" '$1 == t { print $7; exit }' "$ledger" 2>/dev/null || echo 0)" \
|
|
"$(awk -F'\t' -v t="$id" '$1 == t { print $8; exit }' "$ledger" 2>/dev/null || echo 0)"
|
|
done < <(ordered_tasks) > "$map"
|
|
|
|
before="$(wc -l < "$INDEX")"
|
|
|
|
awk -v mapfile="$map" '
|
|
function fmt_tok(n) { return n >= 1000 ? sprintf("%.1fk", n / 1000) : (n ? n : "—") }
|
|
function fmt_usd(n) { return n > 0 ? sprintf("$%.2f", n) : "—" }
|
|
BEGIN {
|
|
FS = "|"; OFS = "|"
|
|
while ((getline line < mapfile) > 0) {
|
|
split(line, a, " ")
|
|
st[a[1]] = a[2]
|
|
tok[a[1]] = a[3] + 0
|
|
usd[a[1]] = a[4] + 0
|
|
p = a[1]; sub(/^T/, "", p); sub(/\..*$/, "", p)
|
|
total[p]++
|
|
if (a[2] == "✅") done[p]++
|
|
else if (a[2] == "🟡") wip[p]++
|
|
else todo[p]++
|
|
gate[p] = a[1] # ordered input => last id in phase is the gate
|
|
ptok[p] += tok[a[1]]
|
|
pusd[p] += usd[a[1]]
|
|
alltok += tok[a[1]]
|
|
allusd += usd[a[1]]
|
|
alltotal++
|
|
}
|
|
}
|
|
# per-task row: | [T0.1](T0.1-foo.md) | Title | S | — | ⬜ |
|
|
$0 ~ /^\| \[T[0-9]+\.[0-9]+\]/ {
|
|
id = $2; sub(/^ *\[/, "", id); sub(/\].*$/, "", id)
|
|
if (id in st) { $6 = " " st[id] " "; print; next }
|
|
}
|
|
# phase summary row: | P0 — Foundations | 9 | 0 | 0 | 9 | ⬜ T0.9 |
|
|
$0 ~ /^\| P[0-8] / {
|
|
p = $2; sub(/^ *P/, "", p); sub(/ .*$/, "", p)
|
|
if (p in total) {
|
|
$3 = " " total[p] " "
|
|
$4 = " " (done[p] + 0) " "
|
|
$5 = " " (wip[p] + 0) " "
|
|
$6 = " " (todo[p] + 0) " "
|
|
$7 = " " st[gate[p]] " " gate[p] " "
|
|
$8 = " " fmt_tok(ptok[p]) " "
|
|
$9 = " " fmt_usd(pusd[p]) " "
|
|
print; next
|
|
}
|
|
}
|
|
# total row: | **Total** | **71** | **0** | **0** | **71** | 0/9 green |
|
|
$0 ~ /^\| \*\*Total\*\*/ {
|
|
gates = 0; green = 0; d = 0; w = 0; t = 0
|
|
for (p in total) {
|
|
gates++
|
|
if (st[gate[p]] == "✅") green++
|
|
d += done[p]; w += wip[p]; t += todo[p]
|
|
}
|
|
$3 = " **" alltotal "** "
|
|
$4 = " **" d "** "
|
|
$5 = " **" w "** "
|
|
$6 = " **" t "** "
|
|
$7 = " " green "/" gates " green "
|
|
$8 = " **" fmt_tok(alltok) "** "
|
|
$9 = " **" fmt_usd(allusd) "** "
|
|
print; next
|
|
}
|
|
{ print }
|
|
' "$INDEX" > "$tmp"
|
|
|
|
after="$(wc -l < "$tmp")"
|
|
if [ "$before" != "$after" ]; then
|
|
echo "[warn] index sync changed line count ($before -> $after); INDEX.md left alone"
|
|
rm -f "$map" "$tmp"
|
|
return 1
|
|
fi
|
|
mv "$tmp" "$INDEX"
|
|
rm -f "$map"
|
|
}
|
|
|
|
mark_done() {
|
|
local file="$1" tmp
|
|
tmp="$(mktemp)"
|
|
sed 's/^| *Status *|.*|$/| Status | ✅ Done |/' "$file" > "$tmp" && mv "$tmp" "$file"
|
|
sync_index
|
|
}
|
|
|
|
print_board() {
|
|
printf '%-7s %-16s %-18s %s\n' TASK STATUS VERDICT FILE
|
|
while read -r f; do
|
|
id="$(task_id "$f")"
|
|
printf '%-7s %-16s %-18s %s\n' \
|
|
"$id" "$(status_of "$f")" "$(verdict_of "$(review_path "$id")")" "$(basename "$f")"
|
|
done < <(ordered_tasks)
|
|
}
|
|
|
|
# ---------------------------------------------------------------------- usage
|
|
|
|
# pi --mode json emits one JSON object per line. Assistant `message_end` events
|
|
# carry the usage for that message; summing them gives the run total. Note that
|
|
# cost.total reports 0.0 under auth that has no per-token price attached — the
|
|
# token counts are still real, so treat dollars as advisory.
|
|
usage_json() {
|
|
jq -s '
|
|
[ .[] | select(.type == "message_end") | .message
|
|
| select(.role == "assistant") | .usage ]
|
|
| { turns: length,
|
|
input: (map(.input) | add // 0),
|
|
output: (map(.output) | add // 0),
|
|
cacheRead: (map(.cacheRead) | add // 0),
|
|
cacheWrite: (map(.cacheWrite) | add // 0),
|
|
tokens: (map(.totalTokens)| add // 0),
|
|
costUsd: (map(.cost.total) | add // 0) }
|
|
' "$1" 2>/dev/null || echo '{}'
|
|
}
|
|
|
|
# Assistant prose only — tool calls and thinking blocks are dropped.
|
|
extract_text() {
|
|
jq -rs '
|
|
[ .[] | select(.type == "message_end") | .message
|
|
| select(.role == "assistant") | .content[]?
|
|
| select(.type == "text") | .text ]
|
|
| join("\n\n")
|
|
' "$1" 2>/dev/null
|
|
}
|
|
|
|
write_cost() {
|
|
local id="$1" dir="$2" cu="$3" ru="$4"
|
|
jq -n --arg id "$id" --arg cm "$CODER_MODEL" --arg rm "$REVIEWER_MODEL" \
|
|
--argjson coder "$cu" --argjson reviewer "$ru" '
|
|
{ task: $id,
|
|
coder: ($coder + { model: $cm }),
|
|
reviewer: ($reviewer + { model: $rm }),
|
|
total: { tokens: ($coder.tokens + $reviewer.tokens),
|
|
costUsd: ($coder.costUsd + $reviewer.costUsd) } }
|
|
' > "$dir/cost.json"
|
|
|
|
local ledger="$ARTIFACTS/cost-ledger.tsv" header tmp
|
|
header="$(printf 'task\tattempts\tcoder_model\tcoder_tokens\treviewer_model\treviewer_tokens\ttotal_tokens\tcost_usd\tfinished')"
|
|
|
|
if [ -f "$ledger" ] && [ "$(head -1 "$ledger")" != "$header" ]; then
|
|
mv "$ledger" "$ledger.$(date -u +%Y%m%dT%H%M%SZ).bak"
|
|
fi
|
|
[ -f "$ledger" ] || printf '%s\n' "$header" > "$ledger"
|
|
|
|
local attempts=1
|
|
if grep -q "^$id " "$ledger"; then
|
|
attempts=$(( $(awk -F'\t' -v t="$id" '$1 == t { print $2; exit }' "$ledger") + 1 ))
|
|
tmp="$(mktemp)"
|
|
awk -F'\t' -v t="$id" '$1 != t' "$ledger" > "$tmp" && mv "$tmp" "$ledger"
|
|
fi
|
|
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$id" "$attempts" \
|
|
"$CODER_MODEL" "$(echo "$cu" | jq -r '.tokens')" \
|
|
"$REVIEWER_MODEL" "$(echo "$ru" | jq -r '.tokens')" \
|
|
"$(jq -r '.total.tokens' "$dir/cost.json")" \
|
|
"$(jq -r '.total.costUsd' "$dir/cost.json")" \
|
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$ledger"
|
|
}
|
|
|
|
# ------------------------------------------------------------------- prompts
|
|
|
|
coder_prompt() {
|
|
local id="$1" file="$2"
|
|
cat <<PROMPT
|
|
Implement $id.
|
|
|
|
Read first, in order: $INDEX lines $GOLDEN_RULE_LINES (golden rule, outranks
|
|
this prompt — includes the caveman output rule), then $file, then $GUIDE.
|
|
|
|
Crate root: $CRATE (Cargo workspace; add new crates to members).
|
|
|
|
Do:
|
|
1. Tests named in Verify BEFORE implementation. Watch them fail for the right
|
|
reason — missing type, not missing test file.
|
|
2. Minimum code satisfying Acceptance. Steps exactly.
|
|
3. Run the Verify Command line.
|
|
|
|
Never: edit $file or $INDEX (driver owns Status); create COMPLETED/VERIFICATION/
|
|
summary files or verify/ scripts. Writes go to source, tests, and Cargo manifests
|
|
under $CRATE only. A Step demanding a generated artifact puts it in $ARTIFACTS/$id/.
|
|
|
|
RULE 0 caveman full applies to this run. Every turn.
|
|
|
|
Report:
|
|
- files changed, one line each
|
|
- Verify Command run, verbatim
|
|
- its output and exit status, verbatim
|
|
- Steps not done, why
|
|
PROMPT
|
|
}
|
|
|
|
reviewer_prompt() {
|
|
local id="$1" file="$2" report="$3"
|
|
cat <<PROMPT
|
|
Review $id. Output ONLY the markdown review document.
|
|
|
|
Board rules: $INDEX lines $GOLDEN_RULE_LINES. Task: $file. Crate: $CRATE.
|
|
Quality bar: $GUIDE.
|
|
|
|
Re-run the Verify command from $ROOT yourself — the report below is a claim, not
|
|
evidence. Audit every False pass and Trap item in the task file.
|
|
|
|
Write no files. Flag any COMPLETED/VERIFICATION/summary file or verify/ script
|
|
the coder left outside $CRATE source and tests as a MINOR finding with its path.
|
|
|
|
RULE 0 caveman full applies to every prose cell. Every turn.
|
|
|
|
--- CODER REPORT ---
|
|
$(head -c 12000 "$report")
|
|
--- END ---
|
|
PROMPT
|
|
}
|
|
|
|
# --------------------------------------------------------------------- runner
|
|
|
|
# A provider/auth failure exits non-zero and produces no assistant text. Bail
|
|
# before spending the next call on a report that does not exist.
|
|
pass_failed() {
|
|
local what="$1" rc="$2" out="$3" err="$4"
|
|
[ "$rc" -eq 0 ] && [ -s "$out" ] && return 1
|
|
echo "[stop] $what pass failed (exit $rc)"
|
|
sed -n '1,3p' "$err" | sed 's/^/ /'
|
|
return 0
|
|
}
|
|
|
|
run_task() {
|
|
local file="$1" id dir review report tmp block vd cu ru rc
|
|
id="$(task_id "$file")"
|
|
dir="$(task_dir "$id")"
|
|
review="$(review_path "$id")"
|
|
report="$dir/coder-report.md"
|
|
|
|
if is_done "$file"; then
|
|
echo "[skip] $id already Done"
|
|
return 0
|
|
fi
|
|
|
|
if block="$(gate_blocks "$(phase_of "$id")")"; then
|
|
echo "[stop] $block"
|
|
return 1
|
|
fi
|
|
|
|
mkdir -p "$dir" "$ARTIFACTS"
|
|
|
|
echo "[code] $id $(date '+%H:%M:%S') $CODER_MODEL -> $dir/"
|
|
( cd "$ROOT" && pi --tools "$CODER_TOOLS" --mode json --model "$CODER_MODEL" \
|
|
--append-system-prompt "$AGENTS/coder.md" \
|
|
--no-session -p "$(coder_prompt "$id" "$file")" ) \
|
|
> "$dir/coder.jsonl" 2> "$dir/coder.stderr"
|
|
rc=$?
|
|
extract_text "$dir/coder.jsonl" > "$report"
|
|
pass_failed coder "$rc" "$report" "$dir/coder.stderr" && return 1
|
|
cu="$(usage_json "$dir/coder.jsonl")"
|
|
echo " coder: $(echo "$cu" | jq -r '"\(.tokens) tokens over \(.turns) turns"')"
|
|
|
|
echo "[revw] $id $(date '+%H:%M:%S') $REVIEWER_MODEL"
|
|
( cd "$ROOT" && pi --tools "$REVIEWER_TOOLS" --mode json --model "$REVIEWER_MODEL" \
|
|
--append-system-prompt "$AGENTS/reviewer.md" \
|
|
--no-session -p "$(reviewer_prompt "$id" "$file" "$report")" ) \
|
|
> "$dir/reviewer.jsonl" 2> "$dir/reviewer.stderr"
|
|
rc=$?
|
|
|
|
# Strip an outer ``` fence if the reviewer wrapped the whole document.
|
|
tmp="$(mktemp)"
|
|
extract_text "$dir/reviewer.jsonl" \
|
|
| sed -e '1{/^```/d;}' -e '${/^```$/d;}' > "$tmp" && mv "$tmp" "$review" && chmod 644 "$review"
|
|
pass_failed reviewer "$rc" "$review" "$dir/reviewer.stderr" && return 1
|
|
ru="$(usage_json "$dir/reviewer.jsonl")"
|
|
echo " reviewer: $(echo "$ru" | jq -r '"\(.tokens) tokens over \(.turns) turns"')"
|
|
|
|
write_cost "$id" "$dir" "$cu" "$ru"
|
|
|
|
vd="$(verdict_of "$review")"
|
|
case "$vd" in
|
|
*APPROVED*)
|
|
mark_done "$file"
|
|
echo "[ ok ] $id verdict=APPROVED status=$(status_of "$file") cost=$(jq -r '"\(.total.tokens) tokens / $\(.total.costUsd)"' "$dir/cost.json")"
|
|
return 0
|
|
;;
|
|
*)
|
|
echo "[stop] $id verdict=${vd:-<none parsed>}"
|
|
echo " artifacts: $dir/"
|
|
echo " review: $review"
|
|
echo " cost: $(jq -r '"\(.total.tokens) tokens / $\(.total.costUsd)"' "$dir/cost.json")"
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# ----------------------------------------------------------------------- main
|
|
|
|
case "${1:-}" in
|
|
--list) print_board; exit 0 ;;
|
|
--sync) sync_index && echo "INDEX.md synced from task files"; exit $? ;;
|
|
--cost) [ -f "$ARTIFACTS/cost-ledger.tsv" ] \
|
|
&& { column -t -s "$(printf '\t')" "$ARTIFACTS/cost-ledger.tsv"
|
|
awk -F'\t' 'NR>1 {t+=$7; c+=$8; a+=$2} END {printf "\nTOTAL %d tokens $%.4f over %d tasks (%d attempts)\n", t, c, NR-1, a}' \
|
|
"$ARTIFACTS/cost-ledger.tsv"; } \
|
|
|| echo "no cost ledger yet"
|
|
exit 0 ;;
|
|
--dry-run) ordered_tasks | while read -r f; do
|
|
is_done "$f" || echo "would run $(task_id "$f")"
|
|
done; exit 0 ;;
|
|
esac
|
|
|
|
command -v pi >/dev/null || { echo "pi not on PATH"; exit 127; }
|
|
command -v jq >/dev/null || { echo "jq not on PATH (needed to parse --mode json)"; exit 127; }
|
|
|
|
# The TypeScript pi silently ignores unknown tools and lacks the Rust flags, so a
|
|
# run against it degrades instead of failing. --list-providers exists only on the
|
|
# Rust port; use it as the discriminator.
|
|
if ! pi --list-providers >/dev/null 2>&1; then
|
|
echo "pi at $(command -v pi) is not pi_agent_rust (reports: $(pi --version 2>&1 | head -1))"
|
|
echo "install the Rust port with install.sh --adopt, or point PATH at it"
|
|
exit 1
|
|
fi
|
|
|
|
[ -f "$AGENTS/coder.md" ] && [ -f "$AGENTS/reviewer.md" ] || {
|
|
echo "missing coder.md / reviewer.md in $AGENTS"; exit 1; }
|
|
|
|
if [ $# -gt 0 ]; then
|
|
for id in "$@"; do
|
|
f="$(file_for_id "$id")"
|
|
[ -n "$f" ] || { echo "no task file for $id"; exit 1; }
|
|
run_task "$f" || exit 1
|
|
done
|
|
echo "[done] requested tasks complete"
|
|
exit 0
|
|
fi
|
|
|
|
while read -r f; do
|
|
run_task "$f" || exit 1
|
|
done < <(ordered_tasks)
|
|
|
|
echo "[done] board complete"
|