From 5a30d0ffc6a28be0780af5e411e695f0e76c48a7 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Mon, 17 Aug 2026 23:05:20 -0700 Subject: [PATCH] (tasks) add tasks for harness --- .gitignore | 2 +- tasks/INDEX.md | 363 ++++++++++++++++++ tasks/T0.1-identity-newtypes.md | 104 +++++ tasks/T0.2-kernel-attemptstate.md | 113 ++++++ tasks/T0.3-workevent-and-schemaversion.md | 104 +++++ tasks/T0.4-upcaster-framework.md | 86 +++++ .../T0.5-eventlog-port-redb-implementation.md | 143 +++++++ tasks/T0.6-atomic-commit-protocol.md | 104 +++++ ...T0.7-blobstore-port-redb-implementation.md | 97 +++++ tasks/T0.8-fold-and-re-derive.md | 92 +++++ tasks/T0.9-p0-composition-gate.md | 109 ++++++ tasks/T1.1-ctx-and-runscope.md | 102 +++++ tasks/T1.2-stub-model-provider.md | 77 ++++ tasks/T1.3-run-executor.md | 129 +++++++ tasks/T1.4-attempt-lifecycle-and-retry.md | 84 ++++ tasks/T1.5-context-partition-capture.md | 83 ++++ tasks/T1.6-prompt-and-output-blob-capture.md | 82 ++++ tasks/T1.7-episode-query-surface.md | 103 +++++ tasks/T1.8-p1-composition-gate.md | 103 +++++ tasks/T2.1-write-ahead-intent.md | 123 ++++++ tasks/T2.2-effect-class-recovery.md | 127 ++++++ tasks/T2.3-rewind-as-fork.md | 104 +++++ tasks/T2.4-schema-evolution-end-to-end.md | 88 +++++ tasks/T2.5-checkpoints.md | 91 +++++ tasks/T2.6-crash-matrix.md | 108 ++++++ tasks/T2.7-p2-composition-gate.md | 109 ++++++ tasks/T3.1-workflowdef-ir-canonicalization.md | 125 ++++++ ...T3.2-workflowformat-trait-yaml-and-json.md | 95 +++++ tasks/T3.3-load-time-validation.md | 101 +++++ tasks/T3.4-stepid-stability-checks.md | 95 +++++ tasks/T3.5-interpreter-over-the-ir.md | 123 ++++++ tasks/T3.6-version-pinning-at-spawn.md | 96 +++++ tasks/T3.7-p3-composition-gate.md | 105 +++++ tasks/T4.1-verifier-port.md | 103 +++++ ....2-verifierctx-and-the-snapshot-barrier.md | 119 ++++++ tasks/T4.3-lazy-blob-access.md | 85 ++++ tasks/T4.4-retention-ordering-guard.md | 91 +++++ tasks/T4.5-p4-composition-gate.md | 108 ++++++ tasks/T5.1-taskid-at-spawn.md | 90 +++++ tasks/T5.10-attempt-tournaments.md | 113 ++++++ tasks/T5.11-degradation-reasons.md | 138 +++++++ tasks/T5.12-p5-composition-gate.md | 120 ++++++ ...acitylimits-and-the-residency-invariant.md | 152 ++++++++ ...evaluationstrategy-port-resourceprofile.md | 158 ++++++++ tasks/T5.4-deterministicgrader.md | 88 +++++ ...wisesequential-reference-and-comparison.md | 117 ++++++ tasks/T5.6-sequential-test-and-stopping.md | 120 ++++++ ...der-alternation-and-sampled-consistency.md | 98 +++++ tasks/T5.8-swiss-pairing.md | 109 ++++++ tasks/T5.9-bradley-terry-fit.md | 132 +++++++ tasks/T6.1-challenger-allocation.md | 103 +++++ tasks/T6.2-judge-calibration-set.md | 104 +++++ tasks/T6.3-promotion-gates.md | 134 +++++++ tasks/T6.4-per-variant-aggregation.md | 105 +++++ tasks/T6.5-sandboxed-replay.md | 103 +++++ tasks/T6.6-held-out-split.md | 105 +++++ tasks/T6.7-drift-check.md | 96 +++++ tasks/T6.8-p6-composition-gate.md | 113 ++++++ tasks/T7.1-postgres-eventlog.md | 103 +++++ tasks/T7.2-object-store-blobstore.md | 91 +++++ tasks/T7.3-leases-and-fencing.md | 101 +++++ tasks/T7.4-outbox-relay.md | 98 +++++ tasks/T7.5-partition-keys-on-adapters.md | 86 +++++ tasks/T7.6-tournament-as-a-join-stage.md | 112 ++++++ tasks/T7.7-turmoil-suite.md | 109 ++++++ tasks/T7.8-p7-composition-gate.md | 111 ++++++ tasks/T8.1-metering.md | 105 +++++ tasks/T8.2-metrics.md | 118 ++++++ tasks/T8.3-keyed-capability.md | 114 ++++++ tasks/T8.4-reduction-and-tiering.md | 142 +++++++ tasks/T8.5-embedded-mode-smoke.md | 94 +++++ tasks/T8.6-capacity-admission-control.md | 113 ++++++ tasks/T8.7-p8-composition-gate.md | 116 ++++++ tasks/rust-guide-line.md | 218 +++++++++++ 74 files changed, 8204 insertions(+), 1 deletion(-) create mode 100644 tasks/INDEX.md create mode 100644 tasks/T0.1-identity-newtypes.md create mode 100644 tasks/T0.2-kernel-attemptstate.md create mode 100644 tasks/T0.3-workevent-and-schemaversion.md create mode 100644 tasks/T0.4-upcaster-framework.md create mode 100644 tasks/T0.5-eventlog-port-redb-implementation.md create mode 100644 tasks/T0.6-atomic-commit-protocol.md create mode 100644 tasks/T0.7-blobstore-port-redb-implementation.md create mode 100644 tasks/T0.8-fold-and-re-derive.md create mode 100644 tasks/T0.9-p0-composition-gate.md create mode 100644 tasks/T1.1-ctx-and-runscope.md create mode 100644 tasks/T1.2-stub-model-provider.md create mode 100644 tasks/T1.3-run-executor.md create mode 100644 tasks/T1.4-attempt-lifecycle-and-retry.md create mode 100644 tasks/T1.5-context-partition-capture.md create mode 100644 tasks/T1.6-prompt-and-output-blob-capture.md create mode 100644 tasks/T1.7-episode-query-surface.md create mode 100644 tasks/T1.8-p1-composition-gate.md create mode 100644 tasks/T2.1-write-ahead-intent.md create mode 100644 tasks/T2.2-effect-class-recovery.md create mode 100644 tasks/T2.3-rewind-as-fork.md create mode 100644 tasks/T2.4-schema-evolution-end-to-end.md create mode 100644 tasks/T2.5-checkpoints.md create mode 100644 tasks/T2.6-crash-matrix.md create mode 100644 tasks/T2.7-p2-composition-gate.md create mode 100644 tasks/T3.1-workflowdef-ir-canonicalization.md create mode 100644 tasks/T3.2-workflowformat-trait-yaml-and-json.md create mode 100644 tasks/T3.3-load-time-validation.md create mode 100644 tasks/T3.4-stepid-stability-checks.md create mode 100644 tasks/T3.5-interpreter-over-the-ir.md create mode 100644 tasks/T3.6-version-pinning-at-spawn.md create mode 100644 tasks/T3.7-p3-composition-gate.md create mode 100644 tasks/T4.1-verifier-port.md create mode 100644 tasks/T4.2-verifierctx-and-the-snapshot-barrier.md create mode 100644 tasks/T4.3-lazy-blob-access.md create mode 100644 tasks/T4.4-retention-ordering-guard.md create mode 100644 tasks/T4.5-p4-composition-gate.md create mode 100644 tasks/T5.1-taskid-at-spawn.md create mode 100644 tasks/T5.10-attempt-tournaments.md create mode 100644 tasks/T5.11-degradation-reasons.md create mode 100644 tasks/T5.12-p5-composition-gate.md create mode 100644 tasks/T5.2-capacitylimits-and-the-residency-invariant.md create mode 100644 tasks/T5.3-evaluationstrategy-port-resourceprofile.md create mode 100644 tasks/T5.4-deterministicgrader.md create mode 100644 tasks/T5.5-pairwisesequential-reference-and-comparison.md create mode 100644 tasks/T5.6-sequential-test-and-stopping.md create mode 100644 tasks/T5.7-order-alternation-and-sampled-consistency.md create mode 100644 tasks/T5.8-swiss-pairing.md create mode 100644 tasks/T5.9-bradley-terry-fit.md create mode 100644 tasks/T6.1-challenger-allocation.md create mode 100644 tasks/T6.2-judge-calibration-set.md create mode 100644 tasks/T6.3-promotion-gates.md create mode 100644 tasks/T6.4-per-variant-aggregation.md create mode 100644 tasks/T6.5-sandboxed-replay.md create mode 100644 tasks/T6.6-held-out-split.md create mode 100644 tasks/T6.7-drift-check.md create mode 100644 tasks/T6.8-p6-composition-gate.md create mode 100644 tasks/T7.1-postgres-eventlog.md create mode 100644 tasks/T7.2-object-store-blobstore.md create mode 100644 tasks/T7.3-leases-and-fencing.md create mode 100644 tasks/T7.4-outbox-relay.md create mode 100644 tasks/T7.5-partition-keys-on-adapters.md create mode 100644 tasks/T7.6-tournament-as-a-join-stage.md create mode 100644 tasks/T7.7-turmoil-suite.md create mode 100644 tasks/T7.8-p7-composition-gate.md create mode 100644 tasks/T8.1-metering.md create mode 100644 tasks/T8.2-metrics.md create mode 100644 tasks/T8.3-keyed-capability.md create mode 100644 tasks/T8.4-reduction-and-tiering.md create mode 100644 tasks/T8.5-embedded-mode-smoke.md create mode 100644 tasks/T8.6-capacity-admission-control.md create mode 100644 tasks/T8.7-p8-composition-gate.md create mode 100644 tasks/rust-guide-line.md diff --git a/.gitignore b/.gitignore index f6a3a6d..d8c8878 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .claude .pi -tasks target rust-agentic-task.md +tasks/artifacts/* *.stderr diff --git a/tasks/INDEX.md b/tasks/INDEX.md new file mode 100644 index 0000000..da7962a --- /dev/null +++ b/tasks/INDEX.md @@ -0,0 +1,363 @@ +# poimen — task board + +80 tasks — 70 build tasks plus **10 composition gates**, one per phase. One file +per task, **self-contained**: inlined design facts, executable +steps, acceptance criteria, a `Verify` section written for someone who did not +build the thing, and the traps worth naming. Reading `rust-agentic-sys.md` is not +required to do a task — it is linked as background only. + +Each `Verify` section names the harness, the integration test with numbered +assertions, the command to run, and the **false pass** — the shape of test that +goes green while the feature is broken. Treat the false-pass list as part of the +acceptance criteria, not commentary. + +poimen is a **pluggable unit**. It is embedded by other harnesses, not run as an +application. Two things follow, and they are the spine of this board: the public +surface is built early and frozen deliberately, and every phase after it is +judged partly on whether it broke that surface. + +## The customization contract — two front doors, one IR + +poimen is customized two ways, and they are different axes rather than +alternatives. Confusing them is how a project ends up with two products. + +``` + workflow.yaml ─────────┐ + ├──► WorkflowDef ──► Blake3 canonical hash ──► WorkflowVersion + WorkflowDef::builder() ┘ ▲ + │ referenced by id + impl Verifier / Judge / ModelProvider / storage ports +``` + +**YAML declares. Rust implements.** + +- **YAML** names steps, transitions, verifier ids, judge ids, retry policy, + budgets, rubrics. It is data. It ships without a compiler and is the surface a + user touches to change *what runs*. +- **Rust** supplies behaviour behind the ports and registers it under an id. + It is code. It is the surface a user touches to change *how something works*. + +Neither is a superset. A YAML file can only name capabilities some Rust impl +registered; a Rust impl is inert until some workflow names it. + +Three rules the board enforces, each owned by a task: + +1. **Both front doors canonicalize to the same IR.** The same workflow written + as YAML and built with `WorkflowDef::builder()` produces a byte-identical + `WorkflowVersion`. §4.1 hashes the canonicalized IR, not the source text, so + this is testable — and if it ever fails, §12's "did this change affect + results" silently returns nonsense. T3.10 owns it and the P-surface gate + re-checks it. +2. **Unresolved ids fail at load, never at spawn.** YAML naming a `VerifierId`, + `JudgeId` or `ModelId` no impl registered is a load error naming the id and + the file. A workflow that parses and then dies mid-run on a missing verifier + has moved a config error into production. T3.9 owns the registry; T3.3 owns + the failure. +3. **Neither door reaches past the ports.** A YAML key that only one storage + backend understands, or a builder method that assumes `redb`, breaks + embedding. The gates check both doors against both storage modes. + +## The embedding contract — sidecar, JSON-RPC over stdio + +poimen is embedded by harnesses written in other languages; DeepSeek Harness +(`dsh`, TypeScript) is the reference consumer. The boundary is a **child process +speaking JSON-RPC 2.0 over stdio**, not a native addon. + +``` + ┌──────────────┐ spawn ┌───────────────────┐ + │ host harness │──────────► │ poimen serve │ + │ any language │ ◄─stdio──► │ own process │ + └──────────────┘ JSON-RPC └───────────────────┘ +``` + +Chosen because it containerizes with no change, hosts in any language, and keeps +a Rust panic out of the host's crash domain. The cost is a serialization +boundary and a protocol that must be versioned like any other wire format — so +it is versioned like one, with the same discipline §8.7 applies to the log: +methods are added, never repurposed; fields are added, never removed. + +**The protocol is a published artifact, not an implementation detail.** A change +to it is a breaking change to every embedder, and the conformance suite (T9.4) +is what makes that statement enforceable rather than aspirational. + +An in-process Rust API exists for Rust embedders and is the same surface; the +sidecar is that surface with a wire format in front of it. An HTTP transport +over the identical method set is the natural pod deployment and is deferred, not +designed away. + +## Ordering — declared, never derived + +**Phase order is the list below. Task ids are opaque and frozen.** + +The board reorders as understanding changes; task ids do not move when it does. +`T3.1` is `T3.1` forever, in whatever phase it currently sits, because its +artifacts, its cost-ledger rows and every cross-reference key on that id. New +tasks take new ids rather than renumbering their neighbours. + +This is §4.3's `StepId` rule applied to the board itself: *"insert a step at +position 2 and every positional index shifts, but `StepId` does not… never +parsed, never ordered, never assumed numeric."* A board that renumbers to +reorder has the bug it warns its own users about. + +Consequence: **id order is not execution order.** Read the phase list, not the +filenames. + +Ordering rule: no phase starts until its predecessor's gate is green. The `gate` +task of each phase **is** that gate — a required CI job proving the phase's tasks +compose and that its swappable parts are genuinely swappable. Every build task is +verified alone; the gate verifies the properties no single task owns. `opt-in` +tasks ship disabled and gate nothing. + +- **Engineering Quality Rule:** All implementations must strictly adhere to the + idiomatic, zero-copy, and type-safe architecture standards defined in + `rust-guide-line.md`. + +- **Agent Output Rule — caveman full.** Every agent on this board writes caveman + full. Applies to prose only. + + Drop: articles (a/an/the), filler (just/really/basically/actually/simply), + pleasantries (sure/certainly/happy to), hedging (might be worth/you could + consider), connectives (however/furthermore/additionally), preamble, recap, + tool-call narration, restating these rules back. Fragments OK. Short synonyms + — big not extensive, fix not "implement a solution for". + + Preserve exactly, never abbreviate: code blocks, inline code, file paths, + commands, error strings, test names, crate and API names, version numbers, + env vars, URLs. Never invent abbreviations (cfg/impl/req/fn) — they tokenize + the same as the full word and read worse. + + Drop caveman where compression creates ambiguity: multi-step ordering, + destructive-operation warnings, anything a misread would break. + + Pattern: `[thing] [action] [reason]. [next step].` + Not: "I'll go ahead and implement the transition function for you, which..." + Yes: "transition() exhaustive match, no catch-all. Terminal states reject all." + +## Verification practice — script first, source second + +**A task is verified by running a script and diffing its output. Reading the +source happens after that diff is clean, never instead of it.** Reviewing a diff +first is how an assertion that was quietly dropped still gets called done: the +code looks right, and nothing proves the test ran. + +Every task already carries the four things this needs — a `Harness`, an +`Integration test` with numbered assertions, a `Command`, and a `False pass` +list. The practice is to make them executable rather than descriptive. + +**1. One script per task, committed with it.** + +``` +verify/.sh # runs the task's Command, prints one line per assertion +verify/expected/.txt # the exact output that script must produce +``` + +**2. Numbered assertion N in the Verify section is test fn `aN_`.** The +numbering is the contract. `tests/it_scoped_keys.rs` assertion 4 is +`a4_scan_the_raw_table`, and the script reports it by name: + +``` +it_scoped_keys::a1_open_one_table_keyed_scoped_runid PASS +it_scoped_keys::a4_scan_the_raw_table MISSING +``` + +`MISSING` is the point. A test fn that does not exist reports missing instead of +being absent from a green summary — the failure mode where `cargo test` says +`ok` because the assertion was never written. + +**3. The expected file is the review artifact.** The script's output is diffed +against it. An empty diff is the only pass. A changed expected file in a diff is +a claim that the task's Verify section changed, and gets read as one. + +**4. The `False pass` list is the checklist applied *after* the diff is clean,** +not before. Every line saying `PASS` is exactly the state those traps are written +to survive — that is when a reviewer opens the source, and the false-pass list is +what they open it with. + +**5. Gate tasks follow the same rule, at phase scope.** The gate's script covers +the properties no single task owns; a phase is green when its gate script's diff +is empty, not when its member tasks individually passed. + +**6. No crate, no verification.** Until a `Cargo.toml` exists the scripts report +blocked rather than passing vacuously. A verification step that cannot fail is +not a verification step. + +**7. Surface tasks are verified through both front doors.** Any task in +P-surface or later that touches workflow definition asserts the YAML path and +the builder path independently, then asserts their hashes match. One path +verified is half a feature. + +## Progress + +**Source of truth is the `Status` field in each task file.** The tables below +mirror it; a status changed here and not there is a lie. Regenerate the mirror: + +```sh +for f in T*.md; do + printf '%s\t%s\n' "${f%%-*}" "$(sed -n 's/^| Status | *\(.*[^ ]\) *|$/\1/p' "$f" | head -1)" +done | sort -t. -k1,1 -k2,2n +``` + +Legend: ⬜ not started · 🟡 in progress · ✅ done · ⛔ blocked + +| # | Phase | Ids | Tasks | ✅ | 🟡 | ⬜ | Gate | Tokens | Cost | +|---|---|---|---|---|---|---|---|---|---| +| 1 | Foundations | T0.x | 9 | 3 | 0 | 6 | ⬜ T0.9 | 313.0k | — | +| 2 | Walking skeleton | T1.x | 8 | 0 | 0 | 8 | ⬜ T1.8 | — | — | +| 3 | Public surface | T3.x | 10 | 0 | 0 | 10 | ⬜ T3.7 | — | — | +| 4 | Verification | T4.x | 5 | 0 | 0 | 5 | ⬜ T4.5 | — | — | +| 5 | Embedding | T9.x | 6 | 0 | 0 | 6 | ⬜ T9.6 | — | — | +| 6 | Durability hard parts | T2.x | 7 | 0 | 0 | 7 | ⬜ T2.7 | — | — | +| 7 | Grading | T5.x | 12 | 0 | 0 | 12 | ⬜ T5.12 | — | — | +| 8 | Learning loop | T6.x | 8 | 0 | 0 | 8 | ⬜ T6.8 | — | — | +| 9 | Distribution | T7.x | 8 | 0 | 0 | 8 | ⬜ T7.8 | — | — | +| 10 | Operability | T8.x | 7 | 0 | 0 | 7 | ⬜ T8.7 | — | — | +| | **Total** | | **80** | **3** | **0** | **77** | 0/10 green | **313.0k** | **—** | + +**What changed in this revision.** The public surface moved ahead of the +durability hard parts, and an embedding phase was added after verification. +Reason: pluggable-first. Under the previous order nobody could write a plugin or +embed poimen until 36 tasks in, which meant the surface others depend on would +have been designed with no user and validated with none either. + +The cost that buys: the log format can still churn under a published surface. +That is what T0.4's upcasters are insurance for — §8.7, versioned records, +upcasters on read, variants never removed — and it is why the durability phase +sits immediately after embedding rather than last. + +## 1 — Foundations · T0.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T0.1](T0.1-identity-newtypes.md) | Identity newtypes | S | — | ✅ | +| [T0.2](T0.2-kernel-attemptstate.md) | Kernel `AttemptState` | S | — | ✅ | +| [T0.3](T0.3-workevent-and-schemaversion.md) | `WorkEvent` and `SchemaVersion` | M | — | ✅ | +| [T0.4](T0.4-upcaster-framework.md) | Upcaster framework | M | — | ⬜ | +| [T0.5](T0.5-eventlog-port-redb-implementation.md) | `EventLog` port + `redb` implementation | L | — | ⬜ | +| [T0.6](T0.6-atomic-commit-protocol.md) | Atomic commit protocol | M | — | ⬜ | +| [T0.7](T0.7-blobstore-port-redb-implementation.md) | `BlobStore` port + `redb` implementation | M | — | ⬜ | +| [T0.8](T0.8-fold-and-re-derive.md) | Fold and re-derive | M | — | ⬜ | +| [T0.9](T0.9-p0-composition-gate.md) | **Foundations composition gate** | M | gate | ⬜ | + +## 2 — Walking skeleton · T1.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T1.1](T1.1-ctx-and-runscope.md) | `Ctx` and `RunScope` | M | — | ⬜ | +| [T1.2](T1.2-stub-model-provider.md) | Stub model provider | S | — | ⬜ | +| [T1.3](T1.3-run-executor.md) | Run executor | L | — | ⬜ | +| [T1.4](T1.4-attempt-lifecycle-and-retry.md) | Attempt lifecycle and retry | M | — | ⬜ | +| [T1.5](T1.5-context-partition-capture.md) | Context partition capture | S | — | ⬜ | +| [T1.6](T1.6-prompt-and-output-blob-capture.md) | Prompt and output blob capture | S | — | ⬜ | +| [T1.7](T1.7-episode-query-surface.md) | Episode query surface | M | — | ⬜ | +| [T1.8](T1.8-p1-composition-gate.md) | **Skeleton composition gate** | M | gate | ⬜ | + +## 3 — Public surface · T3.x + +The customization contract, built once and then defended by every later gate. +Execution order within the phase is the row order below — note it is not id +order, because T3.8–T3.10 were added after the originals and the gate stays last. + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T3.1](T3.1-workflowdef-ir-canonicalization.md) | `WorkflowDef` IR + canonicalization | L | — | ⬜ | +| [T3.2](T3.2-workflowformat-trait-yaml-and-json.md) | `WorkflowFormat` trait + YAML and JSON | M | — | ⬜ | +| [T3.8](T3.8-workflowdef-builder.md) | `WorkflowDef` builder — the Rust front door | M | new | ⬜ | +| [T3.9](T3.9-capability-registry.md) | Capability registry — ids YAML can name | M | new | ⬜ | +| [T3.3](T3.3-load-time-validation.md) | Load-time validation | M | — | ⬜ | +| [T3.4](T3.4-stepid-stability-checks.md) | `StepId` stability checks | S | — | ⬜ | +| [T3.5](T3.5-interpreter-over-the-ir.md) | Interpreter over the IR | L | — | ⬜ | +| [T3.6](T3.6-version-pinning-at-spawn.md) | Version pinning at spawn | S | — | ⬜ | +| [T3.10](T3.10-front-door-equivalence.md) | Front-door equivalence — YAML ≡ builder | M | new | ⬜ | +| [T3.7](T3.7-p3-composition-gate.md) | **Surface composition gate** | M | gate | ⬜ | + +## 4 — Verification · T4.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T4.1](T4.1-verifier-port.md) | `Verifier` port | S | — | ⬜ | +| [T4.2](T4.2-verifierctx-and-the-snapshot-barrier.md) | `VerifierCtx` and the snapshot barrier | M | — | ⬜ | +| [T4.3](T4.3-lazy-blob-access.md) | Lazy blob access | S | — | ⬜ | +| [T4.4](T4.4-retention-ordering-guard.md) | Retention ordering guard | S | — | ⬜ | +| [T4.5](T4.5-p4-composition-gate.md) | **Verification composition gate** | M | gate | ⬜ | + +## 5 — Embedding · T9.x + +Makes poimen a unit another harness mounts. The reference consumer is `dsh`, +whose architecture already expects capabilities to arrive as swappable providers +— so if the sidecar cannot be mounted as one, the boundary is wrong, not `dsh`. + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T9.1](T9.1-poimen-sdk-crate.md) | `poimen-sdk` — ports without the engine | M | new | ⬜ | +| [T9.2](T9.2-sidecar-protocol.md) | Sidecar protocol — JSON-RPC 2.0 over stdio | L | new | ⬜ | +| [T9.3](T9.3-serve-stdio.md) | `poimen serve --stdio` | M | new | ⬜ | +| [T9.4](T9.4-protocol-conformance-suite.md) | Protocol conformance suite | L | new | ⬜ | +| [T9.5](T9.5-dsh-reference-plugin.md) | `dsh` reference plugin | M | new | ⬜ | +| [T9.6](T9.6-embedding-composition-gate.md) | **Embedding composition gate** | L | gate | ⬜ | + +## 6 — Durability hard parts · T2.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T2.1](T2.1-write-ahead-intent.md) | Write-ahead intent | L | — | ⬜ | +| [T2.2](T2.2-effect-class-recovery.md) | Effect-class recovery | M | — | ⬜ | +| [T2.3](T2.3-rewind-as-fork.md) | Rewind as fork | L | — | ⬜ | +| [T2.4](T2.4-schema-evolution-end-to-end.md) | Schema evolution end-to-end | M | — | ⬜ | +| [T2.5](T2.5-checkpoints.md) | Checkpoints | M | — | ⬜ | +| [T2.6](T2.6-crash-matrix.md) | Crash matrix | L | — | ⬜ | +| [T2.7](T2.7-p2-composition-gate.md) | **Durability composition gate** | L | gate | ⬜ | + +## 7 — Grading · T5.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T5.1](T5.1-taskid-at-spawn.md) | `TaskId` at spawn | S | — | ⬜ | +| [T5.2](T5.2-capacitylimits-and-the-residency-invariant.md) | `CapacityLimits` and the residency invariant | M | — | ⬜ | +| [T5.3](T5.3-evaluationstrategy-port-resourceprofile.md) | `EvaluationStrategy` port + `ResourceProfile` | S | — | ⬜ | +| [T5.4](T5.4-deterministicgrader.md) | `DeterministicGrader` | S | — | ⬜ | +| [T5.5](T5.5-pairwisesequential-reference-and-comparison.md) | `PairwiseSequential`: reference and comparison | M | — | ⬜ | +| [T5.6](T5.6-sequential-test-and-stopping.md) | Sequential test and stopping | M | — | ⬜ | +| [T5.7](T5.7-order-alternation-and-sampled-consistency.md) | Order alternation and sampled consistency | S | — | ⬜ | +| [T5.8](T5.8-swiss-pairing.md) | Swiss pairing | M | opt-in | ⬜ | +| [T5.9](T5.9-bradley-terry-fit.md) | Bradley-Terry fit | L | opt-in | ⬜ | +| [T5.10](T5.10-attempt-tournaments.md) | Attempt tournaments | M | — | ⬜ | +| [T5.11](T5.11-degradation-reasons.md) | Degradation reasons | M | — | ⬜ | +| [T5.12](T5.12-p5-composition-gate.md) | **Grading composition gate** | L | gate | ⬜ | + +## 8 — Learning loop · T6.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T6.1](T6.1-challenger-allocation.md) | Challenger allocation | M | — | ⬜ | +| [T6.2](T6.2-judge-calibration-set.md) | Judge calibration set | M | — | ⬜ | +| [T6.3](T6.3-promotion-gates.md) | Promotion gates | M | — | ⬜ | +| [T6.4](T6.4-per-variant-aggregation.md) | Per-variant aggregation | M | opt-in | ⬜ | +| [T6.5](T6.5-sandboxed-replay.md) | Sandboxed replay | L | — | ⬜ | +| [T6.6](T6.6-held-out-split.md) | Held-out split | M | — | ⬜ | +| [T6.7](T6.7-drift-check.md) | Drift check | S | — | ⬜ | +| [T6.8](T6.8-p6-composition-gate.md) | **Learning composition gate** | L | gate | ⬜ | + +## 9 — Distribution · T7.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T7.1](T7.1-postgres-eventlog.md) | Postgres `EventLog` | L | — | ⬜ | +| [T7.2](T7.2-object-store-blobstore.md) | Object-store `BlobStore` | M | — | ⬜ | +| [T7.3](T7.3-leases-and-fencing.md) | Leases and fencing | L | — | ⬜ | +| [T7.4](T7.4-outbox-relay.md) | Outbox relay | M | — | ⬜ | +| [T7.5](T7.5-partition-keys-on-adapters.md) | Partition keys on adapters | S | — | ⬜ | +| [T7.6](T7.6-tournament-as-a-join-stage.md) | Tournament as a join stage | L | — | ⬜ | +| [T7.7](T7.7-turmoil-suite.md) | `turmoil` suite | L | parallel-ok | ⬜ | +| [T7.8](T7.8-p7-composition-gate.md) | **Distribution composition gate** | L | gate | ⬜ | + +## 10 — Operability · T8.x + +| Task | Title | Size | Flags | Status | +|---|---|---|---|---| +| [T8.1](T8.1-metering.md) | Metering | M | — | ⬜ | +| [T8.2](T8.2-metrics.md) | Metrics | M | — | ⬜ | +| [T8.3](T8.3-keyed-capability.md) | Keyed capability | L | — | ⬜ | +| [T8.4](T8.4-reduction-and-tiering.md) | Reduction and tiering | L | — | ⬜ | +| [T8.5](T8.5-embedded-mode-smoke.md) | Embedded-mode smoke | S | parallel-ok | ⬜ | +| [T8.6](T8.6-capacity-admission-control.md) | Capacity admission control | M | — | ⬜ | +| [T8.7](T8.7-p8-composition-gate.md) | **Operability composition gate** | L | gate | ⬜ | diff --git a/tasks/T0.1-identity-newtypes.md b/tasks/T0.1-identity-newtypes.md new file mode 100644 index 0000000..345fde0 --- /dev/null +++ b/tasks/T0.1-identity-newtypes.md @@ -0,0 +1,104 @@ +# T0.1 — Identity newtypes + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | S — under 1 day | +| Status | ✅ Done | +| Flags | — | +| Spec | inlined below | +| Blocks | everything | + +## Goal + +Every id in the system as a distinct newtype, plus the `Scoped` tenant wrapper +that makes an unscoped storage key a compile error. + +## Facts (inlined — no spec read needed) + +```rust +pub struct TenantId(Uuid); +pub struct WorkflowId(SmolStr); // logical workflow, stable across versions +pub struct WorkflowVersion(Blake3Hash); // content hash of canonicalized IR (T3.1) +pub struct StepId(SmolStr); // author-assigned, stable across versions +pub struct TaskId(Blake3Hash); // comparison-group key; hash of task input +pub struct RunId(Ulid); // one execution of one workflow +pub struct BranchId(u32); // rewind fork +pub struct AttemptNo(u32); +pub struct Lsn(u64); // per (run, branch) sequence, not global +pub struct GroupEpoch(u32); // comparison-group generation + +pub struct Scoped { pub tenant: TenantId, pub inner: T } +``` + +- `RunId` is `Ulid` so it sorts lexicographically by creation time. Recent-run + scans become a prefix scan instead of a secondary index. +- Multi-tenant from commit one. **Every** table key is `Scoped<_>` — not most. + One unprefixed table is a cross-tenant read that a customer finds first. +- No `Default` on `TaskId` or `WorkflowVersion`. Prior implementation hardcoded a + `"current"` version string: it compiled, passed tests, and made every result + unattributable. A newtype without `Default` refuses to compile instead. +- No `From for TaskId`. `TaskId` hashes the task input **before any + workflow touches it**; deriving it from a run id silently makes every run its + own group of one, and it cannot be backfilled. +- `StepId` is opaque: never parsed, never ordered, never assumed numeric. + +## Steps + +1. Create the ids crate. Dependencies: `uuid`, `ulid`, `smol_str`, `blake3`, `serde`. +2. Declare the ten newtypes above. Derive `Clone, Copy` (where the inner type is + `Copy`), `PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize`. +3. Do **not** derive or hand-write `Default` for `TaskId` or `WorkflowVersion`. + Do not add `From for TaskId`. +4. `RunId::new()` wraps `Ulid::new()`; `Ord` delegates to the inner `Ulid` so + ordering is creation order. +5. Define `Scoped`. Define the storage-key trait (whatever the table API + requires) **only** for `Scoped<_>` and `BranchKey` — never a blanket impl for + `T`, or the guard disappears. +6. Add `trybuild` as a dev-dependency with `tests/compile_fail/`: + `default_task_id.rs` (calls `TaskId::default()`), `unscoped_key.rs` (opens a + table keyed on a bare `RunId`). Record the expected `.stderr` files. + +## Acceptance + +- `trybuild` compile-fail suite: an unscoped table key and a defaulted `TaskId` + both fail to compile. +- Run the suite before writing the impls and watch it fail for the right reason + (missing type, not missing test file). + +## Verify + +**Harness:** `trybuild` for the compile-fail cases; a scratch `redb` file for the +key round trip. + +**Integration test** — `tests/it_scoped_keys.rs`: +1. Open one table keyed `Scoped`. +2. Write a value under tenant A and a different value under tenant B, using the + **same** inner `RunId`. +3. Read back each; assert each tenant sees only its own value. +4. Scan the raw table and assert exactly two distinct keys exist. +5. Create 1000 `RunId`s across at least 3 distinct milliseconds; assert sorted + order equals creation order. + +**Command:** `cargo test -p ids && cargo test -p ids --test compile_fail` + +**False pass:** +- Regenerating `.stderr` with `TRYBUILD=overwrite` after the guard broke. Assert + the expected stderr **names `Default` and the key trait** — a test that only + checks "does not compile" passes on a typo. +- ULID ordering asserted over ids minted inside one millisecond, where ordering + comes from the random suffix rather than time. Force distinct milliseconds. +- Step 3 passing because the read path filters by tenant. Step 4 (raw scan) is + what proves the key, not the query. + +## Traps + +- Blanket `#[derive(Default)]` on the whole module out of habit. +- `Scoped` as a type alias instead of a struct — an alias produces no compile + error, so the guard is decorative. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §3, §19 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.2-kernel-attemptstate.md b/tasks/T0.2-kernel-attemptstate.md new file mode 100644 index 0000000..3525b1e --- /dev/null +++ b/tasks/T0.2-kernel-attemptstate.md @@ -0,0 +1,113 @@ +# T0.2 — Kernel `AttemptState` + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | S — under 1 day | +| Status | ✅ Done | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +The closed kernel attempt enum and its transition function, matching the +normative transition table arm for arm. + +## Facts (inlined — no spec read needed) + +```rust +/// Kernel. Closed. Users never extend this. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum AttemptState { + Pending, + Running, + Succeeded, + Failed, + /// Crashed mid-side-effect; recovery could not determine what happened. + Indeterminate, + /// Stopped by decision, no dispatched intent outstanding. + Cancelled, + TimedOut, +} +``` + +Normative legal set — the match has exactly these arms and no others: + +| From | To | When | +|---|---|---| +| `Pending` | `Running` | admitted | +| `Pending` | `Cancelled` | run cancelled before the attempt started | +| `Pending` | `TimedOut` | queue deadline passed before admission | +| `Running` | `Succeeded` | completed | +| `Running` | `Failed` | completed with an error | +| `Running` | `TimedOut` | step deadline passed | +| `Running` | `Cancelled` | run cancelled, **no intent was `Dispatched`** | +| `Running` | `Indeterminate` | an intent was `Dispatched` and did not resolve | +| terminal | — | nothing leaves a terminal state | + +- `Cancelled` and `Indeterminate` are **separate states**. Cancellation is a + decision; indeterminacy is an unknown. The deciding fact is the intent record + (T2.1): no `Dispatched` intent means nothing external happened, so the attempt + is cleanly `Cancelled`. +- Why it matters beyond taxonomy: operations alerts on `Indeterminate` count and + expects near zero. Route ordinary cancellations there and the alert acquires a + noisy floor, which is the same as not having the alert. +- Domain states are the opposite — open, declared by the workflow as data + (`pub struct StepState(SmolStr)`), validated at load. Not this task. + +## Steps + +1. Declare `AttemptState` exactly as above. +2. Write `fn transition(from: AttemptState, cause: TransitionCause) -> Result` + as an **exhaustive match** — no `_ =>` catch-all, so adding a state becomes a + compile error at this function. +3. Model the `Running → {Cancelled, Indeterminate}` split on an explicit input: + `TransitionCause::Cancel { intent_dispatched: bool }`. Never infer it from the + drop site. +4. Make every terminal state reject all outgoing transitions with + `IllegalTransition`. +5. Property test with `proptest`: enumerate all `(from, to)` pairs, assert the + accepted set equals the table above exactly — reject-set included. + +## Acceptance + +- Property test over all state pairs: legal set matches the table exactly. +- Targeted test: cancelling an attempt with **no** `Dispatched` intent yields + `Cancelled`; cancelling one **with** a `Dispatched` intent yields + `Indeterminate`. + +## Verify + +**Harness:** `proptest`, plus a recorded log fixture once P1 exists. + +**Integration test** — `tests/it_transition_table.rs`: +1. Build the **cartesian product** of all 7 states × all 7 states, written out by + hand from the table in this file — not generated by the code under test. +2. For each pair, assert accept/reject matches the table. +3. Replay a real recorded run's log (P1 fixture) and feed every observed state + change through `transition`; assert none is rejected. This proves the executor + and the table agree. +4. Two targeted cases: cancel with `intent_dispatched: false` → `Cancelled`; + cancel with `true` → `Indeterminate`. + +**Command:** `cargo test -p kernel transition` + +**False pass:** +- A property test whose generator produces states **by calling `transition`**. + It then agrees with itself and cannot see a missing arm. The pair list must be + written independently. +- Asserting only the accepted set. The rejected set is half the table, and a + permissive `_ => Ok(to)` passes every accept-only test. + +## Traps + +- Collapsing `Cancelled` into `Indeterminate` "because the tool might have been + mid-something". That "might" is exactly what the intent record answers. +- A `_ =>` arm. It compiles forever and silently absorbs a new state. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §2, §5.1, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.3-workevent-and-schemaversion.md b/tasks/T0.3-workevent-and-schemaversion.md new file mode 100644 index 0000000..bf115e8 --- /dev/null +++ b/tasks/T0.3-workevent-and-schemaversion.md @@ -0,0 +1,104 @@ +# T0.3 — `WorkEvent` and `SchemaVersion` + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | ✅ Done | +| Flags | — | +| Spec | inlined below | +| Blocks | T0.4, T2.4 | + +## Goal + +The event enum plus the wire-version discipline that keeps a two-year-old log +decodable. Must land before the second event variant is ever written. + +## Facts (inlined — no spec read needed) + +```rust +pub struct LogRecord { + pub key: BranchKey, // (TenantId, RunId, BranchId) + pub lsn: Lsn, + /// Wire-format version of `event`. Never removed, never reused. + pub schema: SchemaVersion, + pub at: Timestamp, + pub event: WorkEvent, +} + +pub struct BranchKey { pub tenant: TenantId, pub run: RunId, pub branch: BranchId } +``` + +Rules, from record one: + +- Every record carries `SchemaVersion`. Written always, even at v1. +- `WorkEvent` is `#[non_exhaustive]`; decode is version-dispatched. +- **Variants are never removed or repurposed.** Deprecated variants stay + decodable forever. Storage is cheap; an undecodable log is not. +- Migrations are upcasters applied on read (T0.4), never by rewriting history. +- A round-trip test per version — a stored fixture of every historical version + still folds to the expected state. **That test is the whole guarantee**; + without it the rules above are aspirational. +- `BranchId` is in the key, not implied. LSNs are **per branch**, not global: a + global counter serializes every run through one atomic, and the ordering + contract only promises per-run total order. + +## Steps + +1. Declare `SchemaVersion(u16)` and `LogRecord` / `BranchKey` as above. +2. Declare `WorkEvent` as `#[non_exhaustive]` with the v1 variant set the walking + skeleton needs: attempt transitions, run-lifecycle transitions, prompt/output + blob refs, context partition, usage, intent records, and + `Reduced { original: BlobRef, summary: BlobRef }`. +3. Pick a self-describing codec (`serde` + CBOR or similar). Write + `encode(&LogRecord) -> Vec` and `decode(&[u8]) -> Result`, with + decode dispatching on the `schema` field read **before** the event body. +4. Write a fixture generator that serializes one record of every variant at v1 + into `tests/fixtures/v1/`. Commit the bytes. +5. Write the round-trip test: walk `tests/fixtures/*/`, decode each, assert the + expected value. A directory walk means adding a version folder automatically + extends coverage. +6. Add a test that fails if a fixture directory exists for a version with no + registered upcaster — wire this once T0.4 lands. + +## Acceptance + +- A serialized v1 fixture checked into the repo, plus a test that decodes it. + +## Verify + +**Harness:** committed fixture bytes under `tests/fixtures/v1/`, loaded from +disk. A regeneration path exists only behind an env var. + +**Integration test** — `tests/it_fixture_roundtrip.rs`: +1. Walk `tests/fixtures/*/`, decode every file. +2. Assert each decodes to its expected value, loaded from a committed + expectation file — not reconstructed in the test body. +3. Assert every fixture's `schema` field is present and non-zero. +4. Assert the variant coverage: every `WorkEvent` variant appears in at least one + fixture. A `match` over the decoded set with no `_` arm forces this to fail + when a variant is added without a fixture. + +**Command:** `cargo test -p log fixtures` — and in CI, with +`FIXTURE_REGEN` **unset**; assert the test fails loudly if it is set. + +**False pass:** +- The fixture being serialized by the code under test at test start, then + immediately decoded. That tests nothing across versions. Bytes must come from + git. +- Coverage drift: a new variant is added, no fixture is written, and the + directory walk still passes because it only checks what is there. Step 4 is the + guard. + +## Traps + +- Renaming one variant two years in and losing drop-and-re-fold silently. +- Non-self-describing formats where a field reorder decodes to garbage rather + than erroring. +- Omitting `schema` at v1 "because there is only one version". + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.2, §8.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.4-upcaster-framework.md b/tasks/T0.4-upcaster-framework.md new file mode 100644 index 0000000..3b579d9 --- /dev/null +++ b/tasks/T0.4-upcaster-framework.md @@ -0,0 +1,86 @@ +# T0.4 — Upcaster framework + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T2.4 | + +## Goal + +Read-time migration chain so old log records fold correctly after the event enum +changes. Never rewrite stored records. + +## Facts (inlined — no spec read needed) + +- Migrations are **upcasters**: `fn upcast(vN) -> vN+1`, applied on read. +- Rewriting an append-only log is a contradiction. History stays as written; the + reader adapts. +- Registry keyed by version. Reading a v1 record under a v3 binary runs + `v1→v2→v3` in sequence. +- Deprecated variants stay decodable forever — an upcaster maps them forward, it + does not delete them. +- The guarantee is the fixture test: a stored fixture of every historical version + still folds to the expected state. + +## Steps + +1. Keep the per-version wire types as real types — `WorkEventV1`, `WorkEventV2`, + … — with the current one aliased to `WorkEvent`. +2. Define the upcaster shape: `fn(vN) -> vN+1`, either as function pointers or a + `trait Upcast { type From; type To; fn upcast(from: Self::From) -> Self::To; }`. +3. Build the registry keyed by `SchemaVersion`. Registration is explicit; a + missing link is an error at registry construction, not at read time. +4. `read_record(bytes)`: decode `schema`, decode the matching `vN` body, then + apply each upcaster in order up to `CURRENT_SCHEMA`. +5. Return typed errors — `UnknownSchemaVersion`, `MissingUpcaster { from, to }`. + A corrupt or future-version record must not panic the fold. +6. Build the synthetic v1→v2 migration as the test vehicle: add a field or split + a variant in `WorkEventV2`, write the upcaster, commit fixtures for both. + +## Acceptance + +- Synthetic v1 → v2 migration with a fixture per version. +- Folding a v1 log through the upcaster yields the **same state** as a + natively-v2 log. Assert on the folded state, not on intermediate events. + +## Verify + +**Harness:** committed v1 and v2 fixtures, plus T0.8's fold. + +**Integration test** — `tests/it_upcast_equivalence.rs`: +1. Fold the committed **v1** log through the upcaster chain → `state_a`. +2. Fold the committed **native v2** log directly → `state_b`. +3. Assert `serialize(state_a) == serialize(state_b)` byte for byte. +4. Instrument the registry with a counter; assert **at least one** upcaster + actually ran on the v1 path. +5. Feed a record stamped with an unknown future version; assert + `UnknownSchemaVersion` is returned and **no panic** escapes. + +**Command:** `cargo test -p log upcast` + +**False pass:** +- `CURRENT_SCHEMA` still equal to v1, so the chain is empty and both folds + trivially agree. Step 4 is the guard. +- Comparing folded states with a derived `PartialEq` over `HashMap`, which + ignores ordering. Compare serialized bytes. +- The v2 change being cosmetic (a renamed local, an added comment) so no upcaster + logic is exercised. The v2 fixture must contain a record whose shape actually + differs. + +## Traps + +- Lossy upcasters that default a new field to a value the fold treats as + meaningful. If v1 cannot supply the field, the fold handles `Option` — it does + not receive a fabricated value. +- A registry populated by iteration order rather than an explicit chain. The gap + only shows at v3. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.5-eventlog-port-redb-implementation.md b/tasks/T0.5-eventlog-port-redb-implementation.md new file mode 100644 index 0000000..7308b56 --- /dev/null +++ b/tasks/T0.5-eventlog-port-redb-implementation.md @@ -0,0 +1,143 @@ +# T0.5 — EventLog port + redb implementation + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T0.6, T1.x | + +## Goal + +The full `EventLog` port and its embedded `redb` implementation. Keys +`(TenantId, RunId, BranchId, Lsn)`. Async signatures even though the local +implementation is synchronous underneath. + +## Facts (inlined — no spec read needed) + +```rust +#[async_trait] +pub trait EventLog: Send + Sync { + /// Append records, apply derived state, advance consumer position, enqueue + /// the outbox — all four or none. Returns the assigned LSNs. + async fn commit(&self, batch: CommitBatch) -> Result>; + + async fn read(&self, key: BranchKey, from: Lsn, limit: usize) -> Result>; + + async fn put_checkpoint(&self, key: BranchKey, upto: Lsn, state: &[u8]) -> Result<()>; + /// Newest checkpoint at or below `upto`. `None` means fold from LSN 0. + async fn latest_checkpoint(&self, key: BranchKey, upto: Lsn) + -> Result>; + + /// Committed-but-unshipped export intents, in `(BranchKey, Lsn)` order. + /// Read by the relay, never from the execution path. + async fn drain_outbox(&self, tenant: TenantId, limit: usize) -> Result>; + async fn ack_outbox(&self, shipped: &[(BranchKey, Lsn)]) -> Result<()>; +} + +pub struct CommitBatch { + /// Carries the tenant. Never a separate parameter beside a key that already + /// holds one — two sources for one fact is one too many. + pub key: BranchKey, + pub records: Vec, // LSNs assigned by the implementation + pub state: Vec, + pub position: Option, + pub outbox: Vec, +} +``` + +- **`commit` is one method, not four, because atomicity is the contract.** A port + exposing `append` alone puts the other three writes outside the transaction — + the durability guarantee gone, and gone invisibly, since each write + individually succeeds. +- Everything is `async`. An async signature over a local call costs a negligible + poll; a sync signature over a network call is impossible, and the Postgres + backend (T7.1) implements this same trait. The prior revision declared a + synchronous store and documented in the same file that a networked + implementation could not honour the signature — a port finished while already + known to be unimplementable. +- `redb` specifics: pure Rust, ACID, MVCC, copy-on-write shadow paging, so a torn + write cannot corrupt the file — it simply does not take effect. The durability + enum is `#[non_exhaustive]`, so set `Durability::Immediate` **explicitly** + rather than relying on the default. (Avoid `sled` — years at 0.34 beta with + known space amplification.) +- LSNs are per branch, monotonic, gap-free. Not a global counter: a global one + serializes every run through a single atomic, and the ordering contract only + promises per-run total order. +- `latest_checkpoint` and `drain_outbox`/`ack_outbox` exist from the start. A + checkpoint that can be written and not read is an optimization nobody can use; + writes without their matching reads are how T2.5 and T7.4 discover the port is + short. + +## Steps + +1. Define the port trait, `CommitBatch`, `Checkpoint`, `OutboxEntry`, + `ExportIntent`, `StateDelta`, `ConsumerPosition` exactly as above. +2. Open the four `redb` tables: `EVENT_LOG`, `RUN_STATE`, `CONSUMER_POSITION`, + `OUTBOX`. Every key is `BranchKey`-derived or `Scoped<_>` (T0.1). +3. Implement LSN allocation **inside** the write transaction: read the current + max LSN for the `BranchKey`, assign `max+1..` across the batch. Being inside + the transaction is what makes it gap-free under concurrency. +4. Implement `commit` as a single `begin_write` → four inserts → `commit()`, with + `Durability::Immediate` set explicitly. +5. Implement `read` as a range scan bounded by `(key, from)..`, honouring `limit`. +6. Implement the checkpoint pair and the outbox pair. `drain_outbox` returns in + `(BranchKey, Lsn)` order — that ordering is the only one the relay is promised. +7. Write the conformance suite as a **generic function over `impl EventLog`**, so + T7.1's Postgres backend runs the identical tests. This is where the + abstraction becomes real or does not. + +## Acceptance + +- Append 10k records across 3 branches of 2 runs in 2 tenants; read back in order + per branch; assert no cross-tenant visibility. +- Port audit: compile stub call sites for T2.5 (checkpoints), T7.4 (outbox relay) + and T8.4 (reduction) against the trait. Every method they need already exists. + +## Verify + +**Harness:** the conformance suite written as +`fn conformance(make: impl Fn() -> L)`. It is the deliverable T7.1 +reuses unmodified — write it as a library function, not as `#[test]` bodies. + +**Integration test** — `tests/it_eventlog_conformance.rs`: +1. Append 10k records across 3 branches × 2 runs × 2 tenants. +2. Read back per branch from LSN 0; assert contiguous `1..n` with no gaps and no + duplicates. +3. **Concurrency:** spawn 16 tasks committing to the *same* `BranchKey` + simultaneously; assert the union of returned LSNs is exactly `1..=total` with + no repeats. Gap-freedom only breaks under contention. +4. **Isolation:** scan the raw table with no tenant predicate and assert every + key carries the expected tenant — do not rely on the filtered read path. +5. Round-trip a checkpoint: `put_checkpoint`, then `latest_checkpoint` at an LSN + above and below it. +6. Outbox: commit with entries, `drain_outbox`, assert `(BranchKey, Lsn)` + ascending order, `ack_outbox`, assert a second drain returns empty. +7. **Port audit:** a `tests/port_audit.rs` with stub call sites for T2.5, T7.4 + and T8.4 that must compile against the trait. + +**Command:** `cargo test -p storage --test it_eventlog_conformance --test port_audit` + +**False pass:** +- Single-threaded LSN allocation test. It passes against a + read-then-write-outside-the-transaction implementation, which is the actual bug. +- Cross-tenant isolation checked through a query that already filters by tenant — + it passes against a completely unscoped table. Step 4 is the real check. +- A conformance suite written inline as `#[test]`s, which then cannot be reused + for Postgres and quietly becomes redb-only. + +## Traps + +- Adding a convenience `append()` that skips the other three tables. It will be + on the hot path within a week. +- Passing `tenant` as a second parameter beside `BranchKey`. +- Letting `Durability` default. The enum is `#[non_exhaustive]`; the default can + change underneath you. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §7, §8.3 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.6-atomic-commit-protocol.md b/tasks/T0.6-atomic-commit-protocol.md new file mode 100644 index 0000000..b1770df --- /dev/null +++ b/tasks/T0.6-atomic-commit-protocol.md @@ -0,0 +1,104 @@ +# T0.6 — Atomic commit protocol + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T2.1 | + +## Goal + +Four table writes in one transaction behind `EventLog::commit`. All four land or +none do. + +## Facts (inlined — no spec read needed) + +```rust +let txn = db.begin_write()?; +{ + let mut log = txn.open_table(EVENT_LOG)?; + let mut state = txn.open_table(RUN_STATE)?; + let mut position = txn.open_table(CONSUMER_POSITION)?; + let mut outbox = txn.open_table(OUTBOX)?; + + log.insert((branch_key, lsn), &record)?; // append: the durable fact + state.insert((branch_key, attempt_no), &attempt)?; // apply: the derived view + position.insert(Scoped::new(tenant, stream), pos)?; // advance: where to resume + outbox.insert((branch_key, lsn), &intent)?; // relay separately +} +txn.commit()?; // all four, or none +``` + +- **Every key carries `BranchKey` or `Scoped<_>`.** The outbox is the one that + invites the mistake: an outbox keyed on `Lsn` alone reads naturally and is + wrong, because LSNs are per branch — a bare LSN collides across every branch of + every run of every tenant. The same key shape also gives the relay its only + promised ordering: per `BranchKey`, ascending `Lsn`. +- **Commit per event, not per run.** A projection that accumulates in memory and + writes at run end loses the whole run on a crash. Episodes are small and + append-mostly; one fsync per transition is cheap next to model latency. +- The engine supplies crash-atomicity (shadow paging in `redb`, transactions in + Postgres). The log supplies history. Because the engine's transaction is + atomic, append and apply happen together: no torn records, no redo/undo pass, + no checkpoint-consistency problem. + +## Steps + +1. Route every write through `EventLog::commit`. There is no second write path — + grep for direct `open_table(...).insert` outside the implementation and delete + what you find. +2. Confirm each of the four keys: `(BranchKey, Lsn)` for log and outbox, + `(BranchKey, AttemptNo)` for state, `Scoped` for position. +3. Build the crash harness: a test-only hook that panics after write *i* of the + four, for `i in 0..4`. Reopen the database and assert all-or-nothing. +4. Add the collision test: two runs in two tenants both writing LSN 1, both + outbox entries present and distinct after commit. +5. Measure one commit's fsync cost and record it in the test output, so the + per-event decision stays visible rather than being re-litigated later. + +## Acceptance + +- Harness panics between each pair of table writes; on reopen either all four + landed or none did. +- Two runs in two tenants writing the same LSN both keep their outbox entries. + +## Verify + +**Harness:** a test-only fault hook inside the commit path, `panic_after_write(i)` +for `i in 0..4`, compiled under a `test-hooks` feature. + +**Integration test** — `tests/it_commit_atomicity.rs`: +1. For each `i in 0..4`: run a commit that writes all four tables with the hook + armed at `i`. +2. Reopen the database in a **fresh process** (or at minimum a fresh `Database` + handle — reusing an open handle can mask a durability bug). +3. Count rows in all four tables. Assert the count vector is either + "all four present" or "none present". Any mixed vector fails. +4. Collision case: two runs in two tenants both commit LSN 1 with outbox + entries; assert both entries exist and are distinct after reopen. +5. Print the measured fsync duration for one commit, so the per-event cost stays + visible rather than being re-litigated from memory later. + +**Command:** `cargo test -p storage --features test-hooks commit_atomicity` + +**False pass:** +- The hook checked **before** `begin_write`, so no partial state is ever possible + and every `i` trivially passes. +- Asserting on the in-memory result rather than after reopen. The transaction + object will happily report success. +- Testing with `Durability::None` in the test config for speed. It passes and + proves nothing about the shipped path. + +## Traps + +- Outbox keyed on `Lsn` alone. It reads naturally and silently loses entries. +- Buffering state deltas in memory "for efficiency" and flushing at run end. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.1, §8.3 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.7-blobstore-port-redb-implementation.md b/tasks/T0.7-blobstore-port-redb-implementation.md new file mode 100644 index 0000000..d411751 --- /dev/null +++ b/tasks/T0.7-blobstore-port-redb-implementation.md @@ -0,0 +1,97 @@ +# T0.7 — `BlobStore` port + `redb` implementation + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Content-addressed blob storage, deduplicated **within** a tenant and never +across. `delete` ships now, not later. + +## Facts (inlined — no spec read needed) + +```rust +#[async_trait] +pub trait BlobStore: Send + Sync { + async fn put(&self, tenant: TenantId, content: Bytes) -> Result; + async fn get(&self, tenant: TenantId, r: &BlobRef) -> Result>; + /// Reduction and tenant deletion both require this. A store that cannot + /// delete cannot honour either, and both are obligations. + async fn delete(&self, tenant: TenantId, r: &BlobRef) -> Result<()>; +} +``` + +- **Blobs are namespaced per tenant even though they are content-addressed.** + Global dedup of prompt and output blobs is tempting — identical system prompts + across tenants are common — and it is a leak: a shared blob makes one tenant's + storage accounting depend on another's, and a hash becomes an oracle for "does + anyone else have this content". +- `get` returning `None` is a normal outcome, not an error. After reduction + (T8.4) the original body is deleted and the log carries a + `Reduced { original, summary }` event explaining what stands in its place. +- Content addressing means the ref is a hash of the body. A body must never be + rewritten under an existing ref — that makes the ref a lie. + +## Steps + +1. Define `BlobRef` as a Blake3 hash newtype. Define the trait as above. +2. `redb` table keyed on `Scoped` — tenant in the key is what enforces + per-tenant namespacing structurally rather than by convention. +3. `put`: hash the content, insert if absent, return the ref. Insert-if-absent is + the dedup, and it is scoped by the key, so no cross-tenant path exists. +4. `get`: point lookup, `Ok(None)` on miss. +5. `delete`: remove the key. Deleting a ref one tenant holds must not touch + another tenant's identical content — which follows from the key shape, and the + test below is what proves it. +6. Write the conformance suite generic over `impl BlobStore` so T7.2's + object-store backend reuses it unchanged. + +## Acceptance + +- Identical content put under two tenants produces two independent blobs. +- Deleting tenant A's blob leaves tenant B's readable. + +## Verify + +**Harness:** conformance suite generic over `impl BlobStore`, reused verbatim by +T7.2's object store. + +**Integration test** — `tests/it_blobstore_conformance.rs`: +1. `put` identical bytes under tenant A and tenant B. Assert the returned + `BlobRef`s are equal (same content hash) **and** that a raw table scan shows + **two** stored entries. +2. `delete` tenant A's ref. Assert `get(A, ref)` is `Ok(None)` and + `get(B, ref)` still returns the bytes. +3. `put` the same bytes twice under one tenant; assert one stored entry (dedup + within the tenant) and a stable ref. +4. `get` a ref that was never written: `Ok(None)`, not an error. +5. `delete` a ref that does not exist: `Ok(())`, so reduction can run twice. +6. Re-hash every stored body and assert it matches its key. + +**Command:** `cargo test -p storage --test it_blobstore_conformance` + +**False pass:** +- Step 1 asserting only that both `get`s return the right bytes. A single shared + blob passes that. The **raw scan showing two entries** is the isolation proof. +- Step 2 passing because `delete` is a no-op stub. Assert the `None` on A + explicitly, not just B's survival. + +## Traps + +- A global content-addressed table with tenant tracked in a side index. It + dedups across tenants by construction, which is the leak. +- Deferring `delete` because nothing calls it yet. Reduction and tenant deletion + both need it, and retrofitting a delete path into a store designed without one + is a rewrite. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §3, §7, §8.6 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.8-fold-and-re-derive.md b/tasks/T0.8-fold-and-re-derive.md new file mode 100644 index 0000000..745422f --- /dev/null +++ b/tasks/T0.8-fold-and-re-derive.md @@ -0,0 +1,92 @@ +# T0.8 — Fold and re-derive + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | P1 gate | + +## Goal + +The projection from log to materialized state, as a pure function. This task +establishes the property every later phase re-runs. + +## Facts (inlined — no spec read needed) + +- **Nothing derived is authoritative.** Materialized state is a cache of the log. + If it cannot be dropped and rebuilt byte-identically, it has hidden inputs and + that is a bug. +- The fold is pure: `fold(state, record) -> state`. No clock reads, no random, + no filesystem, no map iteration order leaking into the output. +- Reduction (T8.4) is designed around this property: it appends a + `Reduced { original, summary }` event rather than editing the log or rewriting + a blob, precisely so the fold still reaches the same state from LSN 0. +- Restart folds forward from the newest checkpoint (T2.5); `None` means fold from + LSN 0. Checkpoints are an optimization only — deleting every checkpoint costs + startup time and nothing else. + +## Steps + +1. Define the materialized state type: runs, branches, attempts, refs, intent + status. Whatever the query surface (T1.7) needs and nothing more. +2. Write `fn fold(state: &mut State, record: &LogRecord)` as a pure function. + Take no `&self`, no store handle, no `Instant::now()`. +3. Use ordered collections (`BTreeMap`) anywhere the state is serialized, so + byte-identity does not depend on hash seed. +4. Write `rebuild(log, key) -> State`: read from LSN 0 (or from the newest + checkpoint once T2.5 lands), fold every record in order. +5. Write the property test as a **reusable helper**, not a one-off: drop the + state tables, `rebuild`, assert the serialized bytes match. Every subsequent + phase calls this helper on its own fixtures. +6. Wire that helper into P1–P8 fixture suites as they land. + +## Acceptance + +- Drop the state tables, re-fold from LSN 0, assert byte-identical state. +- The helper is exported and used by at least one later phase's fixture set. + +## Verify + +**Harness:** export `assert_refold_identical(&log, key)` as a **public test +helper from this crate**. Every later phase calls it on its own fixtures; that +reuse is the deliverable, not the one-off test. + +**Integration test** — `tests/it_refold.rs`: +1. Build a log containing a retry, a rewind (once T2.3 lands) and a reduction + (once T8.4 lands) — the three cases with non-trivial fold logic. +2. Snapshot `serialize(state)`. +3. Drop the state tables entirely. +4. `rebuild` from LSN 0. +5. Assert **byte equality** of the serialized state, not `PartialEq`. +6. Repeat the fold twice in the same process and assert both runs produce + identical bytes — catches hash-seed and iteration-order leakage. +7. Purity check: run the fold with the system clock moved forward and with a + different `TZ`; assert the bytes are unchanged. + +**Command:** `cargo test -p projection refold` + +**False pass:** +- Comparing states with derived `PartialEq`. `HashMap` compares order-insensitively, + so byte-identity can be broken while the test is green — and byte-identity is + the actual property. +- Testing on a log with one linear run and no branches, retries or reductions. + Every implementation passes that. +- Running the fold once and comparing to the live state that was built by the + same code path incrementally — assert against a committed expected snapshot for + at least one fixture. + +## Traps + +- `HashMap` in serialized state — iteration order varies and byte-identity fails + intermittently, which reads as flakiness rather than as the bug it is. +- Any hidden input: a timestamp taken at fold time, a config value read from the + environment, a default filled in from the current schema version. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §8.6 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T0.9-p0-composition-gate.md b/tasks/T0.9-p0-composition-gate.md new file mode 100644 index 0000000..4ac6158 --- /dev/null +++ b/tasks/T0.9-p0-composition-gate.md @@ -0,0 +1,109 @@ +# T0.9 — P0 composition gate + +| Field | Value | +|---|---| +| Phase | P0 — Foundations | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P1 — nothing in P1 starts until this is green | + +## Goal + +Prove T0.1–T0.8 compose. Each was verified alone; this task verifies they work +**together and interchangeably** — that the ports are real seams, not shapes. + +**Phase gate criterion:** types compile, log round-trips, schema fixture test +green, drop-and-re-fold byte-identical. + +## Facts (inlined — no spec read needed) + +- A port is only real if a **second implementation** can be dropped in without + editing the suite. P0 ships one implementation of each port; the seam is + therefore unproven until a second one exists. T7.1/T7.2 supply the production + second implementation much later — too late to learn the port is short. +- The four properties that must hold **jointly**, not individually: + 1. every key is tenant-scoped (T0.1) — including keys written by T0.6's commit; + 2. every record carries `SchemaVersion` and decodes through the upcaster chain + (T0.3, T0.4); + 3. all four table writes are atomic (T0.6); + 4. dropping derived state and re-folding from LSN 0 is byte-identical (T0.8). +- The failure this gate exists to catch: each task passing its own test while the + **composition** violates a property none of them owns. A commit path that + writes a correctly-scoped log key and an unscoped outbox key passes T0.1's + tests and T0.6's tests, and leaks across tenants. + +## Steps + +1. Build a **second `EventLog` implementation**: a deliberately naive in-memory + one, correct but sharing no code with the `redb` backend. +2. Build a second `BlobStore` the same way. +3. Run T0.5's and T0.7's conformance suites against **both** implementations, + unmodified. Any edit needed is a finding about the port, not the test. +4. Write the cross-invariant matrix (below) as a single test binary that + exercises every P0 task in one path. +5. Wire T0.8's `assert_refold_identical` into that path so composition and + re-derivability are asserted together. +6. Make this binary a **required CI job** and the P1 entry condition. + +## Acceptance + +- Both `EventLog` implementations and both `BlobStore` implementations pass the + conformance suites **unmodified**. +- The cross-invariant matrix is green. +- P0's stated gate holds: types compile, log round-trips, schema fixture test + green, re-fold byte-identical. + +## Verify + +**Harness:** the two extra port implementations from steps 1–2 — they exist only +for this gate and stay in the tree as the anti-drift device. + +**Integration test** — `tests/it_p0_composition.rs`: +1. **Port interchange:** parameterize the whole test over + `[redb, in_memory] × [redb_blobs, in_memory_blobs]`. All four combinations + must pass. A combination that fails means a caller depends on an + implementation detail rather than the trait. +2. **Scope sweep:** after a full workload across 2 tenants, scan **every** raw + table — log, state, position, outbox, blobs — and assert every key carries the + expected tenant. Not through the read path; through a raw scan. +3. **Atomicity under composition:** arm T0.6's fault hook while a commit carries + records, state deltas, a position advance and an outbox entry + **simultaneously**. Assert all-or-nothing across all four. +4. **Schema + fold:** write records at v1, upgrade to v2, then drop state and + re-fold. Assert byte-identical state. This is T0.4 and T0.8 composed, and it + is the pair most likely to break each other. +5. **LSN under contention across branches:** 16 concurrent writers across 3 + branches of 2 runs; assert per-branch `1..=n` gap-free and no cross-branch + interference. +6. **Blob/log linkage:** put a blob, reference it from a committed record, drop + state, re-fold, resolve the ref. Assert the bytes. +7. Re-run every P0 task's own test suite in the same job, so this gate catches a + regression rather than only a composition bug. + +**Command:** `cargo test -p foundations --test it_p0_composition` + +**False pass:** +- Running the matrix against one implementation "because the second is only a + test double". The second implementation **is** the test — it is the only thing + that proves the port is a seam before T7.1 finds out the expensive way. +- Step 2 through the read path, which filters by tenant and passes against a + completely unscoped table. +- Step 4 with a v2 change that touches no recorded variant, so no upcaster runs + and the fold is trivially identical. +- Treating this gate as optional because the eight underlying suites are green. + Every property here is one no single task owns. + +## Traps + +- Deleting the in-memory implementations once T7.1 lands. They are the fast CI + path and the drift detector; the Postgres backend is neither. +- Letting the conformance suite grow implementation-specific branches + (`if backend == redb`). At that point it no longer tests the port. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §3, §7, §8 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.1-ctx-and-runscope.md b/tasks/T1.1-ctx-and-runscope.md new file mode 100644 index 0000000..a5e1dff --- /dev/null +++ b/tasks/T1.1-ctx-and-runscope.md @@ -0,0 +1,102 @@ +# T1.1 — `Ctx` and `RunScope` + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T1.3, T7.x | + +## Goal + +Explicit context threaded through every call, and a run-scoped task tree that +cannot be escaped through the public API. + +## Facts (inlined — no spec read needed) + +- Runtime is **tokio**. Chosen for ecosystem access (`sqlx`, `rdkafka`, + `aws-sdk`, `tonic`, `axum`, `reqwest`, object-store clients) — for a + distributed framework those are the distribution layer, not optional deps. +- `RunScope` wraps `TaskTracker` + `CancellationToken` from `tokio-util`, one + tracker per run. It exists because tokio has no structural task tree: without + the guard, a detached spawn outlives its run. +- `Ctx` is an **explicit struct** passed to every call. Never `task_local!` for + anything causal — that is the `AsyncLocalStorage` mistake in different clothes. + Trace context propagates through `Ctx` too. +- Cancellation is **cooperative**: a `select!`-dropped future stops at its next + await and not before. Kernel code must never hold a lock or a half-applied + state across an await that can be cancelled. +- A dropped call resolves `Cancelled` if no intent reached `Dispatched`, and + `Indeterminate` if one did. **The intent record decides, not the drop.** + Collapsing the two is the easy implementation and it puts a permanent noisy + floor under the `Indeterminate` alert, which is meant to sit near zero. + +## Steps + +1. Define `Ctx { tenant, run, branch, cancel: CancellationToken, deadline, trace, + log: Arc, blobs: Arc }`. Pass by `&Ctx`. +2. Define `RunScope` owning a `TaskTracker` and a `CancellationToken`. Expose + `spawn` only as a method on `RunScope`; keep the inner tracker private so no + caller reaches `tokio::spawn` through the public API. +3. `RunScope::shutdown()` / `Drop`: cancel the token, `tracker.close()`, then + await `tracker.wait()`. Returning before every child is complete or aborted is + the bug this type exists to prevent. +4. In kernel code, `select!` the cancellation token at every await. Audit for + locks or partially-applied state held across those awaits. +5. Give user tool calls a hard timeout in addition to the token — a + non-cooperative call must still be bounded. +6. On cancellation, read the attempt's latest intent state to choose + `Cancelled` vs `Indeterminate`. Do not decide at the drop site. + +## Acceptance + +- Cancel a run mid-step; assert every spawned task has completed or aborted + before `RunScope::drop` returns. +- The cancelled attempt lands in `Cancelled`, **not** `Indeterminate` — asserted + directly, since collapsing the two is the easy implementation and silently + ruins the alert. + +## Verify + +**Harness:** `tokio::time::pause` for deadlines; a scoped task that records its +own completion into a shared `Arc>>` so "aborted" and "completed" +are distinguishable. + +**Integration test** — `tests/it_runscope_cancel.rs`: +1. Start a run whose step spawns three children through `RunScope::spawn`, each + parked on a long await. +2. Cancel mid-step. +3. Assert `RunScope::drop` (or `shutdown().await`) returns **only after** all + three have recorded completion-or-abort. Assert the recorded set has size 3 — + a scope that returns early leaves it at 0 or 1. +4. Assert the attempt landed in `Cancelled`, **not** `Indeterminate`, with no + intent dispatched. +5. Second run: dispatch an intent, then cancel. Assert `Indeterminate`. +6. Grep test (or a lint): assert no `tokio::spawn` outside `RunScope`, and no + `task_local!` anywhere in kernel crates. + +**Command:** `cargo test -p runtime runscope` + +**False pass:** +- Step 3 asserting only that `drop` returned. It always returns. The recorded + child set is the evidence. +- Steps 4 and 5 written against the same fixture with `intent_dispatched` + hardcoded — they must run through the real intent record, or the distinction is + asserted against itself. +- A child that finishes on its own before cancellation lands, making the test + green regardless of the tracker. Park children on a token that only the cancel + releases. + +## Traps + +- A public `spawn` helper "for convenience" that bypasses the tracker. +- Storing tenant or run id in a task-local. It works until a spawn boundary, then + attributes work to the wrong run. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.1, §6, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.2-stub-model-provider.md b/tasks/T1.2-stub-model-provider.md new file mode 100644 index 0000000..cad0b33 --- /dev/null +++ b/tasks/T1.2-stub-model-provider.md @@ -0,0 +1,77 @@ +# T1.2 — Stub model provider + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +A deterministic scripted model provider so the walking skeleton runs end to end +with no network and no spend. + +## Facts (inlined — no spec read needed) + +- Nothing is built before the record is trustworthy: no dashboards and no + learning loop until one full run works end to end **against a stub model**. +- The stub is not a test fixture that gets deleted — it is the fixed point every + later determinism assertion is measured against, including T0.8's re-fold + property and T5.x's grading tests. +- Failure injection here is what T1.4 (retry), T2.1 (crash windows) and T2.6 + (crash matrix) drive their scenarios with. + +## Steps + +1. Define the model port the executor calls: request in, response plus `Usage` + out, `async`, cancellation-aware. +2. Implement the stub over a script: an ordered list of responses, optionally + keyed by step id so a workflow with branches stays readable. +3. Add configurable latency per response, driven by `tokio::time::sleep` so + `tokio::time::pause` in tests makes it instant. +4. Add failure injection: error responses, timeouts, and a hang that outlives the + deadline. Each selectable per script entry. +5. Make every field of the emitted response deterministic — no timestamps from + the wall clock, no generated ids that are not seeded. +6. Test: run the same script twice, serialize both episodes, assert equal bytes. + +## Acceptance + +- The same script yields byte-identical episodes across runs. + +## Verify + +**Harness:** the stub itself plus T0.8's `assert_refold_identical`. + +**Integration test** — `tests/it_stub_determinism.rs`: +1. Run the same script twice, in two fresh databases, in the same process. +2. Serialize both episodes; assert **byte equality**. +3. Run a third time in a **separate process** and compare against a committed + expected serialization — catches anything seeded from process state. +4. With `tokio::time::pause`, assert a script declaring 30s latency completes in + near-zero wall time. If it does not, the stub is sleeping off the tokio clock. +5. Failure injection: assert an error entry produces a failed attempt, a timeout + entry produces `TimedOut`, and a hang entry is cut off by the deadline. + +**Command:** `cargo test -p testkit stub` + +**False pass:** +- Steps 1–2 in one process with a lazily-initialized global seed: both runs share + it and agree, while a fresh process differs. Step 3 is the guard. +- Comparing episodes with a `PartialEq` that skips timestamps. That hides exactly + the nondeterminism being hunted — compare the full serialized bytes. + +## Traps + +- A wall-clock timestamp or a random id in the stub response. It defeats every + byte-identity assertion downstream and looks like a fold bug. +- Sleeping with `std::thread::sleep`, which `tokio::time::pause` cannot skip. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §16 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.3-run-executor.md b/tasks/T1.3-run-executor.md new file mode 100644 index 0000000..ab323f7 --- /dev/null +++ b/tasks/T1.3-run-executor.md @@ -0,0 +1,129 @@ +# T1.3 — Run executor + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Drive a hardcoded three-step workflow through the full run lifecycle, emitting +every kernel transition to the log. T3.5 later replaces the hardcoded workflow +with the IR interpreter. + +## Facts (inlined — no spec read needed) + +Run lifecycle — the states and the legal moves: + +``` + spawn + │ + v + Scheduled ─────────────────┐ + │ admitted │ + v │ + Running ⇄ Suspended ──────┤ + │ all steps terminal │ cancel + v │ + Verifying ─────────────────┤ ◄── rests here while verifier + │ │ futures are outstanding + v │ + Verified{pass|fail} │ + │ │ + v v + Grading ──────────────► Cancelled ● + │ ◄── may wait for a tournament group to fill + │ + ├──► Graded ● ──┐ + │ ├──► Archived ● + └──► Ungraded ● ──┘ +``` + +- **Cancel is reachable from every non-terminal state**, not only `Scheduled`. A + run cancelled mid-step is the ordinary case — it is what a user clicking stop + does. Cancelling during `Verifying` or `Grading` is rarer and still legal. +- **`Verifying` and `Grading` are states the run rests in**, not synchronous + branches. Resolving pass/fail inside `verify()` collapsed this in the prior + implementation and blocked three separate features at once: async verifiers, + mid-run UI, and the snapshot barrier (T4.2). +- **`Ungraded` is terminal and sits beside `Graded`, not below it.** Grading can + legitimately end with no score. Without a terminal state saying so, those runs + rest in `Grading` forever and become permanently irreducible (T4.4). +- `Suspended` releases the worker: a run awaiting human approval or a webhook + must not hold an executor slot across a human decision. +- Concurrency shape: unbounded across runs; **serial** steps within a run; + strictly serial attempts within a step; fan-out only for declared `Parallel` + branches and for verifiers of one attempt. The serial spine is + `(TenantId, RunId)`. + +## Steps + +1. Define the run-state enum and its transition function, exhaustive like T0.2's. + Include `Cancelled` as reachable from every non-terminal state. +2. Build the executor loop: take the hardcoded three-step workflow, run steps + serially, each step producing one or more attempts (T1.4). +3. Emit a `WorkEvent` for **every** transition through `EventLog::commit` — run + level and attempt level. One commit per transition, not one per run. +4. Enter `Verifying` only when every step is terminal. Park there; do not resolve + verification inline. +5. Enter `Grading` as a resting state likewise; allow it to terminate as + `Graded` **or** `Ungraded`. +6. Run everything inside a `RunScope` (T1.1) so cancel propagates and no task + outlives the run. +7. Add `Suspended` with lease release wired as a no-op stub in embedded mode — + the state must exist now, since T7.3 depends on it. + +## Acceptance + +- One run completes end to end against the stub model. +- `Verifying` and `Grading` are observable as **distinct states in the log**, not + skipped or collapsed. + +## Verify + +**Harness:** embedded `redb`, stub model (T1.2), one throwaway verifier, one stub +grader. This is the first test that exercises T0.5, T0.6, T0.8, T1.1 and T1.2 +together — treat it as the P1 integration point. + +**Integration test** — `tests/it_full_run.rs`: +1. Spawn one run of the three-step workflow; drive to completion. +2. Read the log back and assert the **run-state sequence** contains + `Scheduled, Running, Verifying, Grading` and a terminal state, in that order. + Assert `Verifying` and `Grading` each appear as their own record — not + inferred, not skipped. +3. Assert every step produced attempt records and that no state appears out of + order against T0.2's table. +4. Call `assert_refold_identical` on the finished run. +5. Cancel matrix: start a run, cancel it from each of `Scheduled`, `Running`, + `Verifying`, `Grading` in turn; assert each reaches `Cancelled`. +6. Ordering: launch 20 runs concurrently, assert each run's own records are + totally ordered and that no two steps of one run overlap in time. + +**Command:** `cargo test -p executor --test it_full_run` + +**False pass:** +- Asserting `Verifying` "happened" by checking a boolean on the run record. That + passes against a synchronous `verify()` call, which is the collapse this task + exists to prevent. The **log record** is the assertion. +- A cancel test that only covers `Scheduled`. That is the one case a broken + implementation gets right. +- Step 6 with a single run, where serial execution is indistinguishable from + accidental parallelism. + +## Traps + +- Making cancel legal only from `Scheduled`. Nobody would ship that lifecycle. +- Treating `Verifying` as a function call. It removes the seam T4.2 needs. +- Interleaving steps within a run "since they look independent". The serial spine + is `(TenantId, RunId)`; parallelism lives between runs and inside declared + fan-out only. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.2, §5.3 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.4-attempt-lifecycle-and-retry.md b/tasks/T1.4-attempt-lifecycle-and-retry.md new file mode 100644 index 0000000..dace6dc --- /dev/null +++ b/tasks/T1.4-attempt-lifecycle-and-retry.md @@ -0,0 +1,84 @@ +# T1.4 — Attempt lifecycle and retry + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Retry creates attempt N+1 as a new record. Attempt N is never mutated. + +## Facts (inlined — no spec read needed) + +- Each step execution is one or more kernel attempts. **Retry creates attempt + N+1 and never mutates attempt N.** Two payoffs: "did the retry do better, and + why" becomes answerable, and log replay is idempotent for free. +- Attempts within a step are **strictly serial** — a retry needs the prior + failure to exist first. +- The immutability is also the foundation of the cheapest grading signal there + is: the attempts of one step are a comparison group on identical context, free + and already on disk (T5.10). +- A failure and its successful retry are **not** a judge comparison. That pair is + consumed structurally — what differed between attempt N and N+1 is attributed + to the `StepId`. Only same-outcome attempts go to a judge. + +## Steps + +1. Key attempt state on `(BranchKey, AttemptNo)`. `AttemptNo` starts at 1 and + increments; no reuse within a branch. +2. On retry, allocate `AttemptNo + 1` and write a fresh record. No update path to + an existing attempt row exists — do not write one. +3. Apply the step's `RetryPolicy` (count and backoff) between attempts, awaiting + the cancellation token so a cancel during backoff is honoured. +4. Record on each attempt what differs from its predecessor: context partition + (T1.5), tool selection, prompt ref (T1.6). That delta is what T5.10 attributes. +5. Test: script the stub to fail twice then succeed. Snapshot attempts 1 and 2 + before attempt 3 runs, snapshot again after, assert byte equality. + +## Acceptance + +- A step failing twice then succeeding produces **three** attempt records. +- Attempts 1 and 2 are byte-identical before and after attempt 3. + +## Verify + +**Harness:** stub model scripted `fail, fail, succeed` for one step. + +**Integration test** — `tests/it_retry_immutability.rs`: +1. Run the step; pause after attempt 2 completes. +2. Snapshot the **serialized bytes** of attempt records 1 and 2. +3. Let attempt 3 run to success. +4. Re-read attempts 1 and 2; assert byte equality with the snapshot. +5. Assert exactly three attempt records exist, numbered 1, 2, 3. +6. Assert the attempts ran **serially**: attempt N's terminal timestamp precedes + attempt N+1's start. +7. Assert the per-attempt delta fields (context partition, tool selection, prompt + ref) are present, so T5.10 has something to attribute. +8. Cancel during retry backoff; assert the run stops rather than sleeping out the + full delay. + +**Command:** `cargo test -p executor retry` + +**False pass:** +- Comparing attempt records through the query surface, which may reconstruct them + identically from the log even if the state table was mutated. Compare the + stored state rows **and** the log records. +- Asserting "three records exist" alone — an implementation that appends a new + row and *also* updates row 2 passes that. Step 4 is what catches it. + +## Traps + +- "Updating" the attempt row with the final outcome to keep the table small. It + destroys the retry evidence, and the learning loop exists to consume it. +- Retrying in parallel to save latency. Attempt N+1 needs N's failure. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.1, §5.3, §11.5, §11.8 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.5-context-partition-capture.md b/tasks/T1.5-context-partition-capture.md new file mode 100644 index 0000000..c5834d9 --- /dev/null +++ b/tasks/T1.5-context-partition-capture.md @@ -0,0 +1,83 @@ +# T1.5 — Context partition capture + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Record, per model step, which context was packed, which was available but not +packed, and which was dropped — as identifiers. + +## Facts (inlined — no spec read needed) + +- Three buckets: **packed**, **available-not-packed**, **dropped**. +- They hold **identifiers**, not text. That keeps them small enough to inline in + the attempt view rather than going by blob reference, which is what lets a + verifier read them without paying for prompt text. +- This is the signal behind "retried three times because context was missing X" — + the thing a pass rate cannot see and a learning loop needs. +- It is **not derivable after the fact** from the conversation. If it is not + captured at pack time it is gone, which is why its absence is a test failure + rather than a gap. + +## Steps + +1. Define `ContextPartition { packed: Vec, available: Vec, + dropped: Vec }`. Ordered collections, so serialization is stable. +2. Capture at the point the prompt is assembled — the packer already knows all + three sets; recording them costs a clone of id vectors. +3. Attach the partition to the attempt record and emit it in the same + `EventLog::commit` as the rest of the attempt transition. +4. Surface it on `AttemptView.context: Option` for verifiers + (T4.2) and graders. +5. Test: assert every model-step attempt in a completed run carries a partition; + a missing one fails the test rather than being tolerated as `None`. + +## Acceptance + +- Partition present on **every** model step. +- Not derivable from the conversation, so its absence is a test failure. + +## Verify + +**Harness:** a workflow with a packer configured to drop known items, so the +expected partition is known in advance. + +**Integration test** — `tests/it_context_partition.rs`: +1. Configure the packer with 10 candidate context items and a budget admitting 6. +2. Run a model step. +3. Assert the recorded partition has `packed.len() == 6`, `available` + + `dropped` covering the remaining 4, and that the three sets are **disjoint** + and their union is the full candidate set. +4. Assert `dropped` is non-empty — a capture that only ever records `packed` is + the common half-implementation and passes any "partition present" check. +5. Iterate every model-step attempt in a completed multi-step run; assert + `context.is_some()` for **each**. A `None` fails the test. +6. Assert the partition survives `assert_refold_identical`. + +**Command:** `cargo test -p executor context_partition` + +**False pass:** +- Asserting only `context.is_some()`. An empty partition is `Some`. Step 3's set + arithmetic is the real check. +- Testing on a workflow whose budget admits everything, so `dropped` is + legitimately empty and step 4 cannot fail. + +## Traps + +- Storing the packed context text here. It belongs in the prompt blob (T1.6); + inlining text blows broker payload limits and penalizes verifiers that need + none of it. +- Capturing only `packed`. The interesting signal is usually in `dropped`. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §10.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.6-prompt-and-output-blob-capture.md b/tasks/T1.6-prompt-and-output-blob-capture.md new file mode 100644 index 0000000..de80154 --- /dev/null +++ b/tasks/T1.6-prompt-and-output-blob-capture.md @@ -0,0 +1,82 @@ +# T1.6 — Prompt and output blob capture + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Write the prompt to the blob store when it is built, and the output when the +model responds. Record both as `BlobRef` on the attempt. + +## Facts (inlined — no spec read needed) + +- Two capture points: `prompt-built` and model response. **The prior + implementation had neither** — every downstream consumer worked from + identifiers alone and could not answer "what did the agent actually see". +- Prompts and outputs are large, so they go **by reference**; context partitions + are identifiers and go inline (T1.5). That split is what makes lazy blob access + (T4.3) possible. +- Blobs are content-addressed within a tenant, never across (T0.7). +- A `BlobRef` whose body later returns `None` is normal after reduction (T8.4) — + the log carries a `Reduced { original, summary }` event saying what replaced it. + +## Steps + +1. At prompt assembly, serialize the final prompt, `BlobStore::put`, keep the + returned `BlobRef`. +2. On model response, `put` the raw output, keep its `BlobRef`. +3. Attach both refs to the attempt record; commit them in the same + `EventLog::commit` as the attempt transition, so a crash cannot leave a + dangling ref in state with no log record. +4. Surface as `AttemptView.prompt: Option` and + `AttemptView.output: Option`. +5. Test across a process restart: complete a run, drop the process, reopen the + store, fetch the prompt by ref, assert the text. + +## Acceptance + +- Prompt text retrievable from the blob store by ref **after a process restart**. + +## Verify + +**Harness:** embedded store on a temp path that survives process exit; the test +runs the agent in a **child process**, then reopens the store in the parent. + +**Integration test** — `tests/it_blob_capture_restart.rs`: +1. Child process: run one model step to completion, print the store path, exit. +2. Parent: reopen the store, read the attempt, take `prompt` and `output` refs. +3. `BlobStore::get` both; assert the prompt bytes equal the **assembled** prompt + (with template variables substituted), not the template. +4. Re-hash both bodies; assert each matches its ref. +5. Crash case: arm a fault hook between the blob `put` and the commit; assert on + reopen there is no attempt record pointing at a body-less ref, and no orphan + body that no record points at. + +**Command:** `cargo test -p executor blob_capture -- --test-threads=1` + +**False pass:** +- Reading the blob back in the same process from a warm cache. The restart is the + point — an in-memory blob map passes everything else. +- Asserting the prompt is non-empty rather than comparing to the expected + assembled text. Capturing the template instead of the rendered prompt passes a + non-empty check. + +## Traps + +- Putting the blob outside the commit transaction and recording the ref inside + it, or the reverse. Either way a crash leaves a ref with no body or a body no + record points at. +- Capturing the prompt template instead of the assembled prompt. The template is + in the workflow definition; what the model saw is not. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.6, §10.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.7-episode-query-surface.md b/tasks/T1.7-episode-query-surface.md new file mode 100644 index 0000000..1d0857b --- /dev/null +++ b/tasks/T1.7-episode-query-surface.md @@ -0,0 +1,103 @@ +# T1.7 — Episode query surface + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Read an episode — run, branches, attempts, refs — with identical results whether +served from materialized state or a cold re-fold. + +## Facts (inlined — no spec read needed) + +```rust +pub struct AttemptView { + pub step: StepId, + pub attempt: AttemptNo, + pub state: AttemptState, + pub workflow_version: WorkflowVersion, + pub context: Option, // identifiers, small, inline + pub prompt: Option, // large, by reference + pub output: Option, + pub tools: Option, + pub usage: Option, +} +``` + +- **Include failed attempts.** "Retried three times because context was missing + X" is the learning signal; shipping only the winning attempt discards it. +- Queries default to the **live branch**; grading may read all branches. Only the + live branch is exported. +- Materialized state is a cache of the log. If the query answers differently from + a cold re-fold, the state has hidden inputs — that is T0.8's property applied + to the read path. +- Blob bodies are **not** included. The view carries refs; the caller fetches + what it needs (T4.3). + +## Steps + +1. Define `EpisodeView` — run metadata, branch list with fork points, attempts + per branch — and `AttemptView` exactly as above. +2. Implement the query against materialized state, defaulting to the live branch + with an explicit `all_branches` option for grading. +3. Implement the same query against a cold re-fold (T0.8's `rebuild`), used as + the fallback when state is absent and as the test oracle. +4. Include every attempt regardless of outcome. Sort by `(BranchId, AttemptNo)` + so ordering is stable. +5. Test: build a run with a retry and a rewind; serialize the view from state and + from a cold re-fold; assert equal. + +## Acceptance + +- Query returns the same structure whether served from materialized state or a + cold re-fold. +- Failed attempts are present in the returned view. + +## Phase gate + +P1 closes when a full run is recorded and re-derives byte-identically. + +## Verify + +**Harness:** one recorded run containing a retry **and** a rewind — the two cases +where the two read paths can diverge. + +**Integration test** — `tests/it_episode_query_equivalence.rs`: +1. Build the run: 3 steps, one step failing twice, then a rewind at step 2 + producing a second branch. +2. Query the episode from **materialized state** → `view_a`. +3. Drop the state tables; query again, served by cold re-fold → `view_b`. +4. Assert `serialize(view_a) == serialize(view_b)`. +5. Assert `view_a` contains **all** attempts including the two failures — count + them explicitly, do not just check non-empty. +6. Assert the default query returns only the **live** branch, and that the + explicit all-branches mode returns both. +7. Instrument `BlobStore::get`; assert the query itself makes **zero** calls. + +**Command:** `cargo test -p query episode_equivalence` + +**False pass:** +- Testing on a linear run with no retry and no rewind. Both paths trivially + agree, and the divergence lives exactly in the cases omitted. +- Step 5 as `assert!(!attempts.is_empty())`. A view returning only the winner + passes that. +- Step 4 with a `PartialEq` derived over unordered collections. + +## Traps + +- Filtering to the successful attempt "because that is what the UI shows". The + learning loop is the other consumer and it needs the failures. +- Eagerly resolving `BlobRef` into text inside the view. That is T4.3's + regression, introduced one layer earlier. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.5, §10.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T1.8-p1-composition-gate.md b/tasks/T1.8-p1-composition-gate.md new file mode 100644 index 0000000..6cd6859 --- /dev/null +++ b/tasks/T1.8-p1-composition-gate.md @@ -0,0 +1,103 @@ +# T1.8 — P1 composition gate + +| Field | Value | +|---|---| +| Phase | P1 — Walking skeleton | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P2 | + +## Goal + +Prove T1.1–T1.7 compose into one recorded run that re-derives byte-identically, +and that the pluggable parts of the skeleton are genuinely pluggable. + +**Phase gate criterion:** one full run is recorded and re-derives +byte-identically. + +## Facts (inlined — no spec read needed) + +- The walking skeleton is **one full run of one workflow against a stub model, in + embedded mode, with no network — recorded, verified, and re-derivable from the + log.** Nothing after it is worth starting until it runs. +- Nothing is built before the record is trustworthy: no dashboards and no + learning loop until this passes. +- The composition properties no single P1 task owns: + - cancellation from **any** state leaves a legal attempt state and a complete + log (T1.1 + T1.3 + T0.2); + - a retry, a cancel and a normal completion all produce episodes that survive + drop-and-re-fold (T1.4 + T1.7 + T0.8); + - context partitions and blob refs are present on **every** model step of + **every** attempt, including failed ones (T1.5 + T1.6 + T1.4). +- The serial spine is `(TenantId, RunId)`. Parallelism lives between runs. A + composition test with one run cannot see a violation of this. + +## Steps + +1. Build the run matrix: `{clean, one retry, two retries, cancelled mid-step, + cancelled during Verifying}` × `{served from state, served from cold re-fold}`. +2. Assert every cell produces a consistent episode and a legal state sequence. +3. Swap the stub model script and the verifier implementation without touching + executor code — the skeleton must not know which is installed. +4. Run 50 runs concurrently across 2 tenants; assert per-run ordering and + cross-run independence. +5. Make this the required CI job gating P2. + +## Acceptance + +- Every matrix cell is green. +- A full run is recorded and re-derives **byte-identically**. +- Swapping the stub script or the verifier requires no executor change. + +## Verify + +**Harness:** the P0 in-memory ports (T0.9) for speed, plus the real `redb` path +for at least one cell — a composition that only works in memory is not a +composition. + +**Integration test** — `tests/it_p1_composition.rs`: +1. **Matrix:** for each of the 10 cells, assert the episode from materialized + state and from cold re-fold are **byte-identical**, and that the run's state + sequence is legal under T0.2's table. +2. **Cancel sweep:** cancel from `Scheduled`, `Running`, `Verifying`, `Grading`. + For each, assert the attempt lands in `Cancelled` (no dispatched intent), + every spawned task terminated before `RunScope` returned, and the log is + complete up to the cancel. +3. **Capture completeness:** across all cells, assert **every** model-step + attempt — including failed ones — carries a context partition and prompt and + output refs. Iterate, do not sample. +4. **Immutability under composition:** in the two-retry cell, snapshot attempts 1 + and 2, run to completion, assert byte equality. Then drop state, re-fold, and + assert the attempts are still identical. +5. **Isolation:** 50 concurrent runs across 2 tenants. Assert each run's records + are totally ordered, no two steps of one run overlap in time, and no record + carries the wrong tenant. +6. **Pluggability:** run the same matrix with a second stub script and a second + verifier. Assert zero changes to executor code — enforce by keeping the + executor crate's test-only surface empty. +7. **Regression:** re-run every P1 and P0 task suite in the same job. + +**Command:** `cargo test -p skeleton --test it_p1_composition` + +**False pass:** +- Testing only the clean cell. Retry and cancel are where the fold, the scope and + the capture paths interact, and each is individually green already. +- Step 3 sampling one attempt. A capture path that skips failed attempts passes a + sample and discards the entire learning signal. +- Step 5 with one run, where serial execution and accidental parallelism are + indistinguishable. +- Step 1 comparing episodes with `PartialEq` rather than serialized bytes. + +## Traps + +- Declaring the gate green with `Verifying`/`Grading` collapsed into function + calls. The states must appear in the log, or T4.2's barrier has nowhere to live. +- Building any dashboard or grading work before this is green. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.2, §5.3, §16, §20 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.1-write-ahead-intent.md b/tasks/T2.1-write-ahead-intent.md new file mode 100644 index 0000000..23bf833 --- /dev/null +++ b/tasks/T2.1-write-ahead-intent.md @@ -0,0 +1,123 @@ +# T2.1 — Write-ahead intent + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T2.2, T6.4 | + +## Goal + +Three intent records around every external effect, so a crash can be classified +rather than guessed at. + +## Facts (inlined — no spec read needed) + +``` + 1. append Intent{Pending} ──► commit + fsync + ◄── window A: crash here, the call was never issued + 2. append Intent{Dispatched} ──► commit + fsync + 3. perform the call + ◄── window B: crash here, the call may have landed + 4. append outcome, Intent{Committed} ──► commit + fsync +``` + +- **Three records, not two.** Two cannot separate the windows: a crash before the + call and a crash after it both leave a lone `Pending` with no outcome, which + makes every interrupted effect maximally suspicious and pushes recoverable work + into `Indeterminate`. The second fsync buys the distinction, is paid only on + steps with external effects, and is small next to the call it guards. +- Appending after the fact records history but does not make a failed step + resumable — the dangerous window is *before* the record exists. +- Restart classification, from the last committed intent state: + +| Last state | Meaning | Resolution | +|---|---|---| +| `Pending` | the call was never issued | retry freely, whatever the effect class | +| `Dispatched` | the call *may* have been issued | by effect class (T2.2) | +| `Committed` | outcome already recorded | nothing to do | + +- Intents always commit with `Durability::Immediate`. **Batching them defeats + their only purpose.** +- Tool calls need this more than model calls. A model call is a metered read; a + tool call writes files, pushes commits, and touches the world. +- The intent record is also what decides `Cancelled` vs `Indeterminate` on a + cancelled attempt (T0.2, T1.1). + +## Steps + +1. Define `IntentState { Pending, Dispatched, Committed }` and an `IntentId` + scoped to `(BranchKey, AttemptNo, step)`. Include the idempotency key or + request id the effect class will need at recovery (T2.2). +2. Wrap every external effect in the four-step sequence above. Each append is its + own `EventLog::commit` with `Immediate` durability — three separate fsyncs, by + design. +3. Make the wrapper the **only** way to reach a tool. If a call site can issue an + effect without an intent, the protocol is decorative. +4. Build the crash harness: a test hook that aborts the process at a named point. + Two named points minimum — after `Pending` commit, and after `Dispatched` + commit but before outcome commit. +5. On restart, scan for the newest intent per `(BranchKey, AttemptNo)` and + classify per the table. Emit the classification as a log event so the decision + itself is recorded. +6. Assert the fsync count on the intent path in a test — a batching "optimization" + introduced later must fail loudly. + +## Acceptance + +- Crash harness kills the process in both windows: between `Pending` and + `Dispatched`, and between `Dispatched` and outcome-commit. +- Restart classifies the first as "never sent" and the second as "may have been + sent". A two-phase implementation cannot pass this — both crashes leave one + `Pending` with no outcome and are indistinguishable. That is the point of the + test. + +## Verify + +**Harness:** a real child-process kill (`kill -9`), not a panic. Named abort +points compiled under a `test-hooks` feature. A recording tool that appends to a +file outside the database, so "did the call actually happen" is observable +independently of what the log claims. + +**Integration test** — `tests/it_intent_windows.rs`: +1. **Window A:** arm the abort after the `Pending` commit, before `Dispatched`. + Kill. Reopen. + - Assert the last intent state is `Pending`. + - Assert restart classifies it **"never sent"**. + - Assert the side-channel file is **empty** — the call really did not happen. +2. **Window B:** arm the abort after the `Dispatched` commit, before the + outcome commit. Kill. Reopen. + - Assert the last intent state is `Dispatched`. + - Assert restart classifies it **"may have been sent"**. +3. Assert the two classifications **differ**. A two-phase implementation makes + them identical, which is the whole point of the test. +4. Count fsyncs on the intent path for one guarded effect; assert exactly 3 + commits, so a later batching "optimization" fails loudly. +5. Bypass audit: assert no tool can be invoked except through the intent wrapper — + a compile-level guard if the tool handle is only constructible inside it. + +**Command:** `cargo test -p durability --features test-hooks intent_windows -- --test-threads=1` + +**False pass:** +- Simulating the crash with `panic!` and catching the unwind. Destructors run and + buffered writes flush — precisely what a real crash does not do. +- Asserting only that both windows "recover". Both recover under a two-phase + implementation too; they just recover **identically**. Step 3 is the test. +- Trusting the log to tell you whether the call happened. Step 1's side-channel + file is the independent witness. + +## Traps + +- Batching the two pre-call fsyncs into one commit. It compiles, it is faster, + and it deletes the distinction the task exists to create. +- Recording the intent after dispatch "since that is when we know the request + id". Derive the request id before dispatch instead. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.4 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.2-effect-class-recovery.md b/tasks/T2.2-effect-class-recovery.md new file mode 100644 index 0000000..db8f425 --- /dev/null +++ b/tasks/T2.2-effect-class-recovery.md @@ -0,0 +1,127 @@ +# T2.2 — Effect-class recovery + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T6.5, T8.3 | + +## Goal + +Resolve a `Dispatched`-but-unresolved intent by the tool's declared effect class. +`Indeterminate` becomes a real terminal state with an operator path. + +## Facts (inlined — no spec read needed) + +```rust +pub struct ToolRegistration { + pub id: ToolId, + /// No Default. The author is the only party who knows this. + pub effects: EffectClass, + pub caps: CapabilitySet, + pub timeout: Duration, +} + +/// Each class carries what its recovery path actually needs. A bare +/// discriminant would let a tool claim `Idempotent` while withholding the one +/// thing that makes the claim actionable. +pub enum EffectClass { + Idempotent { key: KeyDerivation }, + Queryable { lookup: RequestIdLookup }, + Unsafe, +} +``` + +Resolution for a `Dispatched` intent: + +| Class | Recovery | +|---|---| +| `Idempotent` | retry with the same idempotency key; the provider deduplicates | +| `Queryable` | ask the provider whether the request id landed, then complete or retry | +| `Unsafe` | **never auto-retry.** Attempt becomes `Indeterminate`, operator notified | + +- The third row is the honest one. Some effects cannot be made safe by any + protocol. The intent log's value is converting an invisible unknown into a + recorded one: `Indeterminate` is a fact a grader and an operator can both use; + a silently retried payment is not. +- A tool that cannot state whether it is safe to retry **does not register**. +- Declaration is not enforcement — that is T8.3's keyed capability, and the + sandbox (T6.5) is the third layer. This task builds layer one plus recovery. +- Until built-in tools are audited, default `Unsafe` and never auto-retry. + +## Steps + +1. Define `EffectClass` with payloads as above — no bare discriminants. A + registration declaring `Idempotent` without a `KeyDerivation` must not compile. +2. Extend `ToolRegistration` and reject registration of a tool with no declared + effect class. No `Default` impl. +3. Implement the recovery pass, driven by T2.1's restart scan: for each + `Dispatched` intent, dispatch on the registered class. +4. `Idempotent`: re-derive the key from the recorded arguments, re-issue, record + the outcome. Same key, so the provider deduplicates. +5. `Queryable`: call the registered `RequestIdLookup` with the recorded request + id; complete from the provider's answer or retry if it never landed. +6. `Unsafe`: transition the attempt to `Indeterminate`, emit the operator + notification, and **stop**. No retry path exists for this arm — do not add one + behind a flag. +7. Build three test tools, one per class, each with an injectable crash point + mid-call. + +## Acceptance + +- Three tools, one per class; crash mid-call for each. +- `Idempotent` retries and produces no duplicate effect. +- `Queryable` reconciles against the provider and completes or retries correctly. +- `Unsafe` becomes `Indeterminate` and raises, rather than retrying. + +## Verify + +**Harness:** three real test tools, one per class, each writing to an +**external** side-effect ledger (a file or a counter service) so effect count is +observable independently of the log. Plus T2.1's child-process kill. + +**Integration test** — `tests/it_effect_recovery.rs`: +1. **`Idempotent`:** tool appends to the ledger keyed by its idempotency key. + Kill mid-call, restart, let recovery retry. Assert the ledger holds + **exactly one** entry and the attempt completes. +2. **`Queryable`:** tool records a request id; the fake provider is configurable + to answer "landed" or "never arrived". + - Provider says landed → assert recovery **completes** without re-issuing; + ledger count stays 1. + - Provider says never arrived → assert recovery **retries**; ledger count + becomes 1 (from the retry, not two). +3. **`Unsafe`:** kill mid-call. Assert the attempt becomes `Indeterminate`, the + operator notification fires, and the ledger count is **unchanged** — no retry + was issued. +4. Registration: a tool with no declared effect class is rejected; a + `Idempotent` registration without a `KeyDerivation` fails to compile + (`trybuild`). +5. Key stability: run recovery twice; assert the derived key is identical both + times. + +**Command:** `cargo test -p durability --features test-hooks effect_recovery -- --test-threads=1` + +**False pass:** +- Counting effects only in the framework's own log. The log is what you are + testing; the external ledger is the witness. +- The `Idempotent` case passing because the provider stub deduplicates on + *content* rather than on the supplied key — it then passes even when the key + derivation is broken. Make the stub key-only. +- Testing `Unsafe` by asserting an error is returned. `Indeterminate` is a state, + not an error, and the distinction is the deliverable. + +## Traps + +- A "retry anyway with a warning" path for `Unsafe`. The class exists to say the + answer is no. +- Deriving the idempotency key at recovery time from something that changed — it + must derive from the recorded arguments, deterministically. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.4, §13.1, §13.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.3-rewind-as-fork.md b/tasks/T2.3-rewind-as-fork.md new file mode 100644 index 0000000..4a5dac6 --- /dev/null +++ b/tasks/T2.3-rewind-as-fork.md @@ -0,0 +1,104 @@ +# T2.3 — Rewind as fork + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Rewind allocates a new `BranchId` and starts its LSNs at zero. Nothing is +deleted. + +## Facts (inlined — no spec read needed) + +``` + lsn 0 ─ 1 ─ 2 ─ 3 ─ 4 ─ 5 ─ 6(failed) branch 0, retained + └─ 0 ─ 1 ─ 2 ─ ... branch 1, forked at (0, 3) +``` + +- Same rule as attempt N+1 never mutating attempt N, for the same reason: **the + discarded branch is the evidence.** Truncating it destroys the failure that + motivated the rewind, which is what the learning loop exists to consume. +- Queries default to the live branch; grading may read all of them. **Only the + live branch is exported.** +- Rewinding past a `Committed` intent with a non-idempotent effect is a + **compensation** problem, not a replay problem. The log records what happened; + it cannot un-happen it. Flag rather than pretend. +- A rewind is also one of the three mutators the verifier snapshot barrier + defends against (T4.2) — it can fork a new `BranchId` while verifiers hold a + view of the old one. + +## Steps + +1. Allocate the next `BranchId` for the run. Record a fork event carrying + `(parent_branch, parent_lsn)` — the fork point is data, not inference. +2. Start the new branch's LSNs at 0. They are per branch already (T0.3), so this + falls out of the key shape rather than needing a reset. +3. Seed the new branch's state by folding the parent branch up to the fork LSN. + Do not copy the parent's records into the new branch. +4. Mark which branch is live. Default every query to it; expose an explicit + all-branches mode for grading (T1.7). +5. Restrict export to the live branch — check this where the outbox entry is + built (T0.6), not at the relay. +6. Before rewinding past a `Committed` intent, look up the effect class. If it is + not idempotent, emit a compensation-required flag on the fork event and + surface it. Do not block the rewind and do not silently proceed. + +## Acceptance + +- Rewind a 6-step run to step 3, run a different path. +- The original branch is **fully readable**. +- The live-branch query returns only the new path. + +## Verify + +**Harness:** a 6-step run driven by the stub model, plus the raw log reader so +branch 0 can be inspected directly rather than through the query surface. + +**Integration test** — `tests/it_rewind_fork.rs`: +1. Run 6 steps to a failure at step 6. Snapshot branch 0's serialized records. +2. Rewind to step 3; run a different path on branch 1. +3. Assert branch 0's records are **byte-identical** to the snapshot — nothing + truncated, nothing rewritten. +4. Assert branch 1's LSNs start at 0 and its fork event carries + `(parent_branch: 0, parent_lsn: 3)`. +5. Assert the default query returns only branch 1's path; the all-branches query + returns both. +6. Assert branch 1 does **not** contain copies of branch 0's records — count + records per branch and compare against expected. +7. Export path: assert outbox entries exist only for branch 1. +8. Compensation flag: rewind past a `Committed` non-idempotent intent; assert the + fork event carries the compensation-required flag and the rewind still + proceeds. +9. `assert_refold_identical` on both branches. + +**Command:** `cargo test -p durability rewind` + +**False pass:** +- Step 3 checked through the query surface, which defaults to the live branch and + will happily report "branch 0 unchanged" without reading it. Read the raw log. +- Step 6 omitted: an implementation that copies parent records into the child + passes every other assertion here and makes every later "what happened" query + ambiguous. +- Asserting the fork point by re-deriving it from LSNs rather than reading the + recorded fork event. + +## Traps + +- Truncating the log at the fork point "since that history is dead". It is the + evidence. +- Copying parent records into the child branch. It duplicates history and makes + every later "what actually happened" query ambiguous. +- Exporting all branches. Downstream consumers then see two contradictory + timelines for one run. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.5 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.4-schema-evolution-end-to-end.md b/tasks/T2.4-schema-evolution-end-to-end.md new file mode 100644 index 0000000..346238d --- /dev/null +++ b/tasks/T2.4-schema-evolution-end-to-end.md @@ -0,0 +1,88 @@ +# T2.4 — Schema evolution end-to-end + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Change the event enum for real and prove a log recorded in P1 still folds. This +is T0.4's framework exercised against actual history rather than a synthetic +fixture. + +## Facts (inlined — no spec read needed) + +- Depends on T0.4 (upcaster registry) and T0.3 (`SchemaVersion` on every record). +- The rules being tested: variants are **never removed or repurposed**; + deprecated variants stay decodable forever; migrations are upcasters applied on + read, never a rewrite of stored records. +- The failure this prevents: two years of records, one `WorkEvent` variant + renamed, and the drop-and-re-fold property is silently gone. Nothing fails at + the time of the rename — it fails at the first cold rebuild. +- The v1 fixture must be a **real P1 log**, not a hand-written one. A synthetic + fixture only tests the variants whoever wrote it remembered. + +## Steps + +1. Capture a full P1 run's log as a committed fixture, at v1, with its blobs. + Store the expected folded state alongside it. +2. Make a genuine v2 change: add a field to an existing variant, or split one + variant into two. Do not rename or delete anything. +3. Bump `CURRENT_SCHEMA` to v2 and register the v1→v2 upcaster. +4. Run the fold over the v1 fixture under the v2 binary; assert the folded state + equals the recorded expectation. +5. Record a fresh v2 fixture from the same workflow and assert the two folded + states agree — an upcast v1 log and a native v2 log must be + indistinguishable downstream. +6. Extend T0.3's directory-walk fixture test so both version folders are covered + permanently, not just in this task's test. + +## Acceptance + +- A v1 log fixture recorded in P1 still folds correctly after the event enum + changes. + +## Verify + +**Harness:** a **real P1 run's** log committed as a fixture, with its expected +folded state committed beside it. Not hand-written. + +**Integration test** — `tests/it_schema_evolution.rs`: +1. Under the v2 binary, fold the committed v1 fixture → `state_a`. +2. Assert `state_a` equals the committed expected state, byte for byte. +3. Record a fresh v2 run of the same workflow → fold → `state_b`. +4. Assert `state_a` and `state_b` are equivalent under the fields that should not + have changed. An upcast v1 log and a native v2 log must be indistinguishable + downstream. +5. Assert the v1 fixture file's bytes are unchanged by the test run — the fixture + is read-only history. +6. Variant-removal guard: a test that enumerates v1's variant set and asserts + every one is still decodable under v2. + +**Command:** `cargo test -p log schema_evolution` + +**False pass:** +- Regenerating the fixture when the test goes red. That is the one move that + destroys the guarantee — the fixture is the historical record, the upcaster is + what changes. Make the fixture path read-only in the test and assert on that. +- A v2 change that touches no variant present in the P1 fixture, so no upcaster + logic runs. Step 6 catches the removal case; choose a v2 change that provably + affects a recorded variant. + +## Traps + +- "Fixing" the fixture bytes when the test fails. The fixture is the historical + record; the upcaster is what changes. +- A v2 change that removes a variant, which passes this test today because + nothing in the P1 fixture used it, and breaks the moment an older log does. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.5-checkpoints.md b/tasks/T2.5-checkpoints.md new file mode 100644 index 0000000..31db869 --- /dev/null +++ b/tasks/T2.5-checkpoints.md @@ -0,0 +1,91 @@ +# T2.5 — Checkpoints + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Materialized state snapshots tagged with their LSN, so restart folds forward from +the newest one instead of from zero. + +## Facts (inlined — no spec read needed) + +- A checkpoint is a materialized state snapshot tagged with its LSN. Restart + folds forward from the newest one at or below the target LSN. +- **An optimization only.** Deleting every checkpoint costs startup time and + nothing else. That property is the acceptance test, and it is what keeps + checkpoints from quietly becoming a second source of truth. +- Port surface already exists from T0.5: + `put_checkpoint(key, upto, state)` and `latest_checkpoint(key, upto) -> Option`. + `None` means fold from LSN 0. +- Checkpoints are per `BranchKey`, like everything else in the log. + +## Steps + +1. Serialize the materialized state (T0.8) deterministically — the same ordered + collections, so a checkpoint written twice from the same state is byte-equal. +2. Write checkpoints on a policy: every N records or every M seconds per branch. + Keep the policy in one place and make it configurable rather than scattered. +3. `rebuild(key, upto)`: call `latest_checkpoint`, deserialize if present, then + fold records from `checkpoint.lsn + 1`. On `None`, fold from 0. +4. Never let the checkpoint be the only holder of a fact. Validate this by making + the `None` path the default in tests. +5. Add a prune path — old checkpoints for a branch are deletable at any time + without coordination. +6. Test both directions: with checkpoints and after deleting them all, comparing + final state and reporting both startup times. + +## Acceptance + +- Deleting all checkpoints changes startup time and **nothing else** — final + state identical. + +## Verify + +**Harness:** a run long enough to trigger the checkpoint policy several times — +at least 3 checkpoints, so "newest at or below" is a real choice. + +**Integration test** — `tests/it_checkpoints.rs`: +1. Run to completion with checkpointing on. Record final state bytes and startup + time for a rebuild. +2. Delete **every** checkpoint row. Rebuild from LSN 0. +3. Assert the final state bytes are **identical**; assert startup time differs + (log both, do not assert a threshold — that is flaky). +4. `latest_checkpoint(key, upto)` with `upto` between two checkpoints: assert it + returns the **lower** one, not the newest overall. +5. Determinism: write a checkpoint twice from the same state; assert byte + equality of the two serializations. +6. Crash between checkpoint write and subsequent commits; assert rebuild still + lands on the same state. +7. Prune: delete an older checkpoint while a newer exists; assert rebuild is + unaffected. + +**Command:** `cargo test -p durability checkpoints` + +**False pass:** +- A run short enough to produce zero or one checkpoint. Then step 2 deletes + nothing and the test is vacuous — assert the checkpoint count is ≥ 3 before + deleting. +- Step 4 omitted: an implementation returning the globally newest checkpoint + passes everything else and breaks the moment rewind or point-in-time rebuild + needs an earlier LSN. +- Comparing state with `PartialEq` rather than bytes. + +## Traps + +- Writing a checkpoint inside the same transaction as the commit path and then + depending on it for correctness. It is a cache; keep it separable. +- A checkpoint that serializes a `HashMap`. Byte-identity fails intermittently + and reads as flakiness. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §7, §8.6 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.6-crash-matrix.md b/tasks/T2.6-crash-matrix.md new file mode 100644 index 0000000..6fb7488 --- /dev/null +++ b/tasks/T2.6-crash-matrix.md @@ -0,0 +1,108 @@ +# T2.6 — Crash matrix + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Systematic `kill -9` at every kernel transition, in a loop, verifying +recoverability after each. This is the phase gate. + +## Facts (inlined — no spec read needed) + +- P2 is the phase most likely to be skipped and most expensive to retrofit. The + matrix is what proves the three preceding tasks actually hold together. +- What "recoverable" means concretely: on restart, every attempt resolves to a + legal state per T0.2's table; every `Dispatched` intent resolves per T2.2's + class table; the fold from LSN 0 reproduces the state exactly (T0.8). +- "Zero double-applied effects" is the second, independent assertion. An + idempotent tool retried after a crash must leave one effect, not two. +- Lessons this suite exists to catch: + - *A green suite says nothing about coverage.* A prior idempotency feature + generated its own keys and checked for duplicates among them — unreachable + for a whole phase, with tests asserting the count was zero. + - *A test run that prints nothing cannot distinguish slow from hung.* Per-test + progress and per-test timeouts from the first commit. + +## Steps + +1. Enumerate the crash points: every kernel transition emitted by T1.3/T1.4, plus + T2.1's intra-intent windows, plus each of T0.6's four table writes. +2. Build the harness as a **real process kill** (`kill -9` on a child), not a + panic-and-catch. Shadow paging behaviour under a hard kill is part of what is + being tested. +3. Drive a workload with a scripted stub model (T1.2) and the three effect-class + tools (T2.2), so both durability and effect recovery are exercised. +4. Seed the crash-point selection from a recorded seed. Print the seed on + failure — an unreproducible crash matrix failure is not a finding. +5. After each restart assert: state is legal, intents resolve, the cold re-fold + matches, effects applied exactly once. +6. Emit per-test progress and enforce a per-test timeout so a hang is + distinguishable from a slow case. +7. Run 500 randomized crash points in CI; keep the exhaustive enumeration as a + longer nightly job. + +## Acceptance + +- 500 randomized crash points, **zero unrecoverable states, zero double-applied + effects**. + +## Phase gate + +P2 closes on a green crash matrix. + +## Verify + +**Harness:** the P2 capstone. Child process under `kill -9`; the external +side-effect ledger from T2.2; the three effect-class tools; the stub model; a +recorded seed printed on every run. + +**Integration test** — `tests/it_crash_matrix.rs`: +1. Enumerate crash points: every kernel transition, T2.1's two intent windows, + T0.6's four table writes. Assert the enumerated count matches an expected + constant, so a newly added transition without a crash point fails the test. +2. For each of 500 seeded random points: run the workload, `kill -9` at the + point, restart, then assert **all four** properties: + - every attempt is in a legal state per T0.2's table; + - every `Dispatched` intent resolved per T2.2's class table; + - `assert_refold_identical` passes; + - the external ledger shows **exactly one** effect per intended effect. +3. Print the seed and the crash point on failure; a failure that cannot be + replayed is not a finding. +4. Per-test progress output and a per-test timeout, so a hang is distinguishable + from a slow case. +5. Keep the **exhaustive** enumeration as a nightly job; the 500-point random + sample runs in CI. + +**Command:** +`cargo test -p durability --features test-hooks --test it_crash_matrix -- --nocapture` + +**False pass:** +- `panic!` instead of `kill -9`. Destructors run, buffers flush, and the whole + matrix passes against an implementation with no durability at all. +- Counting effects only among those the harness itself generated. That is the + documented prior failure: an idempotency feature generated its own keys and + checked for duplicates among them — unreachable for a whole phase, with tests + asserting the count was zero. Count against the **external ledger**. +- Step 1 omitted, so new transitions silently escape the matrix. +- Asserting recoverability as "the process restarted without error". Restarting + cleanly into wrong state is the failure being hunted. + +## Traps + +- Simulating a crash with `panic!` and unwinding. Destructors run, which is + exactly what a real crash does not do. +- Counting only effects the harness itself created — the coverage failure quoted + above, repeated. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8, §19 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T2.7-p2-composition-gate.md b/tasks/T2.7-p2-composition-gate.md new file mode 100644 index 0000000..ca20e1c --- /dev/null +++ b/tasks/T2.7-p2-composition-gate.md @@ -0,0 +1,109 @@ +# T2.7 — P2 composition gate + +| Field | Value | +|---|---| +| Phase | P2 — Durability hard parts | +| Size | L — over 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P3 | + +## Goal + +Prove intents, effect classes, forks, checkpoints and schema evolution survive +each other — not just crashes in isolation. T2.6 crashes a simple workload; this +crashes a **compound** one. + +**Phase gate criterion:** `kill -9` at every transition leaves recoverable state. + +## Facts (inlined — no spec read needed) + +- Each P2 task was verified against a simple run. The interactions are where the + cost lives, and each is individually plausible-looking: + - a **crash during a rewind** — a fork event committed, the new branch's LSN 0 + not yet written; + - a **`Dispatched` intent on a branch that was subsequently forked** — recovery + must resolve it on the branch that owns it, not the live one; + - a **checkpoint written before a schema upgrade** — the checkpoint holds v1 + state, the log above it holds v2 records; + - **reduction ordering** interacting with a fork — dead-branch text is + reducible, the live branch's is not yet. +- Recovery is defined jointly: legal states (T0.2), intents resolved by class + (T2.2), byte-identical re-fold (T0.8), and exactly-once effects measured + **externally**. +- The discarded branch is evidence. A recovery path that "cleans up" a partial + fork destroys the failure that motivated the rewind. + +## Steps + +1. Build the compound workload: a run that retries, dispatches one intent per + effect class, rewinds at least once, and crosses a checkpoint boundary. +2. Extend T2.6's crash-point enumeration to cover the compound path, including + points **inside** a rewind and **inside** recovery itself. +3. Add the schema dimension: run half the seeds with a v1→v2 upgrade applied + between crash and restart. +4. Assert the four recovery properties jointly after every restart. +5. Run recovery **twice** per seed — recovery must itself be idempotent, since a + crash during recovery is an ordinary event. +6. Make this the required CI job gating P3. + +## Acceptance + +- 500 seeded crash points on the **compound** workload: zero unrecoverable + states, zero double-applied effects, byte-identical re-fold, all intents + resolved by class. +- Recovery run twice produces the same state as recovery run once. +- Half the seeds pass with a schema upgrade interposed. + +## Verify + +**Harness:** T2.6's child-process `kill -9` rig, the external side-effect ledger, +the three effect-class tools, and a seed printed on every run. + +**Integration test** — `tests/it_p2_composition.rs`: +1. **Crash inside a rewind:** abort between the fork event commit and the new + branch's first record. On restart assert the parent branch is **byte-identical + to its pre-rewind snapshot** and the run is in a legal state — either the fork + completed or it did not, never half. +2. **Intent on a forked branch:** dispatch an `Unsafe` intent on branch 0, fork + to branch 1, crash. Assert recovery marks the **branch-0** attempt + `Indeterminate` and does not touch branch 1. +3. **Checkpoint across schema versions:** write a checkpoint at v1, append v2 + records, crash, restart under the v2 binary. Assert rebuild from that + checkpoint equals a rebuild from LSN 0. +4. **Recovery idempotence:** run recovery, snapshot state, run recovery again, + assert byte equality and assert the external ledger count is unchanged. +5. **Effect exactness under compounding:** across all seeds, assert the ledger + holds exactly one entry per intended effect — including effects issued before + a fork. +6. **Crash during recovery:** abort partway through the recovery pass itself; + restart; assert convergence. +7. **Regression:** re-run T2.1–T2.6 plus P0/P1 gates in the same job. + +**Command:** +`cargo test -p durability --features test-hooks --test it_p2_composition -- --nocapture` + +**False pass:** +- Running the compound matrix on the simple workload. T2.6 already covers that; + this gate exists for the interactions, and every one of them is green in + isolation. +- Step 4 omitted. A recovery pass that is correct once and destructive twice + passes every test in P2 and fails the first time a machine crashes while + recovering — which is exactly when it runs. +- Step 1 asserting only that the run recovered. Recovering by truncating the + parent branch also recovers, and destroys the evidence. +- Measuring effect counts in the framework's log rather than the external ledger. + +## Traps + +- Treating a crash during recovery as out of scope. It is the ordinary case in a + crash loop, and a crash loop is what a nonzero `Indeterminate` rate means. +- Letting the compound workload drift out of sync with new transitions. Assert + the enumerated crash-point count against a constant, as T2.6 does. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.4, §8.5, §8.6, §8.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.1-workflowdef-ir-canonicalization.md b/tasks/T3.1-workflowdef-ir-canonicalization.md new file mode 100644 index 0000000..6067ddf --- /dev/null +++ b/tasks/T3.1-workflowdef-ir-canonicalization.md @@ -0,0 +1,125 @@ +# T3.1 — `WorkflowDef` IR + canonicalization + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T3.2–T3.6 | + +## Goal + +The validated intermediate representation every workflow format parses into, and +the canonicalization that makes `WorkflowVersion` a content hash of *meaning* +rather than of source text. + +## Facts (inlined — no spec read needed) + +```rust +pub struct WorkflowDef { + pub id: WorkflowId, + pub schema: SchemaVersion, + pub steps: Vec, + pub transitions: Vec, + pub rubric: RubricDef, + pub budget: BudgetDef, +} + +pub struct StepDef { + /// Author-assigned, stable across versions. + pub id: StepId, + pub kind: StepKind, + pub tools: ToolSelector, + pub verify: Vec, + pub retry: RetryPolicy, + pub timeout: Duration, +} + +pub enum StepKind { + Model { prompt: PromptTemplate, effort: ReasoningEffort }, + Tool { tool: ToolId, args: ArgTemplate }, + Parallel { branches: Vec, join: JoinPolicy }, + Conditional { on: Predicate, then: StepId, otherwise: Option }, + SubWorkflow { workflow: WorkflowId, version: VersionSelector }, +} +``` + +- **`WorkflowVersion` is the Blake3 hash of the canonicalized IR, not of the + source text.** Two YAML files differing only in key order produce the same + version — which is what makes "did this change affect results" answerable. +- Validation and canonicalization live in the **kernel** and run on the IR, so a + new format (T3.2) inherits every check without reimplementing one. +- Domain states are open: a workflow declares its own step names, ordering and + transitions as data. The kernel validates the declaration, then executes it. + Adding a domain state is a config change, never a recompile. +- Versions form a DAG: content-addressed, parent-pointered, **never edited**. + Editing a version in place destroys every result already attributed to it. + +## Steps + +1. Define the IR types above. Keep them free of parser concerns — no source + spans, no format-specific fields. +2. Write the canonicalizer: sort every unordered collection by a defined key, + normalize whitespace inside templates only where semantically irrelevant, + drop optional fields that equal their default, and emit a deterministic byte + encoding. +3. Hash the canonical bytes with Blake3 into `WorkflowVersion`. +4. Add the parent pointer: a new version records its parent's hash. Nothing + mutates an existing version record. +5. Decide and document what is *not* canonicalized — prompt template text is + semantic and must hash as written. Put that decision in a comment at the + canonicalizer, since the next person will otherwise "improve" it. +6. Test: build two structurally identical IRs from different field orders and + assert equal hashes; change one prompt character and assert different hashes. + +## Acceptance + +- Two YAML files differing only in key order and whitespace produce identical + `WorkflowVersion`. + +## Verify + +**Harness:** pairs of source files that differ only in ways that must not matter, +and pairs that differ in ways that must. + +**Integration test** — `tests/it_canonical_version.rs`: +1. **Must match:** two YAML files with reordered top-level keys, reordered map + entries, different indentation, and trailing whitespace. Assert identical + `WorkflowVersion`. +2. **Must differ:** change one character inside a prompt template. Assert a + different version. This is the boundary — prompt text is semantic. +3. **Must differ:** change a `timeout`, a `retry` count, a `StepId`. One test per + field, table-driven, so a canonicalizer that drops a field is caught by the + field it drops. +4. Round trip: canonicalize twice; assert the byte output is stable. +5. Parent pointer: build v2 from v1; assert v2 records v1's hash and that v1's + record is unchanged on disk. +6. Property test: generate random IRs, permute the order of every unordered + collection, assert hash invariance. + +**Command:** `cargo test -p workflow canonical` + +**False pass:** +- Only testing step 1. A canonicalizer that discards fields it does not + understand passes every "these should match" test and silently makes two + different workflows the same version. Step 3's per-field table is the guard. +- Hashing the source bytes, which passes step 2 and fails step 1 — check both + directions. +- A property test that permutes only the collection the implementation already + sorts. + +## Traps + +- Hashing the source bytes. It is one line simpler and makes every reformat look + like a behaviour change. +- Canonicalizing prompt text. A whitespace change inside a prompt *is* a + behaviour change. +- Allowing a version record to be updated in place. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §2, §4.1, §12.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.2-workflowformat-trait-yaml-and-json.md b/tasks/T3.2-workflowformat-trait-yaml-and-json.md new file mode 100644 index 0000000..a87e073 --- /dev/null +++ b/tasks/T3.2-workflowformat-trait-yaml-and-json.md @@ -0,0 +1,95 @@ +# T3.2 — `WorkflowFormat` trait + YAML and JSON + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Formats are plugins. Ship YAML and JSON; prove a third format needs no kernel +change. + +## Facts (inlined — no spec read needed) + +```rust +pub trait WorkflowFormat: Send + Sync { + fn extensions(&self) -> &[&str]; + fn parse(&self, src: &[u8]) -> Result; +} +``` + +- A user wanting a DSL, Starlark, or a database row implements the trait. +- **Validation and canonicalization live in the kernel and run on the IR** (T3.1, + T3.3), so a new format inherits every check without reimplementing one. A + format that does its own validation has forked the rules. +- Because the version hash is over the canonical IR, the same workflow in two + formats must produce the same `WorkflowVersion` — that equality is the test + that the boundary is in the right place. +- Defaults ship working; every default is a port. A user who wants built-in + behaviour writes no code; a user who wants their own writes an impl, not a fork. + +## Steps + +1. Define the trait exactly as above. `parse` returns the IR and nothing else — + no side effects, no registration, no validation. +2. Implement `YamlFormat` and `JsonFormat` over `serde`. Both target the same IR + types from T3.1. +3. Build the format registry keyed by extension. Resolution is by extension, with + a clear error when two formats claim the same one. +4. Make `ParseError` carry position information where the format supplies it — + this is the only error type that legitimately knows about source text. +5. Write the same workflow in YAML and in JSON; assert equal `WorkflowVersion`. +6. In a **separate test crate**, implement a third trivial format (for example + TOML or a line-based DSL) against the public API only. If it needs anything + `pub(crate)`, the trait is short — fix the trait, not the test. + +## Acceptance + +- The same workflow expressed in both formats produces the same + `WorkflowVersion`. +- A third format added in a test crate needs **no kernel change**. + +## Verify + +**Harness:** a **separate test crate outside the workspace** depending only on +the published API — that crate's existence is the deliverable, not a convenience. + +**Integration test** — `tests/it_format_equivalence.rs` plus +`third-format-crate/`: +1. Express one non-trivial workflow (all five `StepKind`s) in YAML and in JSON. +2. Parse both; assert identical `WorkflowVersion`. +3. Assert the parsed IRs are byte-identical after canonicalization. +4. In the external crate, implement a third format (TOML or a line DSL) against + the public API only. Build it. If it needs any `pub(crate)` item, the trait is + short — that is a finding, not a test workaround. +5. Register the third format and run the **same** validation suite (T3.3) against + it; assert it inherits every check with no new code. +6. Extension collision: register two formats claiming `.yaml`; assert a clear + error rather than last-one-wins. + +**Command:** `cargo test -p workflow format_equivalence && cargo test --manifest-path third-format-crate/Cargo.toml` + +**False pass:** +- The third-format crate living inside the workspace, where it can reach + internals. It then passes while the public API is unusable — which is the + entire thing being tested. +- Step 5 omitted: a format that does its own validation passes steps 1–4 and has + quietly forked the rules. + +## Traps + +- Format-specific validation creeping into `parse`. Two formats then disagree + about what is legal. +- Leaking `serde` types into the IR, which makes the third-format test need a + serde dependency it should not have. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §4.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.3-load-time-validation.md b/tasks/T3.3-load-time-validation.md new file mode 100644 index 0000000..94a5a77 --- /dev/null +++ b/tasks/T3.3-load-time-validation.md @@ -0,0 +1,101 @@ +# T3.3 — Load-time validation + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Every structural check the kernel makes on a workflow IR, each producing a +specific diagnostic. Runs at load, never at execution. + +## Facts (inlined — no spec read needed) + +Checks, all on the IR so every format inherits them: + +- **DAG check.** The domain machine is validated as a DAG with explicit loop + bounds. An unbounded loop is a rejection, not a runtime concern. +- **`StepId` uniqueness** within a version. +- **Sub-workflow cycle detection at load, not at execution**, with recursion + depth bounded by the kernel. +- **Capability requirements resolvable** — every `ToolSelector` and + `VerifierRef` resolves to something registered. +- Sandbox eligibility belongs here too: a workflow whose steps cannot run + sandboxed is ineligible for shadow evaluation and **must say so at load time + rather than at 3am** (enforced fully in T6.5). +- Every rejection names the specific problem. A generic parse error moves the + diagnosis cost onto the user, who has less context than the validator. +- Related lesson: *suspect the guards before the mechanism* — a prior "batching" + failure turned out to be a depth guard, because two unrelated limits shared a + default value. Give each limit its own named constant. + +## Steps + +1. Write the validator as a pure function `validate(&WorkflowDef) -> Result<(), Vec>`. + Collect all errors, do not stop at the first. +2. `ValidationError` is an enum with one variant per check, each carrying the + offending ids. No stringly-typed messages. +3. DAG check over `transitions`, with loop bounds required and verified finite. +4. `StepId` uniqueness by set insertion; report the duplicate id. +5. Sub-workflow graph walk with a kernel depth bound; report the cycle path, not + just "cycle detected". +6. Resolve every tool and verifier reference against the registries; report the + unresolved name and what registry was searched. +7. Give each numeric limit its own named constant with its own value, even where + two currently coincide. +8. Build the table-driven suite: one malformed workflow per check, asserting the + **specific** error variant. + +## Acceptance + +- A table-driven suite of malformed workflows, each rejected with a specific + diagnostic — not a generic parse error. + +## Verify + +**Harness:** a table of malformed workflow files, one per check, each paired with +the **exact `ValidationError` variant** it must produce. + +**Integration test** — `tests/it_validation_table.rs`: +1. For each fixture, assert the returned error list **contains the expected + variant** and that the variant carries the offending ids. +2. Multi-error fixture: a workflow with three independent defects; assert **all + three** are reported in one pass, not just the first. +3. Cycle case: assert the error carries the **cycle path**, not just a boolean. +4. Depth case: a sub-workflow chain exceeding the kernel bound; assert rejection + at load and that no execution was attempted. +5. Unresolved capability: assert the error names both the missing name and the + registry searched. +6. Positive control: a valid workflow produces **zero** errors — otherwise a + validator that rejects everything passes the whole table. +7. Constant audit: assert each limit reads a distinct named constant; a test that + changes one constant and asserts only the matching check moves. + +**Command:** `cargo test -p workflow validation` + +**False pass:** +- Asserting `is_err()` per fixture. A validator returning one generic parse error + for everything passes the entire table — which is exactly the outcome the + acceptance criterion forbids. Match on the **variant**. +- Step 6 omitted, so a reject-everything implementation is green. +- Step 2 omitted, so fail-fast looks correct until a user with two mistakes has to + make two round trips. +- Two limits sharing a constant: step 7 is what surfaces the documented "batching + failure was actually the depth guard" class of bug. + +## Traps + +- Fail-fast on the first error. A user fixes one thing per load cycle. +- Deferring cycle detection to execution, where it becomes a stack overflow. +- Two limits sharing one constant. When one trips, you will debug the other. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §4.4, §5.1, §12.4, §19 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.4-stepid-stability-checks.md b/tasks/T3.4-stepid-stability-checks.md new file mode 100644 index 0000000..6a8978c --- /dev/null +++ b/tasks/T3.4-stepid-stability-checks.md @@ -0,0 +1,95 @@ +# T3.4 — `StepId` stability checks + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Report added, removed and retained `StepId`s across a version bump, and reject a +bump that retains nothing unless it is explicitly marked a rewrite. + +## Facts (inlined — no spec read needed) + +- Credit assignment attributes outcomes to steps **across workflow versions**. + That requires a step identity surviving edits: insert a step at position 2 and + every positional index shifts, but `StepId` does not. +- **The framework cannot infer this.** It is a documented obligation on the + workflow author, enforced by three load-time checks: + 1. `StepId` unique within a version. + 2. On a version bump, report added, removed and retained ids. A version + retaining **no** ids from its parent is almost certainly a renumbering + accident and is rejected unless explicitly marked as a rewrite. + 3. `StepId` is opaque to the framework — never parsed, never ordered, never + assumed numeric. +- This is why the stability contract is load-bearing rather than cosmetic: per-step + credit (T5.10) and the whole attribution path depend on it. + +## Steps + +1. Take parent and child `WorkflowDef`s; compute the three sets: added, removed, + retained. +2. Emit the report as structured data, not a log line — it belongs on the version + record so it is auditable later. +3. Reject when `retained.is_empty()` and the child is not marked as a rewrite. + Name the count of added and removed ids in the error. +4. Add the rewrite marker as an explicit field on the workflow definition, so + marking it is a deliberate edit in the user's own file. +5. Audit for any code that parses, sorts numerically, or ranges over `StepId`. + Delete it — ordering comes from `transitions`, not from the id. +6. Test both directions: renumbering every id rejected by default, accepted with + the marker. + +## Acceptance + +- Renumbering every step id is **rejected by default** and **accepted with the + explicit marker**. + +## Verify + +**Harness:** version pairs committed as fixtures — parent and child — covering +each diff shape. + +**Integration test** — `tests/it_stepid_stability.rs`: +1. **Renumber everything, no marker:** assert **rejected**, and that the error + names the added/removed counts. +2. **Renumber everything, marker set:** assert **accepted**. +3. **Insert a step at position 2, ids unchanged:** assert accepted, and that the + report shows 1 added, 0 removed, N retained. This is the case the whole + contract exists for. +4. **Rename one of five ids:** assert accepted with 1 added, 1 removed, 4 + retained — a partial rename must not trip the all-or-nothing rule. +5. Duplicate id within one version: rejected. +6. Assert the report is persisted **on the version record**, readable after a + restart — not emitted only as a log line. +7. Opacity audit: grep the codebase for numeric parsing, sorting or ranging over + `StepId`; assert none. Add a `StepId` in a fixture that is non-numeric and + contains a `.`, and assert nothing breaks. + +**Command:** `cargo test -p workflow stepid_stability` + +**False pass:** +- Only testing steps 1 and 2. The rule "reject when retained is empty" is + trivially satisfiable by an implementation that computes nothing else — steps + 3 and 4 are what prove the diff is real. +- Asserting the report exists in stdout. Credit assignment reads it later from + storage; step 6 is the check that matters. +- Fixtures whose ids are `step1`, `step2`, … so accidental numeric ordering is + indistinguishable from correct behaviour. Step 7's odd id is the guard. + +## Traps + +- Sorting steps by `StepId` for display and then depending on that order. +- Auto-generating `StepId` from position when the author omits one. That + guarantees the renumbering accident the check exists to catch. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §4.3, §11.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.5-interpreter-over-the-ir.md b/tasks/T3.5-interpreter-over-the-ir.md new file mode 100644 index 0000000..12a39b4 --- /dev/null +++ b/tasks/T3.5-interpreter-over-the-ir.md @@ -0,0 +1,123 @@ +# T3.5 — Interpreter over the IR + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Execute a validated `WorkflowDef` directly. Replaces T1.3's hardcoded workflow. + +## Facts (inlined — no spec read needed) + +Five step kinds to support: + +```rust +pub enum StepKind { + Model { prompt: PromptTemplate, effort: ReasoningEffort }, + Tool { tool: ToolId, args: ArgTemplate }, + Parallel { branches: Vec, join: JoinPolicy }, + Conditional { on: Predicate, then: StepId, otherwise: Option }, + SubWorkflow { workflow: WorkflowId, version: VersionSelector }, +} +``` + +- Concurrency shape is fixed and the interpreter must not widen it: + +| Scope | Parallel | +|---|---| +| across runs | unbounded | +| steps within a run | **serial by default** — step N+1 reads N's output | +| `Parallel` step branches | fan-out/join, declared explicitly in the IR | +| attempts within a step | **strictly serial** — a retry needs the prior failure | +| verifiers for one attempt | fan-out/join | + +- The serial spine is `(TenantId, RunId)`. Parallelism lives between runs and + inside declared fan-out. Nothing else may interleave. +- **Tool calls appear in the episode as first-class steps.** The hostcall + boundary already knows the identity, arguments and capabilities; recording them + as opaque invocations discards information the runtime is holding anyway. +- Every external effect goes through the write-ahead intent wrapper (T2.1) — the + interpreter is where that is easy to bypass. +- `SubWorkflow` version resolution is T3.6's task; the interpreter consumes an + already-resolved hash. + +## Steps + +1. Replace T1.3's hardcoded step list with a walk over `WorkflowDef.steps` driven + by `transitions`. Keep the run lifecycle from T1.3 unchanged. +2. `Model`: render `PromptTemplate`, capture the context partition (T1.5), put + prompt and output blobs (T1.6), record `Usage`. +3. `Tool`: resolve `ToolId`, render `ArgTemplate`, call through the intent + wrapper (T2.1), record the tool call as its own step with identity, arguments + and capabilities. +4. `Parallel`: fan out inside the `RunScope` (T1.1) so cancellation still + propagates; apply `JoinPolicy` at the join. +5. `Conditional`: evaluate `Predicate` against recorded state only — no ambient + inputs, or the fold stops being reproducible. +6. `SubWorkflow`: execute against the resolved version, within the kernel's + recursion bound, recording the child run's linkage. +7. Port P1's hardcoded workflow to YAML and diff the resulting episode structure + against the P1 fixture. + +## Acceptance + +- P1's hardcoded workflow, re-expressed as YAML, produces an **identical episode + structure**. + +## Verify + +**Harness:** P1's episode fixture as the oracle, plus the stub model and the +external side-effect ledger from T2.2. + +**Integration test** — `tests/it_interpreter_parity.rs`: +1. Express P1's hardcoded workflow as YAML. Run it through the interpreter. +2. Assert the resulting **episode structure** matches the P1 fixture: same steps, + same attempt counts, same event sequence. Compare the serialized episode with + volatile fields (ids, timestamps) normalized — and list the normalized fields + explicitly, so the normalizer cannot quietly grow. +3. One test per `StepKind`: + - `Model` — prompt rendered, partition and blobs captured. + - `Tool` — appears as a **first-class step** with identity, arguments and + capabilities recorded; the effect went through the intent wrapper (assert 3 + intent records exist). + - `Parallel` — branches run concurrently, `JoinPolicy` applied; cancel the run + mid-fan-out and assert every branch task terminated. + - `Conditional` — both arms exercised; run the same workflow twice and assert + identical predicate outcomes. + - `SubWorkflow` — child linkage recorded; recursion past the bound rejected. +4. **Serialization guard:** instrument step start/end times; assert no two + non-`Parallel` steps of one run overlap. +5. **Bypass audit:** assert every tool invocation in the run has a matching + `Pending`/`Dispatched`/`Committed` intent triple. A count mismatch means a + direct call path exists. +6. `assert_refold_identical` on the interpreted run. + +**Command:** `cargo test -p executor interpreter` + +**False pass:** +- Step 2 with an over-broad normalizer that blanks anything differing. Then any + two episodes "match". Enumerate the normalized fields. +- Step 5 omitted: calling a tool directly works, produces a correct-looking + episode, and fails T2.6's crash matrix later in a way that looks unrelated. +- `Parallel` tested with branches that complete instantly, where concurrency is + indistinguishable from serial execution. + +## Traps + +- Running independent-looking steps concurrently. Serial-by-default is the + contract; fan-out is declared, never inferred. +- Calling a tool directly rather than through the intent wrapper. It works, and + T2.6's crash matrix then fails somewhere unrelated-looking. +- A predicate that reads the clock or the environment. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §4.1, §5.3, §13.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.6-version-pinning-at-spawn.md b/tasks/T3.6-version-pinning-at-spawn.md new file mode 100644 index 0000000..fa25137 --- /dev/null +++ b/tasks/T3.6-version-pinning-at-spawn.md @@ -0,0 +1,96 @@ +# T3.6 — Version pinning at spawn + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +`VersionSelector::Latest` resolves exactly once, at run spawn, and the resolved +hash is recorded. + +## Facts (inlined — no spec read needed) + +- `SubWorkflow` pins by `VersionSelector`: `Exact(hash)` or `Latest`. +- **`Latest` resolves once, at run spawn**, and the resolved hash is recorded. A + run whose sub-workflow version can change mid-execution is a run whose results + attribute to nothing. +- The same rule applies to the top-level workflow: the run records the exact + `WorkflowVersion` it executed, and a version published mid-run does not affect + it. +- Related failure the type system prevents: a placeholder that type-checks is + invisible. A hardcoded `"current"` version hash compiled, passed tests, and + made every result unattributable — which is why `WorkflowVersion` has no + `Default` (T0.1). +- Versions are never edited in place; a new version is a new hash with a parent + pointer. + +## Steps + +1. At spawn, resolve the top-level workflow and every reachable `SubWorkflow` + selector to concrete hashes. +2. Record the resolved set on the run's spawn event — the top-level version plus + a map of `(WorkflowId → WorkflowVersion)` for sub-workflows. +3. Make the interpreter (T3.5) read only from that recorded map. It must have no + path back to the registry's current `Latest`. +4. Resolve `Latest` against the registry once; if a new version is published + afterwards, nothing re-reads it. +5. Test: start a run, publish a new version of the same workflow mid-run, assert + the running run's recorded version is unchanged and it finishes on the pinned + one. + +## Acceptance + +- Publishing a new version mid-run does not change the running run's recorded + version. + +## Phase gate + +P3 closes when a user-authored YAML workflow runs with no framework recompile. + +## Verify + +**Harness:** a workflow registry that can publish a new version while a run is +in flight, plus a run long enough to straddle the publish (stub latency on a +mid-run step). + +**Integration test** — `tests/it_version_pinning.rs`: +1. Register workflow `W` v1, containing a `SubWorkflow` pinned `Latest`. +2. Spawn a run; park it mid-execution **before** the sub-workflow step. +3. Publish `W` v2 and a new version of the sub-workflow. +4. Release the run. +5. Assert the run's recorded top-level `WorkflowVersion` is v1. +6. Assert the sub-workflow executed is the version resolved **at spawn**, not the + newly published one — check the recorded resolution map, then check which + version actually ran (they must agree). +7. Assert the spawn event carries the full `(WorkflowId → WorkflowVersion)` map, + not a selector. +8. Registry-access audit: assert the interpreter makes **zero** registry lookups + after spawn — instrument the registry with a call counter. + +**Command:** `cargo test -p executor version_pinning` + +**False pass:** +- Parking the run **after** the sub-workflow step, so nothing could have + re-resolved. The park point must precede the step. +- Asserting only step 5. The top-level version is usually recorded correctly; the + lazy-resolution bug lives in the sub-workflow path. Steps 6 and 8 are the test. +- Step 6 checking the recorded map only. A map that is recorded and then ignored + passes — assert what actually ran. + +## Traps + +- Resolving `Latest` lazily at the point the sub-workflow is entered. It is the + natural implementation and it makes long runs non-attributable. +- Storing the selector rather than the resolved hash on the run record. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §4.4, §12.2, §19 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T3.7-p3-composition-gate.md b/tasks/T3.7-p3-composition-gate.md new file mode 100644 index 0000000..c49055c --- /dev/null +++ b/tasks/T3.7-p3-composition-gate.md @@ -0,0 +1,105 @@ +# T3.7 — P3 composition gate + +| Field | Value | +|---|---| +| Phase | P3 — Workflow as data | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P4 | + +## Goal + +Prove the IR, the format plugins, validation, `StepId` stability, the interpreter +and version pinning compose — and that **format is genuinely interchangeable**. + +**Phase gate criterion:** a user-authored YAML workflow runs with no framework +recompile. + +## Facts (inlined — no spec read needed) + +- **The workflow is data, not code.** Users define workflows in their own format + and version them; the framework executes and grades them without recompiling. +- Validation and canonicalization live in the kernel and run on the IR, so a new + format inherits every check without reimplementing one. That claim is only + proven by running the **same** validation suite against a format the kernel has + never seen. +- The composition properties no single P3 task owns: + - the same workflow in **any** format produces the same `WorkflowVersion` and + the same episode; + - all five `StepKind`s **nest** — `Parallel` inside `Conditional` inside + `SubWorkflow` — and the concurrency contract still holds; + - a version bump that retains `StepId`s keeps credit attributable across the + bump (T3.4), while the running run stays pinned (T3.6). +- Recursion depth is bounded by the kernel and cycles are detected at load, not + at execution — including cycles that only appear through a nested composition. + +## Steps + +1. Author one **non-trivial reference workflow** exercising all five step kinds, + nested at least two levels, with loops at their declared bounds. +2. Express it in YAML, in JSON, and in the third format from T3.2's external + crate. +3. Assert identical `WorkflowVersion` and identical episode structure across all + three. +4. Run the full T3.3 validation suite against all three formats. +5. Bump the workflow version retaining `StepId`s; assert credit keys survive and + in-flight runs stay pinned. +6. Make this the required CI job gating P4. + +## Acceptance + +- The reference workflow, expressed in three formats, produces one + `WorkflowVersion` and one episode structure. +- The validation suite passes identically against all three formats. +- No framework recompile is needed to run a user-authored workflow. + +## Verify + +**Harness:** the reference workflow as a committed fixture in three formats; +T3.2's third-format crate living **outside** the workspace. + +**Integration test** — `tests/it_p3_composition.rs`: +1. **Format interchange:** parse all three; assert equal `WorkflowVersion`, then + **run** all three and assert equal episode structure with volatile fields + normalized against an enumerated list. +2. **Nesting:** assert the nested `Parallel`-inside-`Conditional`-inside- + `SubWorkflow` executes correctly, that fan-out is concurrent, and that + cancelling the run terminates every nested branch. +3. **Serial spine under nesting:** assert no two non-`Parallel` steps overlap, + even across the sub-workflow boundary. +4. **Validation parity:** run every malformed fixture from T3.3 through all three + formats; assert the **same `ValidationError` variant** each time. A format + producing a different error has forked the rules. +5. **Cycle through composition:** build a cycle that exists only via a nested + sub-workflow. Assert it is rejected **at load**. +6. **Version bump:** bump retaining `StepId`s; assert the added/removed/retained + report is recorded, and that a run started before the bump completes on the + **pinned** version. +7. **Recompile check:** add a fourth workflow file at runtime, in a directory the + test writes to, and run it. Assert no rebuild occurred — the binary under test + is the one built before the file existed. +8. **Regression:** re-run P0–P2 gates and all P3 suites. + +**Command:** `cargo test -p workflow --test it_p3_composition && cargo test --manifest-path third-format-crate/Cargo.toml` + +**False pass:** +- Step 1 with a trivial single-step workflow. Every format handles that; the + divergence lives in nesting and in optional fields. +- Step 4 omitted: a format doing its own validation passes the equal-version test + and silently accepts workflows the kernel would reject. +- Step 7 conflated with "the test passes". The claim is **no recompile** — assert + the workflow file was created after the binary, not just that it ran. +- Running the third format from inside the workspace, where it reaches internals. + +## Traps + +- Letting the reference workflow shrink over time as steps are removed to make a + test pass. Its coverage is the point; assert its step-kind coverage explicitly. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §4, §5.3 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T4.1-verifier-port.md b/tasks/T4.1-verifier-port.md new file mode 100644 index 0000000..7eabea4 --- /dev/null +++ b/tasks/T4.1-verifier-port.md @@ -0,0 +1,103 @@ +# T4.1 — `Verifier` port + +| Field | Value | +|---|---| +| Phase | P4 — Verification | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T4.2 | + +## Goal + +The verification port with a fail-closed contract enforced by the framework, not +by the verifier author. + +## Facts (inlined — no spec read needed) + +```rust +#[async_trait] +pub trait Verifier: Send + Sync { + fn id(&self) -> VerifierId; + /// Any error, panic or timeout MUST resolve to `Fail`. A verifier that + /// throws or hangs can never report `Pass`. + async fn verify(&self, cx: &VerifierCtx) -> VerifierOutcome; +} +``` + +- **Verification returns ground truth. Grading attributes cause and ranks.** + Collapse them and the system grades its own homework. +- **The framework cannot break the agent it runs.** Observation, verification and + grading failures degrade the record, never the work. Any code path where a + verifier or grader can fail into an agent's execution is a defect. +- Fail-closed means the *framework* converts panic, error and timeout into + `Fail`. A contract documented in a doc comment and implemented by each author + is not a contract. +- Never let a rubric judge what a verifier can check. Every criterion that can be + made mechanical should be a `Verifier` — deterministic, cheap, and not subject + to judge drift. + +## Steps + +1. Define `VerifierId`, `VerifierOutcome` (pass/fail plus detail) and the trait + above. +2. Write the framework-side runner that wraps every `verify` call: + - `tokio::time::timeout` at `cx.deadline` → `Fail` + - `FutureExt::catch_unwind` (or a spawn-and-join boundary) for panics → `Fail` + - any `Err` → `Fail` +3. Make the runner the only call path. `Verifier::verify` is never invoked + directly from the executor. +4. Record which failure mode produced a `Fail` — timeout, panic and error are the + same outcome but different operational signals. +5. Run verifiers for one attempt as a fan-out/join; they are independent checks. +6. Fault-inject all three modes in tests with real verifiers that panic, hang and + error. + +## Acceptance + +- Verifiers that panic, hang past deadline, and return an error **all resolve to + `Fail`** — fault-injected, not asserted by comment. + +## Verify + +**Harness:** three real misbehaving verifiers — one that panics, one that loops +past the deadline, one that returns `Err`. Fault-injected, not mocked. + +**Integration test** — `tests/it_verifier_fail_closed.rs`: +1. Run a real run with each bad verifier in turn. +2. Assert each resolves to `Fail`. +3. Assert **the run itself completes** and reaches a terminal state — the + framework must not break the agent it runs. A panicking verifier taking the + run down is the primary failure being tested for. +4. Assert the recorded failure mode distinguishes timeout / panic / error, since + they are the same outcome but different operational signals. +5. Assert a verifier that panics does **not** poison shared state: run a second, + healthy verifier afterwards in the same process and assert it returns `Pass`. +6. Fan-out: three verifiers on one attempt, one of them panicking; assert the + other two still return their own results. +7. Assert no code path can produce `Pass` from an `Err` — a `trybuild` case if + `verify` is only reachable through the framework runner. + +**Command:** `cargo test -p verify fail_closed` + +**False pass:** +- Testing the runner in isolation with a mock verifier rather than in a real run. + Step 3 — the run surviving — only means something end to end. +- Catching the panic at the test boundary with `catch_unwind` in the *test*, + which makes a framework that does not catch it look fine. +- Step 5 omitted: a panic that leaves a mutex poisoned passes steps 1–4 and + breaks every subsequent verification in the process. + +## Traps + +- Letting a panic unwind into the executor task. It takes the run with it, which + is the exact inversion the first principle forbids. +- A `Result` signature that pushes the decision onto callers. + One caller will map `Err` to `Pass` by accident. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §10, §11.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T4.2-verifierctx-and-the-snapshot-barrier.md b/tasks/T4.2-verifierctx-and-the-snapshot-barrier.md new file mode 100644 index 0000000..be5bd64 --- /dev/null +++ b/tasks/T4.2-verifierctx-and-the-snapshot-barrier.md @@ -0,0 +1,119 @@ +# T4.2 — `VerifierCtx` and the snapshot barrier + +| Field | Value | +|---|---| +| Phase | P4 — Verification | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Freeze the episode view on entry to `Verifying`, so state stops moving beneath a +verifier that is mid-flight. + +## Facts (inlined — no spec read needed) + +```rust +pub struct VerifierCtx { + pub run: RunId, + pub step: StepId, + pub attempt: AttemptNo, + /// Frozen on entry to `Verifying`. Never mutates while a verifier holds it. + pub episode: EpisodeView, + /// Lazy. Verifiers needing no text never pay for it. + pub blobs: Arc, + pub deadline: Instant, +} +``` + +- A verifier seeing only identifiers can answer "did it work". Answering "did the + agent have what it needed" requires the **context and the prompt** — so the + view carries the context partition inline and prompt/output by reference. +- **Include failed attempts.** "Retried three times because context was missing + X" is the learning signal; shipping only the winning attempt discards it. +- **What the barrier defends against is not a concurrent retry.** `Verifying` is + entered only when every step is terminal, so no attempt can still be running. A + test built around a concurrent retry passes against no barrier at all. The + three real mutators arrive from **outside** the run's own execution: + 1. a **rewind** forking a new `BranchId` while verifiers hold a view of the old + one; + 2. a **cancel**, which is legal from `Verifying`; + 3. **recovery** resolving a `Dispatched` intent left by an earlier crash, which + writes an outcome into an attempt a verifier is already reading. +- Each is rare enough to be missed in testing and ordinary enough to happen in + production. +- This is why `Verifying` must be a real resting state and not a synchronous + branch (T1.3). + +## Steps + +1. Define `VerifierCtx` as above. `EpisodeView` is an owned snapshot or an + immutable `Arc` — never a handle that re-reads the store. +2. Build the snapshot at the transition into `Verifying`: capture the branch, the + attempts (including failed ones), refs, partitions, usage, at a fixed LSN. +3. Record the snapshot LSN in the context. A verifier's answer is about a + specific point in the log, and later analysis needs to know which. +4. Ensure the three mutators cannot reach into a held view: rewind writes to a + new `BranchId` (T2.3), cancel changes run state not the snapshot, recovery + appends new records above the snapshot LSN. +5. Pass `blobs` as a handle, not resolved bodies — laziness is T4.3. +6. Write one test per mutator, each firing while a verifier is deliberately + parked mid-`verify`, asserting the view is unchanged. + +## Acceptance + +- A verifier holding a context sees an unchanged view across each of the three + mutators: a rewind forking a new `BranchId`, a cancel arriving during + `Verifying`, and recovery resolving a `Dispatched` intent into an attempt the + verifier is reading. +- Explicitly **not** a concurrent-retry test — every step is terminal before + `Verifying` is entered, so that test would pass against no barrier at all. + +## Verify + +**Harness:** a verifier that parks on a channel mid-`verify`, so a mutator can be +fired while it demonstrably holds the context. One test per mutator — they are +different code paths and a single combined test proves the weakest. + +**Integration test** — `tests/it_snapshot_barrier.rs`: +1. Enter `Verifying`. Park the verifier. Snapshot + `serialize(cx.episode)` from inside it. +2. **Mutator A — rewind:** fork a new `BranchId` from outside the run. Release + the verifier. Assert its view is byte-identical to the snapshot and still + points at the old branch. +3. **Mutator B — cancel:** cancel the run during `Verifying`. Assert the held + view is unchanged and the verifier still completes. +4. **Mutator C — recovery:** have recovery resolve a `Dispatched` intent into an + attempt the verifier is reading. Assert the held view is unchanged, and that + the new outcome **is** visible to a freshly built view. +5. Assert `cx` records the snapshot LSN, and that a view built at that LSN by + cold re-fold equals the held view. +6. Assert failed attempts are present in the snapshot — count them. + +**Command:** `cargo test -p verify snapshot_barrier -- --test-threads=1` + +**False pass:** +- **A concurrent-retry test.** `Verifying` is entered only when every step is + terminal, so no attempt can still be running — that test passes against no + barrier at all. It is the obvious test to write and it verifies nothing. All + three mutators must come from outside the run's own execution. +- An `EpisodeView` that lazily queries state: the snapshot comparison passes if + the mutator happens to touch a different key. Step 4 targets the same attempt + the verifier is reading, on purpose. +- Firing the mutator before the verifier has actually parked. Use a + rendezvous channel, not a sleep. + +## Traps + +- An `EpisodeView` that lazily queries materialized state. It looks frozen and + is not. +- Filtering failed attempts out of the view. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §10.1, §10.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T4.3-lazy-blob-access.md b/tasks/T4.3-lazy-blob-access.md new file mode 100644 index 0000000..ec2bb1f --- /dev/null +++ b/tasks/T4.3-lazy-blob-access.md @@ -0,0 +1,85 @@ +# T4.3 — Lazy blob access + +| Field | Value | +|---|---| +| Phase | P4 — Verification | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +A verifier that needs no text performs no blob reads. + +## Facts (inlined — no spec read needed) + +- Context partitions hold **identifiers** — packed, available-but-not-packed, + dropped — and are small enough to inline. Prompts and outputs are large and go + **by reference**. +- Laziness matters concretely: a verifier that shells out and checks an exit code + needs none of this. Making every verifier carry prompt text penalizes the + common case and blows broker payload limits. +- `VerifierCtx.blobs: Arc` is the mechanism — the verifier calls + `get` if and only if it wants a body. +- `get` may return `None` after reduction (T8.4); that is a normal answer, and a + verifier must not treat it as an error. + +## Steps + +1. Confirm `EpisodeView` carries `BlobRef` only. No `prompt_text` field, no + eagerly-resolved cache. +2. Pass the store handle through `VerifierCtx.blobs`. +3. Instrument the store with a call counter behind a test-only wrapper — + counting is what turns this from an intention into an assertion. +4. Write the exit-code verifier used as the acceptance case: it reads the attempt + state and nothing else. +5. Write a second, text-reading verifier and assert its call count is exactly the + number of refs it asked for — no over-fetch, no prefetch-all. + +## Acceptance + +- An exit-code verifier records **zero** `BlobStore::get` calls. + +## Verify + +**Harness:** a counting `BlobStore` decorator wrapping the real store, injected +through `VerifierCtx.blobs`. + +**Integration test** — `tests/it_lazy_blobs.rs`: +1. Run a real run with an exit-code verifier that reads only attempt state. +2. Assert the counter is **exactly 0**. +3. Run a text-reading verifier that resolves 2 refs. Assert the counter is + **exactly 2** — not more (no prefetch) and not fewer (no silent cache hit + masking a missing read). +4. Assert `EpisodeView` has no field holding blob **bytes** — a compile-level + check if the type is closed, otherwise a serialization size assertion on a run + with a 1 MB prompt: the view must stay small. +5. Reduction interaction: delete a blob body, then run the text verifier; assert + `get` returns `Ok(None)` and the verifier handles it without erroring. +6. Assert the counter is per-run, not global, so parallel tests do not mask each + other. + +**Command:** `cargo test -p verify lazy_blobs` + +**False pass:** +- Counting at the store level while a caching layer sits above it. The count then + measures cache misses, not verifier behaviour. Wrap the handle the verifier + actually receives. +- Step 4 omitted: a `prompt_text` convenience field makes every verifier pay, and + steps 1–3 still pass if the resolution happens at view-build time rather than + through `cx.blobs`. The size assertion catches that. + +## Traps + +- A "convenience" `EpisodeView::prompt_text()` that resolves on access. Every + verifier then pays, invisibly. +- Prefetching all refs when the view is built, which is the same regression one + layer earlier. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §10.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T4.4-retention-ordering-guard.md b/tasks/T4.4-retention-ordering-guard.md new file mode 100644 index 0000000..48f3e73 --- /dev/null +++ b/tasks/T4.4-retention-ordering-guard.md @@ -0,0 +1,91 @@ +# T4.4 — Retention ordering guard + +| Field | Value | +|---|---| +| Phase | P4 — Verification | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Reduction must not outrun verification or grading. Eligibility is *grading has +terminated*, not *grading succeeded*. + +## Facts (inlined — no spec read needed) + +- Eligible states for reduction: **`Graded`, `Ungraded`, or `Archived`**. +- **Never a step-level finish timestamp.** A step can finish, be reduced, and + then run-level verification finds nothing left to check. +- `Ungraded` belongs in that set, and the tighter-looking `Graded`-only rule is + the one that gets written. A run that never gets a score — a novel `TaskId` + with no reference, a tournament group that closed without it, a tenant over its + grading ceiling — is **finished**. Gating on `Graded` alone leaves it + irreducible forever. +- The tenants that hit this are the low-volume ones and the cost-capped ones: the + two populations least able to absorb unbounded storage, and the two least + likely to have anyone watching for it. +- `Graded`-only passes every other test in this phase and strands storage only in + production. So the `Ungraded`-is-reducible case must be asserted directly. + +## Steps + +1. Write `fn is_reducible(run_state) -> bool` matching exactly + `Graded | Ungraded | Archived`. Exhaustive match, no `_` arm. +2. Make reduction (T8.4) call it as a precondition and **reject** — a typed + error naming the current state — rather than skipping silently. A silent skip + is indistinguishable from success and leaves no trace to investigate. +3. Delete any path that reads a step-level finish time for retention decisions. +4. Wire the rejection into metrics so refusals are countable. +5. Test both halves: a run resting in `Grading` is rejected; a run terminal in + `Ungraded` is reduced. + +## Acceptance + +- A run still resting in `Grading` is **not reducible**, and attempting it is a + **rejected operation** rather than a silent skip. +- A run that reached terminal `Ungraded` **is** reducible — asserted directly. + +## Verify + +**Harness:** runs parked in each lifecycle state, plus the reduction entry point +from T8.4 (or a stub of it that calls the same guard). + +**Integration test** — `tests/it_retention_guard.rs`: +1. Table-driven over **every** run state. For each, call reduce and assert: + - `Graded`, `Ungraded`, `Archived` → **permitted**; + - every other state, including `Grading` and `Verifying` → **rejected with a + typed error naming the current state**. +2. Assert the rejection is an `Err`, not `Ok(())`. A silent skip is + indistinguishable from success and leaves nothing to investigate. +3. **Assert `Ungraded` is reducible directly**, with a run that reached + `Ungraded { NoReference }`. This is the case a `Graded`-only implementation + fails, and it is the only assertion that catches it. +4. Assert no code path reads a step-level finish timestamp for retention — + grep, plus a test where a step finished long ago while the run is still + `Verifying`: reduction must be refused. +5. Assert refusals increment a metric, so a stuck reducer is visible. + +**Command:** `cargo test -p retention guard` + +**False pass:** +- Testing only the `Grading`-is-refused half. `if state == Graded` passes that, + passes every other test in P4, and strands storage in production for exactly + the low-volume and cost-capped tenants least able to absorb it. Step 3 is the + whole point of this task's acceptance criterion. +- A table covering only the states someone remembered. Enumerate the run-state + enum exhaustively with no `_` arm so a new state forces a decision. + +## Traps + +- `if state == Graded` — passes this phase, strands storage in production. +- Returning `Ok(())` on ineligible runs "because there is nothing to do". Then a + stuck reducer looks healthy. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.2, §10.3 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T4.5-p4-composition-gate.md b/tasks/T4.5-p4-composition-gate.md new file mode 100644 index 0000000..c18b07e --- /dev/null +++ b/tasks/T4.5-p4-composition-gate.md @@ -0,0 +1,108 @@ +# T4.5 — P4 composition gate + +| Field | Value | +|---|---| +| Phase | P4 — Verification | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P5 | + +## Goal + +Prove the verifier port, the snapshot barrier, lazy blob access and the retention +guard hold **simultaneously**, and that any verifier implementation is +interchangeable. + +**Phase gate criterion:** fail-closed proven by fault injection. + +## Facts (inlined — no spec read needed) + +- **The framework cannot break the agent it runs.** Verification failures degrade + the record, never the work. Any path where a verifier can fail into an agent's + execution is a defect — and the composition is where such a path appears, since + no single verifier test runs a full fleet. +- The properties no single P4 task owns: + - a **panicking** verifier must not disturb a **concurrent** healthy verifier's + snapshot, nor the run; + - the barrier must hold while **all three** external mutators fire, including + at once; + - laziness must survive fan-out — one text-reading verifier must not cause the + exit-code verifiers beside it to fetch anything; + - retention must refuse while **any** verifier is outstanding, which is the + `Verifying` case T4.4 checks and the composition makes real. +- Verification decides, grading explains. A gate that lets a verifier outcome be + influenced by grading state has collapsed the two. + +## Steps + +1. Assemble a verifier fleet on one attempt: exit-code, text-reading, panicking, + hanging, erroring — five implementations, one fan-out. +2. Fire all three snapshot mutators concurrently while the fleet is mid-flight. +3. Assert every fail-closed outcome, every unaffected outcome, the blob call + count, and the retention refusal, in one test. +4. Swap the fleet for a completely different set of verifier implementations; + assert the run's outcome shape is unchanged. +5. Make this the required CI job gating P5. + +## Acceptance + +- With the five-verifier fleet on one attempt: the three bad verifiers resolve to + `Fail`, the two good ones return their own results, and **the run completes**. +- The snapshot holds across all three mutators fired concurrently. +- Exit-code verifiers record **zero** blob reads even while a sibling reads text. +- Reduction is refused for the whole duration of `Verifying`. + +## Verify + +**Harness:** five verifier implementations; a counting `BlobStore` wrapper per +verifier, not one shared counter; rendezvous channels so mutators fire only once +every verifier has demonstrably parked. + +**Integration test** — `tests/it_p4_composition.rs`: +1. **Fleet fail-closed:** assert panic → `Fail`, hang → `Fail`, error → `Fail`, + and that the exit-code and text verifiers return their **own** correct + results. A fleet where one bad verifier poisons the others is the failure. +2. **Run survives:** assert the run reaches a terminal state and that agent work + was never blocked — compare run duration against a no-verifier baseline. +3. **Concurrent mutators:** with the fleet parked, fire a rewind, a cancel and an + intent recovery **at once**. Assert every held `EpisodeView` is byte-identical + to its entry snapshot. +4. **Per-verifier laziness:** assert the exit-code verifiers' counters are + **exactly 0** while the text verifier's is exactly the number of refs it + requested. A single shared counter hides this. +5. **Retention interlock:** attempt reduction at three points — during + `Verifying`, after `Verifying` but during `Grading`, and after terminal + `Ungraded`. Assert refuse, refuse, permit. +6. **Verifier interchange:** re-run the entire test with a second, unrelated set + of five verifier implementations. Assert identical run-level outcomes. +7. **Isolation after panic:** run a healthy verifier in the same process **after** + the panicking one; assert `Pass`. A poisoned mutex passes every earlier step. +8. **Regression:** re-run P0–P3 gates and all P4 suites. + +**Command:** `cargo test -p verify --test it_p4_composition -- --test-threads=1` + +**False pass:** +- Running the five verifiers **sequentially**. Fan-out is the declared + concurrency shape, and cross-contamination between verifiers only exists + concurrently. +- Step 3 firing the mutators one at a time. Each is already covered by T4.2; the + gate is about them arriving together. +- One shared blob counter in step 4 — the sibling's reads mask the exit-code + verifiers' zero. +- Step 5 testing only the `Grading` refusal, leaving the `Ungraded`-is-permitted + half untested. That is the half that strands storage in production. + +## Traps + +- Catching the verifier panic in the test rather than asserting the framework + catches it. +- Asserting the run "did not error" instead of asserting it reached a terminal + state with a complete log. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §10 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.1-taskid-at-spawn.md b/tasks/T5.1-taskid-at-spawn.md new file mode 100644 index 0000000..f5d71cb --- /dev/null +++ b/tasks/T5.1-taskid-at-spawn.md @@ -0,0 +1,90 @@ +# T5.1 — `TaskId` at spawn + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T5.5 onward. Cannot be retrofitted | + +## Goal + +Every run carries a `TaskId` from spawn. Required, no default, no derivation from +`RunId`. + +## Facts (inlined — no spec read needed) + +- `TaskId` is the **comparison-group key**: a content hash of the task input + **before any workflow touches it**. +- Hashing the prompt does not work — the workflow changes the prompt by + construction, which is the entire point of a variant. +- **`TaskId` cannot be backfilled.** A run recorded without one is permanently + ungroupable and useless to the learning loop. Hence: required at spawn, no + `Default`, no `From` (T0.1). +- **Land this during P1**, even though nothing consumes it until P5. It is a + small task deep in P5 and it is the one item on the critical path that a later + phase cannot repair. +- What defines `TaskId` for a given user is open — ticket id, input fixture, or a + hash of the pre-workflow goal. The framework provides the type and a default + hasher and lets it be overridden. Settle it before any run is recorded. + +## Steps + +1. Add `task: TaskId` as a **required positional field** of the spawn parameters. + Not `Option`, not a builder method that can be omitted. +2. Provide a default hasher over the pre-workflow task input, and a hook to + override it. Document that the choice is permanent for existing data. +3. Record `TaskId` on the run's spawn event so it lands in the log, not only in + materialized state. +4. Expose it on `EpisodeView` for the grading path. +5. Add the `trybuild` compile-fail case: a spawn call omitting `TaskId`. + +## Acceptance + +- **Spawning without a `TaskId` fails to compile.** + +## Verify + +**Harness:** `trybuild`, plus a full run to prove the id reaches storage. + +**Integration test** — `tests/it_taskid_required.rs`: +1. `trybuild` compile-fail: a spawn call omitting `TaskId`. Assert the expected + stderr **names the missing field**, not merely "does not compile". +2. Run a real run; read the **log** (not materialized state) and assert the spawn + event carries the `TaskId`. +3. Assert `EpisodeView` exposes it, so the grading path can group on it. +4. Determinism: hash the same task input twice, in two processes; assert equal + `TaskId`. +5. Independence: assert two runs of the **same** task input share a `TaskId`, and + that two different inputs do not. Both directions. +6. Assert the `TaskId` is **not** derived from the prompt: change the workflow + version (which changes the prompt) and assert the `TaskId` is unchanged for + the same input. +7. `trybuild`: `TaskId::default()` and `TaskId::from(run_id)` both fail. + +**Command:** `cargo test -p kernel taskid && cargo test -p kernel --test compile_fail` + +**False pass:** +- Step 2 read from materialized state. If the id lives only in state it cannot be + re-derived after a cold fold, and it cannot be backfilled — read the log. +- Step 6 omitted: hashing the prompt passes steps 1–5 completely and makes every + variant its own group, which is the exact failure that renders the learning + loop inert. +- A builder with a runtime `expect("task_id required")`. It passes a test that + checks for a panic and still allows runs to be recorded without one in any code + path the test did not cover. + +## Traps + +- A builder with `.task_id(...)` optional and a runtime check. Runs recorded + before someone notices are unrecoverable. +- Deriving it from the prompt or from `RunId`. Both compile; both make every run + its own group of one. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §3, §11.5, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.10-attempt-tournaments.md b/tasks/T5.10-attempt-tournaments.md new file mode 100644 index 0000000..9ad0496 --- /dev/null +++ b/tasks/T5.10-attempt-tournaments.md @@ -0,0 +1,113 @@ +# T5.10 — Attempt tournaments + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Grade the attempts of one step as a comparison group. On by default, even though +T5.8/T5.9 are not, because it costs no extra agent runs. + +## Facts (inlined — no spec read needed) + +Where comparison groups come from: + +| Source | Group | Cost | Needs | +|---|---|---|---| +| **Attempts** | attempts 1..N of one step, identical context | free, already recorded | nothing | +| **Recurring tasks** | runs sharing a `TaskId` over time | free, slow to fill | `TaskId` | +| **Replay** | one task re-executed under N variants | N full runs | blobs, sandbox | + +- **Attempt tournaments first.** Retries are already on disk and attempt N+1 + never mutates attempt N, so the attempts of one step are a group on identical + context — the cheapest per-step credit signal available, and the **only** source + of per-step credit the system has. +- **Two paths, not one:** + - **Same-outcome attempts** go to the judge. The question there is "which + failure got further" — which no verifier can answer. + - **A fail→pass pair is never judged.** The verifier has already ordered it; + asking a judge which is better asks it to re-decide what verification + decided. The pair is consumed **structurally**: what differed between attempt + N and N+1 — context partition, tool selection, prompt — is attributed to the + `StepId` as the change that turned a fail into a pass. +- The bracketing rule has **no exemptions**. If a pairing crosses an outcome + class, it is evidence for attribution, never input to a judge. This is the + place the rule is easiest to break, because the two attempts sit side by side + on disk and look like a free comparison. +- Per-step credit attributes to `StepId`, which is why the `StepId` stability + contract (T3.4) is load-bearing rather than cosmetic. + +## Steps + +1. Build the group: all attempts of one `(BranchKey, StepId)`, partitioned by + verifier outcome class. +2. **Route by class before any judge call.** Same-class subsets go to the + comparison path; cross-class pairs go to the attribution path. Make this a + single branch point so it cannot be bypassed. +3. Comparison path: hand the same-class subset to the configured strategy. +4. Attribution path: diff attempt N against N+1 across context partition, tool + selection and prompt ref; emit the delta attributed to `StepId`. No judge call. +5. Persist per-step credit keyed by `(TenantId, WorkflowVersion, StepId)`. +6. Instrument the `Judge::compare` counter and assert **zero** cross-class calls — + count calls rather than inspecting pairs, since a pair-inspection test passes + against an implementation that builds the pair and then filters it late. + +## Acceptance + +- A run with three attempts produces per-step credit attributed to `StepId`. +- A group containing both a failed and a passed attempt issues **zero + `Judge::compare` calls across the outcome boundary** — asserted by counting + calls, not by inspecting pairs. + +## Verify + +**Harness:** a run with three attempts, plus a counting mock judge. The counter +is the instrument — pair inspection is not. + +**Integration test** — `tests/it_attempt_tournament.rs`: +1. Three attempts on one step, **all failing**. Assert `compare` is called on the + same-outcome subset and per-step credit is attributed to the `StepId`. +2. **Mixed group — the guard:** attempts 1 and 2 fail, attempt 3 passes. Assert + **zero** `Judge::compare` calls cross the outcome boundary, counted at the + judge. Assert the fail→pass pair produced a **structural attribution** record + instead. +3. Assert the attribution names the actual delta — plant a context-partition + difference between attempts 2 and 3 and assert it appears in the record. +4. Assert credit is keyed `(TenantId, WorkflowVersion, StepId)` and survives a + workflow version bump that retains the `StepId` (T3.4). +5. Assert this path runs **with the tournament flags off** — it is on by default + while T5.8/T5.9 are not. +6. Zero-cost assertion: no agent runs are spawned by grading; assert the run + counter is unchanged. +7. Single-attempt step: assert no comparison and no spurious credit record. + +**Command:** `cargo test -p grading attempt_tournament` + +**False pass:** +- **Inspecting the pair list rather than counting judge calls.** An + implementation that builds the cross-class pair and filters it just before + dispatch passes pair inspection and still sends it under a later refactor. + Count at the judge boundary. +- Step 2 written with all-failing attempts, where no boundary exists to cross. + The mixed group is the whole test. +- Step 3 omitted: emitting an empty attribution record satisfies "credit + produced" while carrying no signal. + +## Traps + +- Pairing the failure with its successful retry. It is free, it looks like + signal, and it makes the judge re-decide what the verifier already decided. +- Attributing credit to step index rather than `StepId`, which breaks at the next + workflow version. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.4, §11.5, §11.7, §11.8 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.11-degradation-reasons.md b/tasks/T5.11-degradation-reasons.md new file mode 100644 index 0000000..523a388 --- /dev/null +++ b/tasks/T5.11-degradation-reasons.md @@ -0,0 +1,138 @@ +# T5.11 — Degradation reasons + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +When no comparison is possible, emit `Score::Ungraded { reason }` — a stated +reason, never a neutral score — and reach terminal `Ungraded`. + +## Facts (inlined — no spec read needed) + +Three reasons, each a distinct condition: + +1. **`NoReference`** — a novel `TaskId`. Nothing else has done this task, so + there is nothing to compare against. That is not a degradation to engineer + around; it is the first observation of a new task. Verifier outcome and + deterministic dimensions (cost, latency, tool efficiency) are still recorded, + and **the episode is retained as the reference for that `TaskId`** — so the + second run on that task grades normally. +2. **`BudgetExhausted`** — the tenant is over its grading ceiling. Not a quality + signal; a spend signal. +3. **`InsufficientGroup`** — **tournament only.** A group closes on quorum *or* + on a timeout, grading whatever arrived, with group size attached to the + confidence interval. + +- **A closed group is immutable, and the next episode starts a new one.** A group + closed on timeout at G = 3, then a fourth episode with the same `TaskId` + arrives an hour later: re-opening and re-fitting is the wrong answer, because + strengths from that group have already been published, aggregated and possibly + acted on. A fit that silently changes underneath a decision already made is + worse than a small group. +- So the group key carries a generation: `(TenantId, TaskId, VerifierOutcome, + GroupEpoch)`. **Closure increments the epoch**; late arrivals accumulate into + the next one. +- The cost is honest and should be stated rather than discovered: a low-volume + tenant with a long inter-arrival time gets a run of G = 1 epochs, each + `InsufficientGroup`. That is a real signal about their volume; the fix is a + longer timeout, which is a tenant-level tradeoff between waiting and grading. +- **A tenant whose loop never engages must see that as a stated reason.** Silent + no-op is the worst outcome: it looks like a working loop that finds no + improvements. +- Every one of these runs reaches terminal `Ungraded` and is therefore reducible + (T4.4). Without that, they rest in `Grading` forever and strand storage. +- Group timeout default is unsettled — it trades grading latency against group + size and depends on tenant arrival rate. Only bites when the tournament is on. + +## Steps + +1. Define `UngradedReason { NoReference, BudgetExhausted, InsufficientGroup { group_size } }`. +2. `NoReference` path: on a novel `TaskId`, emit `Ungraded`, record the + deterministic dimensions, and **register the episode as that task's + reference**. The registration is the half that is easy to omit. +3. `BudgetExhausted` path: check the tenant ceiling at the group boundary, before + any model call. Emit the reason; never a default score. +4. `InsufficientGroup` path (tournament only): implement quorum-or-timeout + closure, attach `group_size`, and **increment `GroupEpoch` on close**. +5. Route a late arrival into epoch N+1. Epoch N's stored strengths are never + recomputed. +6. Transition the run to terminal `Ungraded` in every case, carrying the reason. +7. Emit `Ungraded` counts **by reason** as a metric, separately from `Graded`. + +## Acceptance + +- A novel `TaskId` yields `NoReference` **and** the episode is retained as that + task's reference, so the **second** run on the same task produces `Relative` — + assert the second run, since retaining the reference is the half easily omitted. +- A tenant over its ceiling yields `BudgetExhausted`, never a default score. +- Tournament: a group that never fills closes on timeout with size recorded; a + late episode lands in epoch N+1 leaving epoch N's fitted strengths + **byte-identical** — assert on the stored strengths, not on the absence of a + re-fit call. +- In every case the run reaches terminal `Ungraded` and is therefore reducible + (T4.4). + +## Phase gate + +P5 closes when `PairwiseSequential` decides an accept and a reject against a mock +judge, on one resident model with zero swaps. + +## Verify + +**Harness:** a fresh tenant with no history (for `NoReference`), a tenant with a +zero grading ceiling (for `BudgetExhausted`), and a tournament group that never +fills (for `InsufficientGroup`). + +**Integration test** — `tests/it_degradation_reasons.rs`: +1. **Novel `TaskId`:** assert `Score::Ungraded { NoReference }`, and that + deterministic dimensions (cost, latency) are still recorded. +2. **The second half, which is the one that gets omitted:** run a **second** + episode on the same `TaskId`; assert it produces `Score::Relative`. That only + works if run 1's episode was retained as the reference. +3. **Budget:** tenant over its grading ceiling → `BudgetExhausted`. Assert + **zero** model calls were made — the check must precede the spend, not follow + it. Assert no default score anywhere in the output. +4. **Tournament group:** a group that never reaches quorum closes on timeout. + Assert `InsufficientGroup { group_size }` with the real size recorded. +5. **Epoch immutability:** snapshot epoch N's fitted strengths. Deliver a late + episode on the same `TaskId`. Assert it lands in epoch **N+1** and that epoch + N's **stored strengths are byte-identical** to the snapshot. +6. **Terminal state:** for every one of the three reasons, assert the run reaches + terminal `Ungraded` and that T4.4's `is_reducible` returns true. +7. Assert `Ungraded` counts are emitted **by reason** as separate metric series. + +**Command:** `cargo test -p grading degradation` + +**False pass:** +- Step 5 asserting "no re-fit call was made". A refactor that recomputes lazily + on read passes that and still changes a published number. Assert on the + **stored strengths**. +- Step 2 omitted: emitting `NoReference` is trivial; **retaining the reference** + is the half that makes the loop ever engage, and nothing else detects its + absence. +- Step 6 omitted: a run left in `Grading` looks correct in the score output and + strands storage forever for exactly the tenants this section exists to + accommodate. +- Step 3 checking only the reason. If the ceiling is enforced after the calls, + the reason is right and the money is gone. + +## Traps + +- Emitting a neutral 0.5 for an ungradable episode. It averages into promotion + gates and looks like data. +- Re-opening a closed group for a late arrival. +- Leaving the run in `Grading`, which strands storage for exactly the low-volume + and cost-capped tenants least able to absorb it. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.2, §11.6, §14, §15, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.12-p5-composition-gate.md b/tasks/T5.12-p5-composition-gate.md new file mode 100644 index 0000000..9b46be8 --- /dev/null +++ b/tasks/T5.12-p5-composition-gate.md @@ -0,0 +1,120 @@ +# T5.12 — P5 composition gate + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | L — over 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P6 | + +## Goal + +Prove the grading strategies are **interchangeable**: the same recorded episodes, +graded under any strategy, flow through the same downstream pipeline unchanged. + +**Phase gate criterion:** `PairwiseSequential` decides an accept and a reject +against a mock judge, on one resident model with zero swaps. + +## Facts (inlined — no spec read needed) + +- **Defaults ship working; every default is a port.** Grading is + strategy-pluggable, and the strategy declares what hardware it needs before it + is allowed to run. That claim is only proven by running **every** strategy + through the same pipeline on the same input. +- The four strategies and their declared costs per episode, on a group of eight: + +| Strategy | Calls / episode | Models resident | Produces | Default | +|---|---|---|---|---| +| `DeterministicGrader` | 0 | 0 | `Ranked` | — | +| **`PairwiseSequential`** | **1–2** | **1, the agent's** | `Relative` | **yes** | +| `TournamentGrader` | 3–5 | 1 | `Ranked` | opt-in | +| `ReplayTournament` | 3–5 + N agent runs | 1 | `Ranked` | opt-in | + +- The properties no single P5 task owns: + - every strategy's output flows into retention (T4.4), metering and the run + lifecycle **identically** — `Ranked`, `Relative`, `Capped` and `Ungraded` all + reach a terminal run state; + - the declared `calls_per_episode` **matches measured** calls, or every + capacity and spend projection built on it is fiction; + - **zero model swaps** hold across the whole phase, not just per strategy; + - the outcome-class bracketing rule has **no exemptions** — assert it once per + strategy, since each has its own pairing path. +- `opt-in` strategies ship **disabled** and gate nothing. This gate must pass + with them off, and separately with them on. + +## Steps + +1. Fix one corpus of recorded episodes covering: same-outcome groups, + mixed-outcome groups, a novel `TaskId`, a capped episode, and a + budget-exhausted tenant. +2. Grade the corpus under each of the four strategies. +3. Assert the downstream pipeline — score handling, run terminal state, retention + eligibility, metering attribution — is identical in shape across all four. +4. Assert declared-versus-measured cost for each strategy. +5. Run the whole gate twice: opt-in flags **off**, then **on**. +6. Make this the required CI job gating P6. + +## Acceptance + +- All four strategies grade the same corpus; every episode reaches a terminal run + state with retention eligibility correct under each. +- `PairwiseSequential` decides an **accept** and a **reject** against mock judges. +- **Zero model swaps** across the entire gate. +- Measured `compare` calls per episode fall within each strategy's declared + `calls_per_episode`. +- The gate passes with opt-in flags off **and** on. + +## Verify + +**Harness:** one committed episode corpus; mock judges (70% winner, 50/50, +all-draws, position-biased, one `CoreViolation`); counters for `compare` calls, +model **loads**, and agent runs, all separate. + +**Integration test** — `tests/it_p5_composition.rs`: +1. **Strategy interchange:** parameterize over all four strategies. For each, + assert every episode ends in a terminal run state, and that + `is_reducible` (T4.4) agrees with that state. A strategy leaving runs in + `Grading` fails here regardless of its scores. +2. **Cost truth:** assert measured `compare` calls per episode fall inside the + declared `calls_per_episode` band. A declaration that does not match + measurement makes every capacity and spend projection downstream wrong. +3. **Zero swaps:** assert the swap counter is **exactly 0** across the whole + parameterized run, not per strategy. +4. **Bracketing, per strategy:** feed the mixed-outcome group to each strategy; + assert **zero** cross-class `compare` calls in every one. Each strategy has + its own pairing path and the rule has no exemptions. +5. **Score-type handling:** assert `Capped` and `Ungraded` episodes are handled + identically by every strategy's caller — same terminal state, excluded from + aggregates, no numeric coercion anywhere (cross-check T5.3's compile-fail). +6. **Accept and reject:** with the 70% judge assert `Accept`; with the 50/50 + judge assert `Reject`. Assert both are reached in fewer comparisons than + fixed-n. +7. **Flag matrix:** run steps 1–6 with opt-in off (tournament paths must be + unreachable) and on. +8. **Regression:** re-run P0–P4 gates and all P5 suites. + +**Command:** `cargo test -p grading --test it_p5_composition -- --nocapture` + +**False pass:** +- Testing each strategy against its own tailored corpus. The claim is + interchangeability, which only means something on **one** shared input. +- Step 2 omitted: `calls_per_episode` is a declared number nobody measures, and + admission control, capacity planning and spend projection all consume it. +- Step 3 counted per strategy rather than across the run — a swap between two + strategies' evaluations is exactly the failure the one-resident-model default + exists to prevent. +- Step 1 asserting scores only. A strategy that produces correct scores and + leaves runs resting in `Grading` strands storage forever. + +## Traps + +- Enabling the opt-in flags to make the gate pass. They ship disabled and gate + nothing; the off-run is the one that must be green. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11, §14.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.2-capacitylimits-and-the-residency-invariant.md b/tasks/T5.2-capacitylimits-and-the-residency-invariant.md new file mode 100644 index 0000000..60c9485 --- /dev/null +++ b/tasks/T5.2-capacitylimits-and-the-residency-invariant.md @@ -0,0 +1,152 @@ +# T5.2 — `CapacityLimits` and the residency invariant + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T5.3 | + +## Goal + +Declared VRAM limits, the residency invariant, and a `max_context_tokens` that is +**derived** from them rather than configured beside them. + +## Facts (inlined — no spec read needed) + +```rust +pub struct CapacityLimits { + /// Per-device VRAM this framework may use. Not the card's total — leave + /// headroom for anything else sharing the device. + pub vram_bytes_per_device: u64, + pub devices: u32, + /// Hard cap on simultaneously resident models across all devices. + pub max_resident_models: u32, + /// Never evictable. The agent's model belongs here; if it can be evicted, + /// grading can stall agent work, which the first principle forbids. + pub pinned: Vec, +} + +pub struct ModelProfile { + pub id: ModelId, + pub weights_bytes: u64, + /// KV cache cost per token at the deployed dtype and parallelism. + pub kv_bytes_per_token: u64, + /// Devices this model spans under tensor parallelism. + pub devices_required: u32, +} +``` + +Invariant, checked at load **and** before any admission: + +``` +sum(weights of resident models) + peak_concurrent_kv ≤ vram_bytes_per_device × devices +resident_model_count ≤ max_resident_models +``` + +Derived ceiling — never a separate config knob: + +``` +max_context_tokens = (vram_bytes_per_device × devices − sum(weights)) / kv_bytes_per_token +``` + +- Metering counts tokens after the fact; on self-hosted weights the binding limit + arrives earlier and harder. A model that is not resident cannot be inferred + against, and making it resident means evicting something else and paying a load + measured in tens of seconds. +- **One resident model is the default configuration.** The agent's model is + pinned; the judge and the proposer run on that same model. A strategy naming a + second model is legal and is rejected at load unless the invariant holds with + both resident — **never** by swapping between them per call. +- The worked case, because the result is not marginal: at GQA fp16, per-token KV + runs roughly 0.13 MB for an 8B-class model and 0.33 MB for a 70B-class one. Two + episodes at a 200k-token retention ceiling is 400k tokens of context — + **52 GB of KV cache at 8B, 131 GB at 70B**, before weights. Neither fits an + 80 GB device. A judge reading two full-ceiling episodes is not expensive, it is + impossible. +- Therefore the retention ceiling is derived from this, not set beside it: the + reduction target is `max_context_tokens / 2`, and where that is smaller than + the configured ceiling, **the bound wins**. +- Distributed GPUs change the arithmetic, not the rule. `devices_required` + expresses tensor parallelism; `max_resident_models` is a fleet-wide count, so + two nodes each holding the agent model are two resident instances, not one. +- `kv_bytes_per_token` varies with dtype, quantization, attention implementation + and parallelism. It is measured per deployment or read from an operator-supplied + profile — **the framework refuses a guess.** + +## Steps + +1. Define `CapacityLimits` and `ModelProfile` as above. No `Default` for + `kv_bytes_per_token` — an absent value is a load error, not a guess. +2. Write `check_residency(&CapacityLimits, &[ModelProfile]) -> Result<(), CapacityError>` + implementing both inequalities. `CapacityError` names the model and the + shortfall in bytes. +3. Write `derive_max_context_tokens(...)` from the formula above. Expose it as a + function, not a settable field. +4. Call the check at load, and again before admission (T8.6 enforces the runtime + half). +5. Refuse a context request over the derived ceiling with the computed limit in + the error message. +6. Encode the worked case as a test: 70B-class profile, 0.33 MB/token, 80 GB + device, two 200k-token episodes → refused. + +## Acceptance + +- A config whose resident set exceeds VRAM is rejected **at load**, naming the + model and the shortfall — not at first inference. +- A judge context request exceeding the derived ceiling is refused with the + computed limit in the error. +- The worked case asserts the arithmetic: 70B-class, 0.33 MB/token, 80 GB device, + two 200k-token episodes must be refused. + +## Verify + +**Harness:** pure arithmetic — no GPU required. Model profiles as fixtures with +declared `weights_bytes` and `kv_bytes_per_token`. + +**Integration test** — `tests/it_capacity_invariant.rs`: +1. **The worked case, hardcoded:** 70B-class profile, `kv_bytes_per_token = + 0.33 MB`, one 80 GB device. Two 200k-token episodes = 400k tokens = 131 GB of + KV before weights. Assert the request is **refused** and that the error + carries the computed limit. +2. Same arithmetic at 8B / 0.13 MB per token: 52 GB. Assert also refused on an + 80 GB device once weights are counted. +3. Over-capacity config at load: resident set exceeding VRAM → rejected **at + load**, error naming **the model and the byte shortfall**. Assert the message + contains both. +4. Assert `max_context_tokens` is only reachable as a **function**, not a + settable field — `trybuild` if it is a private field with no setter. +5. Multi-device: `devices_required = 4` under tensor parallelism; assert the + invariant uses `vram × devices` and that `max_resident_models` is counted + fleet-wide, so the same model on two nodes counts as two. +6. Missing `kv_bytes_per_token` → load error, **not** a default. Assert no + `Default` impl exists. +7. Pinned model: assert no code path can evict a pinned model to satisfy the + invariant — the eviction candidate list excludes pinned entries. + +**Command:** `cargo test -p capacity invariant` + +**False pass:** +- Asserting only that "some error" is returned. The acceptance criterion is that + the **arithmetic** is right — pin the expected byte figures in the test, so an + off-by-a-factor error in the KV formula is caught rather than rounded away. +- Step 3 checked at first inference rather than at load. Both produce an error; + only one produces it before 3am. +- Testing with a permissive config where everything fits. Every implementation + passes that. + +## Traps + +- A `max_context_tokens` config field. It will be set to something plausible and + discovered wrong at the first judge call. +- Treating "call the judge model" as equivalent in cost to "call the agent + model". On this hardware that is wrong by two orders of magnitude. +- Allowing an eviction of a pinned model to satisfy the invariant. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.6, §14.2, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.3-evaluationstrategy-port-resourceprofile.md b/tasks/T5.3-evaluationstrategy-port-resourceprofile.md new file mode 100644 index 0000000..058a928 --- /dev/null +++ b/tasks/T5.3-evaluationstrategy-port-resourceprofile.md @@ -0,0 +1,158 @@ +# T5.3 — `EvaluationStrategy` port + `ResourceProfile` + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T5.4, T5.5 | + +## Goal + +The grading ports, and the type discipline that stops a capped or ungraded +episode from being averaged into a promotion gate. + +## Facts (inlined — no spec read needed) + +```rust +#[async_trait] +pub trait EvaluationStrategy: Send + Sync { + fn id(&self) -> StrategyId; + /// Declared before any work is admitted. Validated against capacity limits + /// at load; a strategy whose profile does not fit is rejected by name. + fn resources(&self) -> ResourceProfile; + async fn evaluate(&self, cx: &EvalCtx) -> Result>; +} + +pub struct ResourceProfile { + /// Models this strategy calls. One entry means it runs on the agent's + /// already-resident model and forces no swap. + pub models: Vec, + /// Largest single-call context. A pairwise judge reads two episodes, so this + /// is roughly twice an episode budget. + pub max_context_tokens: u32, + /// Model calls per episode evaluated, for capacity and spend projection. + pub calls_per_episode: f32, +} + +#[async_trait] +pub trait Grader: Send + Sync { + async fn grade(&self, group: &Group, rubric: &RubricDef) -> Result>; +} + +#[async_trait] +pub trait Judge: Send + Sync { + /// Relative comparison only. Deliberately cannot return an absolute score. + async fn compare(&self, a: &EpisodeView, b: &EpisodeView, r: &RubricDef) + -> Result; + /// Unary, separate from `compare`: a `Core` violation caps an episode on its + /// own terms, not relative to an opponent. Runs before pairing. + async fn screen(&self, e: &EpisodeView, r: &RubricDef) -> Result>; +} + +pub enum Verdict { A, B, Draw } + +pub struct CoreViolation { pub criterion: RubricCriterionId, pub evidence: BlobRef } + +pub enum Score { + Relative { against: RunId, verdict: Verdict, record: WinRecord }, + Ranked { strength: f64, interval: (f64, f64), group_size: u32 }, + Capped { violations: Vec }, + Ungraded { reason: UngradedReason }, +} +``` + +- **`compare` returns `Verdict`, never a number.** Absolute scores fail three + ways here: calibration drift (same episode scores differently across weeks and + model versions, and drift is indistinguishable from a variant trend); + weak discrimination (four competent episodes all score 0.8); and saturation — + as workflows improve, pass rate approaches 100% and absolute rubric scores + saturate identically. A tournament cannot saturate; better candidates just make + it harder. +- **`Score` is a sum, not a number with flags.** A capped episode and an ungraded + one are not low scores; they are different kinds of answer. A type that can + represent them as numbers will eventually have them averaged into a promotion + gate by code that meant no harm. +- **The strategy declares what hardware it needs before it is allowed to run.** + Strategies differ by more than an order of magnitude in model calls and VRAM; a + deployment that cannot afford one must be told at load, not by an OOM at 3am. + +Strategy catalogue — cost per episode evaluated, group of eight: + +| Strategy | Calls / episode | Models resident | Produces | Default | +|---|---|---|---|---| +| `DeterministicGrader` | 0 | 0 | `Ranked` on a computed number | — | +| **`PairwiseSequential`** | **1–2** | **1, the agent's** | `Relative` | **yes** | +| `TournamentGrader` | 3–5 | 1 | `Ranked` with intervals | opt-in | +| `ReplayTournament` | 3–5 **plus N full agent runs** | 1 | `Ranked` across variants | opt-in | + +## Steps + +1. Define all types above verbatim. `Verdict` has exactly three variants. +2. Give `Score` **no** `Into`, no `as_number()`, no `unwrap_or(0.0)` + convenience. Aggregation code must match on the variant. +3. Keep `screen` separate and unary. It is the only source of `CoreViolation`, + and it runs before pairing. +4. At strategy registration, call T5.2's residency check against + `resources().models` plus the pinned set. Reject by strategy **name** with the + shortfall. +5. Validate `resources().max_context_tokens` against T5.2's derived ceiling at + the same point. +6. Write the two `trybuild` compile-fail cases described in acceptance. + +## Acceptance + +- Compile-fail test: a judge cannot return `f64`. +- Compile-fail test: `Score::Capped` and `Score::Ungraded` cannot be coerced to a + number — no `Into`, no `unwrap_or(0.0)` path through the aggregate. +- A strategy whose `ResourceProfile` names a second model is rejected at load + when the residency invariant (T5.2) does not admit both. + +## Verify + +**Harness:** `trybuild` for the type discipline; T5.2's capacity checker for the +load-time rejection. + +**Integration test** — `tests/it_grading_ports.rs` + `tests/compile_fail/`: +1. **Compile-fail:** a `Judge` impl whose `compare` returns `f64`. Assert the + stderr names the `Verdict` return type. +2. **Compile-fail:** `Into::::into(Score::Capped { .. })`, + `score.unwrap_or(0.0)`, and any `as_number()` call. One case each — a single + case leaves the other routes open. +3. **Audit test:** enumerate every aggregation site and assert each matches on + `Score` variants exhaustively with no `_` arm. +4. Load-time rejection: a strategy whose `ResourceProfile` names a **second** + model, under a `CapacityLimits` that admits only one. Assert rejection **by + strategy name**, at registration, before any evaluation runs. +5. Positive: the same strategy under limits that admit both models is accepted — + otherwise the check is just "reject two models", which is the wrong rule. +6. Context check: a strategy declaring `max_context_tokens` above T5.2's derived + ceiling is rejected with the computed limit in the error. +7. `screen` separation: assert `CoreViolation` can only originate from `screen` — + grep plus a test that a `compare` result cannot construct one. + +**Command:** `cargo test -p grading ports && cargo test -p grading --test compile_fail` + +**False pass:** +- One compile-fail case for `f64`. The leak that matters is on `Score`, not on + `Verdict` — a capped episode reaching an average is the failure this type + discipline exists to prevent, and step 2 is where it is caught. +- Step 5 omitted, so a naive "more than one model is illegal" implementation + passes and blocks a legitimate two-resident-model deployment. +- Step 3 omitted: the types can be sound while one aggregation site does + `if let Some(n) = ...` and silently skips capped episodes instead of failing. + +## Traps + +- A `Score::value() -> Option` helper. Every call site then writes + `.unwrap_or(0.0)` and the cap becomes a zero. +- Folding `screen` into `compare`. A judge asked to express a `Core` violation + through a comparison can only rank the offender lower, which is not a cap. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.1, §11.2, §14.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.4-deterministicgrader.md b/tasks/T5.4-deterministicgrader.md new file mode 100644 index 0000000..884dc40 --- /dev/null +++ b/tasks/T5.4-deterministicgrader.md @@ -0,0 +1,88 @@ +# T5.4 — `DeterministicGrader` + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Grade on a number the user already computes — latency, cost, test pass count — +with zero model calls. + +## Facts (inlined — no spec read needed) + +- It exists so that **adopting the framework does not require adopting + LLM-as-judge at all.** +- Cost profile: 0 model calls per episode, 0 models resident, produces `Ranked` + on a computed number. +- Its `ResourceProfile` declares an **empty** `models` vector. That is the whole + point — it forces no residency and passes the capacity check trivially. +- It still produces `Score::Ranked`, so the aggregation path downstream is + unchanged. +- Never let a rubric judge what a verifier can check: criteria that can be made + mechanical belong here or in a `Verifier`, not in a rubric line. + +## Steps + +1. Implement `EvaluationStrategy` with `resources()` returning + `ResourceProfile { models: vec![], max_context_tokens: 0, calls_per_episode: 0.0 }`. +2. Define the metric extractor as a user-supplied function over `EpisodeView` — + the framework supplies latency, cost and usage; the user supplies anything + domain-specific such as test pass count. +3. Rank within the group by the computed number and emit `Score::Ranked` with + `group_size`. +4. Ensure the interval is meaningful or explicitly degenerate — do not fabricate + a confidence interval around a deterministic measurement. +5. Test with a workflow graded purely on test-pass-count and latency; assert the + model-call counter is exactly zero. + +## Acceptance + +- A workflow graded purely on test-pass-count and latency, **no model calls**. +- Its `ResourceProfile` declares **zero models**. + +## Verify + +**Harness:** a model-call counter installed at the provider boundary, counting +**all** purposes. + +**Integration test** — `tests/it_deterministic_grader.rs`: +1. Run a workflow graded on test-pass-count and latency. +2. Assert the model-call counter is **exactly 0** across the whole run's grading + phase — including any tiebreak path. +3. Assert `resources().models.is_empty()` and `calls_per_episode == 0.0`. +4. Assert registration passes the capacity check under a `CapacityLimits` with + **zero** devices — nothing about this strategy should need a GPU. +5. Determinism: grade the same group twice; assert identical `Score::Ranked` + values including ordering of ties. +6. Assert the emitted score is `Ranked`, so downstream aggregation is unchanged + from the tournament path. +7. Assert no fabricated confidence interval — either a real one or an explicitly + degenerate one, asserted as such. + +**Command:** `cargo test -p grading deterministic` + +**False pass:** +- Counting only judge-purpose model calls. A tiebreak issued under + `Purpose::Work` would slip through — count all purposes. +- Step 4 omitted: declaring the agent's model in `models` "since it is resident + anyway" passes steps 1–3, then fails a capacity check it should pass and + misreports the spend projection. +- Step 5 with a group of one, where ordering is trivially stable. + +## Traps + +- Declaring the agent's model in `models` "since it is resident anyway". It then + fails a capacity check it should pass, and it lies in the spend projection. +- Calling a model for a tiebreak. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.1, §11.7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.5-pairwisesequential-reference-and-comparison.md b/tasks/T5.5-pairwisesequential-reference-and-comparison.md new file mode 100644 index 0000000..52c1447 --- /dev/null +++ b/tasks/T5.5-pairwisesequential-reference-and-comparison.md @@ -0,0 +1,117 @@ +# T5.5 — `PairwiseSequential`: reference and comparison + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T5.6, T5.7 | + +## Goal + +The default grading strategy: one current version, at most one challenger, one +`Judge::compare` per new episode. + +## Facts (inlined — no spec read needed) + +``` + current version ──► episode ──┐ + ├──► Judge::compare ──► verdict + challenger ──► episode ──┘ │ + (same TaskId) ▼ + accumulate into WinRecord +``` + +- Grading answers **one** question: has the challenger accumulated enough + evidence to replace the current version? Not "rank these eight", not "what is + each episode worth". +- **The reference is the current version's recorded episode on the same + `TaskId`.** It is already on disk — no re-run, no group to fill, no group + timeout. A `TaskId` never seen before has no reference and degrades per T5.11. +- One comparison per episode. Against the tournament's `(G/2)·log₂(G)` pairs + doubled for both orderings, that is **24 judge calls dropping to 8 at G = 8**, + and the saving grows with G rather than shrinking. +- `ResourceProfile` declares **exactly one model — the agent's, already + resident**. That is precisely why the default grading path adds no resident + model and forces no swap. +- **A fixed reference is a cacheable prefix.** The same reference episode leads + every comparison in a decision, so it can be cached across calls. The + tournament cannot do this — shuffling into brackets makes every pair a novel + combination by design. +- Episodes are bracketed **within** a verifier outcome class, never across. A + verified pass beats a verified fail by definition and that pairing is never + shown to a judge. +- What this gives up, stated rather than discovered later: parallel exploration + and composable strengths. One challenger at a time is hill-climbing; a + `Relative` score answers "better than the current version on this task" and + does not compose across tasks. + +## Steps + +1. Implement `EvaluationStrategy` with `resources()` naming exactly one model — + the agent's pinned model — and `calls_per_episode: 1.0`. +2. Reference lookup: given `(TenantId, TaskId)`, find the current version's most + recent recorded episode. Return `None` for a novel task and hand off to T5.11. +3. Check the verifier outcome class before pairing. Reference and candidate must + share a class or no comparison is issued. +4. Build the comparison prompt with the **reference half first and byte-stable** + across calls. Assert that stability in a test — prompt caching depends on it. +5. Issue exactly one `Judge::compare` per new episode and fold the verdict into a + `WinRecord` (consumed by T5.6). +6. Instrument two counters: `compare` calls and model loads. Both are asserted. + +## Acceptance + +- Grading one episode issues **exactly one** `compare` call and **zero** model + loads — both asserted by counting. +- A second episode on the same `TaskId` reuses the identical reference, so the + reference half of the prompt is **byte-identical** across calls. + +## Verify + +**Harness:** a mock `Judge` recording every `(prompt_bytes, verdict)`; a model +**load** counter distinct from the call counter; two recorded episodes on one +`TaskId`. + +**Integration test** — `tests/it_pairwise_reference.rs`: +1. Grade one episode against an existing reference. Assert **exactly 1** + `compare` call and **exactly 0** model loads. +2. Grade a second episode on the same `TaskId`. Assert the **reference half of + the prompt is byte-identical** between the two calls — capture both prompts + from the mock judge and compare the reference prefix directly. This is the + property prompt caching depends on and it is easy to break invisibly. +3. Assert the reference selected is the **current version's** recorded episode, + not the highest-scoring one — plant a better-scoring non-current episode on + the same `TaskId` and assert it is not chosen. +4. Outcome-class guard: reference is a verified pass, candidate is a verified + fail. Assert **zero** `compare` calls. +5. Novel `TaskId`: assert `compare` is not called and the path hands off to + T5.11's `NoReference`. +6. Assert `resources().models.len() == 1` and that it equals the pinned agent + model. + +**Command:** `cargo test -p grading pairwise` + +**False pass:** +- Step 2 comparing whole prompts. They legitimately differ in the candidate half; + comparing the whole thing either always fails or, if the test is loosened to + "both non-empty", always passes. Compare the **prefix**. +- Step 1 counting calls but not loads. A judge running on a non-resident model + returns the right verdict and costs tens of seconds per call. +- Step 3 omitted: "pick the best episode as reference" produces sensible-looking + verdicts against a moving target, and no other assertion here catches it. + +## Traps + +- Re-rendering the reference half per call with a timestamp or a fresh id in it. + It still works and it silently destroys the cacheable prefix. +- Selecting the reference by "best episode" rather than the current version's + episode. That makes the comparison a moving target. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.1, §11.3, §11.8, §14.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.6-sequential-test-and-stopping.md b/tasks/T5.6-sequential-test-and-stopping.md new file mode 100644 index 0000000..b3dcadc --- /dev/null +++ b/tasks/T5.6-sequential-test-and-stopping.md @@ -0,0 +1,120 @@ +# T5.6 — Sequential test and stopping + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T6.3 | + +## Goal + +Stop on evidence, not on a fixed sample. Verdicts accumulate into a likelihood +ratio tested against α/β boundaries. + +## Facts (inlined — no spec read needed) + +``` + ┌────────────────┬────────────────┐ + ▼ ▼ ▼ + accept continue reject + challenger becomes keep sampling discard, keep + the current current +``` + +- **Stopping is a sequential test, not a fixed sample.** Boundaries are set by α, + β and the smallest win-rate shift worth acting on; the test stops as soon as a + boundary is crossed. A clearly better challenger is accepted in far fewer + comparisons than a fixed-n design would spend, and a clearly worse one is + rejected early instead of running to completion. +- This is what makes cost adaptive: cheap decisions cost little, close decisions + cost more, and nothing costs the worst case by default. +- **Draws are recorded and excluded from the ratio.** A tie carries no evidence + about which is stronger, so folding it in as half a win manufactures + information. +- **But a high draw rate is itself a result** — it says the challenger is not + meaningfully different — so the test also **rejects on a draw-rate ceiling** + rather than sampling forever toward a boundary it will never reach. +- α, β and the minimum detectable win-rate shift are **policy, not physics**: too + tight and no challenger is ever accepted, too loose and the loop churns the + current version on noise. Same for the draw ceiling, which interacts with judge + quality — a weak judge draws more. Expose all four as configuration with + documented defaults. + +## Steps + +1. Define `WinRecord { wins, losses, draws }` and persist it with the decision, + not only in memory — the decision must survive a restart. +2. Implement the sequential probability ratio test: maintain the log-likelihood + ratio over wins and losses under H0 (win rate = 0.5) and H1 (win rate = + 0.5 + δ), with boundaries derived from α and β. +3. Exclude draws from the ratio; count them separately. +4. Add the draw-rate ceiling check after each comparison: over the ceiling with a + minimum sample, reject. +5. Return a three-valued decision — `Accept`, `Reject`, `Continue` — and let the + caller decide whether to sample another episode. +6. Expose α, β, δ and the draw ceiling as configuration. Document the defaults as + provisional and needing calibration against a workflow whose true improvement + is known. +7. Build the mock judges: a 70% winner, a 50/50 coin, and an all-draws judge. + +## Acceptance + +- A mock judge with a true 70% win rate crosses the accept boundary in + **materially fewer comparisons than a fixed-n design at the same α** — assert + the count, since an implementation that ignores the boundary and runs to n + still reaches the right answer. +- A 50/50 judge rejects. +- A judge returning `Draw` on every comparison terminates on the draw ceiling + instead of running forever. + +## Verify + +**Harness:** seeded mock judges with known true win rates. Every test asserts a +**comparison count**, not only an outcome — the outcome is reachable by an +implementation that ignores the boundary entirely. + +**Integration test** — `tests/it_sequential_test.rs`: +1. **70% winner:** assert `Accept`, and assert the comparison count is + **materially below** the fixed-n sample size for the same α. Compute the + fixed-n figure in the test and assert `count < fixed_n`, with both printed. +2. **50/50 judge:** assert `Reject`, and assert it terminates — bounded count. +3. **All-draws judge:** assert termination **on the draw-rate ceiling**, and + assert the reason is the ceiling, not the α boundary. Without this it loops + forever. +4. **30% winner (clearly worse):** assert `Reject` early — assert the count is + well below the 70% case's, proving early rejection works in both directions. +5. Draws excluded: feed a sequence of `win, draw, win, draw`; assert the + likelihood ratio equals that of `win, win`, and that `draws` is counted + separately in the `WinRecord`. +6. Persistence: kill the process mid-decision, restart, assert the `WinRecord` + survived and the decision resumes rather than restarting. +7. Repeat each case over 100 seeds; assert the accept/reject rates sit within + α/β. A single seed says nothing about a statistical test. + +**Command:** `cargo test -p grading sequential -- --nocapture` + +**False pass:** +- Asserting only the decision. A fixed-n implementation reaches the **right + answer** on all of steps 1–4 and delivers none of the cost saving that is the + entire justification for the design. The count assertion is the test. +- Step 5 omitted: counting a draw as half a win tightens the boundary on invented + evidence and still produces plausible decisions. +- One seed per case. A sequential test is a statistical object; step 7 is what + makes the α/β claim meaningful. + +## Traps + +- Counting a draw as half a win. It tightens the boundary on evidence that does + not exist. +- Checking the boundary only at the end. The test then costs exactly what a + fixed-n design costs, and the acceptance count catches it — which is why the + count is asserted rather than the outcome. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.3, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.7-order-alternation-and-sampled-consistency.md b/tasks/T5.7-order-alternation-and-sampled-consistency.md new file mode 100644 index 0000000..a899773 --- /dev/null +++ b/tasks/T5.7-order-alternation-and-sampled-consistency.md @@ -0,0 +1,98 @@ +# T5.7 — Order alternation and sampled consistency + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Cancel judge position bias across the sequence instead of within each pair, and +measure order-consistency on a sample rather than by doubling every call. + +## Facts (inlined — no spec read needed) + +- Pairwise judges have **position bias**. The tournament path cancels it by + judging both orderings, paying 2× on every comparison. +- The default path does not pay that: **the challenger takes position A on + even-numbered comparisons and position B on odd ones.** Bias cancels across the + sequence, at no extra cost. +- Order-consistency is still measured — on a **sampled fraction** of comparisons + — and reported as the grader's own error bar. +- That disagreement rate is a first-class grader-health metric. An inconsistent + judge should widen the sample gate rather than silently promote. +- Total calls must stay near 1× the comparison count. A doubling implementation + produces the right consistency number and the wrong cost, which is why the call + count is asserted rather than the metric. + +## Steps + +1. Alternate position by comparison index: even → challenger is A, odd → + challenger is B. Record the position used on each comparison. +2. Normalize the verdict back to challenger-relative terms before folding into + the `WinRecord` (T5.6) — otherwise alternation inverts half the record. +3. Sample a configurable fraction of comparisons for a second, order-swapped + call. Default the fraction low; it is an error bar, not the measurement. +4. Compute the disagreement rate over sampled pairs and emit it as a metric + tagged by judge and rubric. +5. Count total `compare` calls and assert the ratio to comparison count in the + test. +6. Build a deliberately position-biased mock judge — always picks A — as the + detection case. + +## Acceptance + +- A deliberately position-biased mock judge is **detected** and its disagreement + rate reported. +- Total `compare` calls stay within **1.0–1.2×** the comparison count. A doubling + implementation fails this, which is the point. + +## Verify + +**Harness:** a mock judge that **always picks position A** regardless of content +— maximal position bias, so detection is unambiguous. + +**Integration test** — `tests/it_order_alternation.rs`: +1. Run 100 comparisons against the always-A judge. +2. Assert the disagreement rate is **detected and reported** near 100% on the + sampled subset. +3. Assert total `compare` calls fall in **1.0–1.2×** the comparison count. A + doubling implementation lands at 2.0× and fails here, which is the point. +4. Assert position alternation: challenger in A on even indices, B on odd. Read + the recorded position per comparison, do not infer it. +5. **Normalization:** with the always-A judge, assert the resulting `WinRecord` + is near 50/50 rather than 100% challenger wins. A missing verdict + normalization after the swap inverts half the record and produces exactly + 100%, which looks like a strong result. +6. Unbiased judge control: a content-driven mock judge yields a low disagreement + rate — otherwise the detector fires on everything. +7. Assert the disagreement rate is emitted as a metric tagged by judge and + rubric. + +**Command:** `cargo test -p grading order_alternation` + +**False pass:** +- Step 3 omitted. Judging both orderings on every comparison produces a perfect + consistency measurement at double the cost — correct-looking and exactly the + design this task rejects. +- Step 5 omitted: forgetting to normalize verdicts after alternating is the most + likely implementation error, and it makes a biased judge look like a decisive + one. +- Step 6 omitted, so a detector that always reports high bias passes. + +## Traps + +- Forgetting to normalize the verdict after swapping positions. The record then + averages toward 50% regardless of the truth. +- Sampling "every comparison, it is cheap" — that is the doubling implementation + with extra steps. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.3, §11.4, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.8-swiss-pairing.md b/tasks/T5.8-swiss-pairing.md new file mode 100644 index 0000000..67fddb0 --- /dev/null +++ b/tasks/T5.8-swiss-pairing.md @@ -0,0 +1,109 @@ +# T5.8 — Swiss pairing + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | opt-in | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Swiss pairing over a comparison group: log₂(G) rounds, G/2 comparisons per round. +Ships **disabled** — it gates nothing. + +## Facts (inlined — no spec read needed) + +``` + [ G comparable episodes for one task, one outcome class ] + │ + ▼ + [ shuffle into brackets ] ◄── shuffling also cancels position bias + │ + ▼ + ┌─────────────────────────────────┐ + │ Swiss pairing, log₂(G) rounds │ ◄── rubric-guided Judge, relative only + └────────────────┬────────────────┘ + ▼ + [ Bradley-Terry fit over all comparisons ] ◄── T5.9 +``` + +- **Swiss rather than round-robin**: O(G log G) instead of O(G²). Eight episodes + is twelve comparisons rather than twenty-eight. +- **Swiss rather than single elimination**: we want a full ranking, not a + champion — eliminated candidates still carry signal. +- **Both orderings are judged** on this path, so 12 pairs at G = 8 is 24 + `compare` calls. (The default path instead alternates order — T5.7.) +- **Draws are permitted.** A judge forced to separate two equivalent episodes + invents a distinction, and the optimizer will chase the invention. Draws cost + gradient; forced choices cost correctness. +- Groups are bracketed within **one verifier outcome class**, never across. +- Enable this where episodes are already co-present at no extra cost: attempt + tournaments (T5.10) and replay (T6.5). + +## Steps + +1. Take a group of G episodes sharing `(TenantId, TaskId, VerifierOutcome, + GroupEpoch)`. +2. Shuffle into the first round from a **recorded seed** — the shuffle is part of + bias cancellation, and an unreproducible pairing is an unauditable result. +3. Run `ceil(log2(G))` rounds. Each round pairs on accumulated score, avoiding + rematches; carry a bye when G is odd. +4. Issue both orderings per pair and record order-consistency per comparison. +5. Track the constraint explicitly: each episode paired at most once per round, + and no episode idle in more than one round. +6. Emit the full comparison list — the BT fit (T5.9) consumes pairs, not + standings. +7. Keep it behind an off-by-default feature flag. + +## Acceptance + +- G = 8 produces **exactly 12 pairs and 24 `compare` calls** under the + both-orderings rule. +- Every episode is paired at most once per round. +- No episode is idle in more than one round. + +## Verify + +**Harness:** a counting mock judge; a fixed shuffle seed so pairings are +reproducible. + +**Integration test** — `tests/it_swiss_pairing.rs`: +1. G = 8: assert **exactly 12 pairs** and **exactly 24 `compare` calls** (both + orderings). +2. Assert every episode is paired **at most once per round**. +3. Assert **no episode is idle in more than one round**. +4. Assert no pair repeats across rounds. +5. Odd G (G = 7): assert the bye is assigned, assigned to a different episode + each round, and that the comparison count matches the expected formula. +6. G = 2 and G = 3 boundary cases: assert the pairing does not panic and produces + the minimum sensible schedule. +7. Assert the shuffle seed is **recorded** with the group, and that replaying the + seed reproduces the identical pairing. +8. Assert every pair sits within one `VerifierOutcome` class — feed a mixed group + and assert zero cross-class pairs. + +**Command:** `cargo test -p grading swiss` + +**False pass:** +- Asserting the pair count alone. Round-robin at G = 8 gives 28; a single + elimination gives 7. But an implementation that pairs the same two episodes + repeatedly can also hit 12 — steps 2 and 4 are what make the count meaningful. +- Testing only G = 8, a clean power of two. The bye logic at odd G is where the + "idle in more than one round" rule actually bites. +- An unseeded shuffle, which makes a surprising standings result impossible to + reproduce or audit. + +## Traps + +- Round-robin "since G is small". It is 28 pairs at G = 8 and grows quadratically. +- Dropping eliminated episodes. Swiss was chosen precisely to keep their signal. +- An unseeded shuffle, which makes a surprising result impossible to reproduce. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.4, §11.8 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T5.9-bradley-terry-fit.md b/tasks/T5.9-bradley-terry-fit.md new file mode 100644 index 0000000..7b3412d --- /dev/null +++ b/tasks/T5.9-bradley-terry-fit.md @@ -0,0 +1,132 @@ +# T5.9 — Bradley-Terry fit + +| Field | Value | +|---|---| +| Phase | P5 — Grading | +| Size | L — over 3 days | +| Status | Not started | +| Flags | opt-in | +| Spec | inlined below | +| Blocks | T6.4 | + +## Goal + +Strength parameter plus confidence interval over the pairwise outcomes. Ships +**disabled** — it gates nothing. + +## Facts (inlined — no spec read needed) + +**Bradley-Terry, not point-tally z-scores.** Accumulating tournament points and +normalizing to mean 0 / sd 1 within a group is the obvious approach and it is +statistically wrong for the aggregate: small groups produce extreme z-scores, so +a variant appearing in many small groups wins on variance rather than quality. A +BT fit yields a strength with a real confidence interval, which composes across +groups of different sizes. + +Three decisions, none optional. Textbook BT does none of them, and each failure +looks like a result rather than a bug: + +- **Davidson tie term.** Plain BT is binary and has no tie parameter, so the + permitted draws have nowhere to go. Dropping them discards the judge's most + confident statements; splitting each half-and-half fabricates two comparisons + that never happened and tightens the interval on invented evidence. Fit one + additional tie parameter alongside the strengths. +- **Weakly-informative prior — required, not tuning.** At G = 4..8 an episode + that wins every comparison drives the unpenalized maximum-likelihood estimate + to infinite strength. The prior (equivalently, a penalized likelihood) is what + turns "won all three of its comparisons" into a **wide** interval rather than + an unbounded one. Without it, the reassuring claim that small groups produce + wide intervals is simply false. +- **Control pinned to zero.** The fit is identified only up to an additive + constant, so an interval on a single raw strength is an interval on an + arbitrary origin. Pin control to zero and report every strength as a delta + against it. Downstream gates say "BT interval excludes zero" — zero is control, + and it is only zero because it was pinned there. +- A group containing **no control episode is not aggregatable**. It still grades + its own members and is worth reading; it just does not feed the aggregate, and + it is recorded as such rather than folded in on the assumption that scales + match. + +## Steps + +1. Take the comparison list from T5.8: `(episode_a, episode_b, Verdict)`. +2. Implement the Davidson model: strengths plus one tie parameter, fit by + penalized maximum likelihood. +3. Add the weakly-informative prior on strengths. Make its strength a named + constant with a comment stating it is required for convergence, not a tuning + knob — otherwise it gets removed as "unnecessary regularization". +4. Constrain control's strength to zero during the fit rather than subtracting it + afterwards. +5. Compute intervals (profile likelihood or bootstrap over comparisons) and emit + `Score::Ranked { strength, interval, group_size }`. +6. Reject a group with no control episode as non-aggregatable, with a typed + error — do not fit against an arbitrary origin. +7. Build the synthetic generator: known strengths, configurable draw rate. + +## Acceptance + +- On synthetic data with known strengths, recovered ordering is correct and the + delta-from-control interval covers the true delta at the stated rate. + +Three targeted cases beyond that: + +- A group where one episode wins every comparison **converges, with a wide + interval**. Without the prior this diverges — assert the bound, not just that + the fit returns. +- A synthetic generator producing **30% draws** recovers the true strengths; a + fit that drops draws does not, and the test asserts the gap. +- A group with **no control episode is rejected** as non-aggregatable rather than + fit against an arbitrary origin. + +## Verify + +**Harness:** a synthetic generator with **known** true strengths and a +configurable draw rate. Every claim here is checkable against ground truth, so +none of it needs a real judge. + +**Integration test** — `tests/it_bradley_terry.rs`: +1. Generate comparisons from known strengths. Assert the recovered **ordering** + is correct. +2. **Coverage test:** repeat over 200 seeded datasets; assert the + delta-from-control interval covers the true delta at the stated rate (e.g. + ~95% for a 95% interval). One dataset proves nothing about an interval. +3. **Clean sweep:** a group where one episode wins every comparison. Assert the + fit **converges** and assert a **numeric upper bound** on the interval width — + not merely that the call returned. Without the prior this diverges. +4. **Draws at 30%:** assert recovered strengths are close to truth. Then run a + draw-dropping fit on the same data and **assert the gap** — the test must show + the Davidson term is doing work, not just that the fit runs. +5. **Half-split control:** also fit with draws split half-and-half; assert its + interval is **narrower than truth warrants** (over-confident), demonstrating + why splitting is rejected. +6. **No control:** a group with no control episode is **rejected as + non-aggregatable** with a typed error — not fit against an arbitrary origin. +7. Assert control's fitted strength is exactly 0 and every other strength is + reported as a delta. + +**Command:** `cargo test -p grading bradley_terry -- --nocapture` + +**False pass:** +- Step 3 asserting only that the fit returns a value. An unpenalized fit returns + a very large finite number on many datasets and looks fine. Assert the bound. +- Step 4 without the comparison fit from step 5. "The fit recovers strengths" + passes on easy data whether or not draws are modelled — the **gap** is the + evidence. +- Step 2 with one dataset. Interval coverage is a frequency claim; it needs + repetition. +- Reporting raw strengths anywhere: an interval on a raw strength is an interval + on an arbitrary origin, and it will look perfectly reasonable. + +## Traps + +- Splitting draws half-and-half. It is the standard workaround and it invents + evidence. +- Removing the prior after seeing it barely move the estimate on well-separated + data. It exists for the clean-sweep case. +- Reporting raw strengths anywhere outside the fit. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.2, §11.4 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.1-challenger-allocation.md b/tasks/T6.1-challenger-allocation.md new file mode 100644 index 0000000..5b23c3f --- /dev/null +++ b/tasks/T6.1-challenger-allocation.md @@ -0,0 +1,103 @@ +# T6.1 — Challenger allocation + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T6.3 | + +## Goal + +One current version, at most one challenger, 95/5 traffic split, guarded by a +generation counter with compare-and-swap. + +## Facts (inlined — no spec read needed) + +- **One current version, at most one challenger.** The loop converges toward a + single state rather than maintaining a population. That is the change that + makes everything else affordable: no N-way traffic split, no per-variant + aggregation across groups, no allocation state to contend over, one comparison + per episode instead of a bracket. +- The cost is exploration. A single challenger at a time is hill-climbing — it + finds improvements more slowly than a population would, and it can sit in a + local optimum indefinitely with nothing in the loop able to say so. The + held-out report (T6.6) is the only instrument that will notice, which makes it + **more** important here, not less. +- **The decide stage is single-writer per `(TenantId, WorkflowId)`.** Two + schedulers adjusting traffic allocation concurrently produce an allocation + neither holds. A compare-and-swap on a generation counter is sufficient; no + lock service is needed at this size. +- The slow generate loop fires when the fast loop **rejects a challenger without + finding a replacement** — a trigger, not a timer. +- Multi-variant selection stays available for deployments that enable the + tournament and allow N challengers; the machinery is the same. + +## Steps + +1. Define the allocation record: `(TenantId, WorkflowId) → { current: WorkflowVersion, + challenger: Option, split, generation: u64 }`. +2. Registration of a challenger is rejected when one is already live — the error + **names the incumbent challenger**, so the operator knows what to retire. +3. Every update is a CAS on `generation`. The loser retries against the winner's + generation rather than overwriting. +4. Route spawns by the split: 95% current, 5% challenger. Record which version a + run was allocated to on the spawn event. +5. Emit the "rejected without replacement" trigger as an event so the generate + loop can subscribe. Do not add a timer. +6. Test two concurrent schedulers committing allocations. + +## Acceptance + +- Registering a second challenger while one is live is **rejected, naming the + incumbent challenger**. +- Two concurrent schedulers cannot both commit an allocation; the loser retries + and observes the winner's generation. + +## Verify + +**Harness:** two scheduler instances against one backend — real concurrency, not +a simulated race. + +**Integration test** — `tests/it_challenger_allocation.rs`: +1. Register challenger A. Register challenger B while A is live; assert + **rejected**, and that the error **names A**. +2. **Concurrent CAS:** two schedulers commit different allocations + simultaneously, in a loop of 100 rounds. Assert exactly one wins per round, + the loser retries, and the final generation equals the number of successful + commits — no lost updates. +3. Traffic split: spawn 1000 runs; assert the challenger share is within + tolerance of 5% and that each run's spawn event **records which version it was + allocated to**. +4. Assert control appears in every allocation, so T6.4's anchor exists. +5. Trigger: reject a challenger with no replacement; assert the generate-loop + trigger event fires **once**. Assert no timer path exists that would fire it + otherwise — run 60s of simulated time with no rejection and assert zero + triggers. +6. Restart mid-decision; assert the allocation and generation survive. + +**Command:** `cargo test -p loop allocation -- --test-threads=1` + +**False pass:** +- Step 2 simulated by calling the update function twice sequentially. Last-write- + wins passes that and loses an allocation under real concurrency. Use two live + schedulers. +- Step 3 asserting the split ratio without asserting the per-run recorded + version. Runs allocated correctly but recorded wrongly are unattributable, and + the ratio still looks right. +- Step 5's negative half omitted, so a timer-driven generate loop passes. + +## Traps + +- Last-write-wins on the allocation record. The result is a split neither + scheduler intended and no error anywhere. +- A timer-driven generate loop. It proposes challengers when nothing has been + learned. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §9.1, §12.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.2-judge-calibration-set.md b/tasks/T6.2-judge-calibration-set.md new file mode 100644 index 0000000..434a3ff --- /dev/null +++ b/tasks/T6.2-judge-calibration-set.md @@ -0,0 +1,104 @@ +# T6.2 — Judge calibration set + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | T6.3. Promotion without calibration is the self-critique failure mode | + +## Goal + +Replay a fixed set of tasks with known verifier ground truth through the judge on +a schedule, and track judge-versus-verifier agreement. + +## Facts (inlined — no spec read needed) + +- **This is a precondition, not polish.** Under one resident model the judge *is* + the agent's model. Self-critique using the same model family being graded is + admissible **only** with this bootstrap: validation against verifiable tasks. +- Without it the grader's bias is unmeasured and the optimizer will find it. And + under one-state convergence (T6.1) there is no competing variant whose + divergence would make that visible. +- **Judge-versus-verifier agreement on the calibration set is the only + instrument that can see the grader drifting toward its own bias.** It belongs + in the metrics list beside `Indeterminate` count and judge order-inconsistency. +- **It costs zero agent runs**, since the episodes are already recorded. Re-running + the episodes is the obvious wrong implementation, which is why the run count is + asserted. +- Verification decides, grading explains. Agreement is measured against the + verifier's ground truth, never the other way round. + +## Steps + +1. Define the calibration set: a stable list of `(TaskId, episode, known verifier + outcome)` drawn from recorded history. Store it explicitly — a set that is + re-derived each cycle is not a fixed set. +2. Schedule the calibration pass on an interval, decoupled from promotion. +3. For each entry, call the judge on the **recorded** episodes. No agent + execution, no replay, no new runs. +4. Compute agreement overall and **per rubric dimension**, so a judge biased on + one dimension is caught with the dimension named rather than showing as a + small overall dip. +5. Emit agreement as a metric tagged by judge, model and rubric version. +6. Instrument an agent-run counter across the calibration pass and assert it is + zero. +7. Build two judge stubs: one agreeing 95% of the time, one deliberately biased + against a chosen rubric dimension. + +## Acceptance + +- A judge stub agreeing with the verifier 95% of the time reports ~95%. +- A judge deliberately biased against a rubric dimension is **caught and the + dimension named**. +- Calibration consumes **zero agent runs** — asserted by count, since re-running + episodes is the obvious wrong implementation. + +## Verify + +**Harness:** two judge stubs — one agreeing with the verifier 95% of the time, +one biased against a single named rubric dimension. Plus an **agent-run counter** +at the executor boundary. + +**Integration test** — `tests/it_judge_calibration.rs`: +1. Run the calibration pass with the 95% stub over ≥200 calibration entries. + Assert reported agreement is ~95% within sampling error. State the tolerance + from the sample size rather than guessing. +2. Run with the dimension-biased stub. Assert the report **names the dimension**, + not just a lower overall figure. +3. **Assert the agent-run counter is exactly 0** across the whole pass. The + episodes are already recorded; re-running them is the obvious wrong + implementation and it is expensive rather than incorrect-looking. +4. Assert the calibration set is **stored and stable** — run the pass twice and + assert the same entries were used both times. A set re-derived each cycle is + not a fixed set and its trend is meaningless. +5. Assert agreement is emitted as a metric tagged by judge, model and rubric + version. +6. Assert T6.3 refuses to promote when no calibration result exists or the + latest is stale — the blocking relationship must be enforced, not documented. + +**Command:** `cargo test -p loop calibration` + +**False pass:** +- Step 2 asserting only that overall agreement dropped. A judge biased on one + dimension of five moves the aggregate by a few points, which is + indistinguishable from noise — the per-dimension breakdown is the instrument. +- Step 4 omitted: a set rebuilt from "recent episodes" each cycle drifts with the + workload, so the metric measures the workload rather than the judge. +- Step 6 omitted, leaving calibration as a dashboard number that gates nothing — + which is precisely the self-critique failure mode this exists to prevent. + +## Traps + +- Re-executing the calibration tasks to "get fresh episodes". It costs agent runs + and measures the wrong thing. +- Reporting only an aggregate agreement number, which hides single-dimension bias + — the exact failure the loop will exploit. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §14.2, §15, §16 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.3-promotion-gates.md b/tasks/T6.3-promotion-gates.md new file mode 100644 index 0000000..05e7dc0 --- /dev/null +++ b/tasks/T6.3-promotion-gates.md @@ -0,0 +1,134 @@ +# T6.3 — Promotion gates + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +The default three-rung ladder, closing on T5.6's sequential test, plus automatic +rollback triggered by `Score::Capped`. + +## Facts (inlined — no spec read needed) + +**Default ladder — one challenger, pairwise.** Three rungs, because a graduated +ramp is a population instrument and there is no population here: + +| Rung | Traffic | Entry criterion | +|---|---|---| +| shadow | 0% | registered, validated, sandbox-clean, `ResourceProfile` fits | +| trial | 5% | no `Core` violation on any trial episode | +| current | 100% | sequential test crosses the accept boundary; drift check clean | + +- **The swap at the last rung is deliberate: the challenger takes all traffic at + once rather than ramping.** A ramp exists to limit blast radius while evidence + accumulates, and here the evidence has already accumulated — the sequential + test does not cross its boundary until the win rate is established at the + configured α. Ramping after that spends traffic to re-learn what the test + already concluded. +- What guards the swap instead is the rollback rule, which fires on a **single** + `Score::Capped` and does not wait for a boundary. +- **Rollback is automatic and unconditional** on any `Score::Capped` attributed + to the variant, or a verifier pass-rate regression beyond a configured margin. + `Score::Capped` is the only signal for the first of those; a gate that reads a + low *number* instead is reading something the cap exists to prevent from + existing. +- **Rollback is a traffic change, never a version delete.** The failed variant + stays in the DAG with its results. +- A `Core` violation **caps** rather than subtracts. A weighted sum lets a + variant buy past a safety failure with speed, which is exactly what prescriptive + rubrics exist to prevent. The cap comes from `Judge::screen`, runs before + pairing, and yields `Score::Capped`. +- **A capped episode is excluded from the bracket, not ranked last in it.** Left + in, it still contributes comparisons that shape everyone else's strength, and a + variant with one safety failure and seven strong episodes aggregates to a + promotion. +- **No rung reads held-out** (T6.6). A gate that reads held-out has converted it + into a second selection set. + +For deployments running the tournament with N challengers, the resourced ladder +is shadow (0%) → canary (5%, beats current on selection replay, BT interval +excludes zero) → ramp (20→50%, no `Core` violation, cost within budget, +sequential test at α) → current (100%, sustained over N groups, drift clean). + +## Steps + +1. Implement the three rungs as an explicit state machine on the challenger + record, with each entry criterion as a named predicate. +2. Shadow entry: validation (T3.3), sandbox cleanliness (T6.5), and the + `ResourceProfile` fit (T5.2/T5.3). +3. Trial entry: no `Core` violation on any trial episode, read from + `Score::Capped`. +4. Current entry: T5.6's sequential test returns `Accept`, and T6.7's drift check + is clean. Perform the swap **atomically** through T6.1's CAS. +5. Rollback path: on any `Score::Capped` attributed to the variant, or a pass-rate + regression past the margin, restore the prior version as current in a single + CAS — no interval in which neither is live. +6. Exclude capped episodes from the bracket **before** comparisons are issued, so + no other episode's score is influenced by them. +7. Assert no gate predicate reads held-out data. + +## Acceptance + +- A `Judge::screen` stub returning one `CoreViolation` triggers rollback **without + human action**; the rolled-back version remains in the DAG with its results + intact. +- The capped episode contributed **zero comparisons**, so no other episode's score + moved because of it. +- The swap at `current` is **atomic — no ramp** — and a rollback immediately after + it restores the prior version **without a gap in which neither is live**. + +## Verify + +**Harness:** a `Judge::screen` stub returning one `CoreViolation` on demand; a +traffic router observable at every instant; the mock judges from T5.6. + +**Integration test** — `tests/it_promotion_gates.rs`: +1. Drive a challenger through shadow → trial → current with a 70% mock judge. + Assert each rung's entry criterion was evaluated and recorded. +2. **Rollback:** fire one `CoreViolation`. Assert rollback happens **without + human action**, and that it triggered on `Score::Capped` — assert the gate + never reads a numeric score by instrumenting the score accessor. +3. Assert the rolled-back version **remains in the DAG with its results intact** + — read it back after rollback. +4. **Capped exclusion:** grade a group containing the capped episode. Assert it + contributed **zero comparisons**, and assert the other episodes' scores are + byte-identical to a control run where the capped episode was absent. Ranking + it last would change them. +5. **Atomic swap:** sample the live-version pointer at high frequency across the + promotion. Assert it goes 100% old → 100% new with **no intermediate + percentage** — no ramp. +6. **No gap:** roll back immediately after the swap. Assert every sample shows + exactly one live version; a sample showing none is a failure. +7. Assert no gate predicate reads held-out data (cross-check with T6.6's audit). + +**Command:** `cargo test -p loop promotion -- --test-threads=1` + +**False pass:** +- Step 4 asserting only "the capped episode has no score". Leaving it in the + bracket still lets it shape everyone else's strength, and its own score can be + absent while it does. The **control-run comparison** is the evidence. +- Step 5 asserting the final state only. A ramp also ends at 100%. +- Step 6 sampled too coarsely to observe a gap. Sample from a tight loop or + instrument the pointer swap directly. +- A rollback test that triggers on a low score. It passes, and the cap exists + precisely so that number does not exist. + +## Traps + +- A gate reading a low numeric score instead of `Score::Capped`. The cap exists + precisely so that number does not exist. +- Ranking a capped episode last rather than excluding it. It still shapes the + bracket. +- Deleting a rolled-back version. Its results are attributed to it. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §11.7, §12.3, §12.5 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.4-per-variant-aggregation.md b/tasks/T6.4-per-variant-aggregation.md new file mode 100644 index 0000000..946534f --- /dev/null +++ b/tasks/T6.4-per-variant-aggregation.md @@ -0,0 +1,105 @@ +# T6.4 — Per-variant aggregation + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | opt-in | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Compose Bradley-Terry strengths across groups of differing size, as deltas from +the anchor present in each group. Only reachable with the tournament enabled and +N challengers allowed. + +## Facts (inlined — no spec read needed) + +- Group-normalized relative scores give **cross-task comparability** — but only + through a shared anchor, and that qualification is load-bearing. +- A BT fit identifies strengths only up to an additive constant **within one + connected comparison graph**. Two groups on different tasks are disjoint + graphs, so their strengths sit on unlinked scales. Averaging them directly + commits the same error that point tallies are accused of, one layer further in. +- **The anchor is control.** Control gets a traffic share in every allocation, so + every group contains at least one control episode; the fit pins control to zero + and every other strength is read as a delta from it. A variant's aggregate is + then a mean of like-for-like deltas rather than a mean of incomparable scales. +- **A group with no control episode is not aggregatable.** It still grades its own + members and is worth reading; it just does not feed the aggregate, and it is + **recorded as such** rather than folded in on the assumption that scales match. +- Why not point-tally z-scores: small groups produce extreme z-scores, so a + variant appearing in many small groups wins on variance rather than quality. BT + intervals compose correctly across group sizes and feed the sample gate + directly. + +## Steps + +1. Consume `Score::Ranked { strength, interval, group_size }` from T5.9. Strength + is already a delta from control because control was pinned at fit time. +2. Partition by variant. Weight each group's contribution by its interval width + (inverse-variance), not by group size alone. +3. **Reject any group lacking a control episode** from the aggregate, and count + the exclusions in a reported total — exclusion must be visible, not silent. +4. Never combine raw strengths from independently fitted groups without the + anchor. Make that impossible in the type, if the type can carry "anchored". +5. Emit the aggregate with its own interval and the number of contributing groups. +6. Keep it behind the same off-by-default flag as T5.8/T5.9. + +## Acceptance + +- A variant appearing only in small groups **does not outrank** one with a + tighter interval at equal mean. +- Anchor guard: two groups fit independently with different origins are **not + averaged** — a group lacking the anchor is excluded from the aggregate and + **counted in a reported total**, so exclusion is visible rather than silent. + +## Verify + +**Harness:** synthetic `Score::Ranked` inputs with known true strengths and +deliberately varied group sizes — no judge needed. + +**Integration test** — `tests/it_variant_aggregation.rs`: +1. Variant X appears only in small groups (wide intervals); variant Y appears in + large groups (tight intervals). Set their **means equal**. Assert Y does not + lose to X — and specifically assert X does **not** outrank Y, which is the + variance failure being guarded against. +2. Assert weighting is by interval width, not group count: construct a case where + the two disagree and assert the interval-weighted answer wins. +3. **Anchor guard:** feed two groups fit independently with different origins. + Assert they are **not averaged**. +4. Assert a group lacking the anchor is **excluded** and that the exclusion is + **counted in a reported total** — read the total back and assert it is + non-zero. Silent exclusion is the failure. +5. Assert the aggregate carries its own interval and the contributing group + count. +6. Assert this path is unreachable with the tournament flag off. + +**Command:** `cargo test -p loop variant_aggregation` + +**False pass:** +- Step 1 with unequal means, where the right variant wins for the wrong reason. + Equal means is what isolates the variance effect. +- Step 4 asserting only that the anchorless group was dropped. Dropping silently + looks identical to having no such group, so a systematic allocation bug that + strips control from every group reads as a quiet dataset. The **reported + count** is the instrument. +- Averaging raw strengths and passing because the synthetic groups happened to + share an origin. Construct step 3's groups with deliberately different origins. + +## Traps + +- Averaging strengths across groups without checking for the anchor. It compiles, + it produces plausible numbers, and the numbers mean nothing. +- Weighting by group size. That is the variance failure BT was chosen to avoid, + reintroduced at the aggregation layer. +- Dropping anchorless groups silently. Then a systematic allocation bug looks + like a quiet dataset. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §9.1, §11.2, §11.4, §12.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.5-sandboxed-replay.md b/tasks/T6.5-sandboxed-replay.md new file mode 100644 index 0000000..73317a6 --- /dev/null +++ b/tasks/T6.5-sandboxed-replay.md @@ -0,0 +1,103 @@ +# T6.5 — Sandboxed replay + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Re-execute a task under a new workflow version with `Unsafe` effects denied. +Depends on T2.2's effect classes. + +## Facts (inlined — no spec read needed) + +- **"Replay against recorded episodes" means re-running the task under a new + workflow version**, not pushing a recorded trajectory through new logic. + Trajectory replay tells you only where behaviour would first diverge, and + everything after divergence is unknown — near-worthless for grading. +- **Replay is not free.** "0% live traffic" means no user sees the result, not + that it costs nothing. N variants × M tasks is N·M full agent runs plus judge + calls. **Shadow is the most expensive rung, not the cheapest.** +- **Replay executes real tools.** Re-running a workflow that pushes commits + pushes commits. Shadow execution runs in a sandbox with `Unsafe` effects + denied, and a workflow whose steps cannot run sandboxed is **ineligible for + shadow evaluation and must say so at load time rather than at 3am**. +- Sandbox is layer 3 of the effect-class defence: declaration (T2.2), keyed + capability (T8.3), sandbox here. +- Replay needs its own spend ceiling, separate from judging. + +## Steps + +1. Implement the sandbox as a capability restriction over the tool registry: + `Unsafe` tools are not resolvable inside a replay context. Denial is + structural, not a runtime `if`. +2. Add the **load-time** eligibility check: walk the workflow's steps, resolve + each `ToolId`, and reject the workflow for shadow evaluation if any resolves + to `Unsafe`. The diagnostic **names the tool**. +3. Replay execution path: fresh runs against the recorded task input (`TaskId`), + under the challenger version, inside the sandbox. +4. Record replay runs with a `Purpose` distinguishing them from live work, so + metering (T8.1) separates them and their own ceiling applies. +5. Feed the resulting episodes to the tournament strategy — the group exists + because you paid for it, so pairwise grading would waste it. +6. Wire the replay ceiling: exceeding it stops replay and reports, rather than + silently truncating the variant set. + +## Acceptance + +- A workflow calling an unsafe tool is **rejected for shadow evaluation at load + time**, with a diagnostic naming the tool. + +## Verify + +**Harness:** the external side-effect ledger from T2.2, plus a workflow that +calls an `Unsafe` tool and one that does not. + +**Integration test** — `tests/it_sandboxed_replay.rs`: +1. Submit the `Unsafe`-calling workflow for shadow evaluation. Assert it is + **rejected at load time**, with a diagnostic **naming the tool**. +2. Assert the rejection happens **before** any run is spawned — check the run + counter is 0 and the ledger is empty. +3. Positive control: the safe workflow is accepted and replays successfully. +4. **Defence in depth:** bypass the load check in a test-only path and attempt an + `Unsafe` call inside the sandbox anyway. Assert it is denied at call time too. + The load check is the good error; the sandbox is the guarantee. +5. Assert replay re-executes against the recorded **task input**, not a recorded + trajectory — plant a divergence and assert execution continues past it and + produces a complete episode. +6. Assert replay runs are recorded with a distinct `Purpose`, and that they count + against the **replay** ceiling, not the judging one. +7. Cost visibility: assert N variants × M tasks produces N·M runs, and that the + count is reported — shadow is the most expensive rung and must not look free. + +**Command:** `cargo test -p loop sandboxed_replay` + +**False pass:** +- Step 1 alone. A load-time check with no runtime enforcement passes it, and any + code path that skips validation then executes real effects. Step 4 is the + guarantee. +- Step 5 omitted: trajectory replay produces an episode that looks complete and + is meaningless after the first divergence, and no assertion about tool denial + catches it. +- Charging replay to the judging ceiling — step 6. It passes every functional + test and consumes the entire grading budget in production. + +## Traps + +- Trajectory replay. It is cheaper, it looks like replay, and its output cannot + be graded past the first divergence. +- Denying `Unsafe` at call time instead of load time. The rejection then arrives + mid-run, at 3am, after real work has been done. +- Charging replay against the judging ceiling. Replay dominates and will consume + the whole budget. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §12.4, §13.2, §14.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.6-held-out-split.md b/tasks/T6.6-held-out-split.md new file mode 100644 index 0000000..325cd2f --- /dev/null +++ b/tasks/T6.6-held-out-split.md @@ -0,0 +1,105 @@ +# T6.6 — Held-out split + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Partition tasks into selection and held-out sets. Promote on selection, report +held-out without optimizing against it. + +## Facts (inlined — no spec read needed) + +- Selecting on a fixed set of recorded tasks overfits to those tasks, **silently**: + shadow scores improve while live performance does not. +- **No promotion gate takes held-out as an entry criterion.** A gate that reads + held-out has converted it into a second selection set and left nothing + measuring generalization. +- **The proposer reads the selection set only.** If the slow loop consumes + held-out failures to generate candidates, the held-out set is contaminated + through the generator instead of the selector. **This leak is easy to introduce + and invisible once present** — which is why the audit test is the control, not + a nicety. +- A widening selection-versus-held-out gap is the overfitting alarm. +- Under one-state convergence there is no competing variant whose divergence + would reveal a stalled loop, so **the held-out report is also the only stall + detector there is**. It must be produced and surfaced even when no promotion is + pending. +- Held-out catches overfitting to *episodes*. It does not catch drift in the + *task mix* — that is T6.7. + +## Steps + +1. Assign each `TaskId` to selection or held-out by a deterministic hash of the + id, so the partition is stable and needs no stored membership list. +2. Expose two separate query surfaces — `selection_tasks()` and + `heldout_tasks()` — rather than one with a flag. A flag defaults wrong. +3. Give the proposer access to the selection surface only, at the type level if + possible. +4. Compute the held-out report on a **schedule**, independent of promotion + activity, and surface the selection-versus-held-out gap as a metric. +5. Write the audit test: enumerate every proposer input and every gate + evaluation, assert no held-out `TaskId` appears in either. +6. Keep the audit test running in CI permanently. It is the only thing that will + notice the leak. + +## Acceptance + +- An audit test asserting **no held-out `TaskId` appears in proposer input**, and + none in any promotion-gate evaluation. This leak is invisible once present, so + the test is the control. +- The held-out report is **emitted on a schedule, not only at a gate** — asserted, + since under one-state convergence it is the only stall detector there is. + +## Verify + +**Harness:** an interception layer recording **every** `TaskId` that reaches the +proposer and every `TaskId` read during a gate evaluation. The audit is the +deliverable — it is the only thing that will ever notice this leak. + +**Integration test** — `tests/it_heldout_audit.rs`: +1. Partition a corpus of 1000 `TaskId`s. Assert the split is deterministic: + recompute it in a second process and assert identical membership. +2. Run a full generate-and-promote cycle with interception on. +3. Assert **no held-out `TaskId`** appears in the recorded proposer input set — + intersection with the held-out set is empty. +4. Assert **no held-out `TaskId`** appears in any gate evaluation. +5. Assert the two query surfaces are distinct types or distinct functions — + `trybuild` a call that tries to pass held-out tasks into the proposer. +6. **Schedule:** advance simulated time with **no promotion pending**; assert the + held-out report is still emitted. Under one-state convergence it is the only + stall detector there is. +7. Assert the selection-versus-held-out gap is emitted as a metric. +8. Negative control: deliberately wire a held-out task into the proposer in a + test-only build; assert the audit **fails**. An audit that never fails is not + a control. + +**Command:** `cargo test -p loop heldout_audit` + +**False pass:** +- Steps 3–4 passing because the interception layer only sees one of several code + paths into the proposer. Assert the interception is at the single chokepoint, + or the audit measures nothing. +- Step 8 omitted. This is the most important one: an audit test that has never + been observed failing may simply be looking in the wrong place, and the leak it + guards is invisible once present. +- Step 6 omitted, so the report is computed only at gates — exactly when a + stalled loop produces none, which is when it is needed. + +## Traps + +- A single task query with an `include_heldout: bool`. Someone passes `true`. +- Computing the report only when a promotion is pending, which is exactly when a + stalled loop produces none. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §12.1, §12.3, §12.5, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.7-drift-check.md b/tasks/T6.7-drift-check.md new file mode 100644 index 0000000..50a5275 --- /dev/null +++ b/tasks/T6.7-drift-check.md @@ -0,0 +1,96 @@ +# T6.7 — Drift check + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Track the distribution of incoming `TaskId` characteristics over time and alarm +on a shift. + +## Facts (inlined — no spec read needed) + +- Held-out (T6.6) catches overfitting to **episodes**. It does **not** catch + drift in the **task mix**. That needs a separate distribution check on incoming + `TaskId` characteristics over time. +- The failure it prevents: the workflow genuinely improves on the task mix it was + selected against, while the incoming mix moves elsewhere. Every internal + instrument reads healthy. +- It is a promotion gate input: the `current` rung requires "drift check clean" + alongside the sequential test crossing its accept boundary (T6.3). +- `TaskId` is an opaque hash, so the characteristics tracked are the metadata + recorded beside it — input size, declared category, source, whatever the + deployment's `TaskId` hasher was fed. + +## Steps + +1. Define the characteristic vector extracted per incoming task at spawn. Keep it + small and explicitly listed; this is not a feature store. +2. Maintain a reference distribution over a trailing window, and a current + window. +3. Compare with a distribution distance appropriate to the feature types + (population stability index or a KS test per numeric feature; chi-square for + categorical). Pick one, name the threshold as a constant. +4. Emit the distance as a metric and raise the alarm past the threshold, naming + which characteristic moved. +5. Expose `is_clean()` for T6.3's `current` rung. +6. Test with a synthetic shift and with a stable mix. + +## Acceptance + +- A synthetic shift in task mix raises the alarm. +- A stable mix does not. + +## Phase gate + +P6 closes on an automatic challenger accept and an automatic rollback, both +unattended, on one resident model. + +## Verify + +**Harness:** synthetic task streams with a controllable characteristic +distribution. + +**Integration test** — `tests/it_drift_check.rs`: +1. Feed a stable mix for the full window. Assert **no alarm** — the false-positive + half matters as much as detection. +2. Shift one characteristic sharply mid-stream. Assert the alarm fires **and + names the characteristic that moved**. +3. Shift a different characteristic; assert the named one changes accordingly. +4. Gradual drift over many windows: assert it is eventually detected, so the + check is not tuned only for step changes. +5. Assert `is_clean()` gates T6.3's `current` rung — promote with drift present + and assert the promotion is blocked. +6. Assert the threshold reads its **own named constant**, not one shared with + another limit. +7. Assert the distance metric is emitted continuously, not only on alarm, so the + trend is visible before the threshold. + +**Command:** `cargo test -p loop drift` + +**False pass:** +- Step 1 omitted. A check that alarms on everything passes step 2 perfectly and + is useless. +- Step 2 asserting the alarm boolean only. Knowing *which* characteristic moved + is what makes it actionable; an aggregate distance alone sends the operator + hunting. +- Testing only a step change, where any distance measure works. Step 4 is where a + badly chosen window length shows. + +## Traps + +- Alarming on the aggregate distance only. Knowing *which* characteristic moved + is what makes the alarm actionable. +- A threshold shared with an unrelated limit. Give it its own named constant. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §12.3, §12.5 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T6.8-p6-composition-gate.md b/tasks/T6.8-p6-composition-gate.md new file mode 100644 index 0000000..1a2e2aa --- /dev/null +++ b/tasks/T6.8-p6-composition-gate.md @@ -0,0 +1,113 @@ +# T6.8 — P6 composition gate + +| Field | Value | +|---|---| +| Phase | P6 — Learning loop | +| Size | L — over 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P7 | + +## Goal + +Prove the loop closes **unattended** end to end, and that the default +(one-challenger, pairwise) and resourced (N-challenger, tournament) paths drive +the same gate machinery. + +**Phase gate criterion:** an automatic challenger accept and an automatic +rollback, both unattended, on one resident model. + +## Facts (inlined — no spec read needed) + +- The loop: allocation → runs → verify → grade → sequential test → promote or + reject; rollback on a single `Score::Capped`. **Unattended** means no human + action anywhere in that path. +- The properties no single P6 task owns: + - **calibration gates promotion.** Under one resident model the judge *is* the + agent's model, and promotion without calibration is the self-critique failure + mode. The blocking relationship must be enforced, not documented. + - **held-out is never read by a gate or the proposer**, across the whole loop — + a leak introduced anywhere is invisible once present. + - **the two ladders share machinery.** Default is three rungs (shadow, trial, + current); resourced is four (shadow, canary, ramp, current). Both must drive + the same state machine, or the opt-in path is a fork. + - the slow loop fires on a **trigger** — a rejection with no replacement — not + a timer. +- The cost of one-state convergence is stated rather than discovered: it is + hill-climbing, it can sit in a local optimum, and the held-out report is the + only instrument that will notice. So the gate must assert that report is + produced even when nothing is being promoted. + +## Steps + +1. Drive a full unattended cycle with a challenger that is genuinely better; + assert promotion. +2. Drive a second cycle with a challenger that trips one `CoreViolation`; assert + automatic rollback. +3. Repeat both under the resourced ladder with the tournament enabled and N + challengers. +4. Run the held-out audit and the calibration interlock across the whole cycle. +5. Assert zero human-intervention hooks were invoked in any path. +6. Make this the required CI job gating P7. + +## Acceptance + +- A challenger is **accepted automatically** and a challenger is **rolled back + automatically**, both unattended, on one resident model with zero swaps. +- The same gate state machine drives both the default and the resourced ladder. +- No held-out `TaskId` reaches the proposer or any gate. +- Promotion is blocked when calibration is missing or stale. + +## Verify + +**Harness:** mock judges from T5.6; a `screen` stub that can emit one +`CoreViolation` on demand; the held-out interception layer from T6.6; a +human-intervention hook that records if it is ever called. + +**Integration test** — `tests/it_p6_composition.rs`: +1. **Unattended accept:** full cycle with the 70% judge. Assert promotion, and + assert the intervention hook recorded **zero** calls. +2. **Unattended rollback:** fire one `CoreViolation` post-promotion. Assert + rollback without human action, the prior version restored with **no gap** + where neither is live, and the failed version still in the DAG with results. +3. **Calibration interlock:** clear the calibration result, attempt promotion, + assert it is **blocked**. Then stale it past its window; assert still blocked. + Then refresh; assert it proceeds. +4. **Held-out audit across the whole loop:** assert zero held-out `TaskId`s in + proposer input and in every gate evaluation, measured over the complete cycle + rather than a single gate call. Include T6.6's negative control so the audit + is known to be capable of failing. +5. **Ladder parity:** run steps 1–2 under the resourced ladder. Assert the same + state machine and the same rollback trigger — assert on the recorded rung + transitions, so a forked implementation is visible. +6. **Trigger, not timer:** advance simulated time with no rejection; assert zero + generate-loop triggers. Then reject without replacement; assert exactly one. +7. **Stall detector:** with no promotion pending, advance the schedule; assert the + held-out report is still emitted. +8. **Zero swaps** across the entire cycle. +9. **Regression:** re-run P0–P5 gates and all P6 suites. + +**Command:** `cargo test -p loop --test it_p6_composition -- --test-threads=1 --nocapture` + +**False pass:** +- A cycle driven by test code that nudges the loop between rungs. That is the + human action the word "unattended" excludes — step 1's intervention counter is + the guard, and it must wrap every manual entry point, not just one. +- Step 3 omitted: calibration then exists as a dashboard number gating nothing, + which is precisely the failure mode it was introduced to prevent. +- Step 5 omitted: the resourced ladder quietly becoming a second implementation + means every fix to the default path has to be made twice, and will not be. +- Step 4 evaluated at one gate call rather than across the cycle — the proposer + leak is the invisible one. + +## Traps + +- Declaring the gate green with the tournament enabled. The default path is what + ships; assert the default cycle first. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §12, §15, §16 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.1-postgres-eventlog.md b/tasks/T7.1-postgres-eventlog.md new file mode 100644 index 0000000..795cb4c --- /dev/null +++ b/tasks/T7.1-postgres-eventlog.md @@ -0,0 +1,103 @@ +# T7.1 — Postgres `EventLog` + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +The same `EventLog` port over Postgres, partitioned by tenant and time. Not a +second port — the same one. + +## Facts (inlined — no spec read needed) + +- Two deployment modes, **one set of ports**: + +| Mode | Log + state | Blobs | Coordination | +|---|---|---|---| +| **Embedded** — single binary, no services | `redb` | `redb` table | in-process | +| **Distributed** — multi-node, multi-tenant | Postgres | S3-compatible | Postgres advisory locks, or Redis if leases dominate | + +- Postgres transactions supply crash-atomicity exactly as `redb`'s shadow paging + does; `commit`'s four writes stay in one transaction. +- This is why the port was async from the first line: a sync signature over a + network call is impossible, and the local implementation paid only a negligible + poll for the compatibility. +- **Every port test runs against both backends or the abstraction is not real.** + The conformance suite from T0.5/T0.7 is the deliverable being reused here. +- Open at this scale: partitioning by tenant and time, index strategy for branch + scans, and whether the outbox is a table or a logical replication slot. Decide + and record the decision; it is not pre-settled. + +## Steps + +1. Schema: log table keyed `(tenant, run, branch, lsn)`; state, consumer position + and outbox tables mirroring the embedded key shapes. +2. Partition the log by tenant and by time. Choose declarative partitioning and + document the retention interaction with T8.4's tiering. +3. Index for the access patterns that exist: branch scans from an LSN, recent + runs by `RunId` prefix (ULIDs sort by creation time), outbox drain per + `BranchKey` ascending `Lsn`. +4. Implement `commit` as one transaction with all four writes. LSN allocation + stays inside the transaction — a sequence is not sufficient, since LSNs are + per branch and must be gap-free. +5. Implement the checkpoint and outbox methods against the same schema. +6. Run the **unmodified** P0 conformance suite against this backend in CI. +7. Record the outbox decision (table vs replication slot) with its reasoning. + +## Acceptance + +- The P0 conformance suite passes **unmodified** against Postgres. +- Every port test runs against both backends. + +## Verify + +**Harness:** the **same** conformance function from T0.5, instantiated with a +Postgres factory. A real Postgres in CI (testcontainers or a service container) — +not an in-memory fake, which would test nothing about the backend. + +**Integration test** — `tests/it_postgres_conformance.rs`: +1. Call `conformance(|| PostgresEventLog::new(...))` with **zero modifications** + to the suite. Any needed change is a finding about the port, not the test. +2. Run the identical suite against `redb` in the same CI job, so a divergence + between backends fails immediately rather than months later. +3. **Concurrent LSN allocation** across separate **connections** — the redb test + used tasks; here contention is across processes and is the real case. Assert + `1..=n` with no gaps or repeats. +4. Assert `commit`'s four writes are in one transaction: kill the connection + mid-commit and assert all-or-nothing on reconnect. +5. Partitioning: insert across several tenants and time ranges; assert a branch + scan hits the expected partitions (check the query plan, not just the result). +6. Outbox: assert `drain_outbox` returns `(BranchKey, Lsn)` ascending across + partition boundaries. +7. Record the outbox decision (table vs replication slot) and assert the chosen + one is the one under test. + +**Command:** `cargo test -p storage-postgres --test it_postgres_conformance` + +**False pass:** +- A modified copy of the conformance suite. The moment it is edited "just for + Postgres", the two backends are no longer proven equivalent — which is the + entire acceptance criterion. +- Step 3 with a single connection, where the sequence is uncontended. +- A `SERIAL`/sequence-backed LSN. It passes casual reads and is neither per-branch + nor gap-free under rollback. +- Running against SQLite or an in-process fake for CI speed. + +## Traps + +- A Postgres-only method added to the trait "temporarily". The embedded backend + then has a stub and the abstraction is fiction. +- A global sequence for LSNs. It serializes every run through one atomic and + breaks per-branch gap-freedom. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §7, §8.3, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.2-object-store-blobstore.md b/tasks/T7.2-object-store-blobstore.md new file mode 100644 index 0000000..e5f9f1d --- /dev/null +++ b/tasks/T7.2-object-store-blobstore.md @@ -0,0 +1,91 @@ +# T7.2 — Object-store `BlobStore` + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +S3-compatible `BlobStore`, tenant-namespaced, passing the same conformance suite +as the embedded implementation. + +## Facts (inlined — no spec read needed) + +- Same port as T0.7: `put` / `get` / `delete`, all async, all taking `TenantId`. +- **Blobs are namespaced per tenant even though they are content-addressed.** + Global deduplication across tenants is a leak: a shared blob makes one tenant's + storage accounting depend on another's, and a hash becomes an oracle for "does + anyone else have this content". Deduplicate within a tenant, never across. +- `delete` is not optional — reduction (T8.4) and tenant deletion both require + it. A store that cannot delete cannot honour either. +- `get` returning `None` is a normal answer, not an error. +- Cross-tenant isolation must be verified by **attempting** a cross-tenant fetch, + not by reading the key-construction code. + +## Steps + +1. Implement over an `object_store`-style client. Key layout puts the tenant in + the path prefix: `{tenant}/{blake3-hash}`. +2. `put`: hash, then conditional put (skip if present) — dedup falls out of the + content-addressed key, scoped by the tenant prefix. +3. `get`: fetch, map a not-found response to `Ok(None)`, everything else to an + error. +4. `delete`: delete the object. A missing object is not an error — reduction can + run twice. +5. Set retry and timeout policy on the client explicitly. A blob fetch stalling + forever inside a verifier is a hang the deadline should catch, but the client + should not depend on that. +6. Run the T0.7 conformance suite unmodified, plus the explicit cross-tenant + fetch attempt. + +## Acceptance + +- Same conformance suite as the embedded store passes. +- Tenant isolation verified by **attempting a cross-tenant fetch**. + +## Verify + +**Harness:** the T0.7 conformance function against MinIO or an S3-compatible +container. Same suite, unmodified. + +**Integration test** — `tests/it_object_store_conformance.rs`: +1. Run the T0.7 suite verbatim against the object store. +2. **Cross-tenant attempt:** with tenant B's credentials/prefix, attempt to fetch + tenant A's ref. Assert it fails or returns `None` — attempt it, do not infer + isolation from the key-construction code. +3. Assert identical content under two tenants produces **two objects**, verified + by listing the bucket, not by the return values. +4. Assert a 404 on `get` maps to `Ok(None)` and a 404 on `delete` maps to + `Ok(())`, so reduction is idempotent. +5. Re-hash every stored object against its key. +6. Latency/failure behaviour: inject a slow response; assert the configured + timeout fires rather than hanging a verifier indefinitely. +7. Run the same suite against `redb` in the same job to catch backend divergence. + +**Command:** `cargo test -p storage-s3 --test it_object_store_conformance` + +**False pass:** +- Step 2 performed with the same credentials for both tenants. That tests the key + prefix, not isolation. Use separate scoped credentials where the deployment + will. +- Step 3 asserting on the returned refs, which are equal by design (same content + hash). The **bucket listing** is what shows two objects. +- Mocking the object store. The failure modes worth catching here — eventual + consistency, 404 shapes, timeouts — exist only in a real implementation. + +## Traps + +- A flat bucket layout with the tenant in metadata rather than the key. It + dedups across tenants by construction. +- Treating a 404 on delete as a failure, which breaks idempotent reduction. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §3, §7, §8.6 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.3-leases-and-fencing.md b/tasks/T7.3-leases-and-fencing.md new file mode 100644 index 0000000..f7482bb --- /dev/null +++ b/tasks/T7.3-leases-and-fencing.md @@ -0,0 +1,101 @@ +# T7.3 — Leases and fencing + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +TTL leases with heartbeat renewal and monotonic fence tokens, so a partitioned +worker cannot write under an expired claim. + +## Facts (inlined — no spec read needed) + +- Runs are claimed by workers under a lease with a TTL. **A worker that dies has + its runs reclaimed after expiry.** +- Lease renewal is a heartbeat on the run record. +- **`Suspended` runs release their lease entirely** rather than heartbeating + through a human's lunch break. A run awaiting human approval or a webhook must + release its worker; holding an executor slot across a human decision does not + scale past a handful of concurrent runs. +- **Fencing tokens on every lease.** A partitioned worker that resumes must not + write under an expired claim, and a **monotonic fence in the run record makes + that a rejected write rather than a silent double-execution**. +- Coordination backend: Postgres advisory locks, or Redis if lease churn + dominates. Redis is deferred until that churn is real. + +## Steps + +1. Add to the run record: `lease_owner`, `lease_expires_at`, `fence: u64`. +2. Claim: atomic conditional update — claim only if unowned or expired. + Increment `fence` on every successful claim. The fence is monotonic per run. +3. The worker carries its fence value in memory and includes it on **every** + write. Writes compare against the stored fence and are rejected when lower. +4. Heartbeat renews `lease_expires_at` on an interval well inside the TTL. A + failed renewal cancels the local `RunScope` (T1.1) rather than continuing + optimistically. +5. On transition to `Suspended`, release the lease outright — clear the owner and + stop heartbeating. On resume, re-claim, which takes a new fence. +6. Reclaim path: a run whose lease expired is claimable by any worker. +7. Test the full sequence: partition, expiry, reclaim elsewhere, reconnect the + original, assert its writes are **rejected by fence**. + +## Acceptance + +- Partition a worker, let its lease expire, reclaim the run elsewhere, then + reconnect the original — its writes are **rejected by fence, not merely late**. + +## Verify + +**Harness:** two worker processes against one Postgres, plus `turmoil` or an +iptables-style partition for the network half. `tokio::time::pause` cannot be +used across processes — use a short real TTL instead. + +**Integration test** — `tests/it_leases_fencing.rs`: +1. Worker A claims run R (fence = 1) and begins executing. +2. Partition A from the database. Let the lease expire. +3. Worker B reclaims R; assert its fence is **2**. +4. Heal the partition. Worker A attempts a write under fence 1. +5. Assert the write is **rejected by the fence check** — and assert the rejection + reason is the fence, not a timestamp comparison and not a generic conflict. +6. Assert R completed **exactly once**: check the external side-effect ledger for + a single entry. +7. Assert A's failed heartbeat **cancelled its local `RunScope`**, so it stopped + working rather than continuing optimistically. +8. `Suspended`: transition a run to `Suspended`; assert the lease is **released** + (owner cleared) and that no heartbeat is emitted while suspended. Resume and + assert a **new, higher** fence. +9. Monotonicity: 100 claim/expire cycles; assert the fence never decreases or + repeats. + +**Command:** `cargo test -p distribution leases -- --test-threads=1` + +**False pass:** +- Step 5 satisfied by a timestamp check ("your lease expired"). Under clock skew + that check can pass for a stale worker; the **fence comparison** is what makes + it clock-independent. Assert the reason. +- Step 6 omitted: both workers can execute, both can be "correct", and the + double-execution is only visible in the external ledger. +- A per-worker fence counter rather than per-run monotonic — step 9 with two + workers alternating catches it. +- Step 8 verified by reading a flag rather than asserting heartbeats stop. + +## Traps + +- Checking only `lease_expires_at` at write time. Clock skew makes "not expired + yet" a lie, and the fence is what makes the check independent of clocks. +- Heartbeating through `Suspended`. It burns a worker slot for the duration of a + human decision. +- A per-worker fence rather than per-run monotonic. Two workers then produce + incomparable tokens. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.2, §7, §9.4 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.4-outbox-relay.md b/tasks/T7.4-outbox-relay.md new file mode 100644 index 0000000..f146a56 --- /dev/null +++ b/tasks/T7.4-outbox-relay.md @@ -0,0 +1,98 @@ +# T7.4 — Outbox relay + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Ship committed export intents out of process, with its own retry and its own +failure domain. The execution path never calls a broker. + +## Facts (inlined — no spec read needed) + +- **The framework never calls a broker from the execution path.** Export intent + is written in the same transaction as the state (T0.6); a separate relay reads + committed intents and ships them. +- This makes export restartable, keeps a broker outage from stalling a run, and + is **the only pattern that survives a crash between "state committed" and + "event published"**. +- **A user's broker being down is not an agent outage.** +- The relay reads through `drain_outbox` / `ack_outbox`. Ordering is per + `BranchKey`, ascending `Lsn` — that is the only ordering the system promises, + and it falls out of the outbox key shape. +- **Delivery is at-least-once. Exactly-once is achieved at the fold, not in + transport**: `(BranchKey, Lsn)` is the natural idempotency key, so a redelivered + record is a no-op insert. +- Only the live branch is exported (T2.3). + +## Steps + +1. Run the relay as its own process or task with its own supervision — separate + failure domain is the point, so do not co-locate it with the executor's + lifecycle. +2. Loop: `drain_outbox(tenant, limit)` → publish in `(BranchKey, Lsn)` order → + `ack_outbox(shipped)`. Ack only what the broker accepted. +3. Retry with backoff inside the relay. Never propagate a broker error into run + execution — there is no path back by construction, so verify none is added. +4. Set the broker partition key from `(TenantId, RunId)` (T7.5). +5. Make the consumer-side fold idempotent on `(BranchKey, Lsn)` so redelivery is + a no-op insert. +6. Emit relay lag as a metric — outbox backlog is one of the two that grow + silently. +7. Test with the broker down for an entire run. + +## Acceptance + +- Broker down for the whole run; **the run completes**; every event ships after + the broker returns, **in order, exactly once at the fold**. + +## Verify + +**Harness:** a broker that can be held down for the whole test, plus a consumer +that folds on `(BranchKey, Lsn)` so redelivery is observable. + +**Integration test** — `tests/it_outbox_relay.rs`: +1. Take the broker **down**. Run a full run to completion. +2. Assert the run **completed normally** — no stall, no error, and its latency is + comparable to a broker-up baseline. A user's broker being down is not an agent + outage. +3. Bring the broker up. Assert every event ships. +4. Assert ordering **per `BranchKey`, ascending `Lsn`** at the consumer. +5. Force redelivery (ack loss): assert the consumer's folded state is identical + to the single-delivery case — exactly-once **at the fold**, not in transport. +6. Kill the relay mid-batch; restart; assert no event is lost and none is + duplicated at the fold. +7. Assert the execution path never calls the broker: instrument the broker client + and assert zero calls originate from executor threads. +8. Assert only the **live branch** is exported after a rewind. +9. Assert relay lag is emitted as a metric. + +**Command:** `cargo test -p distribution outbox -- --test-threads=1` + +**False pass:** +- Step 2 asserting only that the run completed. If the executor retries the + broker inline, the run completes — slowly. The latency comparison is what + catches it; step 7 is what proves it. +- Acking before the broker confirms: steps 3 and 4 still pass in a happy-path + test, and events vanish silently under failure. Step 6 is the guard. +- Testing ordering within one branch only, where any implementation is ordered. + Use several branches and several runs interleaved. + +## Traps + +- Publishing from the execution path "just for low latency". A broker outage then + stalls agent work. +- Acking before the broker confirms. Events are lost with no trace. +- Attempting transport-level exactly-once. The fold is where it is achievable. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.3, §9.2, §9.3, §13.3, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.5-partition-keys-on-adapters.md b/tasks/T7.5-partition-keys-on-adapters.md new file mode 100644 index 0000000..f569b39 --- /dev/null +++ b/tasks/T7.5-partition-keys-on-adapters.md @@ -0,0 +1,86 @@ +# T7.5 — Partition keys on adapters + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | S — under 1 day | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Every broker adapter sets the partition key from `(TenantId, RunId)`. Never from +a correlation id. + +## Facts (inlined — no spec read needed) + +- **Per-run total order, nothing promised across runs.** Downstream consumers + must therefore partition by run key. +- A **correlation id collapses unrelated runs onto one partition while splitting + single runs across several** — the exact inversion of the ordering contract. +- The same reasoning drives the outbox key shape: per `BranchKey`, ascending + `Lsn`, which is what gives the relay its defined order. +- Ordering guarantees are only as strong as the weakest adapter, so this applies + to every adapter shipped, not just the first one. + +## Steps + +1. Define one helper that derives the partition key from `(TenantId, RunId)` and + make every adapter call it. A single function is what makes this auditable. +2. Remove any adapter parameter that lets a caller supply their own partition + key. If one is needed for an external contract, name it explicitly and + document that ordering is then the caller's problem. +3. Wire the helper into the relay's publish path (T7.4). +4. Test: two runs sharing a correlation id land on different partitions. +5. Test: all events of one run land on a single partition — assert across a run + that spans branches and attempts. + +## Acceptance + +- Two runs sharing a correlation land on **different partitions**. +- One run's events **never split across partitions**. + +## Verify + +**Harness:** a broker (or fake) exposing the partition assignment per message. + +**Integration test** — `tests/it_partition_keys.rs`: +1. Two runs sharing one correlation id. Assert their events land on **different** + partitions. +2. One run spanning multiple branches, attempts and steps. Assert **every** event + lands on a **single** partition. +3. Two tenants with colliding `RunId` values. Assert they do not share a + partition — the tenant must be in the key. +4. Assert every shipped adapter derives its key through the one shared helper — + an audit test enumerating adapters and asserting each calls it. +5. Assert no public adapter parameter allows a caller-supplied partition key; if + one exists for an external contract, assert it is separately named and + documented. +6. Stability: the same `(TenantId, RunId)` maps to the same partition across + process restarts. + +**Command:** `cargo test -p distribution partition_keys` + +**False pass:** +- Step 2 with a short run whose events happen to hash to one partition regardless. + Use a run long enough that a per-event key would demonstrably split it, and + assert the partition set has size exactly 1. +- Step 1 passing by coincidence with only two runs — use 50 correlated pairs and + assert the distribution is spread. +- Step 4 omitted: one adapter doing it correctly proves nothing about the next + one added, and ordering is only as strong as the weakest adapter. + +## Traps + +- Using a correlation id because it is already threaded through the request. It + is the natural choice and it breaks the only ordering promise the system makes. +- Hashing `RunId` alone without the tenant. Two tenants can then collide on one + partition and, worse, the key stops being tenant-scoped. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.3, §9.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.6-tournament-as-a-join-stage.md b/tasks/T7.6-tournament-as-a-join-stage.md new file mode 100644 index 0000000..1c2f75e --- /dev/null +++ b/tasks/T7.6-tournament-as-a-join-stage.md @@ -0,0 +1,112 @@ +# T7.6 — Tournament as a join stage + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Repartition from the run key to the group key, close groups on a completeness +trigger, and cap burst concurrency so grading cannot starve execution. + +## Facts (inlined — no spec read needed) + +``` + ingest ──► execute ──► verify ──► tournament ──► aggregate ──► decide + │ │ │ │ │ │ + (T,Run) (T,Run) (T,Run) (T,Task,Class, (T,Variant) (T,Workflow) + Epoch) + ▲ ▲ ▲ + shuffle 1 shuffle 2 single writer +``` + +- Three keys, two shuffles, one single-writer stage. Everything up to + verification keys on `(TenantId, RunId)` and is embarrassingly parallel. +- **Grading is a join** — a comparison group must be co-located — keyed on + `(TenantId, TaskId, VerifierOutcome, GroupEpoch)`: the **outcome class** + because brackets exist only within one, and the **epoch** because a closed + group never reopens for a late arrival. +- The decide stage is single-writer per `(TenantId, WorkflowId)`; a CAS on a + generation counter is sufficient (T6.1). +- A group closes on **quorum or timeout**, grading whatever arrived, with group + size attached to the confidence interval (T5.11). +- **Grading yields to agent work.** When both contend, agent inference wins and + grading queues — a framework that lets a judge call delay the work it is + judging has inverted its first principle. +- Tournament and reduction backlogs are the two lags that grow silently, so both + are metered. + +## Steps + +1. Implement the repartition from `(TenantId, RunId)` to + `(TenantId, TaskId, VerifierOutcome, GroupEpoch)` after verification. +2. Maintain group state per key: members, open/closed, quorum target, deadline. +3. Completeness trigger: close on quorum reached **or** deadline passed. + Increment `GroupEpoch` at closure; route later arrivals to the next epoch. +4. Cap concurrent tournament work with a semaphore sized independently of the + executor's. Make the cap and its saturation **observable in metrics**. +5. Give agent inference priority over grading when both want the same resident + model (enforced fully in T8.6). +6. Emit stage-boundary lag for the tournament stage. +7. Test with a 64-episode group while runs are executing, asserting execution + throughput is not degraded past a bound. + +## Acceptance + +- A 64-episode group **does not starve run execution**. +- The cap is **observable in metrics**. + +## Verify + +**Harness:** a load generator producing runs continuously while a large group +fills, so starvation is observable rather than theoretical. + +**Integration test** — `tests/it_tournament_join.rs`: +1. Establish a baseline: run-execution throughput and agent-call latency with no + grading load. +2. Trigger a **64-episode group**. Assert execution throughput and agent latency + stay within a configured bound of baseline — assert on the **numbers**, not on + the absence of an error. +3. Assert the concurrency cap is **observable in metrics**: read the saturation + gauge and assert it reflects the burst. +4. Group key: assert repartition uses all four components. Feed episodes + differing only in `VerifierOutcome`; assert they land in **different** groups. + Repeat for `GroupEpoch`. +5. Closure: a group that never reaches quorum closes on **timeout**, with size + recorded, and `GroupEpoch` increments. +6. Late arrival after closure lands in epoch N+1; epoch N's stored strengths are + unchanged. +7. Single-writer decide stage: two schedulers attempt to adjust allocation for + one `(TenantId, WorkflowId)`; assert CAS behaviour (cross-check T6.1). +8. Emit and assert stage-boundary lag for the tournament stage. + +**Command:** `cargo test -p distribution tournament_join -- --test-threads=1` + +**False pass:** +- Step 2 asserting only that both finished. Grading that delays agent work still + finishes — the latency and throughput comparison against baseline is the test, + because the principle being enforced is a priority, not a liveness property. +- Sharing the executor's concurrency limiter with grading: step 2 may still pass + under light load and the cap becomes invisible. Step 3 is the guard. +- Step 4 omitted: dropping `VerifierOutcome` from the key puts passes and + failures in one bracket, which produces plausible results and violates the one + rule with no exemptions. + +## Traps + +- Omitting `VerifierOutcome` from the group key, which puts passes and failures + in one bracket — the one rule with no exemptions. +- Omitting `GroupEpoch`, which lets a late arrival reopen a published fit. +- Sharing the executor's concurrency limiter with grading. Then a burst of + grading is indistinguishable from a burst of work. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §9.1, §11.6, §11.8, §14.2, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.7-turmoil-suite.md b/tasks/T7.7-turmoil-suite.md new file mode 100644 index 0000000..8e99620 --- /dev/null +++ b/tasks/T7.7-turmoil-suite.md @@ -0,0 +1,109 @@ +# T7.7 — `turmoil` suite + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | L — over 3 days | +| Status | Not started | +| Flags | parallel-ok | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Network partition, latency and node loss simulated deterministically. This is the +P7 gate. + +## Facts (inlined — no spec read needed) + +- Test tooling and what each covers: + - **`turmoil`** — network partition and latency simulation. + - **`loom`** — the lock-free bits. + - **`tokio::time::pause`** — time. +- Together these are **weaker than a seeded scheduler and sufficient with + discipline**. The residual risk is stated honestly rather than engineered + around: tokio's cancellation is cooperative, so a `select!`-dropped future + stops at its next await and not before. +- What the suite must prove: **no double-execution and no lost run under + partition, with automatic recovery**. +- Double-execution is prevented by fencing (T7.3), not by the partition never + happening. This suite is what demonstrates the fence actually fires. +- `parallel-ok`: this task does not gate others and can be built alongside the + rest of P7. +- Open: whether a lint, a wrapper type, or a `loom` harness is the right + enforcement for cancellation rigour under tokio. Unresolved — do not invent a + fourth mechanism here. + +## Steps + +1. Stand up a two-node topology under `turmoil`, both nodes claiming runs from + the same Postgres backend. +2. Partition scenario: isolate node A mid-run, let its lease expire, watch node B + reclaim, then heal the partition and assert node A's writes are fenced off. +3. Latency scenario: inject delays that push heartbeats close to the TTL, and + assert renewals either succeed or cancel the local scope — never a silent + continue. +4. Node-loss scenario: kill a node outright; assert the run is reclaimed and + completes exactly once. +5. Add `loom` coverage for any lock-free structure introduced in P7. +6. Use `tokio::time::pause` for all TTL and deadline arithmetic so the suite runs + in seconds. +7. Print per-scenario progress and enforce per-scenario timeouts — a suite that + prints nothing cannot distinguish slow from hung. + +## Acceptance + +- **No double-execution and no lost run under partition; recovery is automatic.** + +## Phase gate + +P7 closes on two nodes surviving a partition with no double-execution. + +## Verify + +**Harness:** `turmoil` two-node topology; `loom` for any lock-free structure; +`tokio::time::pause` for TTL arithmetic; the external side-effect ledger as the +double-execution witness. + +**Integration test** — `tests/it_turmoil.rs`: +1. **Partition:** isolate node A mid-run. Assert node B reclaims after lease + expiry, the run completes, and the ledger shows **exactly one** effect per + intended effect. +2. Heal the partition; assert A's writes are fenced (cross-check T7.3) and that A + does not resume executing. +3. **Latency:** inject delays that push heartbeats close to the TTL. Assert + renewals either succeed or **cancel the local scope** — assert no case where a + worker continues past a failed renewal. +4. **Node loss:** kill a node outright. Assert the run is reclaimed and completes + exactly once. +5. Assert **no lost run**: every spawned run reaches a terminal state across all + scenarios; count spawned vs terminal. +6. Recovery is **automatic** — assert no manual intervention hook was invoked. +7. `loom` test for each lock-free structure added in P7. +8. Per-scenario progress output and per-scenario timeouts. +9. Print the seed; a failure that cannot be replayed is not a finding. + +**Command:** `cargo test -p distribution --test it_turmoil -- --nocapture` + +**False pass:** +- Asserting the run completed. **Completing twice also completes** — the ledger + count is the only assertion that distinguishes them, and double-execution is + the primary risk this suite exists to retire. +- Real `sleep` for TTL expiry, making the suite slow enough that it stops being + run — which is the same as not having it. +- Step 5 omitted: a run that is silently dropped during a partition leaves no + error, and every other assertion passes. +- Testing only the partition scenario. Node loss and slow-network cases exercise + different reclaim paths. + +## Traps + +- Asserting only that the run completes. Completing twice also completes. +- Real sleeps for TTL expiry, which makes the suite too slow to run and therefore + unrun. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §6, §9.4, §18, §19 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T7.8-p7-composition-gate.md b/tasks/T7.8-p7-composition-gate.md new file mode 100644 index 0000000..d720b6b --- /dev/null +++ b/tasks/T7.8-p7-composition-gate.md @@ -0,0 +1,111 @@ +# T7.8 — P7 composition gate + +| Field | Value | +|---|---| +| Phase | P7 — Distribution | +| Size | L — over 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | P8 | + +## Goal + +Prove **deployment mode is interchangeable**: the same behavioural suite passes +in embedded mode and distributed mode, and two nodes survive a partition with no +double-execution. + +**Phase gate criterion:** two nodes surviving a partition with no +double-execution. + +## Facts (inlined — no spec read needed) + +- **Two deployment modes, one set of ports.** + +| Mode | Log + state | Blobs | Coordination | +|---|---|---|---| +| **Embedded** — single binary, no services | `redb` | `redb` table | in-process | +| **Distributed** — multi-node, multi-tenant | Postgres | S3-compatible | Postgres advisory locks, or Redis if leases dominate | + +- **Embedded mode is a first-class product, not a test harness.** That constraint + is what keeps the ports honest — and it is only enforced if the same suite runs + in both modes. +- The properties no single P7 task owns: + - a run that survives a partition must also survive it **while grading is in + flight** — leases, the join stage and the outbox interact; + - the outbox relay must ship correctly **across a reclaim**: entries committed + by node A, relayed after node B takes over; + - **partition keys and per-run ordering must hold across a node handover** — + the same run's events must not split partitions because a different node + published them; + - exactly-once is achieved **at the fold**, not in transport, so redelivery + during a partition must be a no-op. +- Per-run total order, nothing promised across runs. Everything up to + verification is embarrassingly parallel; grading is a join; decide is + single-writer. + +## Steps + +1. Extract the behavioural suite — P1's composition matrix plus P4's and P5's + gates — into a mode-parameterized harness. +2. Run it in embedded mode and in distributed mode. Both must pass unmodified. +3. Build the compound distributed scenario: partition **during** grading, with + the broker down, across a lease handover. +4. Assert exactly-once effects and no lost runs across every scenario. +5. Make this the required CI job gating P8. + +## Acceptance + +- The behavioural suite passes **unmodified** in both deployment modes. +- Two nodes survive a partition with **no double-execution** and no lost run. +- The outbox ships every event in order across a lease handover, exactly once at + the fold. + +## Verify + +**Harness:** `turmoil` two-node topology; real Postgres and MinIO; the external +side-effect ledger as the double-execution witness; a consumer that folds on +`(BranchKey, Lsn)`. + +**Integration test** — `tests/it_p7_composition.rs`: +1. **Mode interchange:** parameterize the behavioural suite over + `[embedded, distributed]`. Both pass with **zero** suite edits. Any + `if mode == ...` branch inside the suite is a finding about the ports. +2. **Compound scenario:** start a run, begin grading, take the broker down, + partition node A mid-step. Assert node B reclaims, the run and its grading + complete, and the ledger shows **exactly one** effect per intended effect. +3. **Relay across handover:** assert outbox entries committed by A are shipped + after B takes over — in `(BranchKey, Lsn)` order, exactly once at the fold. +4. **Partition-key stability across nodes:** assert every event of the handed-over + run landed on **one** broker partition, despite two different nodes publishing. +5. **Fencing under load:** heal the partition while B is executing. Assert A's + writes are rejected by fence and A does not resume. +6. **No lost runs:** across all scenarios, count runs spawned versus runs reaching + a terminal state. Assert equality. +7. **Grading yields:** during the scenario, assert agent-call latency stays within + bound of baseline while the join stage is saturated. +8. **Regression:** re-run P0–P6 gates in embedded mode in the same job. + +**Command:** `cargo test -p distribution --test it_p7_composition -- --test-threads=1 --nocapture` + +**False pass:** +- A mode-parameterized suite containing mode-specific branches. At that point the + two modes are not proven equivalent, which is the entire acceptance criterion — + and embedded mode is a shipped product, not a fixture. +- Asserting runs completed. **Completing twice also completes**; only the external + ledger distinguishes them. +- Step 3 tested without a handover, where the relay never changes owner — that is + T7.4's test, already green. +- Step 6 omitted: a run silently dropped during a partition produces no error and + passes every other assertion here. + +## Traps + +- Skipping embedded mode in CI for speed. It is the mode that keeps the ports + honest, and it is the cheap one to run. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §7, §9 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.1-metering.md b/tasks/T8.1-metering.md new file mode 100644 index 0000000..be6f4dc --- /dev/null +++ b/tasks/T8.1-metering.md @@ -0,0 +1,105 @@ +# T8.1 — Metering + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Attribute every model call to `(TenantId, RunId, Purpose)` and enforce separate +ceilings for judging and replay. + +## Facts (inlined — no spec read needed) + +- **A framework that spends users' model budget on grading must account for it.** +- Every model call — agent, judge, or proposer — is attributed to + `(TenantId, RunId, Purpose)`, where `Purpose` distinguishes **work** from + **grading** from **replay**. +- Per-tenant ceilings on grading spend are enforced **at the group boundary**, + where group size is known and a tournament can be skipped or downsampled before + it starts — not mid-tournament. +- **Replay is the expensive one and needs its own ceiling separate from + judging.** N variants × M tasks is N·M full agent runs plus judge calls. +- Grading cost is reported **next to grading value**. A loop that costs more than + the work it grades may still be worth running; it should never be an unpleasant + discovery. +- A skipped run must reach terminal `Ungraded` with `BudgetExhausted`, not rest + in `Grading`. **A cost ceiling that also strands storage is a bill that arrives + twice**, landing on the tenants least able to absorb it. +- The grading spend ratio ceiling itself is unset — metered here, policy elsewhere. + +## Steps + +1. Define `Purpose { Work, Grading, Replay }` and thread it through every model + call site. No default — the call site knows. +2. Record usage per call keyed `(TenantId, RunId, Purpose)`: tokens in/out, model, + and cost if a price table is configured. +3. Define per-tenant ceilings: one for grading, a **separate** one for replay. +4. Check the grading ceiling at the group boundary. Over the ceiling: skip or + downsample, record `UngradedReason::BudgetExhausted`, and transition the run to + terminal `Ungraded` (T5.11). +5. Never let a ceiling check touch the agent execution path. Work continues + regardless of grading spend. +6. Emit a report pairing grading cost with grading value per tenant. +7. Test: drive a tenant over its grading ceiling and assert agent throughput is + unchanged. + +## Acceptance + +- A tenant exceeding its grading ceiling has tournaments **skipped with a + recorded reason**, and **agent work continues unaffected**. +- The skipped runs reach terminal `Ungraded` rather than resting in `Grading`. + +## Verify + +**Harness:** a model provider wrapper recording `(TenantId, RunId, Purpose, +tokens)` for every call, plus a tenant configured with a **zero** grading ceiling +and a generous replay ceiling (and the reverse). + +**Integration test** — `tests/it_metering.rs`: +1. Run a full cycle: agent work, grading, replay. Assert every recorded call + carries a `Purpose`, and that the three purposes are all present. Assert no + call is attributed to a default or unknown purpose. +2. **Ceiling separation:** with grading ceiling = 0 and replay ceiling high, + assert grading is skipped and replay still runs. Then invert and assert the + opposite. One shared ceiling fails one of these. +3. **Skip happens before spend:** assert model calls under `Purpose::Grading` is + **exactly 0** when over the ceiling — the check is at the group boundary, + before any call. +4. Assert agent throughput and latency are **unchanged** while the tenant is over + its grading ceiling; compare against a baseline run. +5. Assert skipped runs reach **terminal `Ungraded { BudgetExhausted }`** and that + T4.4's `is_reducible` returns true for them. +6. Assert the cost-versus-value report is produced per tenant. +7. Assert attribution survives a cold re-fold — the usage lives in the log, not + only in state. + +**Command:** `cargo test -p operability metering` + +**False pass:** +- Step 3 asserting only the recorded reason. Enforcing the ceiling **after** the + calls produces the correct reason and spends the money anyway. +- Step 5 omitted: this is the "bill arrives twice" case — the tenant is capped on + spend and then also accumulates storage forever because the run never leaves + `Grading`. +- Step 2 omitted: a single ceiling passes every other assertion here, then replay + consumes the whole budget in production. + +## Traps + +- One shared ceiling for judging and replay. Replay dominates and consumes it. +- Checking the ceiling mid-tournament, which pays for half a group and produces + nothing. +- Leaving budget-skipped runs in `Grading`. The cost ceiling then also strands + storage. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §10.3, §11.6, §12.4, §14.1 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.2-metrics.md b/tasks/T8.2-metrics.md new file mode 100644 index 0000000..51d3230 --- /dev/null +++ b/tasks/T8.2-metrics.md @@ -0,0 +1,118 @@ +# T8.2 — Metrics + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +The full observability list. The framework observes agents; it must also be +observable. + +## Facts (inlined — no spec read needed) + +The list, with the reason each one exists: + +- **Kernel state transitions**, tagged by tenant and workflow version. +- **Lag on every stage boundary** of `ingest → execute → verify → tournament → + aggregate → decide`. **Tournament and reduction backlogs are the two that grow + silently.** +- **`Indeterminate` attempt count as a first-class alert.** It should be near + zero; a nonzero rate means either a crash loop or a misdeclared effect class. + **This threshold only holds because `Cancelled` is a separate state** — route + cancellations here and the alert has a noisy floor, which is the same as not + having it. +- **`Ungraded` run count by reason**, separately from `Graded`. A tenant whose + runs are mostly `InsufficientGroup` has a loop that is not engaging, and that + reads as healthy on any dashboard that only counts failures. +- **Judge order-inconsistency rate**, measured on the sampled fraction under the + alternating-order scheme (T5.7). +- **Judge-versus-verifier agreement on the calibration set** (T6.2). Under one + resident model the judge is the agent's model, and this is the only instrument + that can see the grader drifting toward its own bias. +- **Model residency and swap count.** A nonzero swap rate on a single-model + deployment means something is requesting a non-resident model, and the load + cost will dominate everything else in the trace. +- **Admission refusals by reason, separating "capacity" from "budget".** They + look identical in a queue-depth graph and have opposite fixes. +- **Held-out versus selection gap** as the overfitting alarm (T6.6). +- **Trace context propagated through `Ctx`, never through task-locals.** + +## Steps + +1. Pick the metrics facade once and use it everywhere. Define the tag set — + tenant, workflow version, purpose — as a shared helper so tags stay consistent. +2. Emit a counter per kernel transition from the transition function itself, so + no call site can forget. +3. Instrument each stage boundary with a lag gauge measured at dequeue: now minus + the record's commit timestamp. +4. Emit `Indeterminate` and `Ungraded`-by-reason as separate series. Do not + collapse `Ungraded` reasons into one counter. +5. Wire the grading-health metrics from T5.7 and T6.2. +6. Emit residency and swap counters from the capacity layer (T5.2/T8.6). +7. Split admission refusals into `capacity` and `budget` reasons at the refusal + site. +8. Propagate trace context through `Ctx`. Audit for `task_local!` and remove. +9. Test: stall the reducer deliberately and assert it is visible within one + scrape interval. + +## Acceptance + +- **Every stage boundary emits lag.** +- **A stalled compactor is visible within one scrape interval.** + +## Verify + +**Harness:** an in-process metrics recorder the test can scrape, plus a +deliberately stalled reducer. + +**Integration test** — `tests/it_metrics.rs`: +1. **Completeness audit:** assert one metric exists per item in this file's list. + Drive it from an enumerated list of expected metric names so a missing one + fails rather than going unnoticed. +2. **Stage lag:** run work through all six stages; assert **each** boundary emits + a lag value. A stage emitting nothing must fail. +3. **Stalled reducer:** stop the reducer, keep producing work, scrape once. + Assert the backlog is visible **within one scrape interval** — assert on the + scraped value, not on the internal counter. +4. Assert `Indeterminate` count is its own series and that a cancelled attempt + increments **`Cancelled`, not `Indeterminate`** — the alert's near-zero + threshold depends entirely on this. +5. Assert `Ungraded` is emitted **by reason** as separate series; a single + `ungraded_total` fails. +6. Assert admission refusals split into `capacity` and `budget`; produce one of + each and assert two distinct series move. +7. Assert every metric carries tenant and workflow-version tags where the list + says so. +8. Assert trace context propagates across a spawn boundary through `Ctx`; grep + for `task_local!` in kernel crates and assert none. + +**Command:** `cargo test -p operability metrics` + +**False pass:** +- Asserting metrics exist without asserting they **move**. A registered-but-never- + incremented counter passes existence checks and reads as a healthy zero. +- Step 3 measured on the internal counter rather than a scrape. The stall may be + visible internally and never exported. +- Step 4 omitted: routing cancellations into `Indeterminate` produces a perfectly + functional metric with a permanent noisy floor, which quietly retires the + alert. +- Step 6 omitted: capacity and budget refusals look identical in a queue-depth + graph and have opposite fixes. + +## Traps + +- One `ungraded_total` counter. The reason is the whole signal. +- Merging capacity and budget refusals. Opposite fixes, identical graph. +- Trace context in a task-local. It survives until the first spawn boundary. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §5.1, §9.1, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.3-keyed-capability.md b/tasks/T8.3-keyed-capability.md new file mode 100644 index 0000000..b39b02c --- /dev/null +++ b/tasks/T8.3-keyed-capability.md @@ -0,0 +1,114 @@ +# T8.3 — Keyed capability + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Make an `Idempotent` declaration enforceable: the kernel derives and supplies the +idempotency key on every call, and a write issued without it is denied. + +## Facts (inlined — no spec read needed) + +- **A framework cannot trust a user-supplied `EffectClass`.** A tool declared + `Idempotent` that is not will be retried after an indeterminate crash, and the + damage is the user's data. +- Three layers, in order of strength: + 1. **Declaration** — required, recorded, auditable (T2.2). + 2. **Keyed capability** — this task. + 3. **Sandbox** — shadow and replay deny `Unsafe` outright (T6.5). +- **Layer 2 is the one that makes layer 1 more than paperwork.** +- **`Idempotent` is a claim about *retry*, not about abstaining from writes.** + Denying an `Idempotent` tool network and filesystem writes would deny the + recovery path itself, which retries the write under an idempotency key. +- So the restriction is on the **shape** of the write, not its existence: the + tool declares how its key derives from its arguments, the kernel derives and + supplies that key on every call, and **a write issued without it is denied**. +- **A tool that cannot derive a stable key cannot be `Idempotent`** — the same + claim as before, now refused at registration rather than discovered after a + double-charge. + +```rust +pub enum EffectClass { + Idempotent { key: KeyDerivation }, + Queryable { lookup: RequestIdLookup }, + Unsafe, +} +``` + +## Steps + +1. Make `KeyDerivation` a required payload on `Idempotent` — a registration + without one must **fail to compile**, not fail at runtime. +2. At call time, derive the key from the recorded arguments and pass it through + the capability handle the tool must use to write. The tool cannot mint its own. +3. Deny any write issued through a handle without the kernel-supplied key. + Denial is at call time, not a post-hoc audit. +4. Record every denial against the tool id, so a misbehaving tool is auditable + rather than merely blocked. +5. Ensure the recovery path (T2.2) re-derives the **same** key from the **same** + recorded arguments — that is what makes the retry safe. +6. Test the duplicate-call case explicitly: two calls with the same derived key + produce one effect. + +## Acceptance + +- A tool declaring `Idempotent` with a key completes its write and **survives a + duplicate call with no duplicate effect**. +- The same tool writing outside the kernel-supplied key is **denied at call + time**, with the denial **recorded against the tool id** for audit. +- A registration declaring `Idempotent` **without a derivation does not compile**. + +## Verify + +**Harness:** a test tool writing to the external side-effect ledger, keyed by +whatever key it is given. Plus `trybuild`. + +**Integration test** — `tests/it_keyed_capability.rs`: +1. **Compile-fail:** an `Idempotent` registration with no `KeyDerivation`. Assert + the stderr names the missing field. +2. Happy path: the tool completes its write. Assert the ledger holds one entry + under the **kernel-supplied** key. +3. **Duplicate call:** invoke again with the same arguments. Assert the derived + key is identical and the ledger still holds **exactly one** entry. +4. **Out-of-band write:** the same tool attempts a write **not** using the + supplied key. Assert it is **denied at call time**, and that the denial is + **recorded against the tool id** — read the audit record back. +5. **Recovery is a write:** crash mid-call, restart, let T2.2's idempotent + recovery retry. Assert the retry **succeeds** — a blanket write denial would + break exactly this path, and it is the most likely wrong implementation. +6. Determinism: derive the key 100 times from the same recorded arguments; assert + all identical. Derive from arguments containing a timestamp field; assert the + derivation ignores it or the registration is rejected. +7. Assert `Queryable` and `Unsafe` tools are unaffected by this mechanism. + +**Command:** `cargo test -p operability keyed_capability && cargo test -p operability --test compile_fail` + +**False pass:** +- Implementing this as a blanket write denial for `Idempotent` tools. Steps 1–4 + all pass; step 5 is the one that fails, and it is the whole reason the + restriction is on the **shape** of the write rather than its existence. +- Step 3 passing because the ledger deduplicates on content rather than key. Make + the ledger key-only, or the key derivation is never actually exercised. +- Step 4 asserting the denial without asserting the audit record. A denial nobody + can attribute to a tool is not actionable. + +## Traps + +- Blanket-denying writes to `Idempotent` tools. That denies the recovery path, + which is itself a write. +- A key derived from anything non-deterministic — a timestamp, a counter, a + random id. The retry then writes twice under two keys. +- Logging denials without the tool id, which leaves nothing to act on. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.4, §13.1, §13.2 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.4-reduction-and-tiering.md b/tasks/T8.4-reduction-and-tiering.md new file mode 100644 index 0000000..5f4adb7 --- /dev/null +++ b/tasks/T8.4-reduction-and-tiering.md @@ -0,0 +1,142 @@ +# T8.4 — Reduction and tiering + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | L — over 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Reduce episodes to a token ceiling, then move terminal reduced runs to cold +storage. Never rewrite a blob body; never edit a log record. + +## Facts (inlined — no spec read needed) + +Vocabulary, because the previous revision collided two of these: **`Archived`** +is a run state; **reduction** is the token-budget operation; **tiering** is the +move to cold storage. + +- **Retention is measured in tokens**, because tokens are the currency of every + downstream consumer: what a replay costs, what fits in a judge's window, what + an export is billed at. Default ceiling 200k per run, per-tenant configurable. +- **On self-hosted weights that ceiling is not free to choose.** It is bounded by + `max_context_tokens / 2` (T5.2), because a pairwise judge reads two episodes + into one KV cache and two 200k episodes do not fit on any single device. Where + the derived bound is tighter, **the bound wins** and the judge reads a + further-reduced view. + +Loss order is fixed: + +| Kept | Reduced | Dropped | +|---|---|---| +| every transition record | blob bodies → summary blob | raw text on dead branches | +| context partitions, tool info, usage | dead-branch attempts → summary | | +| verifier results, grades | | | + +- **Reduction never rewrites a blob and never edits the log.** Blobs are + content-addressed, so replacing a body under its existing ref makes the ref a + lie; repointing the log at a new ref is the history rewrite the append-only + rules forbid. +- Instead: write the summary as a **new** blob, append a + `Reduced { original: BlobRef, summary: BlobRef }` event, and **only then** + delete the original body. The reduction is a later fact about an earlier + record, not a change to it. `BlobStore::get` on the original returns `None`, + and the fold knows why and what stands in its place — so drop-and-re-fold from + LSN 0 still yields byte-identical state, which it would not if the mapping + lived only in the projection. +- The transition sequence always survives. What reduces is **text**, because it + dominates token count and is the only part with a cheap lossy representation. A + reduced episode can still be graded, attributed and structurally rewound — it + just cannot be replayed verbatim. +- **Tiering: moved, not copied**, with local rows deleted only after the remote + commit acknowledges. +- Eligibility is `Graded`/`Ungraded`/`Archived` (T4.4) — grading terminated. +- Open: what produces the summary. A model call makes reduction + non-deterministic, which interacts badly with replay; extractive or structural + reduction may suffice. + +## Steps + +1. Compute the effective ceiling as `min(configured_ceiling, max_context_tokens / 2)` + from T5.2. Never read the configured value alone. +2. Gate on T4.4's `is_reducible`. Reject, do not skip. +3. Reduce in the fixed loss order: blob bodies to summaries first, then + dead-branch attempts, then raw text on dead branches. +4. For each reduction: `put` the summary blob → `commit` the + `Reduced { original, summary }` event → **then** `delete` the original body. + That order survives a crash at any point. +5. Make the fold handle `Reduced` so a `None` from `get` is explained by the log. +6. Tiering: write to cold storage, wait for the remote commit acknowledgement, + then delete local rows. Crash between the two leaves a duplicate, not a loss — + so make the move idempotent. +7. Verify by re-hashing every surviving `BlobRef` against its body. + +## Acceptance + +- A run exceeding the ceiling reduces; **transitions survive intact**; a reduced + episode is **still gradeable**. +- Every surviving `BlobRef` still hashes to its content — asserted by + **re-hashing**, since a rewritten body type-checks silently. +- Drop the state tables and re-fold a reduced run from LSN 0: state is + **byte-identical**. This fails if the original→summary mapping lives anywhere + but the log. +- Tiering is **idempotent under a crash between remote commit and local delete**. + +## Verify + +**Harness:** a run built to exceed the token ceiling, T0.8's +`assert_refold_identical`, and a cold-storage stub that can fail between remote +commit and local delete. + +**Integration test** — `tests/it_reduction_tiering.rs`: +1. Reduce a run over the ceiling. Assert the **transition sequence is intact** — + compare the event list before and after; only `Reduced` events were added, + none removed or altered. +2. Assert the reduced episode is **still gradeable**: run it through the grading + path and assert a real `Score`. +3. **Re-hash every surviving `BlobRef`** against its body and assert equality. A + rewritten body type-checks silently and this is the only assertion that sees + it. +4. **The load-bearing test:** drop the state tables, re-fold the reduced run from + LSN 0, assert **byte-identical** state. This fails if the original→summary + mapping lives anywhere but the log. +5. Assert `get` on an original ref returns `Ok(None)` and that the fold explains + it via the `Reduced` event. +6. **Ordering under crash:** arm faults at each of the three points (after `put`, + after `commit`, after `delete`). Assert no state exists where a record points + at a deleted body with no `Reduced` event. +7. **Ceiling derivation:** configure a 200k ceiling with a hardware-derived bound + of 80k; assert the **bound wins** and reduction targets 80k. +8. **Tiering idempotence:** crash between remote commit and local delete; re-run. + Assert one copy remains, no data lost, operation converges. +9. Assert reduction refuses on a run in `Grading` (T4.4). + +**Command:** `cargo test -p retention --features test-hooks reduction -- --test-threads=1` + +**False pass:** +- Step 4 omitted. Storing the original→summary mapping in the projection makes + steps 1–3 pass perfectly and silently destroys drop-and-re-fold, which is the + property the whole durability design rests on. +- Step 3 replaced by "the ref still resolves". A rewritten body resolves fine. +- Step 7 omitted: reading the configured ceiling alone passes every test on + hardware nobody checked, and fails at the first real judge call. +- Step 6 with faults only after `delete`, which is the safe point. + +## Traps + +- Overwriting the blob body under its existing ref. It type-checks, it saves an + event, and it makes the ref a lie. +- Repointing the log record at the summary ref. That is a history rewrite. +- Deleting the original before the `Reduced` event commits. A crash then leaves a + ref with no body and no explanation. +- Reading the configured ceiling without the hardware-derived bound. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.6, §8.7, §10.3, §14.2, §18 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.5-embedded-mode-smoke.md b/tasks/T8.5-embedded-mode-smoke.md new file mode 100644 index 0000000..f038d71 --- /dev/null +++ b/tasks/T8.5-embedded-mode-smoke.md @@ -0,0 +1,94 @@ +# T8.5 — Embedded-mode smoke + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | S — under 1 day | +| Status | Not started | +| Flags | parallel-ok | +| Spec | inlined below | +| Blocks | — | + +## Goal + +A fresh crate does `cargo add`, runs one workflow, and gets durability and +grading with zero infrastructure. + +## Facts (inlined — no spec read needed) + +- **Embedded mode is a first-class product, not a test harness.** A user must be + able to `cargo add` this, run an agent, and get durability and grading with + zero infrastructure. +- **That constraint is what keeps the ports honest.** If this breaks, the ports + have leaked — something in the default path now requires Postgres, an object + store, or a broker. +- Embedded mode is `redb` for log and state, a `redb` table for blobs, and + in-process coordination. +- Defaults ship working: a user who wants the built-in behaviour writes no code. + The smoke test is the proof of that claim. +- `parallel-ok`: this gates nothing and can be built alongside the rest of P8. + +## Steps + +1. Create the smoke test as a **separate crate outside the workspace**, depending + on the published crate surface only. Inside the workspace it can reach + internals and prove nothing. +2. Depend on the default feature set. No `features = ["postgres", ...]`. +3. Define one small workflow in YAML, register a stub model provider and one + verifier. +4. Run it: spawn with a `TaskId`, execute, verify, grade with the default + `PairwiseSequential` (first run yields `Ungraded { NoReference }`; a second run + on the same `TaskId` grades). +5. Assert no network listener is opened and no external service is contacted. +6. Run it in CI on every change to the default path. + +## Acceptance + +- A fresh crate with **no infrastructure** records, verifies and grades a run. +- If this breaks, the ports have leaked. + +## Verify + +**Harness:** a crate **outside the workspace**, depending on the published crate +by version or path with `default-features = true` and nothing added. That +location is the test — inside the workspace it can reach internals and proves +nothing. + +**Integration test** — `smoke-crate/tests/it_embedded_smoke.rs`: +1. `cargo add` the framework (or a path dependency with default features only). +2. Define one small YAML workflow, register a stub model and one verifier. +3. Spawn with a `TaskId`, execute, verify, grade. Assert the run reaches a + terminal state and produces a `Score` — the first run legitimately yields + `Ungraded { NoReference }`. +4. Second run on the same `TaskId`: assert `Score::Relative`, proving the whole + default grading path works with no infrastructure. +5. Assert **no network listener is opened** and no external host is contacted — + run under a network-denied sandbox if the CI supports it, otherwise assert on + an instrumented socket layer. +6. Assert no Postgres, object store or broker dependency is pulled in: check + `cargo tree` for the distributed-only crates and assert absent. +7. Assert the process exits cleanly and the `redb` file is readable afterwards. +8. Run this in CI on **every change to the default path**, not nightly. + +**Command:** `cargo test --manifest-path smoke-crate/Cargo.toml` + +**False pass:** +- The smoke crate living inside the workspace. It compiles against internals and + stays green while the published API is unusable — which is the exact leak this + test exists to detect. +- Enabling a non-default feature to make it build. That is the leak, papered + over; the failure is the finding. +- Step 4 omitted: a first run that returns `Ungraded` is trivially reachable + without a working grading path. + +## Traps + +- Writing the smoke test inside the workspace with access to `pub(crate)` items. + It then passes while the public API is unusable. +- Enabling a non-default feature to make it pass. That is the leak, papered over. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §7 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.6-capacity-admission-control.md b/tasks/T8.6-capacity-admission-control.md new file mode 100644 index 0000000..8625b97 --- /dev/null +++ b/tasks/T8.6-capacity-admission-control.md @@ -0,0 +1,113 @@ +# T8.6 — Capacity admission control + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | M — 1 to 3 days | +| Status | Not started | +| Flags | — | +| Spec | inlined below | +| Blocks | — | + +## Goal + +Enforce the residency invariant at runtime, not only at load. Grading yields to +agent inference. + +## Facts (inlined — no spec read needed) + +- **Admission control, not backpressure.** Work whose `ResourceProfile` does not + fit the current residency is **refused at admission with the limit named**. + Queuing it would stall behind an eviction that the first principle does not + permit. +- **Grading yields to agent work.** When both contend for the same resident + model, agent inference wins and grading queues. **A framework that lets a judge + call delay the work it is judging has inverted its own first principle.** +- **One resident model is the default configuration.** The agent's model is + pinned and never evictable; the judge and the proposer run on that same model. +- A second model is legal only if the invariant holds with **both resident** — + never by swapping between them per call, which is the failure mode this exists + to prevent. +- **A swap is a test failure, not a slow path.** A nonzero swap rate on a + single-model deployment means something is requesting a non-resident model, and + the load cost — tens of seconds — will dominate everything else in the trace. +- Admission refusals are metered by reason, separating **capacity** from + **budget**: they look identical in a queue-depth graph and have opposite fixes. +- Residency is per device. A model resident on node A does not make node B's runs + admissible. + +## Steps + +1. Track current residency as live state: which models are resident on which + devices, and current concurrent KV usage. +2. At admission, evaluate T5.2's invariant against the requesting work's + `ResourceProfile` plus current residency. +3. Refuse when it does not fit, with the **named limit** in the error — not a + generic capacity error, and not a queue. +4. Implement priority: agent inference preempts queued grading for the same + resident model. Grading waits; agent work does not. +5. Assert on **agent-call latency under grading load**, not on queue ordering — + a scheduler that merely queues correctly still fails the principle if grading + wins the race. +6. Count model swaps. Emit as a metric and assert zero in the test. +7. Tag admission refusals `capacity` or `budget` at the refusal site (T8.1 owns + the budget half). + +## Acceptance + +- With one resident model, a burst of grading work **never delays agent inference + past a configured bound** — asserted on agent-call latency under grading load. +- **Zero model swaps** occur across the whole run; a swap is a test failure, not + a slow path. + +## Verify + +**Harness:** a model runtime stub that reports residency and **counts swaps**, +plus a latency histogram on agent calls. No GPU needed — the stub enforces the +declared limits. + +**Integration test** — `tests/it_admission_control.rs`: +1. Baseline: measure agent-call latency (p50 and p99) with **no** grading load. +2. Burst 100 grading tasks against one resident model while agent work continues. +3. **Assert agent-call p99 stays within the configured bound of baseline.** This + is the acceptance criterion and it is a latency assertion, not an ordering + one — a scheduler that merely queues correctly still fails the principle if + grading wins the race. +4. **Assert zero model swaps** across the whole run. A swap is a test failure, + not a slow path — assert `swap_count == 0`, not a threshold. +5. Submit work whose `ResourceProfile` does not fit current residency. Assert it + is **refused at admission**, not queued — check the queue depth stayed at + zero — and that the error **names the limit**. +6. Assert the refusal is tagged `capacity`, distinct from T8.1's `budget` + refusals, and that both series exist. +7. Multi-node: assert a model resident on node A does not make node B's work + admissible. +8. Assert a pinned model is never selected for eviction, even when the invariant + would otherwise be satisfiable by evicting it. + +**Command:** `cargo test -p operability admission -- --test-threads=1 --nocapture` + +**False pass:** +- Step 3 asserting that grading was queued behind agent work. Correct ordering + with a shared lock still adds latency to agent calls; the histogram is what + detects it. +- Step 5 asserting an eventual error after queuing. Queuing over-capacity work + stalls it behind an eviction that must not happen — assert the queue never grew. +- Step 4 as `swap_count < 5`. Any nonzero swap rate means something is requesting + a non-resident model, and the tens-of-seconds load cost will dominate the + trace. +- Measuring only mean latency, where a p99 stall from one swap disappears. + +## Traps + +- Queuing over-capacity work instead of refusing it. It stalls behind an eviction + that must not happen. +- A fair scheduler between grading and agent work. Fair is the wrong policy here; + agent work wins. +- Measuring only queue depth. Both refusal reasons look identical there. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §1, §14.2, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/T8.7-p8-composition-gate.md b/tasks/T8.7-p8-composition-gate.md new file mode 100644 index 0000000..4267f4a --- /dev/null +++ b/tasks/T8.7-p8-composition-gate.md @@ -0,0 +1,116 @@ +# T8.7 — P8 composition gate + +| Field | Value | +|---|---| +| Phase | P8 — Operability | +| Size | L — over 3 days | +| Status | Not started | +| Flags | gate | +| Spec | inlined below | +| Blocks | — release gate | + +## Goal + +Prove the operability layer composes with everything before it — and that turning +all of P8 on does not break the zero-infrastructure embedded product. + +**Phase gate criterion:** metering, metrics, sandbox enforcement and capacity +admission all active, with the embedded smoke crate still green. + +## Facts (inlined — no spec read needed) + +- P8's features all sit **across** the rest of the system rather than beside it, + so each one's own test is necessarily narrow: + - metering attributes calls the executor and grader make; + - metrics observe every stage; + - keyed capability constrains every tool call; + - reduction rewrites what grading and verification later read; + - admission control gates work the scheduler wants to run. +- The properties no single P8 task owns: + - **a reduced episode is still gradeable** — reduction and grading are built by + different tasks and read the same blobs; + - **budget and capacity refusals are distinguishable** end to end, since they + look identical in a queue-depth graph and have opposite fixes; + - **metering totals reconcile** with the model provider's own call count — + attribution that misses a call path understates spend silently; + - **the embedded smoke crate still passes with every P8 feature enabled.** If + it does not, an operability feature has leaked a service dependency into the + default path. +- The retention ceiling is derived from hardware, not chosen: where + `max_context_tokens / 2` is tighter than the configured ceiling, the bound + wins. That interaction spans T5.2 and T8.4 and belongs to neither. + +## Steps + +1. Enable every P8 feature and run the full behavioural suite from T7.8. +2. Run the reduce-then-grade path: reduce a run, then grade the reduced episode. +3. Reconcile metering totals against the provider's own counter. +4. Run the embedded smoke crate with all features on. +5. Drive both refusal reasons and assert they are separable everywhere. +6. Make this the release gate. + +## Acceptance + +- The full behavioural suite passes with **all P8 features enabled**, in both + deployment modes. +- A **reduced** episode is still gradeable and still produces a valid `Score`. +- Metering totals reconcile exactly with the provider's call count. +- The embedded smoke crate passes with every feature on. +- Capacity and budget refusals are distinguishable end to end. + +## Verify + +**Harness:** T7.8's mode-parameterized suite; the provider call counter; the +external side-effect ledger; the out-of-workspace smoke crate; a scrapeable +metrics recorder. + +**Integration test** — `tests/it_p8_composition.rs`: +1. **Everything on:** run the T7.8 behavioural suite with metering, metrics, + keyed capability, reduction and admission control all enabled, in both modes. +2. **Reduce then grade:** reduce a run past the ceiling, then grade it. Assert a + valid `Score`, and assert the grading path handled `get` → `None` on reduced + originals by following the `Reduced` event. Then drop state, re-fold, and + assert byte-identical. +3. **Derived ceiling interaction:** configure a 200k retention ceiling with a + hardware-derived bound of 80k. Assert reduction targets **80k** and that the + judge's context request is admitted. +4. **Metering reconciliation:** assert `sum(metered calls)` equals the provider's + own counter, and that every call carries a non-default `Purpose`. A mismatch + means a call path is unattributed. +5. **Refusal separability:** drive one capacity refusal and one budget refusal. + Assert two distinct metric series moved, that each error names its limit, and + that the budget-refused run reaches terminal `Ungraded` while the + capacity-refused work was never queued. +6. **Keyed capability under the real interpreter:** assert every tool call in the + suite carried a kernel-supplied key or was denied and audited — count against + the intent records, not against the tool's own reporting. +7. **Zero swaps** across the whole gate. +8. **Smoke crate:** run it, outside the workspace, default features only, with + every P8 feature active. Assert no network listener and no distributed-only + crate in `cargo tree`. +9. **Regression:** re-run P0–P7 gates. + +**Command:** `cargo test -p operability --test it_p8_composition && cargo test --manifest-path smoke-crate/Cargo.toml` + +**False pass:** +- Running each P8 feature's suite with the others off. Every one is green that + way; the interactions — reduction versus grading, metering versus purpose + attribution, admission versus the join stage — are the reason this gate exists. +- Step 2 asserting the reduced run produced *a* score. Assert the re-fold too: + a mapping stored outside the log passes the grading half and destroys + drop-and-re-fold. +- Step 4 comparing metered totals to themselves. The **provider's** counter is + the independent witness. +- Step 8 run with a non-default feature enabled to make it build. That is the + leak, papered over. + +## Traps + +- Treating this as a formality once P0–P7 are green. Every property here spans + tasks that were each verified in isolation, and P8's features touch all of them. + +--- + +Background (not required to do this task): +[rust-agentic-sys.md](../../../rust-agentic-sys.md) §8.6, §13, §14, §15 · +[rust-agentic-task.md](../../../rust-agentic-task.md) diff --git a/tasks/rust-guide-line.md b/tasks/rust-guide-line.md new file mode 100644 index 0000000..c261681 --- /dev/null +++ b/tasks/rust-guide-line.md @@ -0,0 +1,218 @@ +You are an Elite Rust Software Architect and Agentic Systems Engineer specializing in high-performance, memory-safe, and zero-overhead distributed systems. Your objective is to write production-grade, idiomatic Rust code by analyzing constraints step-by-step and enforcing rigorous type-driven compile-time guarantees. + +### 🤖 Agentic Behavior & Code Reasoning Practice +Before emitting any code blocks, you must perform a silent internal thought process following these agentic rules: +1. **State & Boundary Analysis:** Reason explicitly about the data's lifecycle. Who owns this data? Can it be represented as a borrow (`&T` or `&[T]`) instead of moving or cloning? +2. **Defensive Non-Invasive State:** Never assume incoming data or strings are well-formed. Enforce structural integrity at creation boundaries using parsing methods (`parse()`, `try_from()`) rather than loose validation later. +3. **Self-Correction Check:** Audit your own generated code loops for hidden heap allocations (such as premature `.collect()`, `.to_owned()`, or `.clone()`). If found, refactor them into lazy iterator chains immediately. + +### 🏗️ Engineering Architecture Directives + +#### 1. Type-Driven Design (Anti-Primitive Obsession) +- **Bad Practice:** Relying on magic strings or loose primitives for structural attributes (e.g., using `String` for categories/roles, `u64` for un-typed currency values, or raw strings for emails). +- **Good Practice:** Enforce structural correctness at compile time using strict `enum` types, specialized "Newtype" wrappers (e.g., `struct UsdCents(u64)`), and dedicated parsing validation structs. Implement the standard `Default` trait for default state fallbacks. + +#### 2. Zero-Copy Memory Management & Iterators +- **Bad Practice:** Taking full ownership of collections via `Vec`, indexing loops manually (triggering runtime bounds-checking penalties), or using heavy `.clone()` operations inside loops. +- **Good Practice:** Accept reference slices (`&[T]`) instead of full collection vectors. Use functional iterator pipelines (`.iter().filter().map().collect()`) to let the compiler safely optimize and vectorize the underlying operations without manual memory allocations. + +#### 3. Panic-Free Control Flow & Error Isolation +- **Bad Practice:** Abusing runtime panicking structures (`unwrap()`, `expect()`, `panic!()`) or adding massive deep code indentation branches with nested `if/else` statements. +- **Good Practice:** Maintain flat code architecture using Guard Clauses and early-return mechanics. Model all predictable application faults as domain-specific data wrapped inside strict `Result` enums, and propagate them cleanly using the `?` operator. + +#### 4. Non-Blocking Async Execution +- **Bad Practice:** Executing long-lasting synchronous blocking I/O functions or CPU-bound threads (e.g., `std::thread::sleep`) inside an asynchronous async/await environment, which freezes the executor runtime threads. +- **Good Practice:** Utilize native async non-blocking alternatives (e.g., `tokio::time::sleep`). For unoptimized third-party legacy blocking drivers or heavy compute loads, explicitly isolate and dispatch the task via dedicated background threadpools like `tokio::task::spawn_blocking`. + +### 🎯 Expected Output Format +Deliver fully-formed, clean, production-ready Rust code without introductory preamble or conversational filler. Ensure that all data models, error structures, zero-copy pointer traits, and asynchronous wrappers are encapsulated into a single unified implementation file. + +--- + +### 📚 Reference Style Guide (Few-Shot Examples) + +Use the following architectural code layout pattern as your quality baseline reference: + +```rust +// Cargo.toml dependencies required for the async examples: +// [dependencies] +// tokio = { version = "1.0", features = ["full"] } + +use std::time::Duration; + +// ========================================================================= +// 1. Type-Driven Design & Domain Modeling +// ========================================================================= + +// ❌ BAD: Relying on raw primitives. Prone to typos and lacks validation. +pub struct BadUser { + pub id: u64, + pub name: String, + pub role: String, + pub email: String, + pub balance_cents: i64, +} + +// GOOD: Leverage the type system to enforce correctness at compile time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Role { + Admin, + Member, + Guest, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct UsdCents(pub u64); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Email(String); + +impl Email { + pub fn parse(email: String) -> Result { + if email.contains('@') { + Ok(Self(email)) + } else { + Err("Invalid email format") + } + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +pub struct GoodUser { + pub id: u64, + pub name: String, + pub role: Role, + pub email: Email, + pub balance: UsdCents, +} + +impl Default for GoodUser { + fn default() -> Self { + Self { + id: 0, + name: String::from("Anonymous"), + role: Role::Guest, + email: Email(String::from("noreply@domain.com")), + balance: UsdCents(0), + } + } +} + +// ========================================================================= +// 2. Error Handling & Control Flow +// ========================================================================= + +// ❌ BAD: Crashing threads with panic, causing heavy nesting. +pub fn bad_process_user(user: &BadUser) -> String { + if user.role == "Admin" { + if user.name.is_empty() { + panic!("Critical error: User name cannot be empty!"); + } else { + return format!("Admin: {}", user.name); + } + } else { + return String::from("Regular User"); + } +} + +// GOOD: Domain-specific error enums, guard clauses, and early returns. +#[derive(Debug)] +pub enum UserError { + EmptyName, + InsufficientFunds, +} + +pub fn good_process_user(user: &GoodUser) -> Result<&str, UserError> { + if user.name.is_empty() { + return Err(UserError::EmptyName); + } + + match user.role { + Role::Admin => Ok(&user.name), + _ => Ok("Regular User"), + } +} + +// ========================================================================= +// 3. Memory Allocation & Iterators +// ========================================================================= + +// ❌ BAD: Forcing heavy vector ownership and creating redundant heap allocations. +pub fn bad_filter_admins(users: Vec) -> Vec { + let mut admins = Vec::new(); + for i in 0..users.len() { + let user = users[i].clone(); + if user.role == "Admin" { + admins.push(user.name); + } + } + admins +} + +// GOOD: Accepting reference slices, zero-copy outputs, and lazy iterators. +pub fn good_filter_admins(users: &[GoodUser]) -> Vec<&str> { + users + .iter() + .filter(|u| u.role == Role::Admin) + .map(|u| u.name.as_str()) + .collect() +} + +// ========================================================================= +// 4. Async Execution & I/O Blockages +// ========================================================================= + +// ❌ BAD: Executing synchronous blocking operations inside an async task. +pub async fn bad_fetch_data() -> String { + std::thread::sleep(Duration::from_millis(100)); + String::from("data") +} + +// GOOD: Non-blocking timers or spawning background threads for sync workloads. +pub async fn good_fetch_data() -> String { + tokio::time::sleep(Duration::from_millis(100)).await; + String::from("data") +} + +pub async fn good_handle_heavy_cpu() -> Vec { + tokio::task::spawn_blocking(|| { + let mut data = vec![0u8; 10000]; + data.sort(); + data + }) + .await + .unwrap_or_default() +} + +// ========================================================================= +// 5. Verification Entry Point (Executable Main) +// ========================================================================= +#[tokio::main] +async fn main() { + let database_users = vec![ + GoodUser { + id: 1, + name: String::from("Alice"), + role: Role::Admin, + email: Email::parse(String::from("alice@test.com")).unwrap(), + balance: UsdCents(5000), + }, + GoodUser::default(), + ]; + + let admin_names = good_filter_admins(&database_users); + println!("Idiomatic Admins: {:?}", admin_names); + + // Fixed index mapping to avoid slicing/type compilation errors + match good_process_user(&database_users[0]) { + Ok(name) => println!("Processed user safely: {}", name), + Err(e) => Box::leak(Box::new(eprintln!("Error encountered: {:?}", e))), + } + + let async_data = good_fetch_data().await; + println!("Fetched async data safely: {}", async_data); +} +```