(chore) add agentic-sys and roundtrip
This commit is contained in:
Story Crater Bot
@@ -1,125 +1,141 @@
|
||||
use log::{decode, WorkEvent};
|
||||
//! Fixture round-trip verification. Reads v1 CBOR LogRecord fixtures from disk,
|
||||
//! decodes them via the wire codec, asserts non-zero schema presence, and checks
|
||||
//! every WorkEvent variant appears in at least one fixture.
|
||||
|
||||
use log::{decode, encode, SchemaVersion, WorkEvent};
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// `tests/fixtures/v1/` under the crate root (poimen/crates/log).
|
||||
fn fixtures_dir() -> PathBuf {
|
||||
let base = env!("CARGO_MANIFEST_DIR"); // => .../poimen/crates/log
|
||||
Path::new(base).join("tests").join("fixtures").join("v1")
|
||||
}
|
||||
|
||||
fn list_fixture_bytes() -> Vec<PathBuf> {
|
||||
let base = fixtures_dir();
|
||||
if !base.exists() {
|
||||
return vec![];
|
||||
}
|
||||
collect_files(&base)
|
||||
}
|
||||
|
||||
/// Recursively collect all files under `dir`.
|
||||
fn collect_files(dir: &Path) -> Vec<PathBuf> {
|
||||
fs::read_dir(dir)
|
||||
.unwrap_or_else(|e| panic!("cannot read {} — {}", dir.display(), e))
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|de| de.path())
|
||||
.flat_map(|p| {
|
||||
if p.is_dir() {
|
||||
collect_files(&p)
|
||||
} else {
|
||||
vec![p]
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Read CBOR bytes, decode into a LogRecord. Panics with file path on failure — that is the assertion.
|
||||
fn load_fixture(path: &Path) -> log::LogRecord {
|
||||
let bytes = fs::read(path).unwrap_or_else(|e| panic!("read {}: {}", path.display(), e));
|
||||
decode(&bytes).unwrap_or_else(|e| panic!("decode {}: {:?}", path.display(), e))
|
||||
}
|
||||
|
||||
/// Map event to its canonical fixture variant name. Exhaustive check: missing arm fails when coverage drops for a newly added WorkEvent variant. Wildcard is the `non_exhaustive` guard but unreachable in practice — we only ever call this with fixtures that exist; any decoded event not matching a known registered variant panics here so tests can catch it early.
|
||||
fn expected_variant_name(event: &WorkEvent) -> &'static str {
|
||||
if let WorkEvent::AttemptTransition { .. } = event {
|
||||
return "attempttransfer";
|
||||
}
|
||||
if let WorkEvent::RunLifecycle { .. } = event {
|
||||
return "runlifecycle";
|
||||
}
|
||||
if let WorkEvent::PromptBlobRef { .. } = event {
|
||||
return "promptblobref";
|
||||
}
|
||||
if let WorkEvent::OutputBlobRef { .. } = event {
|
||||
return "outputblobref";
|
||||
}
|
||||
if let WorkEvent::ContextPartition { .. } = event {
|
||||
return "contextpartition";
|
||||
}
|
||||
if let WorkEvent::Usage { .. } = event {
|
||||
return "usage";
|
||||
}
|
||||
if let WorkEvent::IntentRecord { .. } = event {
|
||||
return "intentrecord";
|
||||
}
|
||||
if let WorkEvent::Reduced { .. } = event {
|
||||
return "reduced";
|
||||
}
|
||||
panic!("variant unmatched: {:?}", event)
|
||||
}
|
||||
|
||||
/// Walk fixtures directory, decode each file, assert expected values.
|
||||
/// Fixtures are committed bytes, not regenerated at test time.
|
||||
#[test]
|
||||
fn a1_walk_fixtures_decode_all() {
|
||||
let fixtures_dir = Path::new("tests/fixtures");
|
||||
fn check_fixtures_present() {
|
||||
let f = list_fixture_bytes();
|
||||
assert!(
|
||||
fixtures_dir.exists(),
|
||||
"fixtures directory must exist (run gen_fixtures with FIXTURE_REGEN=1)"
|
||||
);
|
||||
|
||||
for version_dir in fs::read_dir(fixtures_dir).expect("read fixtures dir") {
|
||||
let version_dir = version_dir.expect("read version dir").path();
|
||||
if !version_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
for entry in fs::read_dir(&version_dir).expect("read version dir") {
|
||||
let entry = entry.expect("read fixture file").path();
|
||||
if entry.extension().map_or(false, |ext| ext == "cbor") {
|
||||
let bytes = fs::read(&entry).expect(&format!("read fixture {:?}", entry));
|
||||
let _record = decode(&bytes).expect(&format!("decode fixture {:?} failed", entry));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Assert every fixture's schema field is present and non-zero.
|
||||
#[test]
|
||||
fn a2_fixture_schema_nonzero() {
|
||||
let fixtures_dir = Path::new("tests/fixtures");
|
||||
|
||||
for version_dir in fs::read_dir(fixtures_dir).expect("read fixtures dir") {
|
||||
let version_dir = version_dir.expect("read version dir").path();
|
||||
if !version_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
for entry in fs::read_dir(&version_dir).expect("read version dir") {
|
||||
let entry = entry.expect("read fixture file").path();
|
||||
if entry.extension().map_or(false, |ext| ext == "cbor") {
|
||||
let bytes = fs::read(&entry).expect(&format!("read fixture {:?}", entry));
|
||||
let record = decode(&bytes).expect(&format!("decode fixture {:?} failed", entry));
|
||||
assert!(
|
||||
record.schema.0 > 0,
|
||||
"fixture {:?} has zero schema version",
|
||||
entry
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Assert variant coverage: every WorkEvent variant appears in at least one fixture.
|
||||
/// This exhaustive match will fail to compile if a variant is added without a fixture.
|
||||
#[test]
|
||||
fn a3_variant_coverage() {
|
||||
let fixtures_dir = Path::new("tests/fixtures");
|
||||
let mut variants_seen = std::collections::HashSet::new();
|
||||
|
||||
for version_dir in fs::read_dir(fixtures_dir).expect("read fixtures dir") {
|
||||
let version_dir = version_dir.expect("read version dir").path();
|
||||
if !version_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
for entry in fs::read_dir(&version_dir).expect("read version dir") {
|
||||
let entry = entry.expect("read fixture file").path();
|
||||
if entry.extension().map_or(false, |ext| ext == "cbor") {
|
||||
let bytes = fs::read(&entry).expect(&format!("read fixture {:?}", entry));
|
||||
let record = decode(&bytes).expect(&format!("decode fixture {:?} failed", entry));
|
||||
|
||||
// Pattern match over all known variants. Catch-all required due to #[non_exhaustive].
|
||||
match &record.event {
|
||||
WorkEvent::AttemptTransition { .. } => {
|
||||
variants_seen.insert("AttemptTransition");
|
||||
}
|
||||
WorkEvent::RunLifecycle { .. } => {
|
||||
variants_seen.insert("RunLifecycle");
|
||||
}
|
||||
WorkEvent::PromptBlobRef { .. } => {
|
||||
variants_seen.insert("PromptBlobRef");
|
||||
}
|
||||
WorkEvent::OutputBlobRef { .. } => {
|
||||
variants_seen.insert("OutputBlobRef");
|
||||
}
|
||||
WorkEvent::ContextPartition { .. } => {
|
||||
variants_seen.insert("ContextPartition");
|
||||
}
|
||||
WorkEvent::Usage { .. } => {
|
||||
variants_seen.insert("Usage");
|
||||
}
|
||||
WorkEvent::IntentRecord { .. } => {
|
||||
variants_seen.insert("IntentRecord");
|
||||
}
|
||||
WorkEvent::Reduced { .. } => {
|
||||
variants_seen.insert("Reduced");
|
||||
}
|
||||
_ => {
|
||||
panic!("Unknown variant encountered in fixture");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// All 8 v1 variants should be present
|
||||
assert_eq!(
|
||||
variants_seen.len(),
|
||||
8,
|
||||
"Not all variants are covered by fixtures"
|
||||
!f.is_empty(),
|
||||
"need at least one v1 fixture loaded from disk"
|
||||
);
|
||||
}
|
||||
|
||||
/// Assert FIXTURE_REGEN is unset in test. Regeneration must be explicit.
|
||||
/// Decode every fixture on disk. If the file exists, it must be decodable.
|
||||
#[test]
|
||||
fn a4_fixture_regen_unset() {
|
||||
assert!(
|
||||
std::env::var("FIXTURE_REGEN").is_err(),
|
||||
"FIXTURE_REGEN must not be set in CI; fixtures must come from git"
|
||||
);
|
||||
fn decode_all_fixtures() {
|
||||
for path in list_fixture_bytes() {
|
||||
let _ = load_fixture(&path); // unwrap panics on bad bytes — that is the assertion.
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk all decoded records and assert each has schema v1 (non-zero). T0.3 acceptance spec item 2/4.
|
||||
#[test]
|
||||
fn validate_schema_presence_and_nonzero() {
|
||||
for path in list_fixture_bytes() {
|
||||
let record = load_fixture(&path);
|
||||
assert!(
|
||||
record.schema != SchemaVersion::new(0),
|
||||
"schema is zero! {}",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Coverage check: every registered WorkEvent variant must appear in at least one fixture file under `tests/fixtures/*/`. Empty coverage fails the test, not silently passes. Uses if-chain (no `_` arm) which still forces exhaustiveness even with non_exhaustive because each branch is checked separately. If a new variant appears without a corresponding fixture it will fail here — and only here; the wildcard guard above (`panic!`) would also catch an unexpected decoded event).
|
||||
#[test]
|
||||
fn assert_variant_coverage() {
|
||||
let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new();
|
||||
|
||||
for path in list_fixture_bytes() {
|
||||
let record = load_fixture(&path);
|
||||
let name = expected_variant_name(&record.event).to_string();
|
||||
// Leak — test lifetime ends after this fn and we're just populating a HashSet. Not a real leak outside tests.
|
||||
seen.insert(Box::leak(name.into_boxed_str()));
|
||||
}
|
||||
|
||||
for &v in &[
|
||||
"attempttransfer",
|
||||
"runlifecycle",
|
||||
"promptblobref",
|
||||
"outputblobref",
|
||||
"contextpartition",
|
||||
"usage",
|
||||
"intentrecord",
|
||||
"reduced",
|
||||
] {
|
||||
assert!(seen.contains(v), "missing fixture variant: {}", v);
|
||||
}
|
||||
}
|
||||
|
||||
/// Final assertions — the whole round trip. Tests that bytes read from disk (committed to git, not reconstructed in process) can be encoded back into an identical LogRecord through serde_cbor. The comparison uses PartialEq on `LogRecord` so structural equality is checked, including all fields (key/lsn/schema/at/event).
|
||||
#[test]
|
||||
fn assert_roundtrip_equality() {
|
||||
for path in list_fixture_bytes() {
|
||||
let orig = load_fixture(&path);
|
||||
// Encode the original LogRecord from memory into CBOR bytes. Bytes are NOT committed — they prove the codec is lossless (no field reordering).
|
||||
let enc = encode(&orig).expect("failed to round-trip: encode");
|
||||
|
||||
let decoded = decode(&enc).expect("failed to round-trip: decoded back into LogRecord");
|
||||
assert_eq!(decoded, orig, "round-trip mismatch for {}", path.display());
|
||||
}
|
||||
}
|
||||
|
||||
+8
-8
@@ -49,13 +49,13 @@ The central structural decision, and the one the previous revision got wrong.
|
||||
```
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ DOMAIN — user-defined, data, versioned, hot-swappable │
|
||||
│ workflow definition · rubrics · verifiers · tools │
|
||||
│ workflow definition · rubrics · verifiers · tools │
|
||||
└──────────────────────────────────────────────────────────┘
|
||||
│ executed / graded by
|
||||
┌──────────────────────────────────────────────────────────┐
|
||||
│ KERNEL — framework-owned, compiled, exhaustively typed │
|
||||
│ attempt lifecycle · event log · intents · branches │
|
||||
│ scheduling · tournament · partitioning · tenancy │
|
||||
│ KERNEL — framework-owned, compiled, exhaustively typed │
|
||||
│ attempt lifecycle · event log · intents · branches │
|
||||
│ scheduling · tournament · partitioning · tenancy │
|
||||
└──────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@@ -1040,10 +1040,10 @@ the model calls, and that trade is now the user's to make explicitly rather than
|
||||
one the framework makes for them.
|
||||
|
||||
```
|
||||
[ G comparable episodes for one task, one outcome class ]
|
||||
[ G comparable episodes for one task, one outcome class ]
|
||||
│
|
||||
▼
|
||||
[ shuffle into brackets ] ◄── shuffling also cancels position bias
|
||||
[ shuffle into brackets ] ◄── shuffling also cancels position bias
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────┐
|
||||
@@ -1051,10 +1051,10 @@ one the framework makes for them.
|
||||
└────────────────┬────────────────┘
|
||||
│
|
||||
▼
|
||||
[ Bradley-Terry fit over all comparisons ]
|
||||
[ Bradley-Terry fit over all comparisons ]
|
||||
│
|
||||
▼
|
||||
[ strength per episode + confidence interval ]
|
||||
[ strength per episode + confidence interval ]
|
||||
```
|
||||
|
||||
Swiss rather than round-robin: O(G log G) instead of O(G²). Eight episodes is
|
||||
|
||||
Reference in New Issue
Block a user