Story Crater Bot
631cbfa3e9
M0.1 - Cargo workspace + crate skeletons - 6-crate workspace with correct dependency direction - CI/CD pipeline with GitHub Actions - Integration tests verifying build and dependency structure M0.2 - Domain types and sha256 identity - Level (L0, L1, L2) enum with proper serde formatting - Role enum (User, Assistant, ToolResult, System) - Record, Chunk, and MemoryNode domain types - Content-hash identity system ensuring rebuild idempotence - Newtypes (ProjectId, QueryId, RunId) with validation - Round-trip serde tests for all types M0.3 - RecordSource trait + ChunkPolicy - RecordSource trait for streaming record sources - Chunk policy with token budgets and boundary modes - TokenCounter trait with CharsOverFourCounter stub - Chunking stream that respects budgets without splitting records - VecSource for testing - Integration tests verifying lossless chunking and budget adherence M0.4 - Tokenizer-backed chunk sizing - Vendored Qwen2 tokenizer with hash verification - QwenTokenCounter implementing proper token counting - Hash guard that fails on modified tokenizer - mem tokens CLI subcommand for token counting - Integration tests with known string counts, hash guards, and budget verification Total: 19 integration tests passing, all phases verified to compose correctly Workspace builds cleanly with no clippy warnings
51 lines
1.5 KiB
Rust
51 lines
1.5 KiB
Rust
use mem_core::Record;
|
|
use futures::stream::Stream;
|
|
|
|
/// A source of records, shaped as a stream from day one.
|
|
/// Sources decide how to produce records; the chunker never learns
|
|
/// whether they came from pi, claude, or a socket.
|
|
pub trait RecordSource {
|
|
fn records(self) -> Box<dyn Stream<Item = Result<Record, String>> + Unpin>;
|
|
}
|
|
|
|
/// A test vector source that produces records from a Vec.
|
|
pub struct VecSource(pub Vec<Record>);
|
|
|
|
impl RecordSource for VecSource {
|
|
fn records(self) -> Box<dyn Stream<Item = Result<Record, String>> + Unpin> {
|
|
Box::new(futures::stream::iter(self.0.into_iter().map(Ok)))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use mem_core::{Provenance, Role};
|
|
use time::macros::datetime;
|
|
use futures::StreamExt;
|
|
|
|
#[tokio::test]
|
|
async fn test_vec_source() {
|
|
let records = vec![
|
|
Record {
|
|
role: Role::User,
|
|
text: "Hello".to_string(),
|
|
timestamp: datetime!(2024-08-20 12:00:00 UTC),
|
|
provenance: Provenance {
|
|
source_id: "session1".to_string(),
|
|
offset: 0,
|
|
},
|
|
},
|
|
];
|
|
|
|
let source = VecSource(records.clone());
|
|
let mut stream = source.records();
|
|
|
|
let result = stream.next().await;
|
|
assert!(result.is_some());
|
|
let record = result.unwrap().unwrap();
|
|
assert_eq!(record.role, Role::User);
|
|
assert_eq!(record.text, "Hello");
|
|
}
|
|
}
|