feat: complete M0.1-M0.4 phases

M0.1 - Cargo workspace + crate skeletons - 6-crate workspace with correct dependency direction - CI/CD pipeline with GitHub Actions - Integration tests verifying build and dependency structure M0.2 - Domain types and sha256 identity - Level (L0, L1, L2) enum with proper serde formatting - Role enum (User, Assistant, ToolResult, System) - Record, Chunk, and MemoryNode domain types - Content-hash identity system ensuring rebuild idempotence - Newtypes (ProjectId, QueryId, RunId) with validation - Round-trip serde tests for all types M0.3 - RecordSource trait + ChunkPolicy - RecordSource trait for streaming record sources - Chunk policy with token budgets and boundary modes - TokenCounter trait with CharsOverFourCounter stub - Chunking stream that respects budgets without splitting records - VecSource for testing - Integration tests verifying lossless chunking and budget adherence M0.4 - Tokenizer-backed chunk sizing - Vendored Qwen2 tokenizer with hash verification - QwenTokenCounter implementing proper token counting - Hash guard that fails on modified tokenizer - mem tokens CLI subcommand for token counting - Integration tests with known string counts, hash guards, and budget verification Total: 19 integration tests passing, all phases verified to compose correctly Workspace builds cleanly with no clippy warnings
2026-08-22 23:13:42 -07:00
parent 144fa33574
commit 631cbfa3e9
36 changed files with 3379 additions and 5 deletions
@@ -0,0 +1,50 @@
+use mem_core::Record;
+use futures::stream::Stream;
+
+/// A source of records, shaped as a stream from day one.
+/// Sources decide how to produce records; the chunker never learns
+/// whether they came from pi, claude, or a socket.
+pub trait RecordSource {
+    fn records(self) -> Box<dyn Stream<Item = Result<Record, String>> + Unpin>;
+}
+
+/// A test vector source that produces records from a Vec.
+pub struct VecSource(pub Vec<Record>);
+
+impl RecordSource for VecSource {
+    fn records(self) -> Box<dyn Stream<Item = Result<Record, String>> + Unpin> {
+        Box::new(futures::stream::iter(self.0.into_iter().map(Ok)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use mem_core::{Provenance, Role};
+    use time::macros::datetime;
+    use futures::StreamExt;
+
+    #[tokio::test]
+    async fn test_vec_source() {
+        let records = vec![
+            Record {
+                role: Role::User,
+                text: "Hello".to_string(),
+                timestamp: datetime!(2024-08-20 12:00:00 UTC),
+                provenance: Provenance {
+                    source_id: "session1".to_string(),
+                    offset: 0,
+                },
+            },
+        ];
+
+        let source = VecSource(records.clone());
+        let mut stream = source.records();
+        
+        let result = stream.next().await;
+        assert!(result.is_some());
+        let record = result.unwrap().unwrap();
+        assert_eq!(record.role, Role::User);
+        assert_eq!(record.text, "Hello");
+    }
+}