diff --git a/Cargo.lock b/Cargo.lock index 1d3c5c4..d4f2dae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2124,6 +2124,7 @@ dependencies = [ "anyhow", "futures", "mem-core", + "mem-ingest", "pgvector", "serde", "serde_json", diff --git a/crates/mem-store/Cargo.toml b/crates/mem-store/Cargo.toml index 507c137..cd550d4 100644 --- a/crates/mem-store/Cargo.toml +++ b/crates/mem-store/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] mem-core = { path = "../mem-core" } +mem-ingest = { path = "../mem-ingest" } tokio = { workspace = true } futures = { workspace = true } serde = { workspace = true } diff --git a/crates/mem-store/src/rebuild.rs b/crates/mem-store/src/rebuild.rs index 1855603..8f99c48 100644 --- a/crates/mem-store/src/rebuild.rs +++ b/crates/mem-store/src/rebuild.rs @@ -4,8 +4,11 @@ use std::fs; use std::path::{Path, PathBuf}; use sha2::{Digest, Sha256}; use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; use crate::{MemoryNode, Level, PgRepo}; +use mem_core::optimizer::ContextOptimizer; +use mem_ingest::OptimizationMetrics; /// Memory record from log (local copy for rebuild purposes) #[derive(Debug, Clone, Serialize, Deserialize)] @@ -88,6 +91,10 @@ impl RebuildEngine { let mut sha_to_level: HashMap = HashMap::new(); let mut sha_to_parents: HashMap> = HashMap::new(); + // Initialize optimizer and metrics for ingest-time context optimization (M3.8.2) + let optimizer = ContextOptimizer::from_env().ok(); + let metrics = Arc::new(Mutex::new(OptimizationMetrics::default())); + for memory in &memories { let sha = Self::memory_sha(&memory.text); let level = match memory.level.as_str() { @@ -98,6 +105,35 @@ impl RebuildEngine { _ => continue, }; + // M3.8.2: Optimize text at ingest time + let optimized_text = if let Some(ref opt) = optimizer { + match opt.optimize(&memory.text) { + Ok(optimized) => { + // Track metrics + let input_bytes = memory.text.len(); + let output_bytes = optimized.compressed.len(); + { + let mut m = metrics.lock().unwrap(); + m.total_records += 1; + m.input_bytes_total += input_bytes; + m.output_bytes_total += output_bytes; + } + optimized.compressed + } + Err(e) => { + // Graceful fallback: use original on optimization failure + tracing::warn!( + error = ?e, + project = &opts.project, + "M3.8.2 optimization failed, using original text" + ); + memory.text.clone() + } + } + } else { + memory.text.clone() + }; + let node = MemoryNode { sha256: sha.clone(), level, @@ -106,7 +142,7 @@ impl RebuildEngine { run_id: memory.run_id.clone(), t: memory.t, source: memory.source.clone(), - text: memory.text.clone(), + text: optimized_text, }; nodes_by_sha.insert(sha.clone(), node); @@ -142,6 +178,12 @@ impl RebuildEngine { // For now, mock stats stats.embeddings_cached = 0; stats.embeddings_computed = 0; + + // Log M3.8.2 optimization metrics + let m = metrics.lock().unwrap(); + if m.total_records > 0 { + m.log_summary(&opts.project); + } } Ok(stats) @@ -240,4 +282,52 @@ mod tests { let parent_sha = RebuildEngine::parent_sha("pi", 42); assert_eq!(parent_sha, "pi-42"); } + + #[test] + fn test_memory_sha_stable_with_optimization() { + // SHA should be computed on original text, not optimized + // This ensures idempotence even when optimization changes + let original = "ERROR: permission denied\nINFO: retrying"; + let sha_before = RebuildEngine::memory_sha(original); + let sha_after = RebuildEngine::memory_sha(original); + + assert_eq!( + sha_before, sha_after, + "SHA must be deterministic for idempotence" + ); + } + + #[test] + fn test_optimization_metrics_initialization() { + // Test that OptimizationMetrics can be created and used + let metrics = OptimizationMetrics::default(); + assert_eq!(metrics.total_records, 0); + assert_eq!(metrics.input_bytes_total, 0); + assert_eq!(metrics.output_bytes_total, 0); + assert_eq!(metrics.compression_ratio(), 0.0); + } + + #[test] + fn test_optimization_metrics_aggregation() { + // Test that metrics can track multiple records + let mut metrics = OptimizationMetrics::default(); + + // Simulate first record + metrics.total_records += 1; + metrics.input_bytes_total += 1000; + metrics.output_bytes_total += 500; + + // Simulate second record + metrics.total_records += 1; + metrics.input_bytes_total += 2000; + metrics.output_bytes_total += 1000; + + assert_eq!(metrics.total_records, 2); + assert_eq!(metrics.input_bytes_total, 3000); + assert_eq!(metrics.output_bytes_total, 1500); + + // Check compression ratio: 1500/3000 = 0.5 = 50% + let ratio = metrics.compression_ratio(); + assert!((ratio - 50.0).abs() < 0.1, "should be 50%, got {}", ratio); + } }