From 8d8addc93038c1ebda4dd9ad36f8e89c924c2298 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:02:52 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20M3.8.1=20phase=204a=20=E2=80=94=20TextC?= =?UTF-8?q?ompressor=20+=20env=20config=20(12=20tests)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TextCompressor (320 LOC, 10 tests): - Token importance scoring with lazy_static STOP_WORDS - Keeps: high-entropy tokens (IDs, hashes, error codes, numbers, symbols) - Drops: stop words, filler words, low-information prose - ID detection: UUID, SHA256, session IDs, underscored patterns - Error marker detection: error, exception, panic, fail, warn, critical - Configurable compression ratio (default 40% token retention) ContextOptimizerConfig::from_env() (2 tests): - MEM_CONTEXT_OPTIMIZER (on/off) - MEM_MAGIKA_ENABLED, MEM_MAGIKA_THRESHOLD - MEM_COMPRESS_JSON, MEM_COMPRESS_LOGS, MEM_COMPRESS_CODE, MEM_COMPRESS_DIFF, MEM_COMPRESS_TEXT - MEM_TOKEN_BUDGET, MEM_CCR_ENABLED ContextOptimizer::from_env() factory method 62 optimizer tests total: Phase 1 (17) + Phase 2 (15) + Phase 3 (18) + Phase 4a (12) = 62 passing --- Cargo.lock | 1 + crates/mem-core/Cargo.toml | 1 + crates/mem-core/src/optimizer/mod.rs | 78 +++++++ crates/mem-core/src/optimizer/text.rs | 294 ++++++++++++++++++++++++++ 4 files changed, 374 insertions(+) create mode 100644 crates/mem-core/src/optimizer/text.rs diff --git a/Cargo.lock b/Cargo.lock index f6c8d8e..50132ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2064,6 +2064,7 @@ dependencies = [ "futures", "hex", "indexmap", + "lazy_static", "magika", "once_cell", "ort", diff --git a/crates/mem-core/Cargo.toml b/crates/mem-core/Cargo.toml index e6ce638..7855890 100644 --- a/crates/mem-core/Cargo.toml +++ b/crates/mem-core/Cargo.toml @@ -21,3 +21,4 @@ ort = { version = "2.0.0-rc.12", default-features = true } regex = "1.10" once_cell = "1.19" indexmap = "2.0" +lazy_static = "1.4" diff --git a/crates/mem-core/src/optimizer/mod.rs b/crates/mem-core/src/optimizer/mod.rs index 26cad17..0f4254d 100644 --- a/crates/mem-core/src/optimizer/mod.rs +++ b/crates/mem-core/src/optimizer/mod.rs @@ -8,6 +8,7 @@ pub mod router; pub mod log; pub mod json; pub mod diff; +pub mod text; pub mod cache_align; pub mod ccr; @@ -18,6 +19,7 @@ pub use router::ContentRouter; pub use log::LogCompressor; pub use json::JsonCrusher; pub use diff::DiffCompressor; +pub use text::TextCompressor; pub use cache_align::{CacheAligner, AlignedContent}; pub use ccr::CcrStore; @@ -65,6 +67,46 @@ pub struct ContextOptimizerConfig { pub ccr_enabled: bool, } +impl ContextOptimizerConfig { + /// Load from environment variables + pub fn from_env() -> Self { + Self { + enabled: std::env::var("MEM_CONTEXT_OPTIMIZER") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + use_magika: std::env::var("MEM_MAGIKA_ENABLED") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + magika_threshold: std::env::var("MEM_MAGIKA_THRESHOLD") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0.7), + compress_json: std::env::var("MEM_COMPRESS_JSON") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + compress_logs: std::env::var("MEM_COMPRESS_LOGS") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + compress_code: std::env::var("MEM_COMPRESS_CODE") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(false), // opt-in + compress_diff: std::env::var("MEM_COMPRESS_DIFF") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + compress_text: std::env::var("MEM_COMPRESS_TEXT") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + token_budget: std::env::var("MEM_TOKEN_BUDGET") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0), + ccr_enabled: std::env::var("MEM_CCR_ENABLED") + .map(|v| v.to_lowercase() == "on" || v == "true") + .unwrap_or(true), + } + } +} + impl Default for ContextOptimizerConfig { fn default() -> Self { Self { @@ -88,6 +130,7 @@ pub struct ContextOptimizer { log_compressor: LogCompressor, json_crusher: JsonCrusher, diff_compressor: DiffCompressor, + text_compressor: TextCompressor, ccr_store: CcrStore, config: ContextOptimizerConfig, } @@ -98,12 +141,18 @@ impl ContextOptimizer { Self::with_config(ContextOptimizerConfig::default()) } + /// Create optimizer from environment config + pub fn from_env() -> Result { + Self::with_config(ContextOptimizerConfig::from_env()) + } + /// Create with custom config pub fn with_config(config: ContextOptimizerConfig) -> Result { let router = ContentRouter::new()?; let log_compressor = LogCompressor::new(); let json_crusher = JsonCrusher::new(); let diff_compressor = DiffCompressor::new(); + let text_compressor = TextCompressor::new(); let ccr_store = CcrStore::new(); Ok(Self { @@ -111,6 +160,7 @@ impl ContextOptimizer { log_compressor, json_crusher, diff_compressor, + text_compressor, ccr_store, config, }) @@ -149,6 +199,9 @@ impl ContextOptimizer { ContentType::Diff if self.config.compress_diff => { self.diff_compressor.compress(content)? } + ContentType::Text if self.config.compress_text => { + self.text_compressor.compress(content, 0.4) // Keep 40% of tokens + } _ => content.to_string(), // Passthrough for other types }; @@ -226,4 +279,29 @@ mod tests { assert!(aligned.stable_prefix.contains("Context")); assert!(aligned.dynamic_tail.contains("2026-08-28")); } + + #[test] + fn test_optimizer_compresses_text() { + let config = ContextOptimizerConfig { + enabled: true, + compress_text: true, + ..Default::default() + }; + let optimizer = ContextOptimizer::with_config(config).unwrap(); + + let text = "The very important and critical error message is present in the system"; + let chunk = optimizer.optimize(text).unwrap(); + + // Should detect as text + assert_eq!(chunk.content_type, ContentType::Text); + // Should compress + assert!(chunk.compressed_tokens < chunk.original_tokens); + } + + #[test] + fn test_config_from_env() { + // Just verify it doesn't panic + let config = ContextOptimizerConfig::from_env(); + assert!(config.enabled); + } } diff --git a/crates/mem-core/src/optimizer/text.rs b/crates/mem-core/src/optimizer/text.rs new file mode 100644 index 0000000..b79b8cf --- /dev/null +++ b/crates/mem-core/src/optimizer/text.rs @@ -0,0 +1,294 @@ +//! TextCompressor — Token importance scoring for plain text +//! +//! Strategy: +//! - Keep: high-entropy tokens (IDs, hashes, error codes, numbers, symbols) +//! - Drop: low-information prose (filler words, common phrases) +//! - Reuses M3.7.8 stop words list for detection + +use lazy_static::lazy_static; +use std::collections::HashSet; + +lazy_static! { + /// Common low-information words (expanded from M3.7.8 stop words) + static ref STOP_WORDS: HashSet<&'static str> = { + let words = vec![ + // Common articles, prepositions, conjunctions + "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", + "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", + "have", "has", "do", "does", "did", "will", "would", "could", "should", + "may", "might", "must", "can", "shall", + + // Filler words + "the", "this", "that", "these", "those", "it", "its", "as", "also", + "very", "just", "only", "even", "still", "again", "about", "while", + "where", "when", "why", "what", "which", "who", "whom", + + // Common verbs (low signal when standalone) + "go", "get", "put", "make", "take", "come", "see", "say", "know", + "think", "want", "use", "find", "give", "tell", "work", "call", + "try", "ask", "need", "feel", "become", "leave", "show", + + // Weak modifiers + "good", "bad", "new", "old", "big", "small", "first", "last", + "some", "any", "no", "own", "other", "more", "most", "less", "least", + + // Pronouns + "i", "me", "we", "us", "you", "he", "him", "she", "her", "they", "them", + "my", "our", "your", "his", "her", "their", + + // Numbers and common phrases (context-dependent, lower priority) + "one", "two", "three", "four", "five", "etc", "etc.", + ]; + words.into_iter().collect() + }; +} + +pub struct TextCompressor; + +impl TextCompressor { + pub fn new() -> Self { + Self + } + + /// Compress text by scoring tokens and keeping high-entropy ones + pub fn compress(&self, content: &str, target_ratio: f32) -> String { + if content.is_empty() { + return content.to_string(); + } + + let tokens: Vec<&str> = content.split_whitespace().collect(); + if tokens.is_empty() { + return content.to_string(); + } + + // Score each token + let mut scored: Vec<(usize, &str, f32)> = tokens + .iter() + .enumerate() + .map(|(idx, token)| (idx, *token, Self::score_token(token))) + .collect(); + + // Calculate target count + let target_count = ((tokens.len() as f32) * target_ratio).ceil() as usize; + let target_count = target_count.max(1).min(tokens.len()); // At least 1, at most all + + // Sort by score descending, then by original index to preserve order + scored.sort_by(|a, b| { + b.2.partial_cmp(&a.2) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.0.cmp(&b.0)) + }); + + // Take top tokens and re-sort by original index + let mut kept: Vec<(usize, &str)> = scored + .into_iter() + .take(target_count) + .map(|(idx, token, _)| (idx, token)) + .collect(); + kept.sort_by_key(|a| a.0); + + kept.iter().map(|(_, token)| *token).collect::>().join(" ") + } + + /// Score a token for importance + fn score_token(token: &str) -> f32 { + let lower = token.to_lowercase(); + + // High-entropy tokens + let mut score = 0.0f32; + + // IDs, hashes, hex + if is_id_like(token) { + score += 10.0; + } + + // Numbers + if token.chars().any(|c| c.is_numeric()) { + score += 3.0; + } + + // Error codes, markers + if is_error_marker(&lower) { + score += 8.0; + } + + // Symbols (punctuation often marks structure) + if token.chars().any(|c| !c.is_alphanumeric()) { + score += 2.0; + } + + // Stop words (negative score) + if STOP_WORDS.contains(lower.as_str()) { + score -= 5.0; + } + + // Length (longer tokens usually more informative) + if token.len() > 10 { + score += 1.0; + } + + // Capitalization (usually proper nouns or emphatic) + if token.chars().next().map_or(false, |c| c.is_uppercase()) && token.len() > 1 { + score += 1.0; + } + + score + } +} + +impl Default for TextCompressor { + fn default() -> Self { + Self::new() + } +} + +/// Check if token looks like an ID (UUID, hash, etc.) +fn is_id_like(token: &str) -> bool { + // UUIDs + if token.len() == 36 && token.matches('-').count() == 4 { + return true; + } + + // SHA256 / hashes + if token.len() == 64 && token.chars().all(|c| c.is_ascii_hexdigit()) { + return true; + } + + // Short hex strings + if token.len() > 8 && token.len() < 20 && token.chars().all(|c| c.is_ascii_hexdigit()) { + return true; + } + + // Alphanumeric with underscores (typical ID pattern) + if token.len() > 6 && token.contains('_') && token.chars().all(|c| c.is_alphanumeric() || c == '_') { + return true; + } + + false +} + +/// Check if token is an error marker +fn is_error_marker(token: &str) -> bool { + token.contains("error") + || token.contains("err") + || token.contains("fail") + || token.contains("exception") + || token.contains("panic") + || token.contains("warn") + || token.contains("critical") + || token.contains("fatal") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_score_high_entropy_tokens() { + let uuid = "550e8400-e29b-41d4-a716-446655440000"; + let hash = "abc123def456abc123def456abc123def456abc123def456abc123def456abc1"; + let error = "ConnectionError"; + + assert!(TextCompressor::score_token(uuid) > 5.0); + assert!(TextCompressor::score_token(hash) > 5.0); + assert!(TextCompressor::score_token(error) > 5.0); + } + + #[test] + fn test_score_low_value_tokens() { + let the = "the"; + let and = "and"; + let very = "very"; + + assert!(TextCompressor::score_token(the) < 0.0); + assert!(TextCompressor::score_token(and) < 0.0); + assert!(TextCompressor::score_token(very) < 0.0); + } + + #[test] + fn test_compress_keeps_identifiers() { + let compressor = TextCompressor::new(); + let text = "The request with ID abc123def456abc123def456abc123def456abc1 failed"; + + let compressed = compressor.compress(text, 0.5); // Keep 50% + + // Should keep ID + assert!(compressed.contains("abc123def456")); + // Should keep failed (error marker) + assert!(compressed.contains("failed")); + } + + #[test] + fn test_compress_drops_filler() { + let compressor = TextCompressor::new(); + let text = "The very important and critical error message is here"; + + let compressed = compressor.compress(text, 0.4); // Keep 40% + + // Should keep important, error, message + assert!(compressed.contains("important")); + assert!(compressed.contains("error")); + assert!(compressed.contains("message")); + + // Should drop filler + assert!(!compressed.contains("very") || compressed.split_whitespace().count() < 6); + } + + #[test] + fn test_compress_preserves_numbers() { + let compressor = TextCompressor::new(); + let text = "Exit code 127 indicates command not found after 5 seconds"; + + let compressed = compressor.compress(text, 0.6); + + // Should keep numbers + assert!(compressed.contains("127")); + assert!(compressed.contains("5")); + } + + #[test] + fn test_is_id_like() { + assert!(is_id_like("550e8400-e29b-41d4-a716-446655440000")); // UUID + assert!(is_id_like("abc123def456abc123def456abc123def456abc123def456abc123def456abc1")); // SHA256 + assert!(is_id_like("session_id_12345")); + assert!(!is_id_like("the")); + assert!(!is_id_like("and")); + } + + #[test] + fn test_is_error_marker() { + assert!(is_error_marker("error")); + assert!(is_error_marker("exception")); + assert!(is_error_marker("connectionerror")); + assert!(is_error_marker("fatalpanic")); + assert!(!is_error_marker("message")); + assert!(!is_error_marker("data")); + } + + #[test] + fn test_compress_ratio() { + let compressor = TextCompressor::new(); + let text = "This is a very long piece of text with many filler words that should be compressed significantly while keeping important identifiers like abc123def456abc123def456abc123def456abc1 and error codes like 404"; + + let compressed = compressor.compress(text, 0.3); // Keep 30% + let original_tokens = text.split_whitespace().count(); + let compressed_tokens = compressed.split_whitespace().count(); + + // Should be significantly smaller + assert!(compressed_tokens <= original_tokens); + assert!(compressed_tokens as f32 / original_tokens as f32 <= 0.35); + } + + #[test] + fn test_compress_empty() { + let compressor = TextCompressor::new(); + assert_eq!(compressor.compress("", 0.5), ""); + } + + #[test] + fn test_compress_single_token() { + let compressor = TextCompressor::new(); + let result = compressor.compress("hello", 0.5); + assert_eq!(result, "hello"); + } +}