feat: M3.8.1 phase 4a — TextCompressor + env config (12 tests)

TextCompressor (320 LOC, 10 tests):
- Token importance scoring with lazy_static STOP_WORDS
- Keeps: high-entropy tokens (IDs, hashes, error codes, numbers, symbols)
- Drops: stop words, filler words, low-information prose
- ID detection: UUID, SHA256, session IDs, underscored patterns
- Error marker detection: error, exception, panic, fail, warn, critical
- Configurable compression ratio (default 40% token retention)

ContextOptimizerConfig::from_env() (2 tests):
- MEM_CONTEXT_OPTIMIZER (on/off)
- MEM_MAGIKA_ENABLED, MEM_MAGIKA_THRESHOLD
- MEM_COMPRESS_JSON, MEM_COMPRESS_LOGS, MEM_COMPRESS_CODE, MEM_COMPRESS_DIFF, MEM_COMPRESS_TEXT
- MEM_TOKEN_BUDGET, MEM_CCR_ENABLED

ContextOptimizer::from_env() factory method

62 optimizer tests total:
Phase 1 (17) + Phase 2 (15) + Phase 3 (18) + Phase 4a (12) = 62 passing

Cargo.lock

@@ -2064,6 +2064,7 @@ dependencies = [
  "futures",
  "hex",
  "indexmap",
+ "lazy_static",
  "magika",
  "once_cell",
  "ort",

crates/mem-core/Cargo.toml

@@ -21,3 +21,4 @@ ort = { version = "2.0.0-rc.12", default-features = true }
 regex = "1.10"
 once_cell = "1.19"
 indexmap = "2.0"
+lazy_static = "1.4"

crates/mem-core/src/optimizer/mod.rs

@@ -8,6 +8,7 @@ pub mod router;
 pub mod log;
 pub mod json;
 pub mod diff;
+pub mod text;
 pub mod cache_align;
 pub mod ccr;
 
@@ -18,6 +19,7 @@ pub use router::ContentRouter;
 pub use log::LogCompressor;
 pub use json::JsonCrusher;
 pub use diff::DiffCompressor;
+pub use text::TextCompressor;
 pub use cache_align::{CacheAligner, AlignedContent};
 pub use ccr::CcrStore;
 
@@ -65,6 +67,46 @@ pub struct ContextOptimizerConfig {
     pub ccr_enabled: bool,
 }
 
+impl ContextOptimizerConfig {
+    /// Load from environment variables
+    pub fn from_env() -> Self {
+        Self {
+            enabled: std::env::var("MEM_CONTEXT_OPTIMIZER")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            use_magika: std::env::var("MEM_MAGIKA_ENABLED")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            magika_threshold: std::env::var("MEM_MAGIKA_THRESHOLD")
+                .ok()
+                .and_then(|v| v.parse().ok())
+                .unwrap_or(0.7),
+            compress_json: std::env::var("MEM_COMPRESS_JSON")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            compress_logs: std::env::var("MEM_COMPRESS_LOGS")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            compress_code: std::env::var("MEM_COMPRESS_CODE")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(false), // opt-in
+            compress_diff: std::env::var("MEM_COMPRESS_DIFF")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            compress_text: std::env::var("MEM_COMPRESS_TEXT")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+            token_budget: std::env::var("MEM_TOKEN_BUDGET")
+                .ok()
+                .and_then(|v| v.parse().ok())
+                .unwrap_or(0),
+            ccr_enabled: std::env::var("MEM_CCR_ENABLED")
+                .map(|v| v.to_lowercase() == "on" || v == "true")
+                .unwrap_or(true),
+        }
+    }
+}
+
 impl Default for ContextOptimizerConfig {
     fn default() -> Self {
         Self {
@@ -88,6 +130,7 @@ pub struct ContextOptimizer {
     log_compressor: LogCompressor,
     json_crusher: JsonCrusher,
     diff_compressor: DiffCompressor,
+    text_compressor: TextCompressor,
     ccr_store: CcrStore,
     config: ContextOptimizerConfig,
 }
@@ -98,12 +141,18 @@ impl ContextOptimizer {
         Self::with_config(ContextOptimizerConfig::default())
     }
 
+    /// Create optimizer from environment config
+    pub fn from_env() -> Result<Self> {
+        Self::with_config(ContextOptimizerConfig::from_env())
+    }
+
     /// Create with custom config
     pub fn with_config(config: ContextOptimizerConfig) -> Result<Self> {
         let router = ContentRouter::new()?;
         let log_compressor = LogCompressor::new();
         let json_crusher = JsonCrusher::new();
         let diff_compressor = DiffCompressor::new();
+        let text_compressor = TextCompressor::new();
         let ccr_store = CcrStore::new();
 
         Ok(Self {
@@ -111,6 +160,7 @@ impl ContextOptimizer {
             log_compressor,
             json_crusher,
             diff_compressor,
+            text_compressor,
             ccr_store,
             config,
         })
@@ -149,6 +199,9 @@ impl ContextOptimizer {
             ContentType::Diff if self.config.compress_diff => {
                 self.diff_compressor.compress(content)?
             }
+            ContentType::Text if self.config.compress_text => {
+                self.text_compressor.compress(content, 0.4) // Keep 40% of tokens
+            }
             _ => content.to_string(), // Passthrough for other types
         };
 
@@ -226,4 +279,29 @@ mod tests {
         assert!(aligned.stable_prefix.contains("Context"));
         assert!(aligned.dynamic_tail.contains("2026-08-28"));
     }
+
+    #[test]
+    fn test_optimizer_compresses_text() {
+        let config = ContextOptimizerConfig {
+            enabled: true,
+            compress_text: true,
+            ..Default::default()
+        };
+        let optimizer = ContextOptimizer::with_config(config).unwrap();
+
+        let text = "The very important and critical error message is present in the system";
+        let chunk = optimizer.optimize(text).unwrap();
+
+        // Should detect as text
+        assert_eq!(chunk.content_type, ContentType::Text);
+        // Should compress
+        assert!(chunk.compressed_tokens < chunk.original_tokens);
+    }
+
+    #[test]
+    fn test_config_from_env() {
+        // Just verify it doesn't panic
+        let config = ContextOptimizerConfig::from_env();
+        assert!(config.enabled);
+    }
 }

crates/mem-core/src/optimizer/text.rs

@@ -0,0 +1,294 @@
+//! TextCompressor — Token importance scoring for plain text
+//!
+//! Strategy:
+//! - Keep: high-entropy tokens (IDs, hashes, error codes, numbers, symbols)
+//! - Drop: low-information prose (filler words, common phrases)
+//! - Reuses M3.7.8 stop words list for detection
+
+use lazy_static::lazy_static;
+use std::collections::HashSet;
+
+lazy_static! {
+    /// Common low-information words (expanded from M3.7.8 stop words)
+    static ref STOP_WORDS: HashSet<&'static str> = {
+        let words = vec![
+            // Common articles, prepositions, conjunctions
+            "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for",
+            "of", "with", "by", "from", "is", "are", "was", "were", "be", "been",
+            "have", "has", "do", "does", "did", "will", "would", "could", "should",
+            "may", "might", "must", "can", "shall",
+            
+            // Filler words
+            "the", "this", "that", "these", "those", "it", "its", "as", "also",
+            "very", "just", "only", "even", "still", "again", "about", "while",
+            "where", "when", "why", "what", "which", "who", "whom",
+            
+            // Common verbs (low signal when standalone)
+            "go", "get", "put", "make", "take", "come", "see", "say", "know",
+            "think", "want", "use", "find", "give", "tell", "work", "call",
+            "try", "ask", "need", "feel", "become", "leave", "show",
+            
+            // Weak modifiers
+            "good", "bad", "new", "old", "big", "small", "first", "last",
+            "some", "any", "no", "own", "other", "more", "most", "less", "least",
+            
+            // Pronouns
+            "i", "me", "we", "us", "you", "he", "him", "she", "her", "they", "them",
+            "my", "our", "your", "his", "her", "their",
+            
+            // Numbers and common phrases (context-dependent, lower priority)
+            "one", "two", "three", "four", "five", "etc", "etc.",
+        ];
+        words.into_iter().collect()
+    };
+}
+
+pub struct TextCompressor;
+
+impl TextCompressor {
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Compress text by scoring tokens and keeping high-entropy ones
+    pub fn compress(&self, content: &str, target_ratio: f32) -> String {
+        if content.is_empty() {
+            return content.to_string();
+        }
+
+        let tokens: Vec<&str> = content.split_whitespace().collect();
+        if tokens.is_empty() {
+            return content.to_string();
+        }
+
+        // Score each token
+        let mut scored: Vec<(usize, &str, f32)> = tokens
+            .iter()
+            .enumerate()
+            .map(|(idx, token)| (idx, *token, Self::score_token(token)))
+            .collect();
+
+        // Calculate target count
+        let target_count = ((tokens.len() as f32) * target_ratio).ceil() as usize;
+        let target_count = target_count.max(1).min(tokens.len()); // At least 1, at most all
+
+        // Sort by score descending, then by original index to preserve order
+        scored.sort_by(|a, b| {
+            b.2.partial_cmp(&a.2)
+                .unwrap_or(std::cmp::Ordering::Equal)
+                .then_with(|| a.0.cmp(&b.0))
+        });
+
+        // Take top tokens and re-sort by original index
+        let mut kept: Vec<(usize, &str)> = scored
+            .into_iter()
+            .take(target_count)
+            .map(|(idx, token, _)| (idx, token))
+            .collect();
+        kept.sort_by_key(|a| a.0);
+
+        kept.iter().map(|(_, token)| *token).collect::<Vec<_>>().join(" ")
+    }
+
+    /// Score a token for importance
+    fn score_token(token: &str) -> f32 {
+        let lower = token.to_lowercase();
+
+        // High-entropy tokens
+        let mut score = 0.0f32;
+
+        // IDs, hashes, hex
+        if is_id_like(token) {
+            score += 10.0;
+        }
+
+        // Numbers
+        if token.chars().any(|c| c.is_numeric()) {
+            score += 3.0;
+        }
+
+        // Error codes, markers
+        if is_error_marker(&lower) {
+            score += 8.0;
+        }
+
+        // Symbols (punctuation often marks structure)
+        if token.chars().any(|c| !c.is_alphanumeric()) {
+            score += 2.0;
+        }
+
+        // Stop words (negative score)
+        if STOP_WORDS.contains(lower.as_str()) {
+            score -= 5.0;
+        }
+
+        // Length (longer tokens usually more informative)
+        if token.len() > 10 {
+            score += 1.0;
+        }
+
+        // Capitalization (usually proper nouns or emphatic)
+        if token.chars().next().map_or(false, |c| c.is_uppercase()) && token.len() > 1 {
+            score += 1.0;
+        }
+
+        score
+    }
+}
+
+impl Default for TextCompressor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Check if token looks like an ID (UUID, hash, etc.)
+fn is_id_like(token: &str) -> bool {
+    // UUIDs
+    if token.len() == 36 && token.matches('-').count() == 4 {
+        return true;
+    }
+
+    // SHA256 / hashes
+    if token.len() == 64 && token.chars().all(|c| c.is_ascii_hexdigit()) {
+        return true;
+    }
+
+    // Short hex strings
+    if token.len() > 8 && token.len() < 20 && token.chars().all(|c| c.is_ascii_hexdigit()) {
+        return true;
+    }
+
+    // Alphanumeric with underscores (typical ID pattern)
+    if token.len() > 6 && token.contains('_') && token.chars().all(|c| c.is_alphanumeric() || c == '_') {
+        return true;
+    }
+
+    false
+}
+
+/// Check if token is an error marker
+fn is_error_marker(token: &str) -> bool {
+    token.contains("error")
+        || token.contains("err")
+        || token.contains("fail")
+        || token.contains("exception")
+        || token.contains("panic")
+        || token.contains("warn")
+        || token.contains("critical")
+        || token.contains("fatal")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_score_high_entropy_tokens() {
+        let uuid = "550e8400-e29b-41d4-a716-446655440000";
+        let hash = "abc123def456abc123def456abc123def456abc123def456abc123def456abc1";
+        let error = "ConnectionError";
+
+        assert!(TextCompressor::score_token(uuid) > 5.0);
+        assert!(TextCompressor::score_token(hash) > 5.0);
+        assert!(TextCompressor::score_token(error) > 5.0);
+    }
+
+    #[test]
+    fn test_score_low_value_tokens() {
+        let the = "the";
+        let and = "and";
+        let very = "very";
+
+        assert!(TextCompressor::score_token(the) < 0.0);
+        assert!(TextCompressor::score_token(and) < 0.0);
+        assert!(TextCompressor::score_token(very) < 0.0);
+    }
+
+    #[test]
+    fn test_compress_keeps_identifiers() {
+        let compressor = TextCompressor::new();
+        let text = "The request with ID abc123def456abc123def456abc123def456abc1 failed";
+
+        let compressed = compressor.compress(text, 0.5); // Keep 50%
+
+        // Should keep ID
+        assert!(compressed.contains("abc123def456"));
+        // Should keep failed (error marker)
+        assert!(compressed.contains("failed"));
+    }
+
+    #[test]
+    fn test_compress_drops_filler() {
+        let compressor = TextCompressor::new();
+        let text = "The very important and critical error message is here";
+
+        let compressed = compressor.compress(text, 0.4); // Keep 40%
+
+        // Should keep important, error, message
+        assert!(compressed.contains("important"));
+        assert!(compressed.contains("error"));
+        assert!(compressed.contains("message"));
+
+        // Should drop filler
+        assert!(!compressed.contains("very") || compressed.split_whitespace().count() < 6);
+    }
+
+    #[test]
+    fn test_compress_preserves_numbers() {
+        let compressor = TextCompressor::new();
+        let text = "Exit code 127 indicates command not found after 5 seconds";
+
+        let compressed = compressor.compress(text, 0.6);
+
+        // Should keep numbers
+        assert!(compressed.contains("127"));
+        assert!(compressed.contains("5"));
+    }
+
+    #[test]
+    fn test_is_id_like() {
+        assert!(is_id_like("550e8400-e29b-41d4-a716-446655440000")); // UUID
+        assert!(is_id_like("abc123def456abc123def456abc123def456abc123def456abc123def456abc1")); // SHA256
+        assert!(is_id_like("session_id_12345"));
+        assert!(!is_id_like("the"));
+        assert!(!is_id_like("and"));
+    }
+
+    #[test]
+    fn test_is_error_marker() {
+        assert!(is_error_marker("error"));
+        assert!(is_error_marker("exception"));
+        assert!(is_error_marker("connectionerror"));
+        assert!(is_error_marker("fatalpanic"));
+        assert!(!is_error_marker("message"));
+        assert!(!is_error_marker("data"));
+    }
+
+    #[test]
+    fn test_compress_ratio() {
+        let compressor = TextCompressor::new();
+        let text = "This is a very long piece of text with many filler words that should be compressed significantly while keeping important identifiers like abc123def456abc123def456abc123def456abc1 and error codes like 404";
+
+        let compressed = compressor.compress(text, 0.3); // Keep 30%
+        let original_tokens = text.split_whitespace().count();
+        let compressed_tokens = compressed.split_whitespace().count();
+
+        // Should be significantly smaller
+        assert!(compressed_tokens <= original_tokens);
+        assert!(compressed_tokens as f32 / original_tokens as f32 <= 0.35);
+    }
+
+    #[test]
+    fn test_compress_empty() {
+        let compressor = TextCompressor::new();
+        assert_eq!(compressor.compress("", 0.5), "");
+    }
+
+    #[test]
+    fn test_compress_single_token() {
+        let compressor = TextCompressor::new();
+        let result = compressor.compress("hello", 0.5);
+        assert_eq!(result, "hello");
+    }
+}