feat: M3.8.1 phase 4a — TextCompressor + env config (12 tests)
TextCompressor (320 LOC, 10 tests): - Token importance scoring with lazy_static STOP_WORDS - Keeps: high-entropy tokens (IDs, hashes, error codes, numbers, symbols) - Drops: stop words, filler words, low-information prose - ID detection: UUID, SHA256, session IDs, underscored patterns - Error marker detection: error, exception, panic, fail, warn, critical - Configurable compression ratio (default 40% token retention) ContextOptimizerConfig::from_env() (2 tests): - MEM_CONTEXT_OPTIMIZER (on/off) - MEM_MAGIKA_ENABLED, MEM_MAGIKA_THRESHOLD - MEM_COMPRESS_JSON, MEM_COMPRESS_LOGS, MEM_COMPRESS_CODE, MEM_COMPRESS_DIFF, MEM_COMPRESS_TEXT - MEM_TOKEN_BUDGET, MEM_CCR_ENABLED ContextOptimizer::from_env() factory method 62 optimizer tests total: Phase 1 (17) + Phase 2 (15) + Phase 3 (18) + Phase 4a (12) = 62 passing
This commit is contained in:
@@ -8,6 +8,7 @@ pub mod router;
|
||||
pub mod log;
|
||||
pub mod json;
|
||||
pub mod diff;
|
||||
pub mod text;
|
||||
pub mod cache_align;
|
||||
pub mod ccr;
|
||||
|
||||
@@ -18,6 +19,7 @@ pub use router::ContentRouter;
|
||||
pub use log::LogCompressor;
|
||||
pub use json::JsonCrusher;
|
||||
pub use diff::DiffCompressor;
|
||||
pub use text::TextCompressor;
|
||||
pub use cache_align::{CacheAligner, AlignedContent};
|
||||
pub use ccr::CcrStore;
|
||||
|
||||
@@ -65,6 +67,46 @@ pub struct ContextOptimizerConfig {
|
||||
pub ccr_enabled: bool,
|
||||
}
|
||||
|
||||
impl ContextOptimizerConfig {
|
||||
/// Load from environment variables
|
||||
pub fn from_env() -> Self {
|
||||
Self {
|
||||
enabled: std::env::var("MEM_CONTEXT_OPTIMIZER")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
use_magika: std::env::var("MEM_MAGIKA_ENABLED")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
magika_threshold: std::env::var("MEM_MAGIKA_THRESHOLD")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(0.7),
|
||||
compress_json: std::env::var("MEM_COMPRESS_JSON")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
compress_logs: std::env::var("MEM_COMPRESS_LOGS")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
compress_code: std::env::var("MEM_COMPRESS_CODE")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(false), // opt-in
|
||||
compress_diff: std::env::var("MEM_COMPRESS_DIFF")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
compress_text: std::env::var("MEM_COMPRESS_TEXT")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
token_budget: std::env::var("MEM_TOKEN_BUDGET")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(0),
|
||||
ccr_enabled: std::env::var("MEM_CCR_ENABLED")
|
||||
.map(|v| v.to_lowercase() == "on" || v == "true")
|
||||
.unwrap_or(true),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ContextOptimizerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
@@ -88,6 +130,7 @@ pub struct ContextOptimizer {
|
||||
log_compressor: LogCompressor,
|
||||
json_crusher: JsonCrusher,
|
||||
diff_compressor: DiffCompressor,
|
||||
text_compressor: TextCompressor,
|
||||
ccr_store: CcrStore,
|
||||
config: ContextOptimizerConfig,
|
||||
}
|
||||
@@ -98,12 +141,18 @@ impl ContextOptimizer {
|
||||
Self::with_config(ContextOptimizerConfig::default())
|
||||
}
|
||||
|
||||
/// Create optimizer from environment config
|
||||
pub fn from_env() -> Result<Self> {
|
||||
Self::with_config(ContextOptimizerConfig::from_env())
|
||||
}
|
||||
|
||||
/// Create with custom config
|
||||
pub fn with_config(config: ContextOptimizerConfig) -> Result<Self> {
|
||||
let router = ContentRouter::new()?;
|
||||
let log_compressor = LogCompressor::new();
|
||||
let json_crusher = JsonCrusher::new();
|
||||
let diff_compressor = DiffCompressor::new();
|
||||
let text_compressor = TextCompressor::new();
|
||||
let ccr_store = CcrStore::new();
|
||||
|
||||
Ok(Self {
|
||||
@@ -111,6 +160,7 @@ impl ContextOptimizer {
|
||||
log_compressor,
|
||||
json_crusher,
|
||||
diff_compressor,
|
||||
text_compressor,
|
||||
ccr_store,
|
||||
config,
|
||||
})
|
||||
@@ -149,6 +199,9 @@ impl ContextOptimizer {
|
||||
ContentType::Diff if self.config.compress_diff => {
|
||||
self.diff_compressor.compress(content)?
|
||||
}
|
||||
ContentType::Text if self.config.compress_text => {
|
||||
self.text_compressor.compress(content, 0.4) // Keep 40% of tokens
|
||||
}
|
||||
_ => content.to_string(), // Passthrough for other types
|
||||
};
|
||||
|
||||
@@ -226,4 +279,29 @@ mod tests {
|
||||
assert!(aligned.stable_prefix.contains("Context"));
|
||||
assert!(aligned.dynamic_tail.contains("2026-08-28"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_optimizer_compresses_text() {
|
||||
let config = ContextOptimizerConfig {
|
||||
enabled: true,
|
||||
compress_text: true,
|
||||
..Default::default()
|
||||
};
|
||||
let optimizer = ContextOptimizer::with_config(config).unwrap();
|
||||
|
||||
let text = "The very important and critical error message is present in the system";
|
||||
let chunk = optimizer.optimize(text).unwrap();
|
||||
|
||||
// Should detect as text
|
||||
assert_eq!(chunk.content_type, ContentType::Text);
|
||||
// Should compress
|
||||
assert!(chunk.compressed_tokens < chunk.original_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_from_env() {
|
||||
// Just verify it doesn't panic
|
||||
let config = ContextOptimizerConfig::from_env();
|
||||
assert!(config.enabled);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user