diff --git a/Cargo.lock b/Cargo.lock index 50132ce..1d3c5c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2061,6 +2061,7 @@ name = "mem-core" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "futures", "hex", "indexmap", diff --git a/crates/mem-core/Cargo.toml b/crates/mem-core/Cargo.toml index 7855890..3353c59 100644 --- a/crates/mem-core/Cargo.toml +++ b/crates/mem-core/Cargo.toml @@ -22,3 +22,4 @@ regex = "1.10" once_cell = "1.19" indexmap = "2.0" lazy_static = "1.4" +async-trait = "0.1.92" diff --git a/crates/mem-core/src/optimizer/builtin.rs b/crates/mem-core/src/optimizer/builtin.rs new file mode 100644 index 0000000..ab56323 --- /dev/null +++ b/crates/mem-core/src/optimizer/builtin.rs @@ -0,0 +1,300 @@ +//! Built-in optimizer and format implementations +//! Bridges existing ContextOptimizer to pluggable system + +use async_trait::async_trait; +use super::plugin::{OptimizerPlugin, FormatHandler, OptimizationResult, PluginMetrics}; +use super::ContextOptimizer; +use std::sync::{Arc, Mutex}; + +/// Built-in context optimizer plugin +pub struct BuiltinOptimizer { + optimizer: Arc, + metrics: Arc>, +} + +impl BuiltinOptimizer { + pub fn new(optimizer: Arc) -> Self { + Self { + optimizer, + metrics: Arc::new(Mutex::new(PluginMetrics::default())), + } + } +} + +#[async_trait] +impl OptimizerPlugin for BuiltinOptimizer { + fn name(&self) -> &str { + "builtin-optimizer" + } + + fn supported_types(&self) -> Vec<&str> { + vec![ + "text/plain", + "text/x-log", + "application/json", + "text/x-diff", + "application/x-yaml", + ] + } + + async fn optimize(&self, content: &str) -> Result { + let start = std::time::Instant::now(); + let original_len = content.len(); + + match self.optimizer.optimize(content) { + Ok(chunk) => { + let optimized_len = chunk.compressed.len(); + let ratio = optimized_len as f32 / original_len as f32; + + // Update metrics + { + let mut m = self.metrics.lock().unwrap(); + m.total_optimizations += 1; + m.total_bytes_input += original_len as u64; + m.total_bytes_output += optimized_len as u64; + m.avg_latency_ms = start.elapsed().as_secs_f32() * 1000.0; + } + + Ok(OptimizationResult { + original: content.to_string(), + optimized: chunk.compressed, + ratio, + plugin: self.name().to_string(), + metadata: std::collections::HashMap::new(), + }) + } + Err(e) => { + let mut m = self.metrics.lock().unwrap(); + m.errors += 1; + Err(format!("Optimization failed: {}", e)) + } + } + } + + fn metrics(&self) -> PluginMetrics { + self.metrics.lock().unwrap().clone() + } +} + +// ============================================================================ +// FORMAT HANDLERS +// ============================================================================ + +/// JSON format handler +pub struct JsonFormatter; + +#[async_trait] +impl FormatHandler for JsonFormatter { + fn name(&self) -> &str { + "json" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + serde_json::to_vec(result).map_err(|e| format!("JSON serialization failed: {}", e)) + } + + async fn parse(&self, data: &[u8]) -> Result { + serde_json::from_slice(data) + .map_err(|e| format!("JSON deserialization failed: {}", e)) + } +} + +/// JSONL (newline-delimited JSON) formatter +pub struct JsonlFormatter; + +#[async_trait] +impl FormatHandler for JsonlFormatter { + fn name(&self) -> &str { + "jsonl" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + let mut output = serde_json::to_string(result) + .map_err(|e| format!("JSONL serialization failed: {}", e))?; + output.push('\n'); + Ok(output.into_bytes()) + } + + async fn parse(&self, data: &[u8]) -> Result { + let line = String::from_utf8(data.to_vec()) + .map_err(|e| format!("UTF-8 decode failed: {}", e))?; + serde_json::from_str(line.trim()) + .map_err(|e| format!("JSONL deserialization failed: {}", e)) + } +} + +/// Raw text format (just optimized content) +pub struct RawFormatter; + +#[async_trait] +impl FormatHandler for RawFormatter { + fn name(&self) -> &str { + "raw" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + Ok(result.optimized.as_bytes().to_vec()) + } + + async fn parse(&self, _data: &[u8]) -> Result { + Err("Raw format does not support deserialization".to_string()) + } +} + +/// CSV format (for metrics) +pub struct CsvFormatter; + +#[async_trait] +impl FormatHandler for CsvFormatter { + fn name(&self) -> &str { + "csv" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + let output = format!( + "{},{},{},{}\n", + escape_csv(&result.plugin), + result.original.len(), + result.optimized.len(), + format!("{:.2}", result.ratio) + ); + Ok(output.into_bytes()) + } + + async fn parse(&self, _data: &[u8]) -> Result { + Err("CSV format does not support deserialization".to_string()) + } +} + +/// YAML format +pub struct YamlFormatter; + +#[async_trait] +impl FormatHandler for YamlFormatter { + fn name(&self) -> &str { + "yaml" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + let yaml = format!( + "plugin: {}\nratio: {:.2}\noriginal_bytes: {}\noptimized_bytes: {}\n", + result.plugin, + result.ratio, + result.original.len(), + result.optimized.len() + ); + Ok(yaml.into_bytes()) + } + + async fn parse(&self, _data: &[u8]) -> Result { + Err("YAML format does not support deserialization".to_string()) + } +} + +// ============================================================================ +// HELPERS +// ============================================================================ + +fn escape_csv(s: &str) -> String { + if s.contains(',') || s.contains('"') || s.contains('\n') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_builtin_optimizer_logs() { + let optimizer = Arc::new(ContextOptimizer::new().unwrap()); + let plugin = BuiltinOptimizer::new(optimizer); + + let content = "ERROR: failed\nDEBUG: info\nERROR: error"; + let result = plugin.optimize(content).await.unwrap(); + + assert_eq!(result.plugin, "builtin-optimizer"); + assert!(result.ratio <= 1.0); + } + + #[tokio::test] + async fn test_json_formatter() { + let formatter = JsonFormatter; + let result = OptimizationResult { + original: "test".to_string(), + optimized: "tst".to_string(), + ratio: 0.75, + plugin: "test".to_string(), + metadata: Default::default(), + }; + + let formatted = formatter.format(&result).await.unwrap(); + let parsed = formatter.parse(&formatted).await.unwrap(); + + assert_eq!(parsed.original, "test"); + } + + #[tokio::test] + async fn test_jsonl_formatter() { + let formatter = JsonlFormatter; + let result = OptimizationResult { + original: "test".to_string(), + optimized: "tst".to_string(), + ratio: 0.75, + plugin: "test".to_string(), + metadata: Default::default(), + }; + + let formatted = formatter.format(&result).await.unwrap(); + assert!(String::from_utf8(formatted).unwrap().ends_with('\n')); + } + + #[tokio::test] + async fn test_raw_formatter() { + let formatter = RawFormatter; + let result = OptimizationResult { + original: "original content".to_string(), + optimized: "optimized".to_string(), + ratio: 0.5, + plugin: "test".to_string(), + metadata: Default::default(), + }; + + let formatted = formatter.format(&result).await.unwrap(); + assert_eq!(formatted, b"optimized"); + } + + #[tokio::test] + async fn test_csv_formatter() { + let formatter = CsvFormatter; + let result = OptimizationResult { + original: "test".to_string(), + optimized: "tst".to_string(), + ratio: 0.75, + plugin: "my-plugin".to_string(), + metadata: Default::default(), + }; + + let formatted = formatter.format(&result).await.unwrap(); + let csv = String::from_utf8(formatted).unwrap(); + assert!(csv.contains("my-plugin")); + } + + #[tokio::test] + async fn test_yaml_formatter() { + let formatter = YamlFormatter; + let result = OptimizationResult { + original: "original".to_string(), + optimized: "opt".to_string(), + ratio: 0.33, + plugin: "test".to_string(), + metadata: Default::default(), + }; + + let formatted = formatter.format(&result).await.unwrap(); + let yaml = String::from_utf8(formatted).unwrap(); + assert!(yaml.contains("ratio:")); + } +} diff --git a/crates/mem-core/src/optimizer/mod.rs b/crates/mem-core/src/optimizer/mod.rs index 0f4254d..b000f74 100644 --- a/crates/mem-core/src/optimizer/mod.rs +++ b/crates/mem-core/src/optimizer/mod.rs @@ -11,6 +11,8 @@ pub mod diff; pub mod text; pub mod cache_align; pub mod ccr; +pub mod plugin; +pub mod builtin; use anyhow::Result; use serde::{Deserialize, Serialize}; @@ -22,6 +24,13 @@ pub use diff::DiffCompressor; pub use text::TextCompressor; pub use cache_align::{CacheAligner, AlignedContent}; pub use ccr::CcrStore; +pub use plugin::{ + OptimizerPlugin, FormatHandler, Registry, OptimizationResult, PluginMetrics, + SimpleRegistry, PluginLocator, DefaultLocator, OptimizerService, OptimizerServiceBuilder, +}; +pub use builtin::{ + BuiltinOptimizer, JsonFormatter, JsonlFormatter, RawFormatter, CsvFormatter, YamlFormatter, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct OptimizedChunk { diff --git a/crates/mem-core/src/optimizer/plugin.rs b/crates/mem-core/src/optimizer/plugin.rs new file mode 100644 index 0000000..e4859ba --- /dev/null +++ b/crates/mem-core/src/optimizer/plugin.rs @@ -0,0 +1,430 @@ +//! M3.8 Pluggable Optimizer Architecture (DRY + SOLID) +//! +//! Enables custom optimizers and formats without recompiling. +//! Follows SOLID principles for extensibility. + +use async_trait::async_trait; +use std::collections::HashMap; +use std::sync::Arc; +use serde::{Serialize, Deserialize}; + +// ============================================================================ +// ABSTRACTIONS (Interface Segregation + Open/Closed) +// ============================================================================ + +/// Trait for content optimizers +/// Single Responsibility: optimize content +#[async_trait] +pub trait OptimizerPlugin: Send + Sync { + fn name(&self) -> &str; + fn supported_types(&self) -> Vec<&str>; + fn can_handle(&self, content_type: &str) -> bool { + self.supported_types().contains(&content_type) + } + async fn optimize(&self, content: &str) -> Result; + fn metrics(&self) -> PluginMetrics; +} + +/// Trait for output format handlers +/// Single Responsibility: format/parse data +#[async_trait] +pub trait FormatHandler: Send + Sync { + fn name(&self) -> &str; + async fn format(&self, result: &OptimizationResult) -> Result, String>; + async fn parse(&self, data: &[u8]) -> Result; +} + +/// Trait for registries (DRY: generic registration pattern) +/// Single Responsibility: store and retrieve plugins/formats +pub trait Registry: Send + Sync { + fn register(&mut self, item: Arc); + fn get(&self, name: &str) -> Option>; + fn list(&self) -> Vec; +} + +// ============================================================================ +// DATA STRUCTURES +// ============================================================================ + +/// Result of optimization +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OptimizationResult { + pub original: String, + pub optimized: String, + pub ratio: f32, + pub plugin: String, + pub metadata: HashMap, +} + +/// Plugin metrics +#[derive(Debug, Clone, Default)] +pub struct PluginMetrics { + pub total_optimizations: u64, + pub total_bytes_input: u64, + pub total_bytes_output: u64, + pub errors: u64, + pub avg_latency_ms: f32, +} + +// ============================================================================ +// GENERIC REGISTRY (DRY: avoid duplication) +// ============================================================================ + +/// Generic registry implementation for any plugin type +pub struct SimpleRegistry { + items: HashMap>, +} + +impl SimpleRegistry { + pub fn new() -> Self { + Self { + items: HashMap::new(), + } + } +} + +impl Default for SimpleRegistry { + fn default() -> Self { + Self::new() + } +} + +impl Registry for SimpleRegistry { + fn register(&mut self, item: Arc) { + self.items.insert(item.name().to_string(), item); + } + + fn get(&self, name: &str) -> Option> { + self.items.get(name).cloned() + } + + fn list(&self) -> Vec { + self.items.keys().cloned().collect() + } +} + +impl Registry for SimpleRegistry { + fn register(&mut self, item: Arc) { + self.items.insert(item.name().to_string(), item); + } + + fn get(&self, name: &str) -> Option> { + self.items.get(name).cloned() + } + + fn list(&self) -> Vec { + self.items.keys().cloned().collect() + } +} + +// ============================================================================ +// LOCATOR (Dependency Injection) +// ============================================================================ + +/// Trait for locating plugins (Strategy pattern) +/// Allows different lookup strategies (e.g., by name, by type, by priority) +pub trait PluginLocator: Send + Sync { + fn find_optimizer( + &self, + registry: &SimpleRegistry, + content_type: &str, + ) -> Result, String>; + + fn find_format( + &self, + registry: &SimpleRegistry, + name: &str, + ) -> Result, String>; +} + +/// Default locator: first match by type +pub struct DefaultLocator; + +impl PluginLocator for DefaultLocator { + fn find_optimizer( + &self, + registry: &SimpleRegistry, + content_type: &str, + ) -> Result, String> { + registry + .list() + .iter() + .find_map(|name| registry.get(name).filter(|opt| opt.can_handle(content_type))) + .ok_or_else(|| { + format!( + "No optimizer for type {}. Available: {:?}", + content_type, + registry.list() + ) + }) + } + + fn find_format( + &self, + registry: &SimpleRegistry, + name: &str, + ) -> Result, String> { + registry.get(name).ok_or_else(|| { + format!("Format {} not found. Available: {:?}", name, registry.list()) + }) + } +} + +// ============================================================================ +// SERVICE (Composition + Dependency Inversion) +// ============================================================================ + +/// Optimizer service: composes registry + locator +/// Single Responsibility: orchestrate optimization workflow +pub struct OptimizerService { + optimizer_registry: Arc>, + format_registry: Arc>, + locator: Arc, + default_format: String, +} + +impl OptimizerService { + /// Create service with injected dependencies + pub fn new( + optimizer_registry: Arc>, + format_registry: Arc>, + locator: Arc, + default_format: String, + ) -> Result { + if format_registry.get(&default_format).is_none() { + return Err(format!("Default format {} not registered", default_format)); + } + Ok(Self { + optimizer_registry, + format_registry, + locator, + default_format, + }) + } + + /// Optimize with auto-format conversion + pub async fn optimize( + &self, + content: &str, + content_type: &str, + format: Option<&str>, + ) -> Result, String> { + // Find optimizer + let optimizer = self + .locator + .find_optimizer(&self.optimizer_registry, content_type)?; + + // Optimize + let result = optimizer.optimize(content).await?; + + // Find formatter + let format_name = format.unwrap_or(&self.default_format); + let formatter = self + .locator + .find_format(&self.format_registry, format_name)?; + + // Format output + formatter.format(&result).await + } + + /// Get all metrics (avoid duplication with generic collect) + pub fn get_metrics(&self) -> HashMap { + self.optimizer_registry + .list() + .iter() + .filter_map(|name| { + self.optimizer_registry + .get(name) + .map(|opt| (name.clone(), opt.metrics())) + }) + .collect() + } + + /// List available optimizers + pub fn optimizers(&self) -> Vec { + self.optimizer_registry.list() + } + + /// List available formats + pub fn formats(&self) -> Vec { + self.format_registry.list() + } +} + +// ============================================================================ +// BUILDER (Make service creation ergonomic) +// ============================================================================ + +pub struct OptimizerServiceBuilder { + optimizer_registry: SimpleRegistry, + format_registry: SimpleRegistry, + locator: Arc, + default_format: String, +} + +impl OptimizerServiceBuilder { + pub fn new() -> Self { + Self { + optimizer_registry: SimpleRegistry::new(), + format_registry: SimpleRegistry::new(), + locator: Arc::new(DefaultLocator), + default_format: "json".to_string(), + } + } + + pub fn with_optimizer(mut self, optimizer: Arc) -> Self { + self.optimizer_registry.register(optimizer); + self + } + + pub fn with_format(mut self, format: Arc) -> Self { + self.format_registry.register(format); + self + } + + pub fn with_locator(mut self, locator: Arc) -> Self { + self.locator = locator; + self + } + + pub fn with_default_format(mut self, format: String) -> Self { + self.default_format = format; + self + } + + pub fn build(self) -> Result { + OptimizerService::new( + Arc::new(self.optimizer_registry), + Arc::new(self.format_registry), + self.locator, + self.default_format, + ) + } +} + +impl Default for OptimizerServiceBuilder { + fn default() -> Self { + Self::new() + } +} + +// ============================================================================ +// TESTS +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + struct MockOptimizer; + + #[async_trait] + impl OptimizerPlugin for MockOptimizer { + fn name(&self) -> &str { + "mock" + } + + fn supported_types(&self) -> Vec<&str> { + vec!["text/plain"] + } + + async fn optimize(&self, content: &str) -> Result { + Ok(OptimizationResult { + original: content.to_string(), + optimized: content.to_lowercase(), + ratio: 0.8, + plugin: self.name().to_string(), + metadata: Default::default(), + }) + } + + fn metrics(&self) -> PluginMetrics { + PluginMetrics::default() + } + } + + struct MockFormat; + + #[async_trait] + impl FormatHandler for MockFormat { + fn name(&self) -> &str { + "json" + } + + async fn format(&self, result: &OptimizationResult) -> Result, String> { + let json = serde_json::to_string(result).map_err(|e| e.to_string())?; + Ok(json.into_bytes()) + } + + async fn parse(&self, data: &[u8]) -> Result { + serde_json::from_slice(data).map_err(|e| e.to_string()) + } + } + + #[test] + fn test_registry_register_and_get() { + let mut registry: SimpleRegistry = SimpleRegistry::new(); + let opt: Arc = Arc::new(MockOptimizer); + registry.register(opt); + + assert!(registry.get("mock").is_some()); + } + + #[test] + fn test_registry_list() { + let mut registry: SimpleRegistry = SimpleRegistry::new(); + registry.register(Arc::new(MockOptimizer) as Arc); + + assert_eq!(registry.list(), vec!["mock"]); + } + + #[test] + fn test_locator_find_by_type() { + let mut registry: SimpleRegistry = SimpleRegistry::new(); + registry.register(Arc::new(MockOptimizer) as Arc); + + let locator = DefaultLocator; + let found = locator.find_optimizer(®istry, "text/plain"); + assert!(found.is_ok()); + } + + #[test] + fn test_locator_not_found() { + let registry: SimpleRegistry = SimpleRegistry::new(); + let locator = DefaultLocator; + let result = locator.find_optimizer(®istry, "unknown/type"); + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_service_optimize() { + let service = OptimizerServiceBuilder::new() + .with_optimizer(Arc::new(MockOptimizer) as Arc) + .with_format(Arc::new(MockFormat) as Arc) + .build() + .unwrap(); + + let result = service.optimize("HELLO", "text/plain", None).await; + assert!(result.is_ok()); + } + + #[test] + fn test_builder_creates_service() { + let service = OptimizerServiceBuilder::new() + .with_optimizer(Arc::new(MockOptimizer) as Arc) + .with_format(Arc::new(MockFormat) as Arc) + .build(); + + assert!(service.is_ok()); + } + + #[test] + fn test_builder_fails_on_missing_format() { + let service = OptimizerServiceBuilder::new() + .with_optimizer(Arc::new(MockOptimizer) as Arc) + .with_default_format("missing".to_string()) + .build(); + + assert!(service.is_err()); + } +}