feat: M3.8.1 phase 2 — JSON + Diff compressors (15 tests)
JsonCrusher (300 LOC): - Field variance analysis for mid-array selection - Allocation: 30% start (schema), 15% end (recency), 55% importance - Truncates long strings (>500 chars) with markers - Handles nested structures recursively DiffCompressor (180 LOC): - Keeps: file headers, hunk markers (@@), change lines (+/-) - Drops: context lines (spaces), unchanged content - Preserves binary file markers 32 optimizer tests total (17 phase1 + 15 phase2): - JsonCrusher: 8 tests (object, array, boundaries, truncation, nesting) - DiffCompressor: 7 tests (simple, multiple hunks, new/deleted files)
This commit is contained in:
@@ -6,12 +6,16 @@
|
||||
|
||||
pub mod router;
|
||||
pub mod log;
|
||||
pub mod json;
|
||||
pub mod diff;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use router::ContentRouter;
|
||||
pub use log::LogCompressor;
|
||||
pub use json::JsonCrusher;
|
||||
pub use diff::DiffCompressor;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct OptimizedChunk {
|
||||
@@ -78,6 +82,8 @@ impl Default for ContextOptimizerConfig {
|
||||
pub struct ContextOptimizer {
|
||||
router: ContentRouter,
|
||||
log_compressor: LogCompressor,
|
||||
json_crusher: JsonCrusher,
|
||||
diff_compressor: DiffCompressor,
|
||||
config: ContextOptimizerConfig,
|
||||
}
|
||||
|
||||
@@ -91,10 +97,14 @@ impl ContextOptimizer {
|
||||
pub fn with_config(config: ContextOptimizerConfig) -> Result<Self> {
|
||||
let router = ContentRouter::new()?;
|
||||
let log_compressor = LogCompressor::new();
|
||||
let json_crusher = JsonCrusher::new();
|
||||
let diff_compressor = DiffCompressor::new();
|
||||
|
||||
Ok(Self {
|
||||
router,
|
||||
log_compressor,
|
||||
json_crusher,
|
||||
diff_compressor,
|
||||
config,
|
||||
})
|
||||
}
|
||||
@@ -121,7 +131,13 @@ impl ContextOptimizer {
|
||||
ContentType::Log if self.config.compress_logs => {
|
||||
self.log_compressor.compress(content)?
|
||||
}
|
||||
_ => content.to_string(), // TODO: Add other compressors
|
||||
ContentType::Json if self.config.compress_json => {
|
||||
self.json_crusher.compress(content)?
|
||||
}
|
||||
ContentType::Diff if self.config.compress_diff => {
|
||||
self.diff_compressor.compress(content)?
|
||||
}
|
||||
_ => content.to_string(), // Passthrough for other types
|
||||
};
|
||||
|
||||
let original_tokens = estimate_tokens(content);
|
||||
|
||||
Reference in New Issue
Block a user