feat: M3.8.1 phase 2 — JSON + Diff compressors (15 tests)

JsonCrusher (300 LOC):
- Field variance analysis for mid-array selection
- Allocation: 30% start (schema), 15% end (recency), 55% importance
- Truncates long strings (>500 chars) with markers
- Handles nested structures recursively

DiffCompressor (180 LOC):
- Keeps: file headers, hunk markers (@@), change lines (+/-)
- Drops: context lines (spaces), unchanged content
- Preserves binary file markers

32 optimizer tests total (17 phase1 + 15 phase2):
- JsonCrusher: 8 tests (object, array, boundaries, truncation, nesting)
- DiffCompressor: 7 tests (simple, multiple hunks, new/deleted files)
This commit is contained in:
Story Crater Bot
2026-08-28 09:36:17 -07:00
parent d985c59921
commit fd4ca2a17a
3 changed files with 593 additions and 1 deletions
+17 -1
View File
@@ -6,12 +6,16 @@
pub mod router;
pub mod log;
pub mod json;
pub mod diff;
use anyhow::Result;
use serde::{Deserialize, Serialize};
pub use router::ContentRouter;
pub use log::LogCompressor;
pub use json::JsonCrusher;
pub use diff::DiffCompressor;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OptimizedChunk {
@@ -78,6 +82,8 @@ impl Default for ContextOptimizerConfig {
pub struct ContextOptimizer {
router: ContentRouter,
log_compressor: LogCompressor,
json_crusher: JsonCrusher,
diff_compressor: DiffCompressor,
config: ContextOptimizerConfig,
}
@@ -91,10 +97,14 @@ impl ContextOptimizer {
pub fn with_config(config: ContextOptimizerConfig) -> Result<Self> {
let router = ContentRouter::new()?;
let log_compressor = LogCompressor::new();
let json_crusher = JsonCrusher::new();
let diff_compressor = DiffCompressor::new();
Ok(Self {
router,
log_compressor,
json_crusher,
diff_compressor,
config,
})
}
@@ -121,7 +131,13 @@ impl ContextOptimizer {
ContentType::Log if self.config.compress_logs => {
self.log_compressor.compress(content)?
}
_ => content.to_string(), // TODO: Add other compressors
ContentType::Json if self.config.compress_json => {
self.json_crusher.compress(content)?
}
ContentType::Diff if self.config.compress_diff => {
self.diff_compressor.compress(content)?
}
_ => content.to_string(), // Passthrough for other types
};
let original_tokens = estimate_tokens(content);