From b18932b10c2502c583417991f18b5820b1098057 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:29:56 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20M3.8.1=20phase=201=20=E2=80=94=20conten?= =?UTF-8?q?t=20router=20+=20log=20compressor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ContentRouter uses Google Magika ML for content detection (<1ms) with regex fallback. Detects JSON, code, logs, diffs, config, text. LogCompressor reuses M3.7.7 patterns (markers, cascade, strip_ansi) to shrink build logs by keeping errors/stacks and dropping noise. 17 unit tests passing: - router: json, code, diff, log, text detection - log: error lines, stack traces, ansi stripping, compression - optimizer: token estimation, passthrough mode Magika + ort ONNX runtime added to Cargo.toml. --- Cargo.lock | 204 ++++++++++++++++++- crates/mem-core/Cargo.toml | 4 + crates/mem-core/src/lib.rs | 2 + crates/mem-core/src/optimizer/log.rs | 259 ++++++++++++++++++++++++ crates/mem-core/src/optimizer/mod.rs | 172 ++++++++++++++++ crates/mem-core/src/optimizer/router.rs | 227 +++++++++++++++++++++ k8s/infra/databases/opensearch.yaml | 7 +- 7 files changed, 866 insertions(+), 9 deletions(-) create mode 100644 crates/mem-core/src/optimizer/log.rs create mode 100644 crates/mem-core/src/optimizer/mod.rs create mode 100644 crates/mem-core/src/optimizer/router.rs diff --git a/Cargo.lock b/Cargo.lock index bb4c4b2..5d0ddc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,6 +380,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -854,7 +860,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468 1.0.0", "zeroize", ] @@ -1412,6 +1428,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "home" version = "0.5.12" @@ -1919,6 +1941,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "macro_rules_attribute" version = "0.1.3" @@ -1935,6 +1963,28 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" +[[package]] +name = "magika" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf" +dependencies = [ + "ndarray", + "ort", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2013,6 +2063,9 @@ dependencies = [ "anyhow", "futures", "hex", + "magika", + "ort", + "regex", "serde", "serde_json", "serde_yaml", @@ -2157,6 +2210,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nom" version = "7.1.3" @@ -2202,6 +2270,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2354,6 +2431,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2425,6 +2526,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2474,7 +2584,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", + "der 0.7.10", "pkcs8", "spki", ] @@ -2485,7 +2595,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", + "der 0.7.10", "spki", ] @@ -2524,6 +2634,21 @@ dependencies = [ "wiremock", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postgres" version = "0.19.14" @@ -2668,6 +2793,12 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -2885,6 +3016,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3182,6 +3322,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.9.9" @@ -3198,7 +3349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -3976,6 +4127,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "der 0.8.1", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http 1.5.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3994,6 +4175,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4159,6 +4346,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/crates/mem-core/Cargo.toml b/crates/mem-core/Cargo.toml index 59436e1..40ca688 100644 --- a/crates/mem-core/Cargo.toml +++ b/crates/mem-core/Cargo.toml @@ -2,6 +2,7 @@ name = "mem-core" version = "0.1.0" edition = "2021" +publish = false [dependencies] tokio = { workspace = true } @@ -15,3 +16,6 @@ sha2 = { workspace = true } tracing = { workspace = true } hex = "0.4" time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] } +magika = "1.1.0" +ort = { version = "2.0.0-rc.12", default-features = true } +regex = "1.10" diff --git a/crates/mem-core/src/lib.rs b/crates/mem-core/src/lib.rs index 3b77e01..2346387 100644 --- a/crates/mem-core/src/lib.rs +++ b/crates/mem-core/src/lib.rs @@ -6,6 +6,7 @@ pub mod prompt; pub mod gate_parser; pub mod gated_loop; pub mod query_executor; +pub mod optimizer; pub use gate_parser::{GateResponse, ParseError, parse_gate_response}; @@ -19,3 +20,4 @@ pub use lesson::{ pub use query::{Query, QuerySet, SynthesisQuery}; pub use prompt::{PromptBuilder, PromptMessages}; pub use symptom_projection::{project_symptom, SymptomVector}; +pub use optimizer::{ContextOptimizer, ContextOptimizerConfig, ContentType, OptimizedChunk}; diff --git a/crates/mem-core/src/optimizer/log.rs b/crates/mem-core/src/optimizer/log.rs new file mode 100644 index 0000000..97f177d --- /dev/null +++ b/crates/mem-core/src/optimizer/log.rs @@ -0,0 +1,259 @@ +//! LogCompressor — Build log compression using M3.7.7 patterns +//! +//! Reuses failure signature extraction from lesson.rs: +//! - markers() for error line detection +//! - is_cascade() for noise suppression +//! - strip_ansi() for ANSI code cleanup + +use anyhow::Result; + +pub struct LogCompressor; + +impl LogCompressor { + pub fn new() -> Self { + Self + } + + /// Compress logs by keeping errors, stack traces, and dropping noise + pub fn compress(&self, content: &str) -> Result { + // Step 1: Strip ANSI codes + let cleaned = strip_ansi(content); + + // Step 2: Filter to keep only meaningful lines + let lines: Vec<&str> = cleaned.lines().collect(); + let mut result_lines = Vec::new(); + + for (i, line) in lines.iter().enumerate() { + let trimmed = line.trim(); + + // Skip empty lines + if trimmed.is_empty() { + continue; + } + + // Always keep error lines + if is_error_line(line) { + result_lines.push(*line); + // Look ahead for stack trace lines + if i + 1 < lines.len() { + let next = lines[i + 1]; + if is_stack_line(next) { + result_lines.push(next); + } + } + continue; + } + + // Skip noisy patterns + if is_noise_line(trimmed) { + continue; + } + + // Keep important lines + if is_important_line(line) { + result_lines.push(*line); + } + } + + // Step 3: Remove cascade duplicates + let deduped = remove_cascade(result_lines); + + Ok(deduped.join("\n")) + } +} + +impl Default for LogCompressor { + fn default() -> Self { + Self::new() + } +} + +/// Strip ANSI color/style codes +fn strip_ansi(s: &str) -> String { + // Simple ANSI code removal: \x1b[...m patterns + let ansi_re = regex::Regex::new(r"\x1b\[[0-9;]*m").unwrap_or_else(|_| { + regex::Regex::new(r"\[0m").unwrap() + }); + ansi_re.replace_all(s, "").to_string() +} + +/// Check if line is an error marker +fn is_error_line(line: &str) -> bool { + let line_lower = line.to_lowercase(); + + line_lower.contains("error:") + || line_lower.contains("err!") + || line_lower.contains("panic:") + || line_lower.contains("fatal:") + || line_lower.contains("exception") + || line.contains("FAIL") + || line.contains("Error") + || line.contains("ERROR") + || line.contains("panic") + || line.contains("error[") + || line.contains("at line") +} + +/// Check if line is part of a stack trace (indented) +fn is_stack_line(line: &str) -> bool { + // Stack trace lines are usually indented and contain location info + (line.starts_with('\t') || line.starts_with(" ")) + && (line.contains("at ") + || line.contains("in file ") + || line.contains("line ") + || line.contains(".rs:") // Rust + || line.contains(".py:") // Python + || line.contains(".js:") // JavaScript + || line.contains(".go:") // Go + || line.contains(".java:")) // Java +} + +/// Check if line is noisy and should be dropped +fn is_noise_line(line: &str) -> bool { + let line_lower = line.to_lowercase(); + + // Drop common noise patterns + line_lower.contains("warn") + || line_lower.contains("notice") + || line_lower.contains("optional dependency") + || line_lower.contains("skipping") + || line_lower.contains("deprecated") + || line_lower.contains("compiling") + || line_lower.contains("finished") + || line_lower.contains("compiling") + || line_lower.contains("warning:") + || line_lower.contains("created a lockfile") + || line_lower.contains("you should commit") + || line.contains(">>>") +} + +/// Check if line contains important exit info +fn is_important_line(line: &str) -> bool { + let line_lower = line.to_lowercase(); + + line_lower.contains("exit code:") + || line_lower.contains("exited with") + || line_lower.contains("failed with") + || line_lower.contains("status:") + || line_lower.contains("error found") + || line_lower.contains("aborting due to") +} + +/// Remove cascade (repeated identical lines) +fn remove_cascade(lines: Vec<&str>) -> Vec { + let mut result = Vec::new(); + let mut prev = ""; + + for line in lines { + if !line.trim().is_empty() && line != prev { + result.push(line.to_string()); + prev = line; + } + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_strip_ansi() { + let input = "Normal text \x1b[31mRED\x1b[0m normal"; + let cleaned = strip_ansi(input); + assert!(!cleaned.contains("\x1b")); + } + + #[test] + fn test_error_line_detection() { + assert!(is_error_line("ERROR: connection timeout")); + assert!(is_error_line("panic: index out of bounds")); + assert!(is_error_line("npm ERR! 404 not found")); + assert!(!is_error_line("Info: compiling module")); + } + + #[test] + fn test_stack_line_detection() { + assert!(is_stack_line(" at main.rs:42")); + assert!(is_stack_line(" in file test.py line 10")); + assert!(is_stack_line("\tat String.js:5")); + assert!(!is_stack_line("Normal text")); + } + + #[test] + fn test_cascade_removal() { + let lines = vec!["error", "error", "warning", "warning", "info"]; + let result = remove_cascade(lines); + assert_eq!(result, vec!["error", "warning", "info"]); + } + + #[test] + fn test_compress_npm_error_log() { + let log = r#" +npm WARN deprecated info@0.8.0: package no longer supported +npm notice created a lockfile as package-lock.json. You should commit this file. +npm WARN optional SKIPPING OPTIONAL DEPENDENCY: fsevents@2.3.2 (node_modules/fsevents): +npm ERR! 404 Not Found - GET https://registry.npmjs.org/react@99.0.0 +npm ERR! 404 +npm ERR! 404 Not Found - GET https://registry.npmjs.org/react@99.0.0 +npm ERR! It is likely you are currently behind a proxy or have bad network connectivity. +npm ERR! If you are behind a proxy, please make sure that the proxy url is correct. +npm ERR! A complete log of this error can be found in: +npm ERR! /home/user/.npm/_logs/2026-08-28T09_15_00_000Z-debug-0.log + "#; + + let compressor = LogCompressor::new(); + let result = compressor.compress(log).unwrap(); + + // Should keep error lines + assert!(result.to_lowercase().contains("npm err!")); + assert!(result.contains("404")); + + // Should drop warning/info noise + assert!(!result.contains("WARN deprecated")); + assert!(!result.contains("notice created")); + + // Result should be smaller (some lines dropped) + assert!(result.len() < log.len()); + } + + #[test] + fn test_compress_rust_error_log() { + let log = r#" + Compiling myapp v0.1.0 (/home/user/myapp) +error[E0425]: cannot find value `unknown_var` in this scope + --> src/main.rs:42:5 + | + 42 | unknown_var = 42; + | ^^^^^^^^^^^ not found in this scope +error: aborting due to 1 previous error + +warning: build succeeded with warnings + "#; + + let compressor = LogCompressor::new(); + let result = compressor.compress(log).unwrap(); + + // Should keep error lines + assert!(result.contains("error[E0425]")); + assert!(result.contains("src/main.rs:42")); + + // Should drop compilation success noise + assert!(!result.contains("Compiling myapp")); + } + + #[test] + fn test_compression_ratio() { + let log = "INFO: Starting process\n".repeat(100) + + "ERROR: failed to connect\n" + + "ERROR: retry attempt 1\n" + + "ERROR: retry attempt 2\n"; + + let compressor = LogCompressor::new(); + let compressed = compressor.compress(&log).unwrap(); + + // Should be much smaller (dropped 100 INFO lines) + assert!(compressed.len() < log.len() / 10); + } +} diff --git a/crates/mem-core/src/optimizer/mod.rs b/crates/mem-core/src/optimizer/mod.rs new file mode 100644 index 0000000..673c986 --- /dev/null +++ b/crates/mem-core/src/optimizer/mod.rs @@ -0,0 +1,172 @@ +//! Context Optimizer — Pre-LLM compression pipeline +//! +//! Sits between hybrid search retrieval and LLM gateway. +//! Compresses evidence chunks to reduce token costs and stabilize KV cache hits. +//! Search indexes remain untouched at full fidelity. + +pub mod router; +pub mod log; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +pub use router::ContentRouter; +pub use log::LogCompressor; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OptimizedChunk { + /// Compressed content + pub compressed: String, + /// Original token count (estimated) + pub original_tokens: usize, + /// Compressed token count (estimated) + pub compressed_tokens: usize, + /// Detected content type + pub content_type: ContentType, + /// CCR hash for retrieving original (if compressed) + pub ccr_hash: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ContentType { + Json, + Code, + Log, + Diff, + Config, + Text, +} + +#[derive(Debug, Clone)] +pub struct ContextOptimizerConfig { + /// Enable/disable optimizer entirely + pub enabled: bool, + /// Enable Magika ML detection + pub use_magika: bool, + /// Confidence threshold for Magika (0.0-1.0) + pub magika_threshold: f32, + /// Per-compressor toggles + pub compress_json: bool, + pub compress_logs: bool, + pub compress_code: bool, + pub compress_diff: bool, + pub compress_text: bool, + /// Target token budget per chunk (0 = no budget) + pub token_budget: usize, + /// Enable CCR (Compress-Cache-Retrieve) + pub ccr_enabled: bool, +} + +impl Default for ContextOptimizerConfig { + fn default() -> Self { + Self { + enabled: true, + use_magika: true, + magika_threshold: 0.7, + compress_json: true, + compress_logs: true, + compress_code: false, + compress_diff: true, + compress_text: true, + token_budget: 0, + ccr_enabled: true, + } + } +} + +/// Main context optimizer +pub struct ContextOptimizer { + router: ContentRouter, + log_compressor: LogCompressor, + config: ContextOptimizerConfig, +} + +impl ContextOptimizer { + /// Create a new optimizer with default config + pub fn new() -> Result { + Self::with_config(ContextOptimizerConfig::default()) + } + + /// Create with custom config + pub fn with_config(config: ContextOptimizerConfig) -> Result { + let router = ContentRouter::new()?; + let log_compressor = LogCompressor::new(); + + Ok(Self { + router, + log_compressor, + config, + }) + } + + /// Optimize a chunk of content + pub fn optimize(&self, content: &str) -> Result { + if !self.config.enabled { + // Passthrough mode + let token_estimate = estimate_tokens(content); + return Ok(OptimizedChunk { + compressed: content.to_string(), + original_tokens: token_estimate, + compressed_tokens: token_estimate, + content_type: ContentType::Text, + ccr_hash: None, + }); + } + + // Detect content type + let content_type = self.router.detect(content)?; + + // Compress based on type + let compressed = match content_type { + ContentType::Log if self.config.compress_logs => { + self.log_compressor.compress(content)? + } + _ => content.to_string(), // TODO: Add other compressors + }; + + let original_tokens = estimate_tokens(content); + let compressed_tokens = estimate_tokens(&compressed); + + Ok(OptimizedChunk { + compressed, + original_tokens, + compressed_tokens, + content_type, + ccr_hash: None, // TODO: Implement CCR + }) + } +} + +impl Default for ContextOptimizer { + fn default() -> Self { + Self::new().expect("failed to create optimizer") + } +} + +/// Simple token estimation (1 token ~= 4 chars, 1 space-separated word) +fn estimate_tokens(text: &str) -> usize { + (text.len() / 4).max(text.split_whitespace().count()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_estimate() { + let text = "hello world this is a test"; + let tokens = estimate_tokens(text); + assert!(tokens > 0); + } + + #[test] + fn test_optimizer_passthrough_when_disabled() { + let config = ContextOptimizerConfig { + enabled: false, + ..Default::default() + }; + let optimizer = ContextOptimizer::with_config(config).unwrap(); + let chunk = optimizer.optimize("test content").unwrap(); + assert_eq!(chunk.compressed, "test content"); + } +} diff --git a/crates/mem-core/src/optimizer/router.rs b/crates/mem-core/src/optimizer/router.rs new file mode 100644 index 0000000..371692e --- /dev/null +++ b/crates/mem-core/src/optimizer/router.rs @@ -0,0 +1,227 @@ +//! Content Router — Magika ML + regex fallback detection + +use anyhow::Result; +use magika::Session; +use std::sync::Mutex; + +use super::ContentType; + +pub struct ContentRouter { + magika: Mutex, + confidence_threshold: f32, +} + +impl ContentRouter { + /// Create router with default Magika session + pub fn new() -> Result { + let magika = Session::new()?; + Ok(Self { + magika: Mutex::new(magika), + confidence_threshold: 0.7, + }) + } + + /// Detect content type using Magika ML first, then regex fallback + pub fn detect(&self, content: &str) -> Result { + // Try Magika ML classification + let mut magika = self.magika.lock().map_err(|e| anyhow::anyhow!("mutex lock failed: {}", e))?; + if let Ok(result) = magika.identify_content_sync(content.as_bytes()) { + let label = result.info().label; + let score = result.score(); + + if score >= self.confidence_threshold { + if let Some(ct) = self.map_magika_label(label) { + return Ok(ct); + } + } + } + + // Fallback to regex heuristics + Ok(self.regex_fallback(content)) + } + + /// Map Magika label to our ContentType + fn map_magika_label(&self, label: &str) -> Option { + match label { + "json" | "jsonl" => Some(ContentType::Json), + "python" + | "javascript" + | "typescript" + | "rust" + | "go" + | "shell" + | "bash" + | "java" + | "cpp" + | "csharp" + | "sql" => Some(ContentType::Code), + "diff" | "patch" => Some(ContentType::Diff), + "yaml" | "toml" | "ini" | "xml" => Some(ContentType::Config), + "markdown" | "txt" => None, // Fallback to regex for better detection + _ => None, + } + } + + /// Regex-based fallback detection + fn regex_fallback(&self, content: &str) -> ContentType { + if is_json(content) { + return ContentType::Json; + } + if is_log(content) { + return ContentType::Log; + } + if is_diff(content) { + return ContentType::Diff; + } + if is_code(content) { + return ContentType::Code; + } + ContentType::Text + } +} + +/// Check if content is valid JSON +fn is_json(content: &str) -> bool { + let trimmed = content.trim(); + if !((trimmed.starts_with('{') || trimmed.starts_with('['))) { + return false; + } + serde_json::from_str::(trimmed).is_ok() +} + +/// Check if content looks like logs (timestamps, log levels, errors) +fn is_log(content: &str) -> bool { + let content_lower = content.to_lowercase(); + + // Log level markers + let has_log_level = content_lower.contains("error:") + || content_lower.contains("warn:") + || content_lower.contains("info:") + || content_lower.contains("debug:") + || content_lower.contains("err!") + || content.contains("ERROR") + || content.contains("WARN") + || content.contains("INFO") + || content.contains("FAIL"); + + // ISO timestamp pattern + let has_timestamp = regex_contains( + content, + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", + ); + + // Common error markers + let has_error_markers = content_lower.contains("exception") + || content_lower.contains("stack trace") + || content_lower.contains("at line") + || content.contains("npm ERR!") + || content.contains("error[") + || content.contains("panic:"); + + has_log_level || (has_timestamp && has_error_markers) +} + +/// Check if content looks like a unified diff +fn is_diff(content: &str) -> bool { + let has_diff_markers = content.contains("---") && content.contains("+++") + || content.contains("@@"); + + has_diff_markers && (content.contains("+") || content.contains("-")) +} + +/// Check if content looks like source code +fn is_code(content: &str) -> bool { + let lines: Vec<&str> = content.lines().collect(); + + // Check for code-like patterns + let import_use_pattern = + regex_contains(content, r"^(import|use|require|include|from|package|class|def|fn|public|private|const|let|var|function)\b"); + let has_brackets = + content.contains('{') && content.contains('}') || content.contains('[') && content.contains(']'); + let has_indentation = lines.iter().any(|line| line.starts_with('\t') || line.starts_with(" ")); + + (import_use_pattern || has_brackets) && has_indentation +} + +/// Helper to check if text contains a regex pattern +fn regex_contains(text: &str, pattern: &str) -> bool { + if let Ok(re) = regex::Regex::new(pattern) { + re.is_match(text) + } else { + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_json_detection() { + let json = r#"{"key": "value", "number": 42}"#; + assert!(is_json(json)); + } + + #[test] + fn test_json_array_detection() { + let json = r#"[1, 2, 3, {"nested": true}]"#; + assert!(is_json(json)); + } + + #[test] + fn test_invalid_json_rejected() { + let not_json = r#"{"key": "value", invalid}"#; + assert!(!is_json(not_json)); + } + + #[test] + fn test_log_detection_with_error_level() { + let log = "2026-08-28T09:15:00Z ERROR: connection timeout"; + assert!(is_log(log)); + } + + #[test] + fn test_log_detection_with_npm_err() { + let log = "npm ERR! 404 Not Found - react@18.2.5"; + assert!(is_log(log)); + } + + #[test] + fn test_diff_detection() { + let diff = "--- a/file.txt\n+++ b/file.txt\n@@ -1,3 +1,3 @@\n-old\n+new"; + assert!(is_diff(diff)); + } + + #[test] + fn test_code_detection() { + let code = r#" +fn main() { + println!("hello"); +} + "#; + assert!(is_code(code)); + } + + #[test] + fn test_text_detection_fallback() { + let text = "This is just plain English text without any special structure."; + assert_eq!(regex_fallback_helper(text), ContentType::Text); + } +} + +// Helper for test fallback +fn regex_fallback_helper(content: &str) -> ContentType { + if is_json(content) { + return ContentType::Json; + } + if is_log(content) { + return ContentType::Log; + } + if is_diff(content) { + return ContentType::Diff; + } + if is_code(content) { + return ContentType::Code; + } + ContentType::Text +} diff --git a/k8s/infra/databases/opensearch.yaml b/k8s/infra/databases/opensearch.yaml index bd35eba..03288b8 100644 --- a/k8s/infra/databases/opensearch.yaml +++ b/k8s/infra/databases/opensearch.yaml @@ -285,9 +285,6 @@ data: opensearch.password: "admin" opensearch.ssl.verificationMode: none - # Dashboards index - opensearch_dashboards.index: ".opensearch_dashboards" - # Logging logging.appenders.default.type: console logging.appenders.default.layout.type: pattern @@ -355,7 +352,7 @@ spec: # Liveness probe livenessProbe: httpGet: - path: /api/status + path: / port: 5601 initialDelaySeconds: 30 periodSeconds: 10 @@ -365,7 +362,7 @@ spec: # Readiness probe readinessProbe: httpGet: - path: /api/status + path: / port: 5601 initialDelaySeconds: 15 periodSeconds: 5