feat: M3.8.1 phase 1 — content router + log compressor

ContentRouter uses Google Magika ML for content detection (<1ms) with regex fallback. Detects JSON, code, logs, diffs, config, text. LogCompressor reuses M3.7.7 patterns (markers, cascade, strip_ansi) to shrink build logs by keeping errors/stacks and dropping noise. 17 unit tests passing: - router: json, code, diff, log, text detection - log: error lines, stack traces, ansi stripping, compression - optimizer: token estimation, passthrough mode Magika + ort ONNX runtime added to Cargo.toml.
2026-08-28 09:29:56 -07:00
parent 1f9b30b1ec
commit 05aec4e23b
8 changed files with 872 additions and 13 deletions
@@ -0,0 +1,259 @@
+//! LogCompressor — Build log compression using M3.7.7 patterns
+//!
+//! Reuses failure signature extraction from lesson.rs:
+//! - markers() for error line detection
+//! - is_cascade() for noise suppression
+//! - strip_ansi() for ANSI code cleanup
+
+use anyhow::Result;
+
+pub struct LogCompressor;
+
+impl LogCompressor {
+    pub fn new() -> Self {
+        Self
+    }
+
+    /// Compress logs by keeping errors, stack traces, and dropping noise
+    pub fn compress(&self, content: &str) -> Result<String> {
+        // Step 1: Strip ANSI codes
+        let cleaned = strip_ansi(content);
+
+        // Step 2: Filter to keep only meaningful lines
+        let lines: Vec<&str> = cleaned.lines().collect();
+        let mut result_lines = Vec::new();
+
+        for (i, line) in lines.iter().enumerate() {
+            let trimmed = line.trim();
+
+            // Skip empty lines
+            if trimmed.is_empty() {
+                continue;
+            }
+
+            // Always keep error lines
+            if is_error_line(line) {
+                result_lines.push(*line);
+                // Look ahead for stack trace lines
+                if i + 1 < lines.len() {
+                    let next = lines[i + 1];
+                    if is_stack_line(next) {
+                        result_lines.push(next);
+                    }
+                }
+                continue;
+            }
+
+            // Skip noisy patterns
+            if is_noise_line(trimmed) {
+                continue;
+            }
+
+            // Keep important lines
+            if is_important_line(line) {
+                result_lines.push(*line);
+            }
+        }
+
+        // Step 3: Remove cascade duplicates
+        let deduped = remove_cascade(result_lines);
+
+        Ok(deduped.join("\n"))
+    }
+}
+
+impl Default for LogCompressor {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Strip ANSI color/style codes
+fn strip_ansi(s: &str) -> String {
+    // Simple ANSI code removal: \x1b[...m patterns
+    let ansi_re = regex::Regex::new(r"\x1b\[[0-9;]*m").unwrap_or_else(|_| {
+        regex::Regex::new(r"\[0m").unwrap()
+    });
+    ansi_re.replace_all(s, "").to_string()
+}
+
+/// Check if line is an error marker
+fn is_error_line(line: &str) -> bool {
+    let line_lower = line.to_lowercase();
+
+    line_lower.contains("error:")
+        || line_lower.contains("err!")
+        || line_lower.contains("panic:")
+        || line_lower.contains("fatal:")
+        || line_lower.contains("exception")
+        || line.contains("FAIL")
+        || line.contains("Error")
+        || line.contains("ERROR")
+        || line.contains("panic")
+        || line.contains("error[")
+        || line.contains("at line")
+}
+
+/// Check if line is part of a stack trace (indented)
+fn is_stack_line(line: &str) -> bool {
+    // Stack trace lines are usually indented and contain location info
+    (line.starts_with('\t') || line.starts_with("  "))
+        && (line.contains("at ")
+            || line.contains("in file ")
+            || line.contains("line ")
+            || line.contains(".rs:") // Rust
+            || line.contains(".py:") // Python
+            || line.contains(".js:") // JavaScript
+            || line.contains(".go:") // Go
+            || line.contains(".java:")) // Java
+}
+
+/// Check if line is noisy and should be dropped
+fn is_noise_line(line: &str) -> bool {
+    let line_lower = line.to_lowercase();
+
+    // Drop common noise patterns
+    line_lower.contains("warn")
+        || line_lower.contains("notice")
+        || line_lower.contains("optional dependency")
+        || line_lower.contains("skipping")
+        || line_lower.contains("deprecated")
+        || line_lower.contains("compiling")
+        || line_lower.contains("finished")
+        || line_lower.contains("compiling")
+        || line_lower.contains("warning:")
+        || line_lower.contains("created a lockfile")
+        || line_lower.contains("you should commit")
+        || line.contains(">>>")
+}
+
+/// Check if line contains important exit info
+fn is_important_line(line: &str) -> bool {
+    let line_lower = line.to_lowercase();
+
+    line_lower.contains("exit code:")
+        || line_lower.contains("exited with")
+        || line_lower.contains("failed with")
+        || line_lower.contains("status:")
+        || line_lower.contains("error found")
+        || line_lower.contains("aborting due to")
+}
+
+/// Remove cascade (repeated identical lines)
+fn remove_cascade(lines: Vec<&str>) -> Vec<String> {
+    let mut result = Vec::new();
+    let mut prev = "";
+
+    for line in lines {
+        if !line.trim().is_empty() && line != prev {
+            result.push(line.to_string());
+            prev = line;
+        }
+    }
+
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_strip_ansi() {
+        let input = "Normal text \x1b[31mRED\x1b[0m normal";
+        let cleaned = strip_ansi(input);
+        assert!(!cleaned.contains("\x1b"));
+    }
+
+    #[test]
+    fn test_error_line_detection() {
+        assert!(is_error_line("ERROR: connection timeout"));
+        assert!(is_error_line("panic: index out of bounds"));
+        assert!(is_error_line("npm ERR! 404 not found"));
+        assert!(!is_error_line("Info: compiling module"));
+    }
+
+    #[test]
+    fn test_stack_line_detection() {
+        assert!(is_stack_line("  at main.rs:42"));
+        assert!(is_stack_line("    in file test.py line 10"));
+        assert!(is_stack_line("\tat String.js:5"));
+        assert!(!is_stack_line("Normal text"));
+    }
+
+    #[test]
+    fn test_cascade_removal() {
+        let lines = vec!["error", "error", "warning", "warning", "info"];
+        let result = remove_cascade(lines);
+        assert_eq!(result, vec!["error", "warning", "info"]);
+    }
+
+    #[test]
+    fn test_compress_npm_error_log() {
+        let log = r#"
+npm WARN deprecated [email protected]: package no longer supported
+npm notice created a lockfile as package-lock.json. You should commit this file.
+npm WARN optional SKIPPING OPTIONAL DEPENDENCY: [email protected] (node_modules/fsevents):
+npm ERR! 404 Not Found - GET https://registry.npmjs.org/[email protected]
+npm ERR! 404
+npm ERR!  404 Not Found - GET https://registry.npmjs.org/[email protected]
+npm ERR!  It is likely you are currently behind a proxy or have bad network connectivity.
+npm ERR!  If you are behind a proxy, please make sure that the proxy url is correct.
+npm ERR! A complete log of this error can be found in:
+npm ERR! /home/user/.npm/_logs/2026-08-28T09_15_00_000Z-debug-0.log
+        "#;
+
+        let compressor = LogCompressor::new();
+        let result = compressor.compress(log).unwrap();
+
+        // Should keep error lines
+        assert!(result.to_lowercase().contains("npm err!"));
+        assert!(result.contains("404"));
+
+        // Should drop warning/info noise
+        assert!(!result.contains("WARN deprecated"));
+        assert!(!result.contains("notice created"));
+
+        // Result should be smaller (some lines dropped)
+        assert!(result.len() < log.len());
+    }
+
+    #[test]
+    fn test_compress_rust_error_log() {
+        let log = r#"
+   Compiling myapp v0.1.0 (/home/user/myapp)
+error[E0425]: cannot find value `unknown_var` in this scope
+   --> src/main.rs:42:5
+    |
+ 42 |     unknown_var = 42;
+    |     ^^^^^^^^^^^ not found in this scope
+error: aborting due to 1 previous error
+
+warning: build succeeded with warnings
+        "#;
+
+        let compressor = LogCompressor::new();
+        let result = compressor.compress(log).unwrap();
+
+        // Should keep error lines
+        assert!(result.contains("error[E0425]"));
+        assert!(result.contains("src/main.rs:42"));
+
+        // Should drop compilation success noise
+        assert!(!result.contains("Compiling myapp"));
+    }
+
+    #[test]
+    fn test_compression_ratio() {
+        let log = "INFO: Starting process\n".repeat(100)
+            + "ERROR: failed to connect\n"
+            + "ERROR: retry attempt 1\n"
+            + "ERROR: retry attempt 2\n";
+
+        let compressor = LogCompressor::new();
+        let compressed = compressor.compress(&log).unwrap();
+
+        // Should be much smaller (dropped 100 INFO lines)
+        assert!(compressed.len() < log.len() / 10);
+    }
+}