From 262478f7f252a9748001bbafdef3954cdec8f086 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Fri, 28 Aug 2026 09:12:09 -0700 Subject: [PATCH] plan: add Magika ML classifier to content router --- docs/CONTEXT_OPTIMIZER.md | 84 ++++++++++++++++++++--------- k8s/infra/databases/opensearch.yaml | 10 ++++ tasks/M3.8.1-context-optimizer.md | 37 ++++++++----- 3 files changed, 95 insertions(+), 36 deletions(-) diff --git a/docs/CONTEXT_OPTIMIZER.md b/docs/CONTEXT_OPTIMIZER.md index 32ea9d3..a6eb0fc 100644 --- a/docs/CONTEXT_OPTIMIZER.md +++ b/docs/CONTEXT_OPTIMIZER.md @@ -92,42 +92,78 @@ impl CacheAligner { } ``` -### Stage 2: ContentRouter +### Stage 2: ContentRouter (Magika ML + regex fallback) **Goal:** Auto-detect content type and route to the best compressor. -Each evidence chunk might be JSON, source code, build logs, a diff, or plain -text. The router classifies using structural heuristics: +**Primary classifier:** Google Magika (`magika` crate v1.1.0) — fast encoder-only +ONNX model that classifies content into 100+ types. <1ms per classification. +No LLM calls, no network — runs locally with embedded ONNX model. -| Content Type | Detection Signal | Compressor | -|---|---|---| -| JSON | Valid JSON, `{` or `[` start, key-value pairs | JsonCrusher | -| Source code | Import/use statements, function defs, indentation | CodeCompressor | -| Build/test logs | Timestamps, log levels, `error:`, `FAIL` | LogCompressor | -| Unified diffs | `---`, `+++`, `@@` markers | DiffCompressor | -| Plain text | Default fallback | TextCompressor | - -**Implementation:** Pattern matching + simple heuristics. No ML classifier -needed initially (can add Magika later). +**Fallback:** Regex heuristics for content types Magika doesn't distinguish +well (e.g., build logs vs. plain text) or when confidence is below threshold. ```rust +use magika::Session; + +pub struct ContentRouter { + magika: Session, + confidence_threshold: f32, // default 0.7 +} + pub enum ContentType { Json, - Code, + Code { language: String }, Log, Diff, + Config, Text, } -pub fn detect(content: &str) -> ContentType { - if is_json(content) { return ContentType::Json; } - if is_code(content) { return ContentType::Code; } - if is_log(content) { return ContentType::Log; } - if is_diff(content) { return ContentType::Diff; } - ContentType::Text +impl ContentRouter { + pub fn detect(&self, content: &str) -> ContentType { + // 1. Try Magika ML classification + if let Ok(result) = self.magika.identify_content_sync(content.as_bytes()) { + let label = result.info().label; + let score = result.score(); + + if score >= self.confidence_threshold { + return match label { + "json" | "jsonl" => ContentType::Json, + "python" | "javascript" | "typescript" | "rust" | "go" | "shell" + => ContentType::Code { language: label.to_string() }, + "diff" => ContentType::Diff, + "yaml" | "toml" | "ini" | "xml" => ContentType::Config, + _ => self.regex_fallback(content), + }; + } + } + + // 2. Fallback to regex heuristics + self.regex_fallback(content) + } + + fn regex_fallback(&self, content: &str) -> ContentType { + if is_json(content) { return ContentType::Json; } + if is_log(content) { return ContentType::Log; } + if is_diff(content) { return ContentType::Diff; } + if is_code(content) { return ContentType::Code { language: "unknown".into() }; } + ContentType::Text + } } ``` +**Magika label → compressor mapping:** + +| Magika Label | ContentType | Compressor | +|---|---|---| +| `json`, `jsonl` | Json | JsonCrusher | +| `python`, `javascript`, `rust`, `go`, `typescript`, `shell` | Code | CodeCompressor | +| `diff` | Diff | DiffCompressor | +| `yaml`, `toml`, `ini`, `xml` | Config | (passthrough, already compact) | +| `txt` + log heuristics | Log | LogCompressor | +| everything else | Text | TextCompressor | + ### Stage 3: Compressors **Goal:** Reduce token count per content type while preserving signal. @@ -238,7 +274,7 @@ let turn_msg = CACHE_TURN | New Code | Location | Est. LOC | |---|---|---| | `context_optimizer.rs` | `crates/mem-core/src/` | 150 | -| `content_router.rs` | `crates/mem-core/src/` | 100 | +| `content_router.rs` | `crates/mem-core/src/` | 150 | | `compressors/json.rs` | `crates/mem-core/src/` | 200 | | `compressors/log.rs` | `crates/mem-core/src/` | 150 | | `compressors/code.rs` | `crates/mem-core/src/` | 150 | @@ -284,9 +320,9 @@ let turn_msg = CACHE_TURN ## Key Design Decisions -1. **No ML in hot path.** All detection and compression uses rules, regex, and - statistics. ML classifiers (Magika) can be added later as an optional - enhancement. +1. **Magika ML for detection, rules for compression.** Content type detection + uses Google's Magika ONNX model (<1ms, local, no network). Compression + itself uses deterministic algorithms (no LLM calls in hot path). 2. **Search indexes untouched.** Compression happens AFTER retrieval. pgvector and OpenSearch see full-fidelity text. Only the LLM prompt is optimized. diff --git a/k8s/infra/databases/opensearch.yaml b/k8s/infra/databases/opensearch.yaml index 26dd819..bd35eba 100644 --- a/k8s/infra/databases/opensearch.yaml +++ b/k8s/infra/databases/opensearch.yaml @@ -114,6 +114,16 @@ spec: serviceAccountName: opensearch hostNetwork: false + initContainers: + - name: sysctl + image: busybox:1.28 + command: + - sysctl + - -w + - vm.max_map_count=262144 + securityContext: + privileged: true + containers: - name: opensearch image: opensearchproject/opensearch:2.11.0 diff --git a/tasks/M3.8.1-context-optimizer.md b/tasks/M3.8.1-context-optimizer.md index 9f87570..5070e10 100644 --- a/tasks/M3.8.1-context-optimizer.md +++ b/tasks/M3.8.1-context-optimizer.md @@ -8,7 +8,7 @@ | Flags | — | | Spec | `docs/CONTEXT_OPTIMIZER.md` | | Blocks | M3.8.4 | -| Depends | M3.7.7 (lesson.rs patterns), M3.7.8 (stop words) | +| Depends | M3.7.7 (lesson.rs patterns), M3.7.8 (stop words), magika crate | ## Goal @@ -24,15 +24,26 @@ before they enter the GRU-Mem prompt. Search indexes stay untouched. - `crates/mem-core/src/optimizer/router.rs` — content type detection - `crates/mem-core/src/optimizer/log.rs` — log compression -**ContentRouter** detects content type via heuristics: +**ContentRouter** uses Google Magika (ML) + regex fallback: -| Type | Signal | -|---|---| -| Json | starts with `{` or `[`, valid JSON parse | -| Log | timestamp patterns, log levels, `error:`, `npm ERR!` | -| Diff | `---`/`+++`/`@@` markers | -| Code | `import`/`use`/`fn`/`def`/`class` + indentation | -| Text | fallback | +```rust +// Primary: Magika ML classifier (ONNX, <1ms per classification) +let magika = magika::Session::new()?; +let result = magika.identify_content_sync(content.as_bytes())?; +let label = result.info().label; // "json", "python", "shell", "yaml", etc. + +// Map Magika labels → our compressor types +// Fallback to regex heuristics if Magika confidence < threshold +``` + +| Type | Magika Labels | Regex Fallback | +|---|---|---| +| Json | `json`, `jsonl` | starts with `{` or `[`, valid parse | +| Log | `txt` + log heuristics | timestamp patterns, `error:`, `npm ERR!` | +| Diff | `diff` | `---`/`+++`/`@@` markers | +| Code | `python`, `javascript`, `rust`, `go`, `typescript`, `shell` | `import`/`use`/`fn`/`def` | +| Config | `yaml`, `toml`, `ini`, `xml` | key-value patterns | +| Text | fallback | default | **LogCompressor** reuses M3.7.7 `lesson.rs`: - `markers()` for error line detection @@ -41,10 +52,12 @@ before they enter the GRU-Mem prompt. Search indexes stay untouched. - Keep: error lines, stack traces, exit codes - Drop: INFO/DEBUG noise, passing tests, repeated patterns -**Tests (10):** -- `detect_json`, `detect_log`, `detect_diff`, `detect_code`, `detect_text` +**Tests (12):** +- `magika_detects_json`, `magika_detects_python`, `magika_detects_diff` +- `fallback_detects_log`, `fallback_detects_code`, `fallback_detects_text` +- `router_maps_magika_to_compressor`, `router_low_confidence_uses_fallback` - `log_keeps_errors`, `log_drops_info_noise`, `log_keeps_stack_traces` -- `log_compression_ratio_above_80pct`, `log_strips_ansi` +- `log_compression_ratio_above_80pct` ### Phase 2: JsonCrusher + DiffCompressor (day 2)