From 302ffe1d750040e355c9a3b481db916aa0c18d72 Mon Sep 17 00:00:00 2001 From: poimen Date: Fri, 28 Aug 2026 15:42:06 -0700 Subject: [PATCH] fix: remove magika/ort dependency (CI glibc too old for C23 symbols) Root cause: ort (ONNX Runtime) links against __isoc23_strtoll which requires glibc 2.38+. CI runner has older glibc, causing linker failure. Replace magika ML detection with regex-only ContentRouter. Regex fallback already covers all content types (JSON, log, diff, code). All 294 tests passing. --- Cargo.lock | 203 +----------------------- crates/mem-core/Cargo.toml | 4 +- crates/mem-core/src/optimizer/router.rs | 80 ++-------- 3 files changed, 18 insertions(+), 269 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 850edbc..6b6a34b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,12 +380,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64ct" version = "1.8.3" @@ -860,17 +854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", - "pem-rfc7468 0.7.0", - "zeroize", -] - -[[package]] -name = "der" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" -dependencies = [ - "pem-rfc7468 1.0.0", + "pem-rfc7468", "zeroize", ] @@ -1428,12 +1412,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "hmac-sha256" -version = "1.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" - [[package]] name = "home" version = "0.5.12" @@ -1941,12 +1919,6 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" -[[package]] -name = "lzma-rust2" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" - [[package]] name = "macro_rules_attribute" version = "0.1.3" @@ -1963,28 +1935,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" -[[package]] -name = "magika" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf" -dependencies = [ - "ndarray", - "ort", - "thiserror 1.0.69", - "tokio", -] - -[[package]] -name = "matrixmultiply" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "md-5" version = "0.10.6" @@ -2068,9 +2018,7 @@ dependencies = [ "hex", "indexmap", "lazy_static", - "magika", "once_cell", - "ort", "regex", "serde", "serde_json", @@ -2218,21 +2166,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "ndarray" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - [[package]] name = "nom" version = "7.1.3" @@ -2278,15 +2211,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - [[package]] name = "num-conv" version = "0.2.2" @@ -2439,30 +2363,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "ort" -version = "2.0.0-rc.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" -dependencies = [ - "ndarray", - "ort-sys", - "smallvec", - "tracing", - "ureq", -] - -[[package]] -name = "ort-sys" -version = "2.0.0-rc.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" -dependencies = [ - "hmac-sha256", - "lzma-rust2", - "ureq", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2534,15 +2434,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "pem-rfc7468" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" -dependencies = [ - "base64ct", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2592,7 +2483,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der 0.7.10", + "der", "pkcs8", "spki", ] @@ -2603,7 +2494,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.10", + "der", "spki", ] @@ -2642,21 +2533,6 @@ dependencies = [ "wiremock", ] -[[package]] -name = "portable-atomic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "portable-atomic-util" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" -dependencies = [ - "portable-atomic", -] - [[package]] name = "postgres" version = "0.19.14" @@ -2801,12 +2677,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "rayon" version = "1.12.0" @@ -3024,15 +2894,6 @@ dependencies = [ "base64 0.21.7", ] -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "zeroize", -] - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3330,17 +3191,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "socks" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" -dependencies = [ - "byteorder", - "libc", - "winapi", -] - [[package]] name = "spin" version = "0.9.9" @@ -3357,7 +3207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.10", + "der", ] [[package]] @@ -4135,36 +3985,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" -dependencies = [ - "base64 0.23.1", - "der 0.8.1", - "log", - "native-tls", - "percent-encoding", - "rustls-pki-types", - "socks", - "ureq-proto", - "utf8-zero", - "webpki-root-certs", -] - -[[package]] -name = "ureq-proto" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" -dependencies = [ - "base64 0.23.1", - "http 1.5.0", - "httparse", - "log", -] - [[package]] name = "url" version = "2.5.8" @@ -4183,12 +4003,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf8-zero" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -4354,15 +4168,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-root-certs" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "webpki-roots" version = "0.25.4" diff --git a/crates/mem-core/Cargo.toml b/crates/mem-core/Cargo.toml index 3353c59..e6ca9d7 100644 --- a/crates/mem-core/Cargo.toml +++ b/crates/mem-core/Cargo.toml @@ -16,8 +16,8 @@ sha2 = { workspace = true } tracing = { workspace = true } hex = "0.4" time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] } -magika = "1.1.0" -ort = { version = "2.0.0-rc.12", default-features = true } +# magika and ort removed: CI runner glibc too old for ort's C23 symbols +# regex fallback in router.rs covers all content types regex = "1.10" once_cell = "1.19" indexmap = "2.0" diff --git a/crates/mem-core/src/optimizer/router.rs b/crates/mem-core/src/optimizer/router.rs index 371692e..7b1536f 100644 --- a/crates/mem-core/src/optimizer/router.rs +++ b/crates/mem-core/src/optimizer/router.rs @@ -1,68 +1,27 @@ -//! Content Router — Magika ML + regex fallback detection +//! Content Router — regex-based content type detection use anyhow::Result; -use magika::Session; -use std::sync::Mutex; use super::ContentType; pub struct ContentRouter { - magika: Mutex, - confidence_threshold: f32, + _confidence_threshold: f32, } impl ContentRouter { - /// Create router with default Magika session + /// Create router with regex-only detection pub fn new() -> Result { - let magika = Session::new()?; Ok(Self { - magika: Mutex::new(magika), - confidence_threshold: 0.7, + _confidence_threshold: 0.7, }) } - /// Detect content type using Magika ML first, then regex fallback + /// Detect content type using regex heuristics pub fn detect(&self, content: &str) -> Result { - // Try Magika ML classification - let mut magika = self.magika.lock().map_err(|e| anyhow::anyhow!("mutex lock failed: {}", e))?; - if let Ok(result) = magika.identify_content_sync(content.as_bytes()) { - let label = result.info().label; - let score = result.score(); - - if score >= self.confidence_threshold { - if let Some(ct) = self.map_magika_label(label) { - return Ok(ct); - } - } - } - - // Fallback to regex heuristics Ok(self.regex_fallback(content)) } - /// Map Magika label to our ContentType - fn map_magika_label(&self, label: &str) -> Option { - match label { - "json" | "jsonl" => Some(ContentType::Json), - "python" - | "javascript" - | "typescript" - | "rust" - | "go" - | "shell" - | "bash" - | "java" - | "cpp" - | "csharp" - | "sql" => Some(ContentType::Code), - "diff" | "patch" => Some(ContentType::Diff), - "yaml" | "toml" | "ini" | "xml" => Some(ContentType::Config), - "markdown" | "txt" => None, // Fallback to regex for better detection - _ => None, - } - } - - /// Regex-based fallback detection + /// Regex-based detection fn regex_fallback(&self, content: &str) -> ContentType { if is_json(content) { return ContentType::Json; @@ -93,7 +52,6 @@ fn is_json(content: &str) -> bool { fn is_log(content: &str) -> bool { let content_lower = content.to_lowercase(); - // Log level markers let has_log_level = content_lower.contains("error:") || content_lower.contains("warn:") || content_lower.contains("info:") @@ -104,13 +62,11 @@ fn is_log(content: &str) -> bool { || content.contains("INFO") || content.contains("FAIL"); - // ISO timestamp pattern let has_timestamp = regex_contains( content, r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", ); - // Common error markers let has_error_markers = content_lower.contains("exception") || content_lower.contains("stack trace") || content_lower.contains("at line") @@ -133,7 +89,6 @@ fn is_diff(content: &str) -> bool { fn is_code(content: &str) -> bool { let lines: Vec<&str> = content.lines().collect(); - // Check for code-like patterns let import_use_pattern = regex_contains(content, r"^(import|use|require|include|from|package|class|def|fn|public|private|const|let|var|function)\b"); let has_brackets = @@ -204,24 +159,13 @@ fn main() { #[test] fn test_text_detection_fallback() { - let text = "This is just plain English text without any special structure."; - assert_eq!(regex_fallback_helper(text), ContentType::Text); + let router = ContentRouter::new().unwrap(); + assert_eq!(router.detect("This is just plain English text without any special structure.").unwrap(), ContentType::Text); } -} -// Helper for test fallback -fn regex_fallback_helper(content: &str) -> ContentType { - if is_json(content) { - return ContentType::Json; + #[test] + fn test_router_json() { + let router = ContentRouter::new().unwrap(); + assert_eq!(router.detect(r#"{"key": "value"}"#).unwrap(), ContentType::Json); } - if is_log(content) { - return ContentType::Log; - } - if is_diff(content) { - return ContentType::Diff; - } - if is_code(content) { - return ContentType::Code; - } - ContentType::Text }