fix: remove magika/ort dependency (CI glibc too old for C23 symbols)
Root cause: ort (ONNX Runtime) links against __isoc23_strtoll which requires glibc 2.38+. CI runner has older glibc, causing linker failure. Replace magika ML detection with regex-only ContentRouter. Regex fallback already covers all content types (JSON, log, diff, code). All 294 tests passing.
This commit is contained in:
Generated
+4
-199
@@ -380,12 +380,6 @@ version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.23.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||
|
||||
[[package]]
|
||||
name = "base64ct"
|
||||
version = "1.8.3"
|
||||
@@ -860,17 +854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
||||
dependencies = [
|
||||
"const-oid 0.9.6",
|
||||
"pem-rfc7468 0.7.0",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||
dependencies = [
|
||||
"pem-rfc7468 1.0.0",
|
||||
"pem-rfc7468",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -1428,12 +1412,6 @@ dependencies = [
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hmac-sha256"
|
||||
version = "1.1.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f"
|
||||
|
||||
[[package]]
|
||||
name = "home"
|
||||
version = "0.5.12"
|
||||
@@ -1941,12 +1919,6 @@ version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619"
|
||||
|
||||
[[package]]
|
||||
name = "macro_rules_attribute"
|
||||
version = "0.1.3"
|
||||
@@ -1963,28 +1935,6 @@ version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
||||
|
||||
[[package]]
|
||||
name = "magika"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf"
|
||||
dependencies = [
|
||||
"ndarray",
|
||||
"ort",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md-5"
|
||||
version = "0.10.6"
|
||||
@@ -2068,9 +2018,7 @@ dependencies = [
|
||||
"hex",
|
||||
"indexmap",
|
||||
"lazy_static",
|
||||
"magika",
|
||||
"once_cell",
|
||||
"ort",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -2218,21 +2166,6 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ndarray"
|
||||
version = "0.17.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
|
||||
dependencies = [
|
||||
"matrixmultiply",
|
||||
"num-complex",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"portable-atomic",
|
||||
"portable-atomic-util",
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
@@ -2278,15 +2211,6 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-complex"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
@@ -2439,30 +2363,6 @@ dependencies = [
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ort"
|
||||
version = "2.0.0-rc.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133"
|
||||
dependencies = [
|
||||
"ndarray",
|
||||
"ort-sys",
|
||||
"smallvec",
|
||||
"tracing",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ort-sys"
|
||||
version = "2.0.0-rc.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90"
|
||||
dependencies = [
|
||||
"hmac-sha256",
|
||||
"lzma-rust2",
|
||||
"ureq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.5"
|
||||
@@ -2534,15 +2434,6 @@ dependencies = [
|
||||
"base64ct",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "percent-encoding"
|
||||
version = "2.3.2"
|
||||
@@ -2592,7 +2483,7 @@ version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"der",
|
||||
"pkcs8",
|
||||
"spki",
|
||||
]
|
||||
@@ -2603,7 +2494,7 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||
dependencies = [
|
||||
"der 0.7.10",
|
||||
"der",
|
||||
"spki",
|
||||
]
|
||||
|
||||
@@ -2642,21 +2533,6 @@ dependencies = [
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||
|
||||
[[package]]
|
||||
name = "portable-atomic-util"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "postgres"
|
||||
version = "0.19.14"
|
||||
@@ -2801,12 +2677,6 @@ version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||
|
||||
[[package]]
|
||||
name = "rawpointer"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.12.0"
|
||||
@@ -3024,15 +2894,6 @@ dependencies = [
|
||||
"base64 0.21.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.101.7"
|
||||
@@ -3330,17 +3191,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socks"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.9"
|
||||
@@ -3357,7 +3207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"der 0.7.10",
|
||||
"der",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4135,36 +3985,6 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "ureq"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"der 0.8.1",
|
||||
"log",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"rustls-pki-types",
|
||||
"socks",
|
||||
"ureq-proto",
|
||||
"utf8-zero",
|
||||
"webpki-root-certs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ureq-proto"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613"
|
||||
dependencies = [
|
||||
"base64 0.23.1",
|
||||
"http 1.5.0",
|
||||
"httparse",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.8"
|
||||
@@ -4183,12 +4003,6 @@ version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-zero"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -4354,15 +4168,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.25.4"
|
||||
|
||||
@@ -16,8 +16,8 @@ sha2 = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
hex = "0.4"
|
||||
time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] }
|
||||
magika = "1.1.0"
|
||||
ort = { version = "2.0.0-rc.12", default-features = true }
|
||||
# magika and ort removed: CI runner glibc too old for ort's C23 symbols
|
||||
# regex fallback in router.rs covers all content types
|
||||
regex = "1.10"
|
||||
once_cell = "1.19"
|
||||
indexmap = "2.0"
|
||||
|
||||
@@ -1,68 +1,27 @@
|
||||
//! Content Router — Magika ML + regex fallback detection
|
||||
//! Content Router — regex-based content type detection
|
||||
|
||||
use anyhow::Result;
|
||||
use magika::Session;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::ContentType;
|
||||
|
||||
pub struct ContentRouter {
|
||||
magika: Mutex<Session>,
|
||||
confidence_threshold: f32,
|
||||
_confidence_threshold: f32,
|
||||
}
|
||||
|
||||
impl ContentRouter {
|
||||
/// Create router with default Magika session
|
||||
/// Create router with regex-only detection
|
||||
pub fn new() -> Result<Self> {
|
||||
let magika = Session::new()?;
|
||||
Ok(Self {
|
||||
magika: Mutex::new(magika),
|
||||
confidence_threshold: 0.7,
|
||||
_confidence_threshold: 0.7,
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect content type using Magika ML first, then regex fallback
|
||||
/// Detect content type using regex heuristics
|
||||
pub fn detect(&self, content: &str) -> Result<ContentType> {
|
||||
// Try Magika ML classification
|
||||
let mut magika = self.magika.lock().map_err(|e| anyhow::anyhow!("mutex lock failed: {}", e))?;
|
||||
if let Ok(result) = magika.identify_content_sync(content.as_bytes()) {
|
||||
let label = result.info().label;
|
||||
let score = result.score();
|
||||
|
||||
if score >= self.confidence_threshold {
|
||||
if let Some(ct) = self.map_magika_label(label) {
|
||||
return Ok(ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to regex heuristics
|
||||
Ok(self.regex_fallback(content))
|
||||
}
|
||||
|
||||
/// Map Magika label to our ContentType
|
||||
fn map_magika_label(&self, label: &str) -> Option<ContentType> {
|
||||
match label {
|
||||
"json" | "jsonl" => Some(ContentType::Json),
|
||||
"python"
|
||||
| "javascript"
|
||||
| "typescript"
|
||||
| "rust"
|
||||
| "go"
|
||||
| "shell"
|
||||
| "bash"
|
||||
| "java"
|
||||
| "cpp"
|
||||
| "csharp"
|
||||
| "sql" => Some(ContentType::Code),
|
||||
"diff" | "patch" => Some(ContentType::Diff),
|
||||
"yaml" | "toml" | "ini" | "xml" => Some(ContentType::Config),
|
||||
"markdown" | "txt" => None, // Fallback to regex for better detection
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Regex-based fallback detection
|
||||
/// Regex-based detection
|
||||
fn regex_fallback(&self, content: &str) -> ContentType {
|
||||
if is_json(content) {
|
||||
return ContentType::Json;
|
||||
@@ -93,7 +52,6 @@ fn is_json(content: &str) -> bool {
|
||||
fn is_log(content: &str) -> bool {
|
||||
let content_lower = content.to_lowercase();
|
||||
|
||||
// Log level markers
|
||||
let has_log_level = content_lower.contains("error:")
|
||||
|| content_lower.contains("warn:")
|
||||
|| content_lower.contains("info:")
|
||||
@@ -104,13 +62,11 @@ fn is_log(content: &str) -> bool {
|
||||
|| content.contains("INFO")
|
||||
|| content.contains("FAIL");
|
||||
|
||||
// ISO timestamp pattern
|
||||
let has_timestamp = regex_contains(
|
||||
content,
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}",
|
||||
);
|
||||
|
||||
// Common error markers
|
||||
let has_error_markers = content_lower.contains("exception")
|
||||
|| content_lower.contains("stack trace")
|
||||
|| content_lower.contains("at line")
|
||||
@@ -133,7 +89,6 @@ fn is_diff(content: &str) -> bool {
|
||||
fn is_code(content: &str) -> bool {
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
|
||||
// Check for code-like patterns
|
||||
let import_use_pattern =
|
||||
regex_contains(content, r"^(import|use|require|include|from|package|class|def|fn|public|private|const|let|var|function)\b");
|
||||
let has_brackets =
|
||||
@@ -204,24 +159,13 @@ fn main() {
|
||||
|
||||
#[test]
|
||||
fn test_text_detection_fallback() {
|
||||
let text = "This is just plain English text without any special structure.";
|
||||
assert_eq!(regex_fallback_helper(text), ContentType::Text);
|
||||
let router = ContentRouter::new().unwrap();
|
||||
assert_eq!(router.detect("This is just plain English text without any special structure.").unwrap(), ContentType::Text);
|
||||
}
|
||||
}
|
||||
|
||||
// Helper for test fallback
|
||||
fn regex_fallback_helper(content: &str) -> ContentType {
|
||||
if is_json(content) {
|
||||
return ContentType::Json;
|
||||
#[test]
|
||||
fn test_router_json() {
|
||||
let router = ContentRouter::new().unwrap();
|
||||
assert_eq!(router.detect(r#"{"key": "value"}"#).unwrap(), ContentType::Json);
|
||||
}
|
||||
if is_log(content) {
|
||||
return ContentType::Log;
|
||||
}
|
||||
if is_diff(content) {
|
||||
return ContentType::Diff;
|
||||
}
|
||||
if is_code(content) {
|
||||
return ContentType::Code;
|
||||
}
|
||||
ContentType::Text
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user