fix: remove magika/ort dependency (CI glibc too old for C23 symbols)
Root cause: ort (ONNX Runtime) links against __isoc23_strtoll which requires glibc 2.38+. CI runner has older glibc, causing linker failure. Replace magika ML detection with regex-only ContentRouter. Regex fallback already covers all content types (JSON, log, diff, code). All 294 tests passing.
This commit is contained in:
@@ -16,8 +16,8 @@ sha2 = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
hex = "0.4"
|
||||
time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] }
|
||||
magika = "1.1.0"
|
||||
ort = { version = "2.0.0-rc.12", default-features = true }
|
||||
# magika and ort removed: CI runner glibc too old for ort's C23 symbols
|
||||
# regex fallback in router.rs covers all content types
|
||||
regex = "1.10"
|
||||
once_cell = "1.19"
|
||||
indexmap = "2.0"
|
||||
|
||||
@@ -1,68 +1,27 @@
|
||||
//! Content Router — Magika ML + regex fallback detection
|
||||
//! Content Router — regex-based content type detection
|
||||
|
||||
use anyhow::Result;
|
||||
use magika::Session;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::ContentType;
|
||||
|
||||
pub struct ContentRouter {
|
||||
magika: Mutex<Session>,
|
||||
confidence_threshold: f32,
|
||||
_confidence_threshold: f32,
|
||||
}
|
||||
|
||||
impl ContentRouter {
|
||||
/// Create router with default Magika session
|
||||
/// Create router with regex-only detection
|
||||
pub fn new() -> Result<Self> {
|
||||
let magika = Session::new()?;
|
||||
Ok(Self {
|
||||
magika: Mutex::new(magika),
|
||||
confidence_threshold: 0.7,
|
||||
_confidence_threshold: 0.7,
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect content type using Magika ML first, then regex fallback
|
||||
/// Detect content type using regex heuristics
|
||||
pub fn detect(&self, content: &str) -> Result<ContentType> {
|
||||
// Try Magika ML classification
|
||||
let mut magika = self.magika.lock().map_err(|e| anyhow::anyhow!("mutex lock failed: {}", e))?;
|
||||
if let Ok(result) = magika.identify_content_sync(content.as_bytes()) {
|
||||
let label = result.info().label;
|
||||
let score = result.score();
|
||||
|
||||
if score >= self.confidence_threshold {
|
||||
if let Some(ct) = self.map_magika_label(label) {
|
||||
return Ok(ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to regex heuristics
|
||||
Ok(self.regex_fallback(content))
|
||||
}
|
||||
|
||||
/// Map Magika label to our ContentType
|
||||
fn map_magika_label(&self, label: &str) -> Option<ContentType> {
|
||||
match label {
|
||||
"json" | "jsonl" => Some(ContentType::Json),
|
||||
"python"
|
||||
| "javascript"
|
||||
| "typescript"
|
||||
| "rust"
|
||||
| "go"
|
||||
| "shell"
|
||||
| "bash"
|
||||
| "java"
|
||||
| "cpp"
|
||||
| "csharp"
|
||||
| "sql" => Some(ContentType::Code),
|
||||
"diff" | "patch" => Some(ContentType::Diff),
|
||||
"yaml" | "toml" | "ini" | "xml" => Some(ContentType::Config),
|
||||
"markdown" | "txt" => None, // Fallback to regex for better detection
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Regex-based fallback detection
|
||||
/// Regex-based detection
|
||||
fn regex_fallback(&self, content: &str) -> ContentType {
|
||||
if is_json(content) {
|
||||
return ContentType::Json;
|
||||
@@ -93,7 +52,6 @@ fn is_json(content: &str) -> bool {
|
||||
fn is_log(content: &str) -> bool {
|
||||
let content_lower = content.to_lowercase();
|
||||
|
||||
// Log level markers
|
||||
let has_log_level = content_lower.contains("error:")
|
||||
|| content_lower.contains("warn:")
|
||||
|| content_lower.contains("info:")
|
||||
@@ -104,13 +62,11 @@ fn is_log(content: &str) -> bool {
|
||||
|| content.contains("INFO")
|
||||
|| content.contains("FAIL");
|
||||
|
||||
// ISO timestamp pattern
|
||||
let has_timestamp = regex_contains(
|
||||
content,
|
||||
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}",
|
||||
);
|
||||
|
||||
// Common error markers
|
||||
let has_error_markers = content_lower.contains("exception")
|
||||
|| content_lower.contains("stack trace")
|
||||
|| content_lower.contains("at line")
|
||||
@@ -133,7 +89,6 @@ fn is_diff(content: &str) -> bool {
|
||||
fn is_code(content: &str) -> bool {
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
|
||||
// Check for code-like patterns
|
||||
let import_use_pattern =
|
||||
regex_contains(content, r"^(import|use|require|include|from|package|class|def|fn|public|private|const|let|var|function)\b");
|
||||
let has_brackets =
|
||||
@@ -204,24 +159,13 @@ fn main() {
|
||||
|
||||
#[test]
|
||||
fn test_text_detection_fallback() {
|
||||
let text = "This is just plain English text without any special structure.";
|
||||
assert_eq!(regex_fallback_helper(text), ContentType::Text);
|
||||
let router = ContentRouter::new().unwrap();
|
||||
assert_eq!(router.detect("This is just plain English text without any special structure.").unwrap(), ContentType::Text);
|
||||
}
|
||||
}
|
||||
|
||||
// Helper for test fallback
|
||||
fn regex_fallback_helper(content: &str) -> ContentType {
|
||||
if is_json(content) {
|
||||
return ContentType::Json;
|
||||
#[test]
|
||||
fn test_router_json() {
|
||||
let router = ContentRouter::new().unwrap();
|
||||
assert_eq!(router.detect(r#"{"key": "value"}"#).unwrap(), ContentType::Json);
|
||||
}
|
||||
if is_log(content) {
|
||||
return ContentType::Log;
|
||||
}
|
||||
if is_diff(content) {
|
||||
return ContentType::Diff;
|
||||
}
|
||||
if is_code(content) {
|
||||
return ContentType::Code;
|
||||
}
|
||||
ContentType::Text
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user