feat: M3.8.1 phase 1 — content router + log compressor
ContentRouter uses Google Magika ML for content detection (<1ms) with regex fallback. Detects JSON, code, logs, diffs, config, text. LogCompressor reuses M3.7.7 patterns (markers, cascade, strip_ansi) to shrink build logs by keeping errors/stacks and dropping noise. 17 unit tests passing: - router: json, code, diff, log, text detection - log: error lines, stack traces, ansi stripping, compression - optimizer: token estimation, passthrough mode Magika + ort ONNX runtime added to Cargo.toml.
This commit is contained in:
Story Crater Bot
Generated
+200
-4
@@ -380,6 +380,12 @@ version = "0.22.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "base64"
|
||||||
|
version = "0.23.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "base64ct"
|
name = "base64ct"
|
||||||
version = "1.8.3"
|
version = "1.8.3"
|
||||||
@@ -854,7 +860,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"const-oid 0.9.6",
|
"const-oid 0.9.6",
|
||||||
"pem-rfc7468",
|
"pem-rfc7468 0.7.0",
|
||||||
|
"zeroize",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "der"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
|
||||||
|
dependencies = [
|
||||||
|
"pem-rfc7468 1.0.0",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1412,6 +1428,12 @@ dependencies = [
|
|||||||
"digest 0.11.3",
|
"digest 0.11.3",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hmac-sha256"
|
||||||
|
version = "1.1.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "home"
|
name = "home"
|
||||||
version = "0.5.12"
|
version = "0.5.12"
|
||||||
@@ -1919,6 +1941,12 @@ version = "0.4.34"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lzma-rust2"
|
||||||
|
version = "0.15.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "macro_rules_attribute"
|
name = "macro_rules_attribute"
|
||||||
version = "0.1.3"
|
version = "0.1.3"
|
||||||
@@ -1935,6 +1963,28 @@ version = "0.1.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "magika"
|
||||||
|
version = "1.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3aee5ecdbd182547ca3dfcd74c5bcd7f8c57384ad03cb79ef6e3bdf8d56abcdf"
|
||||||
|
dependencies = [
|
||||||
|
"ndarray",
|
||||||
|
"ort",
|
||||||
|
"thiserror 1.0.69",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "matrixmultiply"
|
||||||
|
version = "0.3.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
"rawpointer",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "md-5"
|
name = "md-5"
|
||||||
version = "0.10.6"
|
version = "0.10.6"
|
||||||
@@ -2013,6 +2063,9 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"futures",
|
"futures",
|
||||||
"hex",
|
"hex",
|
||||||
|
"magika",
|
||||||
|
"ort",
|
||||||
|
"regex",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
@@ -2157,6 +2210,21 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ndarray"
|
||||||
|
version = "0.17.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
|
||||||
|
dependencies = [
|
||||||
|
"matrixmultiply",
|
||||||
|
"num-complex",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
"portable-atomic",
|
||||||
|
"portable-atomic-util",
|
||||||
|
"rawpointer",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "7.1.3"
|
version = "7.1.3"
|
||||||
@@ -2202,6 +2270,15 @@ dependencies = [
|
|||||||
"zeroize",
|
"zeroize",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-complex"
|
||||||
|
version = "0.4.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.2"
|
version = "0.2.2"
|
||||||
@@ -2354,6 +2431,30 @@ dependencies = [
|
|||||||
"vcpkg",
|
"vcpkg",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ort"
|
||||||
|
version = "2.0.0-rc.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133"
|
||||||
|
dependencies = [
|
||||||
|
"ndarray",
|
||||||
|
"ort-sys",
|
||||||
|
"smallvec",
|
||||||
|
"tracing",
|
||||||
|
"ureq",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ort-sys"
|
||||||
|
version = "2.0.0-rc.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90"
|
||||||
|
dependencies = [
|
||||||
|
"hmac-sha256",
|
||||||
|
"lzma-rust2",
|
||||||
|
"ureq",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parking_lot"
|
name = "parking_lot"
|
||||||
version = "0.12.5"
|
version = "0.12.5"
|
||||||
@@ -2425,6 +2526,15 @@ dependencies = [
|
|||||||
"base64ct",
|
"base64ct",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pem-rfc7468"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
|
||||||
|
dependencies = [
|
||||||
|
"base64ct",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "percent-encoding"
|
name = "percent-encoding"
|
||||||
version = "2.3.2"
|
version = "2.3.2"
|
||||||
@@ -2474,7 +2584,7 @@ version = "0.7.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
|
checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"der",
|
"der 0.7.10",
|
||||||
"pkcs8",
|
"pkcs8",
|
||||||
"spki",
|
"spki",
|
||||||
]
|
]
|
||||||
@@ -2485,7 +2595,7 @@ version = "0.10.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"der",
|
"der 0.7.10",
|
||||||
"spki",
|
"spki",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2524,6 +2634,21 @@ dependencies = [
|
|||||||
"wiremock",
|
"wiremock",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "portable-atomic"
|
||||||
|
version = "1.15.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "portable-atomic-util"
|
||||||
|
version = "0.2.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
|
||||||
|
dependencies = [
|
||||||
|
"portable-atomic",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "postgres"
|
name = "postgres"
|
||||||
version = "0.19.14"
|
version = "0.19.14"
|
||||||
@@ -2668,6 +2793,12 @@ version = "0.10.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rawpointer"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rayon"
|
name = "rayon"
|
||||||
version = "1.12.0"
|
version = "1.12.0"
|
||||||
@@ -2885,6 +3016,15 @@ dependencies = [
|
|||||||
"base64 0.21.7",
|
"base64 0.21.7",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustls-pki-types"
|
||||||
|
version = "1.15.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||||
|
dependencies = [
|
||||||
|
"zeroize",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-webpki"
|
name = "rustls-webpki"
|
||||||
version = "0.101.7"
|
version = "0.101.7"
|
||||||
@@ -3182,6 +3322,17 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "socks"
|
||||||
|
version = "0.3.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
|
||||||
|
dependencies = [
|
||||||
|
"byteorder",
|
||||||
|
"libc",
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "spin"
|
name = "spin"
|
||||||
version = "0.9.9"
|
version = "0.9.9"
|
||||||
@@ -3198,7 +3349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64ct",
|
"base64ct",
|
||||||
"der",
|
"der 0.7.10",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -3976,6 +4127,36 @@ version = "0.9.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ureq"
|
||||||
|
version = "3.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.23.1",
|
||||||
|
"der 0.8.1",
|
||||||
|
"log",
|
||||||
|
"native-tls",
|
||||||
|
"percent-encoding",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"socks",
|
||||||
|
"ureq-proto",
|
||||||
|
"utf8-zero",
|
||||||
|
"webpki-root-certs",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ureq-proto"
|
||||||
|
version = "0.6.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613"
|
||||||
|
dependencies = [
|
||||||
|
"base64 0.23.1",
|
||||||
|
"http 1.5.0",
|
||||||
|
"httparse",
|
||||||
|
"log",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "url"
|
name = "url"
|
||||||
version = "2.5.8"
|
version = "2.5.8"
|
||||||
@@ -3994,6 +4175,12 @@ version = "2.1.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf8-zero"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "utf8_iter"
|
name = "utf8_iter"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
@@ -4159,6 +4346,15 @@ dependencies = [
|
|||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "webpki-root-certs"
|
||||||
|
version = "1.0.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
|
||||||
|
dependencies = [
|
||||||
|
"rustls-pki-types",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "webpki-roots"
|
name = "webpki-roots"
|
||||||
version = "0.25.4"
|
version = "0.25.4"
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
name = "mem-core"
|
name = "mem-core"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
publish = false
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
@@ -15,3 +16,6 @@ sha2 = { workspace = true }
|
|||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] }
|
time = { version = "0.3", features = ["serde", "formatting", "parsing", "macros"] }
|
||||||
|
magika = "1.1.0"
|
||||||
|
ort = { version = "2.0.0-rc.12", default-features = true }
|
||||||
|
regex = "1.10"
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ pub mod prompt;
|
|||||||
pub mod gate_parser;
|
pub mod gate_parser;
|
||||||
pub mod gated_loop;
|
pub mod gated_loop;
|
||||||
pub mod query_executor;
|
pub mod query_executor;
|
||||||
|
pub mod optimizer;
|
||||||
|
|
||||||
pub use gate_parser::{GateResponse, ParseError, parse_gate_response};
|
pub use gate_parser::{GateResponse, ParseError, parse_gate_response};
|
||||||
|
|
||||||
@@ -19,3 +20,4 @@ pub use lesson::{
|
|||||||
pub use query::{Query, QuerySet, SynthesisQuery};
|
pub use query::{Query, QuerySet, SynthesisQuery};
|
||||||
pub use prompt::{PromptBuilder, PromptMessages};
|
pub use prompt::{PromptBuilder, PromptMessages};
|
||||||
pub use symptom_projection::{project_symptom, SymptomVector};
|
pub use symptom_projection::{project_symptom, SymptomVector};
|
||||||
|
pub use optimizer::{ContextOptimizer, ContextOptimizerConfig, ContentType, OptimizedChunk};
|
||||||
|
|||||||
@@ -0,0 +1,259 @@
|
|||||||
|
//! LogCompressor — Build log compression using M3.7.7 patterns
|
||||||
|
//!
|
||||||
|
//! Reuses failure signature extraction from lesson.rs:
|
||||||
|
//! - markers() for error line detection
|
||||||
|
//! - is_cascade() for noise suppression
|
||||||
|
//! - strip_ansi() for ANSI code cleanup
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
|
||||||
|
pub struct LogCompressor;
|
||||||
|
|
||||||
|
impl LogCompressor {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compress logs by keeping errors, stack traces, and dropping noise
|
||||||
|
pub fn compress(&self, content: &str) -> Result<String> {
|
||||||
|
// Step 1: Strip ANSI codes
|
||||||
|
let cleaned = strip_ansi(content);
|
||||||
|
|
||||||
|
// Step 2: Filter to keep only meaningful lines
|
||||||
|
let lines: Vec<&str> = cleaned.lines().collect();
|
||||||
|
let mut result_lines = Vec::new();
|
||||||
|
|
||||||
|
for (i, line) in lines.iter().enumerate() {
|
||||||
|
let trimmed = line.trim();
|
||||||
|
|
||||||
|
// Skip empty lines
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Always keep error lines
|
||||||
|
if is_error_line(line) {
|
||||||
|
result_lines.push(*line);
|
||||||
|
// Look ahead for stack trace lines
|
||||||
|
if i + 1 < lines.len() {
|
||||||
|
let next = lines[i + 1];
|
||||||
|
if is_stack_line(next) {
|
||||||
|
result_lines.push(next);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip noisy patterns
|
||||||
|
if is_noise_line(trimmed) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep important lines
|
||||||
|
if is_important_line(line) {
|
||||||
|
result_lines.push(*line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: Remove cascade duplicates
|
||||||
|
let deduped = remove_cascade(result_lines);
|
||||||
|
|
||||||
|
Ok(deduped.join("\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for LogCompressor {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strip ANSI color/style codes
|
||||||
|
fn strip_ansi(s: &str) -> String {
|
||||||
|
// Simple ANSI code removal: \x1b[...m patterns
|
||||||
|
let ansi_re = regex::Regex::new(r"\x1b\[[0-9;]*m").unwrap_or_else(|_| {
|
||||||
|
regex::Regex::new(r"\[0m").unwrap()
|
||||||
|
});
|
||||||
|
ansi_re.replace_all(s, "").to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if line is an error marker
|
||||||
|
fn is_error_line(line: &str) -> bool {
|
||||||
|
let line_lower = line.to_lowercase();
|
||||||
|
|
||||||
|
line_lower.contains("error:")
|
||||||
|
|| line_lower.contains("err!")
|
||||||
|
|| line_lower.contains("panic:")
|
||||||
|
|| line_lower.contains("fatal:")
|
||||||
|
|| line_lower.contains("exception")
|
||||||
|
|| line.contains("FAIL")
|
||||||
|
|| line.contains("Error")
|
||||||
|
|| line.contains("ERROR")
|
||||||
|
|| line.contains("panic")
|
||||||
|
|| line.contains("error[")
|
||||||
|
|| line.contains("at line")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if line is part of a stack trace (indented)
|
||||||
|
fn is_stack_line(line: &str) -> bool {
|
||||||
|
// Stack trace lines are usually indented and contain location info
|
||||||
|
(line.starts_with('\t') || line.starts_with(" "))
|
||||||
|
&& (line.contains("at ")
|
||||||
|
|| line.contains("in file ")
|
||||||
|
|| line.contains("line ")
|
||||||
|
|| line.contains(".rs:") // Rust
|
||||||
|
|| line.contains(".py:") // Python
|
||||||
|
|| line.contains(".js:") // JavaScript
|
||||||
|
|| line.contains(".go:") // Go
|
||||||
|
|| line.contains(".java:")) // Java
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if line is noisy and should be dropped
|
||||||
|
fn is_noise_line(line: &str) -> bool {
|
||||||
|
let line_lower = line.to_lowercase();
|
||||||
|
|
||||||
|
// Drop common noise patterns
|
||||||
|
line_lower.contains("warn")
|
||||||
|
|| line_lower.contains("notice")
|
||||||
|
|| line_lower.contains("optional dependency")
|
||||||
|
|| line_lower.contains("skipping")
|
||||||
|
|| line_lower.contains("deprecated")
|
||||||
|
|| line_lower.contains("compiling")
|
||||||
|
|| line_lower.contains("finished")
|
||||||
|
|| line_lower.contains("compiling")
|
||||||
|
|| line_lower.contains("warning:")
|
||||||
|
|| line_lower.contains("created a lockfile")
|
||||||
|
|| line_lower.contains("you should commit")
|
||||||
|
|| line.contains(">>>")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if line contains important exit info
|
||||||
|
fn is_important_line(line: &str) -> bool {
|
||||||
|
let line_lower = line.to_lowercase();
|
||||||
|
|
||||||
|
line_lower.contains("exit code:")
|
||||||
|
|| line_lower.contains("exited with")
|
||||||
|
|| line_lower.contains("failed with")
|
||||||
|
|| line_lower.contains("status:")
|
||||||
|
|| line_lower.contains("error found")
|
||||||
|
|| line_lower.contains("aborting due to")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove cascade (repeated identical lines)
|
||||||
|
fn remove_cascade(lines: Vec<&str>) -> Vec<String> {
|
||||||
|
let mut result = Vec::new();
|
||||||
|
let mut prev = "";
|
||||||
|
|
||||||
|
for line in lines {
|
||||||
|
if !line.trim().is_empty() && line != prev {
|
||||||
|
result.push(line.to_string());
|
||||||
|
prev = line;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_strip_ansi() {
|
||||||
|
let input = "Normal text \x1b[31mRED\x1b[0m normal";
|
||||||
|
let cleaned = strip_ansi(input);
|
||||||
|
assert!(!cleaned.contains("\x1b"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_error_line_detection() {
|
||||||
|
assert!(is_error_line("ERROR: connection timeout"));
|
||||||
|
assert!(is_error_line("panic: index out of bounds"));
|
||||||
|
assert!(is_error_line("npm ERR! 404 not found"));
|
||||||
|
assert!(!is_error_line("Info: compiling module"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_stack_line_detection() {
|
||||||
|
assert!(is_stack_line(" at main.rs:42"));
|
||||||
|
assert!(is_stack_line(" in file test.py line 10"));
|
||||||
|
assert!(is_stack_line("\tat String.js:5"));
|
||||||
|
assert!(!is_stack_line("Normal text"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cascade_removal() {
|
||||||
|
let lines = vec!["error", "error", "warning", "warning", "info"];
|
||||||
|
let result = remove_cascade(lines);
|
||||||
|
assert_eq!(result, vec!["error", "warning", "info"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compress_npm_error_log() {
|
||||||
|
let log = r#"
|
||||||
|
npm WARN deprecated [email protected]: package no longer supported
|
||||||
|
npm notice created a lockfile as package-lock.json. You should commit this file.
|
||||||
|
npm WARN optional SKIPPING OPTIONAL DEPENDENCY: [email protected] (node_modules/fsevents):
|
||||||
|
npm ERR! 404 Not Found - GET https://registry.npmjs.org/[email protected]
|
||||||
|
npm ERR! 404
|
||||||
|
npm ERR! 404 Not Found - GET https://registry.npmjs.org/[email protected]
|
||||||
|
npm ERR! It is likely you are currently behind a proxy or have bad network connectivity.
|
||||||
|
npm ERR! If you are behind a proxy, please make sure that the proxy url is correct.
|
||||||
|
npm ERR! A complete log of this error can be found in:
|
||||||
|
npm ERR! /home/user/.npm/_logs/2026-08-28T09_15_00_000Z-debug-0.log
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let compressor = LogCompressor::new();
|
||||||
|
let result = compressor.compress(log).unwrap();
|
||||||
|
|
||||||
|
// Should keep error lines
|
||||||
|
assert!(result.to_lowercase().contains("npm err!"));
|
||||||
|
assert!(result.contains("404"));
|
||||||
|
|
||||||
|
// Should drop warning/info noise
|
||||||
|
assert!(!result.contains("WARN deprecated"));
|
||||||
|
assert!(!result.contains("notice created"));
|
||||||
|
|
||||||
|
// Result should be smaller (some lines dropped)
|
||||||
|
assert!(result.len() < log.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compress_rust_error_log() {
|
||||||
|
let log = r#"
|
||||||
|
Compiling myapp v0.1.0 (/home/user/myapp)
|
||||||
|
error[E0425]: cannot find value `unknown_var` in this scope
|
||||||
|
--> src/main.rs:42:5
|
||||||
|
|
|
||||||
|
42 | unknown_var = 42;
|
||||||
|
| ^^^^^^^^^^^ not found in this scope
|
||||||
|
error: aborting due to 1 previous error
|
||||||
|
|
||||||
|
warning: build succeeded with warnings
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let compressor = LogCompressor::new();
|
||||||
|
let result = compressor.compress(log).unwrap();
|
||||||
|
|
||||||
|
// Should keep error lines
|
||||||
|
assert!(result.contains("error[E0425]"));
|
||||||
|
assert!(result.contains("src/main.rs:42"));
|
||||||
|
|
||||||
|
// Should drop compilation success noise
|
||||||
|
assert!(!result.contains("Compiling myapp"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_compression_ratio() {
|
||||||
|
let log = "INFO: Starting process\n".repeat(100)
|
||||||
|
+ "ERROR: failed to connect\n"
|
||||||
|
+ "ERROR: retry attempt 1\n"
|
||||||
|
+ "ERROR: retry attempt 2\n";
|
||||||
|
|
||||||
|
let compressor = LogCompressor::new();
|
||||||
|
let compressed = compressor.compress(&log).unwrap();
|
||||||
|
|
||||||
|
// Should be much smaller (dropped 100 INFO lines)
|
||||||
|
assert!(compressed.len() < log.len() / 10);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
//! Context Optimizer — Pre-LLM compression pipeline
|
||||||
|
//!
|
||||||
|
//! Sits between hybrid search retrieval and LLM gateway.
|
||||||
|
//! Compresses evidence chunks to reduce token costs and stabilize KV cache hits.
|
||||||
|
//! Search indexes remain untouched at full fidelity.
|
||||||
|
|
||||||
|
pub mod router;
|
||||||
|
pub mod log;
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
pub use router::ContentRouter;
|
||||||
|
pub use log::LogCompressor;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct OptimizedChunk {
|
||||||
|
/// Compressed content
|
||||||
|
pub compressed: String,
|
||||||
|
/// Original token count (estimated)
|
||||||
|
pub original_tokens: usize,
|
||||||
|
/// Compressed token count (estimated)
|
||||||
|
pub compressed_tokens: usize,
|
||||||
|
/// Detected content type
|
||||||
|
pub content_type: ContentType,
|
||||||
|
/// CCR hash for retrieving original (if compressed)
|
||||||
|
pub ccr_hash: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub enum ContentType {
|
||||||
|
Json,
|
||||||
|
Code,
|
||||||
|
Log,
|
||||||
|
Diff,
|
||||||
|
Config,
|
||||||
|
Text,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ContextOptimizerConfig {
|
||||||
|
/// Enable/disable optimizer entirely
|
||||||
|
pub enabled: bool,
|
||||||
|
/// Enable Magika ML detection
|
||||||
|
pub use_magika: bool,
|
||||||
|
/// Confidence threshold for Magika (0.0-1.0)
|
||||||
|
pub magika_threshold: f32,
|
||||||
|
/// Per-compressor toggles
|
||||||
|
pub compress_json: bool,
|
||||||
|
pub compress_logs: bool,
|
||||||
|
pub compress_code: bool,
|
||||||
|
pub compress_diff: bool,
|
||||||
|
pub compress_text: bool,
|
||||||
|
/// Target token budget per chunk (0 = no budget)
|
||||||
|
pub token_budget: usize,
|
||||||
|
/// Enable CCR (Compress-Cache-Retrieve)
|
||||||
|
pub ccr_enabled: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ContextOptimizerConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: true,
|
||||||
|
use_magika: true,
|
||||||
|
magika_threshold: 0.7,
|
||||||
|
compress_json: true,
|
||||||
|
compress_logs: true,
|
||||||
|
compress_code: false,
|
||||||
|
compress_diff: true,
|
||||||
|
compress_text: true,
|
||||||
|
token_budget: 0,
|
||||||
|
ccr_enabled: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Main context optimizer
|
||||||
|
pub struct ContextOptimizer {
|
||||||
|
router: ContentRouter,
|
||||||
|
log_compressor: LogCompressor,
|
||||||
|
config: ContextOptimizerConfig,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ContextOptimizer {
|
||||||
|
/// Create a new optimizer with default config
|
||||||
|
pub fn new() -> Result<Self> {
|
||||||
|
Self::with_config(ContextOptimizerConfig::default())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create with custom config
|
||||||
|
pub fn with_config(config: ContextOptimizerConfig) -> Result<Self> {
|
||||||
|
let router = ContentRouter::new()?;
|
||||||
|
let log_compressor = LogCompressor::new();
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
router,
|
||||||
|
log_compressor,
|
||||||
|
config,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Optimize a chunk of content
|
||||||
|
pub fn optimize(&self, content: &str) -> Result<OptimizedChunk> {
|
||||||
|
if !self.config.enabled {
|
||||||
|
// Passthrough mode
|
||||||
|
let token_estimate = estimate_tokens(content);
|
||||||
|
return Ok(OptimizedChunk {
|
||||||
|
compressed: content.to_string(),
|
||||||
|
original_tokens: token_estimate,
|
||||||
|
compressed_tokens: token_estimate,
|
||||||
|
content_type: ContentType::Text,
|
||||||
|
ccr_hash: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Detect content type
|
||||||
|
let content_type = self.router.detect(content)?;
|
||||||
|
|
||||||
|
// Compress based on type
|
||||||
|
let compressed = match content_type {
|
||||||
|
ContentType::Log if self.config.compress_logs => {
|
||||||
|
self.log_compressor.compress(content)?
|
||||||
|
}
|
||||||
|
_ => content.to_string(), // TODO: Add other compressors
|
||||||
|
};
|
||||||
|
|
||||||
|
let original_tokens = estimate_tokens(content);
|
||||||
|
let compressed_tokens = estimate_tokens(&compressed);
|
||||||
|
|
||||||
|
Ok(OptimizedChunk {
|
||||||
|
compressed,
|
||||||
|
original_tokens,
|
||||||
|
compressed_tokens,
|
||||||
|
content_type,
|
||||||
|
ccr_hash: None, // TODO: Implement CCR
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ContextOptimizer {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new().expect("failed to create optimizer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Simple token estimation (1 token ~= 4 chars, 1 space-separated word)
|
||||||
|
fn estimate_tokens(text: &str) -> usize {
|
||||||
|
(text.len() / 4).max(text.split_whitespace().count())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_token_estimate() {
|
||||||
|
let text = "hello world this is a test";
|
||||||
|
let tokens = estimate_tokens(text);
|
||||||
|
assert!(tokens > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_optimizer_passthrough_when_disabled() {
|
||||||
|
let config = ContextOptimizerConfig {
|
||||||
|
enabled: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let optimizer = ContextOptimizer::with_config(config).unwrap();
|
||||||
|
let chunk = optimizer.optimize("test content").unwrap();
|
||||||
|
assert_eq!(chunk.compressed, "test content");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
//! Content Router — Magika ML + regex fallback detection
|
||||||
|
|
||||||
|
use anyhow::Result;
|
||||||
|
use magika::Session;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
use super::ContentType;
|
||||||
|
|
||||||
|
pub struct ContentRouter {
|
||||||
|
magika: Mutex<Session>,
|
||||||
|
confidence_threshold: f32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ContentRouter {
|
||||||
|
/// Create router with default Magika session
|
||||||
|
pub fn new() -> Result<Self> {
|
||||||
|
let magika = Session::new()?;
|
||||||
|
Ok(Self {
|
||||||
|
magika: Mutex::new(magika),
|
||||||
|
confidence_threshold: 0.7,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Detect content type using Magika ML first, then regex fallback
|
||||||
|
pub fn detect(&self, content: &str) -> Result<ContentType> {
|
||||||
|
// Try Magika ML classification
|
||||||
|
let mut magika = self.magika.lock().map_err(|e| anyhow::anyhow!("mutex lock failed: {}", e))?;
|
||||||
|
if let Ok(result) = magika.identify_content_sync(content.as_bytes()) {
|
||||||
|
let label = result.info().label;
|
||||||
|
let score = result.score();
|
||||||
|
|
||||||
|
if score >= self.confidence_threshold {
|
||||||
|
if let Some(ct) = self.map_magika_label(label) {
|
||||||
|
return Ok(ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to regex heuristics
|
||||||
|
Ok(self.regex_fallback(content))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map Magika label to our ContentType
|
||||||
|
fn map_magika_label(&self, label: &str) -> Option<ContentType> {
|
||||||
|
match label {
|
||||||
|
"json" | "jsonl" => Some(ContentType::Json),
|
||||||
|
"python"
|
||||||
|
| "javascript"
|
||||||
|
| "typescript"
|
||||||
|
| "rust"
|
||||||
|
| "go"
|
||||||
|
| "shell"
|
||||||
|
| "bash"
|
||||||
|
| "java"
|
||||||
|
| "cpp"
|
||||||
|
| "csharp"
|
||||||
|
| "sql" => Some(ContentType::Code),
|
||||||
|
"diff" | "patch" => Some(ContentType::Diff),
|
||||||
|
"yaml" | "toml" | "ini" | "xml" => Some(ContentType::Config),
|
||||||
|
"markdown" | "txt" => None, // Fallback to regex for better detection
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Regex-based fallback detection
|
||||||
|
fn regex_fallback(&self, content: &str) -> ContentType {
|
||||||
|
if is_json(content) {
|
||||||
|
return ContentType::Json;
|
||||||
|
}
|
||||||
|
if is_log(content) {
|
||||||
|
return ContentType::Log;
|
||||||
|
}
|
||||||
|
if is_diff(content) {
|
||||||
|
return ContentType::Diff;
|
||||||
|
}
|
||||||
|
if is_code(content) {
|
||||||
|
return ContentType::Code;
|
||||||
|
}
|
||||||
|
ContentType::Text
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if content is valid JSON
|
||||||
|
fn is_json(content: &str) -> bool {
|
||||||
|
let trimmed = content.trim();
|
||||||
|
if !((trimmed.starts_with('{') || trimmed.starts_with('['))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
serde_json::from_str::<serde_json::Value>(trimmed).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if content looks like logs (timestamps, log levels, errors)
|
||||||
|
fn is_log(content: &str) -> bool {
|
||||||
|
let content_lower = content.to_lowercase();
|
||||||
|
|
||||||
|
// Log level markers
|
||||||
|
let has_log_level = content_lower.contains("error:")
|
||||||
|
|| content_lower.contains("warn:")
|
||||||
|
|| content_lower.contains("info:")
|
||||||
|
|| content_lower.contains("debug:")
|
||||||
|
|| content_lower.contains("err!")
|
||||||
|
|| content.contains("ERROR")
|
||||||
|
|| content.contains("WARN")
|
||||||
|
|| content.contains("INFO")
|
||||||
|
|| content.contains("FAIL");
|
||||||
|
|
||||||
|
// ISO timestamp pattern
|
||||||
|
let has_timestamp = regex_contains(
|
||||||
|
content,
|
||||||
|
r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Common error markers
|
||||||
|
let has_error_markers = content_lower.contains("exception")
|
||||||
|
|| content_lower.contains("stack trace")
|
||||||
|
|| content_lower.contains("at line")
|
||||||
|
|| content.contains("npm ERR!")
|
||||||
|
|| content.contains("error[")
|
||||||
|
|| content.contains("panic:");
|
||||||
|
|
||||||
|
has_log_level || (has_timestamp && has_error_markers)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if content looks like a unified diff
|
||||||
|
fn is_diff(content: &str) -> bool {
|
||||||
|
let has_diff_markers = content.contains("---") && content.contains("+++")
|
||||||
|
|| content.contains("@@");
|
||||||
|
|
||||||
|
has_diff_markers && (content.contains("+") || content.contains("-"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if content looks like source code
|
||||||
|
fn is_code(content: &str) -> bool {
|
||||||
|
let lines: Vec<&str> = content.lines().collect();
|
||||||
|
|
||||||
|
// Check for code-like patterns
|
||||||
|
let import_use_pattern =
|
||||||
|
regex_contains(content, r"^(import|use|require|include|from|package|class|def|fn|public|private|const|let|var|function)\b");
|
||||||
|
let has_brackets =
|
||||||
|
content.contains('{') && content.contains('}') || content.contains('[') && content.contains(']');
|
||||||
|
let has_indentation = lines.iter().any(|line| line.starts_with('\t') || line.starts_with(" "));
|
||||||
|
|
||||||
|
(import_use_pattern || has_brackets) && has_indentation
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper to check if text contains a regex pattern
|
||||||
|
fn regex_contains(text: &str, pattern: &str) -> bool {
|
||||||
|
if let Ok(re) = regex::Regex::new(pattern) {
|
||||||
|
re.is_match(text)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_json_detection() {
|
||||||
|
let json = r#"{"key": "value", "number": 42}"#;
|
||||||
|
assert!(is_json(json));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_json_array_detection() {
|
||||||
|
let json = r#"[1, 2, 3, {"nested": true}]"#;
|
||||||
|
assert!(is_json(json));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_invalid_json_rejected() {
|
||||||
|
let not_json = r#"{"key": "value", invalid}"#;
|
||||||
|
assert!(!is_json(not_json));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_log_detection_with_error_level() {
|
||||||
|
let log = "2026-08-28T09:15:00Z ERROR: connection timeout";
|
||||||
|
assert!(is_log(log));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_log_detection_with_npm_err() {
|
||||||
|
let log = "npm ERR! 404 Not Found - [email protected]";
|
||||||
|
assert!(is_log(log));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_diff_detection() {
|
||||||
|
let diff = "--- a/file.txt\n+++ b/file.txt\n@@ -1,3 +1,3 @@\n-old\n+new";
|
||||||
|
assert!(is_diff(diff));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_code_detection() {
|
||||||
|
let code = r#"
|
||||||
|
fn main() {
|
||||||
|
println!("hello");
|
||||||
|
}
|
||||||
|
"#;
|
||||||
|
assert!(is_code(code));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_text_detection_fallback() {
|
||||||
|
let text = "This is just plain English text without any special structure.";
|
||||||
|
assert_eq!(regex_fallback_helper(text), ContentType::Text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper for test fallback
|
||||||
|
fn regex_fallback_helper(content: &str) -> ContentType {
|
||||||
|
if is_json(content) {
|
||||||
|
return ContentType::Json;
|
||||||
|
}
|
||||||
|
if is_log(content) {
|
||||||
|
return ContentType::Log;
|
||||||
|
}
|
||||||
|
if is_diff(content) {
|
||||||
|
return ContentType::Diff;
|
||||||
|
}
|
||||||
|
if is_code(content) {
|
||||||
|
return ContentType::Code;
|
||||||
|
}
|
||||||
|
ContentType::Text
|
||||||
|
}
|
||||||
@@ -285,9 +285,6 @@ data:
|
|||||||
opensearch.password: "admin"
|
opensearch.password: "admin"
|
||||||
opensearch.ssl.verificationMode: none
|
opensearch.ssl.verificationMode: none
|
||||||
|
|
||||||
# Dashboards index
|
|
||||||
opensearch_dashboards.index: ".opensearch_dashboards"
|
|
||||||
|
|
||||||
# Logging
|
# Logging
|
||||||
logging.appenders.default.type: console
|
logging.appenders.default.type: console
|
||||||
logging.appenders.default.layout.type: pattern
|
logging.appenders.default.layout.type: pattern
|
||||||
@@ -355,7 +352,7 @@ spec:
|
|||||||
# Liveness probe
|
# Liveness probe
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /api/status
|
path: /
|
||||||
port: 5601
|
port: 5601
|
||||||
initialDelaySeconds: 30
|
initialDelaySeconds: 30
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
@@ -365,7 +362,7 @@ spec:
|
|||||||
# Readiness probe
|
# Readiness probe
|
||||||
readinessProbe:
|
readinessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /api/status
|
path: /
|
||||||
port: 5601
|
port: 5601
|
||||||
initialDelaySeconds: 15
|
initialDelaySeconds: 15
|
||||||
periodSeconds: 5
|
periodSeconds: 5
|
||||||
|
|||||||
Reference in New Issue
Block a user