diff --git a/Cargo.lock b/Cargo.lock index 129e6a2..bb4c4b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2516,6 +2516,7 @@ dependencies = [ "mem-ingest", "mem-llm", "mem-store", + "regex", "serde_json", "time", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 3396aa4..8a71aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ sqlx = { version = "0.7", features = ["postgres", "runtime-tokio-rustls", "chron pgvector = { version = "0.2", features = ["sqlx"] } base64 = "0.21" jsonwebtoken = "9.2" +regex = "1.10" [dev-dependencies] toml = { workspace = true } @@ -60,6 +61,7 @@ actix-web = { workspace = true } actix-rt = { workspace = true } wiremock = "0.6" chrono = { version = "0.4", features = ["serde"] } +regex = { workspace = true } [profile.release] opt-level = 3 diff --git a/crates/mem-cli/src/main.rs b/crates/mem-cli/src/main.rs index c1017a7..0d2ec21 100644 --- a/crates/mem-cli/src/main.rs +++ b/crates/mem-cli/src/main.rs @@ -136,6 +136,16 @@ enum Commands { #[arg(long)] database_url: Option, }, + + /// Extract and explain failure signature + Sig { + /// Tool name (e.g. npm, cargo, kubectl) + #[arg(long, value_name = "TOOL")] + tool: String, + /// Read failure log from file (stdin if not specified) + #[arg(long, value_name = "FILE")] + file: Option, + }, } #[tokio::main] @@ -191,6 +201,9 @@ async fn main() -> anyhow::Result<()> { }; cmd_verify(&project, db, log, log_dir, output_format, &database_url).await? } + Commands::Sig { tool, file } => { + cmd_sig(&tool, file.as_ref())? + } } Ok(()) @@ -356,3 +369,36 @@ async fn cmd_verify( Ok(()) } + +fn cmd_sig(tool: &str, file: Option<&PathBuf>) -> anyhow::Result<()> { + use mem_core::lesson; + use std::io::Read; + + // Read failure log from file or stdin + let mut output = String::new(); + if let Some(file_path) = file { + output = fs::read_to_string(file_path)?; + } else { + std::io::stdin().read_to_string(&mut output)?; + } + + // Extract signature + match lesson::extract(tool, &output) { + Some(sig) => { + println!("=== Failure Signature ==="); + println!("Tool: {}", sig.tool); + println!("Rule: {}", sig.rule); + println!("Hash (SHA256): {}", sig.sig_sha); + println!("\n=== Raw Error ==="); + println!("{}", sig.raw); + println!("\n=== Normalised Form ==="); + println!("{}", sig.normalised); + } + None => { + eprintln!("Failed to extract signature for tool: {}", tool); + std::process::exit(1); + } + } + + Ok(()) +} diff --git a/fixtures/failures/cargo-different.txt b/fixtures/failures/cargo-different.txt new file mode 100644 index 0000000..0ce4a1d --- /dev/null +++ b/fixtures/failures/cargo-different.txt @@ -0,0 +1,12 @@ + Compiling mem-core v0.1.0 (/home/runner/work/Poimen/memory/crates/mem-core) +error[E0433]: cannot find function `extract` in this scope + --> crates/mem-core/src/lesson.rs:456:23 + | +456 | let sig = extract(&self.output)?; + | ^^^^^^^ not found in this scope + | +help: consider importing this function + | +456 | use crate::extract; + +error: could not compile `mem-core` due to 1 previous error diff --git a/fixtures/failures/cargo-run-a.txt b/fixtures/failures/cargo-run-a.txt new file mode 100644 index 0000000..20fbb9d --- /dev/null +++ b/fixtures/failures/cargo-run-a.txt @@ -0,0 +1,12 @@ + Compiling mem-cli v0.1.0 (/home/runner/work/Poimen/memory/crates/mem-cli) +error[E0599]: no method named `unwrap_or` found for struct `Result` in this scope + --> crates/mem-cli/src/main.rs:456:78 + | +456 | let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string())); + | ^^^^^^^^^^^^^^^ method not found in `Result` + | + = note: the following candidate methods were found: + as core::ops::try::Try>::into_ok + as core::ops::try::Try>::into_err + +error: could not compile `mem-cli` due to 1 previous error diff --git a/fixtures/failures/cargo-run-b.txt b/fixtures/failures/cargo-run-b.txt new file mode 100644 index 0000000..644403e --- /dev/null +++ b/fixtures/failures/cargo-run-b.txt @@ -0,0 +1,12 @@ + Compiling mem-cli v0.1.0 (/home/ubuntu/workplace/Poimen/memory/crates/mem-cli) +error[E0599]: no method named `unwrap_or` found for struct `Result` in this scope + --> crates/mem-cli/src/main.rs:456:78 + | +456 | let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string())); + | ^^^^^^^^^^^^^^^ method not found in `Result` + | + = note: the following candidate methods were found: + as core::ops::try::Try>::into_ok + as core::ops::try::Try>::into_err + +error: could not compile `mem-cli` due to 1 previous error diff --git a/fixtures/failures/kubectl-apply-a.txt b/fixtures/failures/kubectl-apply-a.txt new file mode 100644 index 0000000..6ebe4b4 --- /dev/null +++ b/fixtures/failures/kubectl-apply-a.txt @@ -0,0 +1,4 @@ +2026-08-21T10:02:11.482Z +kubectl apply -f /home/runner/work/Poimen/memory/k8s/app/opensearch.yaml +error: error validating "/home/runner/work/Poimen/memory/k8s/app/opensearch.yaml": error validating data: [ValidationError(PersistentVolumeClaim.metadata): unknown field "storageClassName" in io.k8s.api.core.v1.ObjectMeta, ValidationError(PersistentVolumeClaim.metadata): unknown field "capacity" in io.k8s.api.core.v1.ObjectMeta] +The server is rejecting the request. (422) diff --git a/fixtures/failures/kubectl-apply-b.txt b/fixtures/failures/kubectl-apply-b.txt new file mode 100644 index 0000000..c694b1c --- /dev/null +++ b/fixtures/failures/kubectl-apply-b.txt @@ -0,0 +1,4 @@ +2026-08-22T14:32:45.923Z +kubectl apply -f /home/ubuntu/workplace/Poimen/memory/k8s/app/opensearch.yaml +error: error validating "/home/ubuntu/workplace/Poimen/memory/k8s/app/opensearch.yaml": error validating data: [ValidationError(PersistentVolumeClaim.metadata): unknown field "storageClassName" in io.k8s.api.core.v1.ObjectMeta, ValidationError(PersistentVolumeClaim.metadata): unknown field "capacity" in io.k8s.api.core.v1.ObjectMeta] +The server is rejecting the request. (422) diff --git a/fixtures/failures/kubectl-different.txt b/fixtures/failures/kubectl-different.txt new file mode 100644 index 0000000..8ca137e --- /dev/null +++ b/fixtures/failures/kubectl-different.txt @@ -0,0 +1,3 @@ +2026-08-21T08:15:22.100Z +kubectl port-forward -n poimen svc/opensearch 9200:9200 +error: error forwarding port after handlers/portforward: Timeout occured, check if the service/pod is running and accessible diff --git a/fixtures/failures/npm-different.txt b/fixtures/failures/npm-different.txt new file mode 100644 index 0000000..5cd7308 --- /dev/null +++ b/fixtures/failures/npm-different.txt @@ -0,0 +1,21 @@ +> mem@1.0.0 test +> jest --coverage + +FAIL src/utils.test.ts + ● Test suite failed to compile + + TypeError: Cannot find module 'typescript' + at Function.Module._load (internal/modules/require.js:497:11) + at Module.load (internal/modules/require.js:387:81) + at Object. (/home/runner/work/Poimen/memory/src/setup.ts:1:1) + +npm ERR! code ELIFECYCLE +npm ERR! errno 1 +npm ERR! mem@1.0.0 test: `jest --coverage` +npm ERR! Exit status 1 +npm ERR! +npm ERR! Failed at the mem@1.0.0 test stage. +npm ERR! This is probably not a problem with npm. +npm ERR! There is likely additional logging output above. + +npm WARN optional optional dependency failed, continuing fsevents@2.3.2 diff --git a/fixtures/failures/npm-run-a.txt b/fixtures/failures/npm-run-a.txt new file mode 100644 index 0000000..e6e4747 --- /dev/null +++ b/fixtures/failures/npm-run-a.txt @@ -0,0 +1,23 @@ +> mem@1.0.0 build +> cargo build --release + +Compiling mem-cli v0.1.0 (/home/runner/work/Poimen/memory/crates/mem-cli) +error: unresolved import `mem_core` + --> crates/mem-cli/src/main.rs:15:5 + | +15 | use mem_core::{Record, Provenance, Role}; + | ^^^^^^^^ could not find `mem_core` in `extern prelude` + | + = note: consider adding `extern crate mem_core` to use the crate + +error: could not compile `mem-cli` due to previous error + +npm ERR! code ELIFECYCLE +npm ERR! errno 1 +npm ERR! mem@1.0.0 build: `cargo build --release` +npm ERR! Exit status 1 +npm ERR! +npm ERR! Failed at the mem@1.0.0 build stage. +npm ERR! Make sure you have the latest version of node.js and npm installed. + +npm WARN optional optional dependency failed, continuing fsevents@2.3.2 diff --git a/fixtures/failures/npm-run-b.txt b/fixtures/failures/npm-run-b.txt new file mode 100644 index 0000000..65029a4 --- /dev/null +++ b/fixtures/failures/npm-run-b.txt @@ -0,0 +1,23 @@ +> mem@1.0.0 build +> cargo build --release + +Compiling mem-cli v0.1.0 (/home/ubuntu/workspace/Poimen/memory/crates/mem-cli) +error: unresolved import `mem_core` + --> crates/mem-cli/src/main.rs:15:5 + | +15 | use mem_core::{Record, Provenance, Role}; + | ^^^^^^^^ could not find `mem_core` in `extern prelude` + | + = note: consider adding `extern crate mem_core` to use the crate + +error: could not compile `mem-cli` due to previous error + +npm ERR! code ELIFECYCLE +npm ERR! errno 1 +npm ERR! mem@1.0.0 build: `cargo build --release` +npm ERR! Exit status 1 +npm ERR! +npm ERR! Failed at the mem@1.0.0 build stage. +npm ERR! Make sure you have the latest version of node.js and npm installed. + +npm WARN optional optional dependency failed, continuing fsevents@2.3.2 diff --git a/tests/it_signature.rs b/tests/it_signature.rs new file mode 100644 index 0000000..cd8274f --- /dev/null +++ b/tests/it_signature.rs @@ -0,0 +1,207 @@ +//! M3.7.7 integration tests: failure signature extraction + normalisation +//! +//! Verifies that: +//! 1. Same failure from different runs produces identical hash +//! 2. Different failures produce different hashes +//! 3. Normalisation strips volatile data (timestamps, paths, SHAs) +//! 4. Cascading failures pick the root error +//! 5. Unknown tools still produce signatures +//! 6. No model calls are made +//! 7. Extraction is fast (< 50ms for 50KB logs) +//! 8. Tool is part of the signature identity +//! 9. `mem sig explain` names the rule + +use mem_core::lesson; +use std::fs; +use std::path::Path; +use std::time::Instant; + +fn load_fixture(name: &str) -> String { + let path = format!("fixtures/failures/{}.txt", name); + fs::read_to_string(&path).expect(&format!("Failed to load fixture: {}", path)) +} + +#[test] +fn a1_same_failure_same_hash() { + // For each tool, two runs of the same failure should produce identical sig_sha + let tools_and_fixtures = vec![ + ("npm", "npm-run-a", "npm-run-b"), + ("cargo", "cargo-run-a", "cargo-run-b"), + ("kubectl", "kubectl-apply-a", "kubectl-apply-b"), + ]; + + for (tool, fixture_a, fixture_b) in tools_and_fixtures { + let log_a = load_fixture(fixture_a); + let log_b = load_fixture(fixture_b); + + let sig_a = lesson::extract(tool, &log_a) + .expect(&format!("Failed to extract signature from {}", fixture_a)); + let sig_b = lesson::extract(tool, &log_b) + .expect(&format!("Failed to extract signature from {}", fixture_b)); + + assert_eq!( + sig_a.sig_sha, sig_b.sig_sha, + "Same failure ({}) from different runs should have identical hash. a={}, b={}", + tool, sig_a.sig_sha, sig_b.sig_sha + ); + } +} + +#[test] +fn a2_different_failure_different_hash() { + // Different failures from the same tool should produce different hashes + let tools_and_fixtures = vec![ + ("npm", "npm-run-a", "npm-different"), + ("cargo", "cargo-run-a", "cargo-different"), + ("kubectl", "kubectl-apply-a", "kubectl-different"), + ]; + + for (tool, fixture_same, fixture_diff) in tools_and_fixtures { + let log_same = load_fixture(fixture_same); + let log_diff = load_fixture(fixture_diff); + + let sig_same = lesson::extract(tool, &log_same) + .expect(&format!("Failed to extract signature from {}", fixture_same)); + let sig_diff = lesson::extract(tool, &log_diff) + .expect(&format!("Failed to extract signature from {}", fixture_diff)); + + assert_ne!( + sig_same.sig_sha, sig_diff.sig_sha, + "Different failures ({}) should have different hashes", + tool + ); + } +} + +#[test] +fn a3_normalisation_removes_volatiles() { + // The normalised form should not contain timestamps, paths, SHAs, line numbers + let log = load_fixture("npm-run-a"); + let sig = lesson::extract("npm", &log).expect("Failed to extract signature"); + + // These patterns should NOT appear in the normalised form + let patterns = vec![ + (r"2026-08-\d{2}T\d{2}:\d{2}:\d{2}", "ISO timestamp"), + (r"/home/runner/work", "workspace path"), + (r"\d{7,40}", "git SHA"), + (r":\d+:\d+", "line:col"), + ]; + + for (pattern, name) in patterns { + let re = regex::Regex::new(pattern).unwrap(); + if re.is_match(&sig.normalised) { + panic!( + "Normalised signature should not contain {} but found in: {}", + name, sig.normalised + ); + } + } +} + +#[test] +fn a4_cascade_picks_first() { + // When multiple error lines exist, pick the root (first), not the consequence + let log = r#"error: some root cause +error: consequence of root cause +error: further consequence + +Error summary +"#; + + let sig = lesson::extract("cargo", log).expect("Failed to extract signature"); + assert!( + sig.raw.contains("root cause"), + "Should pick first error, not consequence" + ); +} + +#[test] +fn a5_unknown_tool_fallback() { + // An unrecognised tool should still produce a signature + let log = load_fixture("npm-run-a"); + let sig = lesson::extract("unknown-tool", &log).expect("Failed to extract signature for unknown tool"); + + // Should have a fallback rule + assert_eq!(sig.tool, "unknown-tool"); + assert!(!sig.normalised.is_empty(), "Should produce normalised form for unknown tool"); +} + +#[test] +fn a6_no_model_calls() { + // Extraction should work with no HTTP requests to embedding/LLM services + // (This test passes if extraction succeeds without panicking) + let fixtures = vec!["npm-run-a", "npm-different", "cargo-run-a", "kubectl-apply-a"]; + + for fixture in fixtures { + let log = load_fixture(fixture); + let tool = fixture.split('-').next().unwrap(); + let sig = lesson::extract(tool, &log); + assert!(sig.is_some(), "Extraction should succeed for {}", fixture); + } +} + +#[test] +fn a7_latency_under_50ms() { + // Extraction on a 50KB log should complete in under 50ms + // Create a 50KB synthetic log + let base_log = load_fixture("npm-run-a"); + let mut large_log = base_log.clone(); + while large_log.len() < 50_000 { + large_log.push_str(&base_log); + } + large_log.truncate(50_000); + + let start = Instant::now(); + let _sig = lesson::extract("npm", &large_log).expect("Failed to extract from 50KB log"); + let elapsed = start.elapsed(); + + assert!( + elapsed.as_millis() < 50, + "Extraction should complete in < 50ms, took {}ms", + elapsed.as_millis() + ); +} + +#[test] +fn a8_tool_in_identity() { + // The same normalised text under two different tools should hash differently + let log = r#"error: connection refused +Error: failed to connect +"#; + + let sig_npm = lesson::extract("npm", log).expect("Failed npm extraction"); + let sig_cargo = lesson::extract("cargo", log).expect("Failed cargo extraction"); + + assert_ne!( + sig_npm.sig_sha, sig_cargo.sig_sha, + "Same error text under different tools should produce different hashes" + ); +} + +#[test] +fn a9_explain_output() { + // Verify that extract() returns the rule name + let fixtures = vec![ + ("npm", "npm-run-a"), + ("cargo", "cargo-run-a"), + ("kubectl", "kubectl-apply-a"), + ]; + + for (tool, fixture) in fixtures { + let log = load_fixture(fixture); + let sig = lesson::extract(tool, &log).expect(&format!("Failed to extract {}", fixture)); + + // Rule should be set (not empty or "unknown") + assert!(!sig.rule.is_empty(), "Rule should be set for {}", tool); + assert_ne!(sig.rule, "unknown", "Rule should be specific for {}", tool); + + // Tool should be in rule (e.g., "npm_error_line") + let rule_lower = sig.rule.to_lowercase(); + let tool_lower = tool.to_lowercase(); + assert!( + rule_lower.contains(&tool_lower) || rule_lower == "fallback", + "Rule should mention tool {} or be fallback, got: {}", + tool, sig.rule + ); + } +}