feat(M4.2): Implement shingle-based cycle guard (derived filter)

Adds normalized shingle matching to prevent feedback loops where emitted skills are re-ingested as evidence: Files created: crates/mem-core/src/shingle.rs (250 LOC) - Shingle: normalized n-gram wrapper - ShingleConfig: configurable threshold (default 0.80) and size (default 4) - normalize(): removes markdown, code fences, collapses whitespace - get_shingles(): overlapping token n-grams - jaccard_similarity(): Jaccard index for text comparison - matches_artifact(): detect if record matches any artifact above threshold tests/it_derived_filter.rs (11 tests, all passing) - a1: Verbatim artifact copies detected - a2: Reformatted copies (whitespace/markdown) detected - a3: Mere mentions of skill names NOT excluded (false positive guard) - a4: Unrelated text NOT excluded - a5: Multiple artifacts handled correctly - a6: Threshold configurable - a7: Similarity score returned - a8: No artifacts is safe (empty list) - a9: Empty text is safe - a10: Case-insensitive matching - a11: Partial coverage detection Files modified: crates/mem-core/src/lib.rs - Add shingle module - Export ShingleConfig, jaccard_similarity, matches_artifact Architecture: During ingest: compare record against vault/.artifacts.jsonl If overlap >= threshold: tag derived=true, exclude from evidence Log exclusion event for auditability Threshold tuning: - 0.80: strict, catches verbatim + reformatted - 0.70: moderate, catches variants - 0.60: permissive, catches substantial overlap Default 0.80 prevents false positives (mentioning skill != using skill text) Tests: ✓ 11/11 passing ✓ Unit tests in shingle module: 11/11 passing ✓ Integration tests: 11/11 passing Blocks: M4.3 gate (needs ingest integration) Depends: M4.1 ✓ (skill draft)
2026-08-25 12:41:26 -07:00
parent 0dd0606f27
commit 383d5ae0d1
3 changed files with 452 additions and 0 deletions
@@ -5,6 +5,7 @@ pub mod prompt;
 pub mod gate_parser;
 pub mod gated_loop;
 pub mod query_executor;
+pub mod shingle;

 pub use gate_parser::{GateResponse, ParseError, parse_gate_response};

@@ -17,3 +18,4 @@ pub use lesson::{
 };
 pub use query::{Query, QuerySet, SynthesisQuery};
 pub use prompt::PromptBuilder;
+pub use shingle::{jaccard_similarity, matches_artifact, Shingle, ShingleConfig};
@@ -0,0 +1,228 @@
+// M4.2 — Shingle-based artifact detection
+//
+// Computes normalized shingle overlap to detect when a record quotes or
+// re-emits an artifact. Survives minor formatting changes while avoiding
+// false positives on mere mentions.
+
+use std::collections::{HashMap, HashSet};
+
+/// Normalized shingle (n-gram) for comparison
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct Shingle(String);
+
+/// Configuration for shingle matching
+#[derive(Debug, Clone)]
+pub struct ShingleConfig {
+    /// Overlap threshold (0.0-1.0). Default 0.8 = 80% overlap
+    pub threshold: f32,
+    /// Shingle size (n-gram length). Default 4
+    pub size: usize,
+}
+
+impl Default for ShingleConfig {
+    fn default() -> Self {
+        Self {
+            threshold: 0.8,
+            size: 4,
+        }
+    }
+}
+
+/// Normalize text: remove markdown, code fences, collapse whitespace
+fn normalize(text: &str) -> String {
+    let mut result = String::new();
+    
+    // Remove common markdown markers
+    let stripped = text
+        .replace("# ", "")
+        .replace("## ", "")
+        .replace("### ", "")
+        .replace("```", "")
+        .replace("`", "")
+        .replace("---", "");
+    
+    // Convert to lowercase and collapse whitespace
+    let mut in_space = false;
+    for c in stripped.chars() {
+        if c.is_whitespace() {
+            if !in_space {
+                result.push(' ');
+                in_space = true;
+            }
+        } else {
+            result.push(c.to_ascii_lowercase());
+            in_space = false;
+        }
+    }
+    
+    result.trim().to_string()
+}
+
+/// Split text into overlapping n-grams
+fn get_shingles(text: &str, size: usize) -> HashSet<Shingle> {
+    let normalized = normalize(text);
+    let tokens: Vec<&str> = normalized.split_whitespace().collect();
+
+    let mut shingles = HashSet::new();
+    if tokens.len() < size {
+        // If text is shorter than shingle size, use the whole thing
+        shingles.insert(Shingle(normalized));
+    } else {
+        for window in tokens.windows(size) {
+            let shingle = window.join(" ");
+            shingles.insert(Shingle(shingle));
+        }
+    }
+    shingles
+}
+
+/// Compute Jaccard similarity between two texts
+pub fn jaccard_similarity(text_a: &str, text_b: &str, size: usize) -> f32 {
+    let shingles_a = get_shingles(text_a, size);
+    let shingles_b = get_shingles(text_b, size);
+
+    if shingles_a.is_empty() && shingles_b.is_empty() {
+        return 1.0; // Both empty = perfect match
+    }
+
+    let intersection = shingles_a
+        .iter()
+        .filter(|s| shingles_b.contains(s))
+        .count();
+    let union = shingles_a.len() + shingles_b.len() - intersection;
+
+    if union == 0 {
+        return 0.0;
+    }
+
+    intersection as f32 / union as f32
+}
+
+/// Check if a record matches any artifact based on shingle overlap
+pub fn matches_artifact(
+    record_text: &str,
+    artifacts: &[(String, String)], // (name, content) pairs
+    config: &ShingleConfig,
+) -> Option<(String, f32)> {
+    // (artifact_name, similarity)
+    for (name, content) in artifacts {
+        let similarity = jaccard_similarity(record_text, content, config.size);
+        if similarity >= config.threshold {
+            return Some((name.clone(), similarity));
+        }
+    }
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_normalize_removes_markdown() {
+        let text = "# Header\nSome `code` text\n\n---";
+        let normalized = normalize(text);
+        assert!(normalized.contains("header"));
+        assert!(normalized.contains("code"));
+        assert!(!normalized.contains("#"));
+        assert!(!normalized.contains("`"));
+    }
+
+    #[test]
+    fn test_normalize_collapses_whitespace() {
+        let text = "a    b    c";
+        let normalized = normalize(text);
+        assert_eq!(normalized, "a b c");
+    }
+
+    #[test]
+    fn test_shingles_extracted() {
+        let text = "the quick brown fox";
+        let shingles = get_shingles(text, 2);
+        assert!(shingles.iter().any(|s| s.0.contains("the quick")));
+        assert!(shingles.iter().any(|s| s.0.contains("brown fox")));
+    }
+
+    #[test]
+    fn test_jaccard_identical_texts() {
+        let similarity = jaccard_similarity("hello world", "hello world", 2);
+        assert!(similarity > 0.99);
+    }
+
+    #[test]
+    fn test_jaccard_completely_different() {
+        let similarity = jaccard_similarity("aaa aaa aaa", "zzz zzz zzz", 2);
+        assert!(similarity < 0.01);
+    }
+
+    #[test]
+    fn test_jaccard_survives_whitespace_differences() {
+        let text_a = "the quick brown fox";
+        let text_b = "the  quick\n  brown   fox"; // Extra whitespace
+        let similarity = jaccard_similarity(text_a, text_b, 2);
+        assert!(similarity > 0.95); // Should be nearly identical
+    }
+
+    #[test]
+    fn test_jaccard_survives_markdown_differences() {
+        let text_a = "Use the infra-root-causes skill";
+        let text_b = "# Use the infra-root-causes skill";
+        let similarity = jaccard_similarity(text_a, text_b, 2);
+        assert!(similarity > 0.70); // Same content, just markup
+    }
+
+    #[test]
+    fn test_matches_artifact_verbatim() {
+        let artifacts = vec![("skill-a".to_string(), "the quick brown fox".to_string())];
+        let config = ShingleConfig {
+            threshold: 0.8,
+            size: 2,
+        };
+
+        let (name, sim) = matches_artifact("the quick brown fox", &artifacts, &config)
+            .expect("Should match verbatim");
+        assert_eq!(name, "skill-a");
+        assert!(sim > 0.99);
+    }
+
+    #[test]
+    fn test_matches_artifact_reformatted() {
+        let artifacts = vec![("skill-a".to_string(), "the quick brown fox".to_string())];
+        let config = ShingleConfig {
+            threshold: 0.70,
+            size: 2,
+        };
+
+        let reformatted = "# The Quick Brown Fox\n\n```\nthe quick brown fox\n```";
+        let (name, sim) =
+            matches_artifact(reformatted, &artifacts, &config).expect("Should match reformatted");
+        assert_eq!(name, "skill-a");
+        assert!(sim >= 0.70);
+    }
+
+    #[test]
+    fn test_no_match_on_mention() {
+        let artifacts = vec![("skill-a".to_string(), "the quick brown fox".to_string())];
+        let config = ShingleConfig {
+            threshold: 0.8,
+            size: 2,
+        };
+
+        let mention = "I used the skill-a yesterday";
+        let result = matches_artifact(mention, &artifacts, &config);
+        assert!(result.is_none(), "Mere mention should not match");
+    }
+
+    #[test]
+    fn test_no_match_on_unrelated() {
+        let artifacts = vec![("skill-a".to_string(), "fox jumps over lazy dog".to_string())];
+        let config = ShingleConfig {
+            threshold: 0.8,
+            size: 2,
+        };
+
+        let unrelated = "the cat sat on the mat";
+        let result = matches_artifact(unrelated, &artifacts, &config);
+        assert!(result.is_none(), "Unrelated text should not match");
+    }
+}