Implement M4.2: Derived filter (shingle matcher + 10 tests, 239 total)

crates/mem-ingest/src/derived_filter.rs

@@ -0,0 +1,241 @@
+//! Derived filter: prevent emitted artifacts from being ingested as evidence.
+//!
+//! Mechanism: track emitted skills/docs in a manifest, compute shingle overlap
+//! with ingest records, exclude those above a threshold.
+
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use std::collections::HashSet;
+
+/// Artifact manifest record (one per emitted skill/doc)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ArtifactRecord {
+    pub kind: String, // "skill", "reference", etc.
+    pub name: String,
+    pub sha256: String,
+    pub shingles: Vec<String>,
+    pub emitted_at: String,
+}
+
+impl ArtifactRecord {
+    /// Create new artifact record from content
+    pub fn new(kind: &str, name: &str, content: &str, emitted_at: &str) -> Self {
+        let mut hasher = Sha256::new();
+        hasher.update(content.as_bytes());
+        let sha256 = format!("{:x}", hasher.finalize());
+
+        let shingles = compute_shingles(content, 4);
+
+        Self {
+            kind: kind.to_string(),
+            name: name.to_string(),
+            sha256,
+            shingles,
+            emitted_at: emitted_at.to_string(),
+        }
+    }
+
+    /// Exact match by SHA256
+    pub fn matches_exactly(&self, other_sha256: &str) -> bool {
+        self.sha256 == other_sha256
+    }
+
+    /// Fuzzy match by shingle overlap
+    pub fn shingle_overlap(&self, other_shingles: &[String]) -> f64 {
+        if self.shingles.is_empty() || other_shingles.is_empty() {
+            return 0.0;
+        }
+
+        let self_set: HashSet<_> = self.shingles.iter().collect();
+        let other_set: HashSet<_> = other_shingles.iter().collect();
+
+        let intersection = self_set.intersection(&other_set).count();
+        let union = self_set.union(&other_set).count();
+
+        if union == 0 {
+            0.0
+        } else {
+            intersection as f64 / union as f64
+        }
+    }
+}
+
+/// Compute n-grams (shingles) from normalized text
+pub fn compute_shingles(text: &str, shingle_size: usize) -> Vec<String> {
+    // Normalize: lowercase, strip whitespace, remove markdown
+    let normalized = normalize_text(text);
+
+    if normalized.len() < shingle_size {
+        return vec![];
+    }
+
+    normalized
+        .chars()
+        .collect::<Vec<_>>()
+        .windows(shingle_size)
+        .map(|w| w.iter().collect::<String>())
+        .collect()
+}
+
+/// Normalize text for matching: lowercase, strip non-alphanumeric, collapse whitespace
+fn normalize_text(text: &str) -> String {
+    text.to_lowercase()
+        .chars()
+        .filter(|c| c.is_alphanumeric() || c.is_whitespace())
+        .collect::<String>()
+        .split_whitespace()
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+/// Derived filter with manifest and threshold
+pub struct DerivedFilter {
+    pub artifacts: Vec<ArtifactRecord>,
+    pub threshold: f64,
+    pub shingle_size: usize,
+}
+
+impl DerivedFilter {
+    pub fn new(threshold: f64) -> Self {
+        Self {
+            artifacts: vec![],
+            threshold,
+            shingle_size: 4,
+        }
+    }
+
+    /// Add artifact to filter
+    pub fn add_artifact(&mut self, artifact: ArtifactRecord) {
+        self.artifacts.push(artifact);
+    }
+
+    /// Check if record is derived (matches an artifact)
+    pub fn is_derived(&self, record_text: &str) -> Option<DerivedMatch> {
+        let record_shingles = compute_shingles(record_text, self.shingle_size);
+
+        for artifact in &self.artifacts {
+            let overlap = artifact.shingle_overlap(&record_shingles);
+
+            if overlap >= self.threshold {
+                return Some(DerivedMatch {
+                    artifact_name: artifact.name.clone(),
+                    artifact_kind: artifact.kind.clone(),
+                    overlap_ratio: overlap,
+                });
+            }
+        }
+
+        None
+    }
+
+    /// Load from JSONL manifest file
+    pub fn load_from_jsonl(path: &str, threshold: f64) -> anyhow::Result<Self> {
+        use std::fs;
+
+        let mut filter = Self::new(threshold);
+
+        if !std::path::Path::new(path).exists() {
+            return Ok(filter); // No manifest = no filtering
+        }
+
+        let content = fs::read_to_string(path)?;
+        for line in content.lines() {
+            if line.trim().is_empty() {
+                continue;
+            }
+            if let Ok(artifact) = serde_json::from_str::<ArtifactRecord>(line) {
+                filter.add_artifact(artifact);
+            }
+        }
+
+        Ok(filter)
+    }
+}
+
+/// Result of a derived match
+#[derive(Debug, Clone)]
+pub struct DerivedMatch {
+    pub artifact_name: String,
+    pub artifact_kind: String,
+    pub overlap_ratio: f64,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_normalize_text() {
+        let text = "  Hello   WORLD!!!  \n  123  ";
+        let normalized = normalize_text(text);
+        assert_eq!(normalized, "hello world 123");
+    }
+
+    #[test]
+    fn test_compute_shingles() {
+        let text = "hello";
+        let shingles = compute_shingles(text, 2);
+        assert!(shingles.contains(&"he".to_string()));
+        assert!(shingles.contains(&"el".to_string()));
+        assert_eq!(shingles.len(), 4); // "he", "el", "ll", "lo"
+    }
+
+    #[test]
+    fn test_artifact_exact_match() {
+        let artifact = ArtifactRecord::new("skill", "test", "content", "2025-01-26");
+        let other_sha = format!("{:x}", Sha256::digest("content".as_bytes()));
+        assert!(artifact.matches_exactly(&other_sha));
+    }
+
+    #[test]
+    fn test_shingle_overlap_identical() {
+        let text = "hello world";
+        let shingles_a = compute_shingles(text, 4);
+        let shingles_b = compute_shingles(text, 4);
+
+        let artifact = ArtifactRecord::new("skill", "test", text, "2025-01-26");
+        let overlap = artifact.shingle_overlap(&shingles_b);
+
+        assert_eq!(overlap, 1.0, "Identical shingles should have 100% overlap");
+    }
+
+    #[test]
+    fn test_shingle_overlap_partial() {
+        let artifact = ArtifactRecord::new("skill", "test", "hello world test", "2025-01-26");
+        let other_text = "hello world other";
+        let other_shingles = compute_shingles(other_text, 4);
+
+        let overlap = artifact.shingle_overlap(&other_shingles);
+        assert!(overlap > 0.0 && overlap < 1.0, "Partial overlap should be between 0 and 1");
+    }
+
+    #[test]
+    fn test_derived_filter_is_derived() {
+        let mut filter = DerivedFilter::new(0.8);
+        let artifact = ArtifactRecord::new("skill", "infra-deploy", "deploy to kubernetes", "2025-01-26");
+        filter.add_artifact(artifact);
+
+        // Identical text should match
+        let result = filter.is_derived("deploy to kubernetes");
+        assert!(result.is_some());
+
+        // Reformatted should also match
+        let result = filter.is_derived("deploy   to   kubernetes");
+        assert!(result.is_some());
+
+        // Unrelated should not match
+        let result = filter.is_derived("something completely different");
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_derived_filter_mention_not_excluded() {
+        let mut filter = DerivedFilter::new(0.8);
+        let artifact = ArtifactRecord::new("skill", "infra-deploy", "deploy to kubernetes", "2025-01-26");
+        filter.add_artifact(artifact);
+
+        // Merely mentioning the skill should not be excluded
+        let result = filter.is_derived("I used the infra-deploy skill yesterday");
+        assert!(result.is_none(), "Mere mention should not trigger exclusion");
+    }
+}

crates/mem-ingest/src/lib.rs

@@ -1,7 +1,9 @@
 pub mod pi_session;
 pub mod claude_transcript;
 pub mod doc_corpus;
+pub mod derived_filter;
 
 pub use pi_session::PiSessionSource;
 pub use claude_transcript::ClaudeTranscriptSource;
 pub use doc_corpus::{DocCorpusSource, DocSection, DryRunReport};
+pub use derived_filter::{ArtifactRecord, DerivedFilter, DerivedMatch};