Implement M4.2: Derived filter (shingle matcher + 10 tests, 239 total)

tests/it_derived_filter.rs

@@ -0,0 +1,202 @@
+//! Integration tests for derived filter
+//!
+//! Verifies that emitted skills/docs are excluded from ingest when they
+//! reappear verbatim or reformatted.
+
+use mem_ingest::{ArtifactRecord, DerivedFilter};
+
+/// Test 1: Verbatim match is excluded
+#[test]
+fn a1_verbatim_excluded() {
+    let mut filter = DerivedFilter::new(0.8);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "deploy-to-prod",
+        "Deploy service to production using kubectl apply",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    let record = "Deploy service to production using kubectl apply";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_some(), "Verbatim match should be excluded");
+    if let Some(m) = result {
+        assert_eq!(m.artifact_name, "deploy-to-prod");
+        assert!(m.overlap_ratio >= 0.8);
+    }
+}
+
+/// Test 2: Reformatted (whitespace, line breaks) is excluded
+#[test]
+fn a2_reformatted_excluded() {
+    let mut filter = DerivedFilter::new(0.8);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "deploy-to-prod",
+        "Deploy service to production using kubectl apply",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    // Same content, different whitespace
+    let record = "Deploy   service   to   production   using   kubectl   apply";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_some(), "Reformatted match should be excluded");
+}
+
+/// Test 3: Mere mention is NOT excluded
+#[test]
+fn a3_mention_not_excluded() {
+    let mut filter = DerivedFilter::new(0.8);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "deploy-to-prod",
+        "Deploy service to production using kubectl apply",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    let record = "I used the deploy-to-prod skill yesterday to update the service";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_none(), "Mere mention should NOT be excluded");
+}
+
+/// Test 4: Unrelated text is not excluded
+#[test]
+fn a4_unrelated_not_excluded() {
+    let mut filter = DerivedFilter::new(0.8);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "deploy-to-prod",
+        "Deploy service to production using kubectl apply",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    let record = "I went to the grocery store today and bought some milk and bread";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_none(), "Unrelated text should not be excluded");
+}
+
+/// Test 5: Exclusion is logged with match metadata
+#[test]
+fn a5_exclusion_logged() {
+    let mut filter = DerivedFilter::new(0.8);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "test-skill",
+        "Test content here",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    let record = "Test content here";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_some());
+    if let Some(m) = result {
+        assert_eq!(m.artifact_name, "test-skill");
+        assert_eq!(m.artifact_kind, "skill");
+        assert!(m.overlap_ratio > 0.0, "Overlap ratio should be logged");
+    }
+}
+
+/// Test 6: Threshold is configurable
+#[test]
+fn a6_threshold_configurable() {
+    let filter_strict = DerivedFilter::new(0.9);
+    let filter_loose = DerivedFilter::new(0.5);
+
+    assert_eq!(filter_strict.threshold, 0.9);
+    assert_eq!(filter_loose.threshold, 0.5);
+}
+
+/// Test 7: No manifest is safe (doesn't fail, just no filtering)
+#[test]
+fn a7_no_manifest_is_safe() {
+    // Non-existent path should not panic
+    let filter = DerivedFilter::load_from_jsonl("/nonexistent/path.jsonl", 0.8);
+
+    assert!(filter.is_ok(), "Missing manifest should be safe");
+    if let Ok(f) = filter {
+        assert_eq!(f.artifacts.len(), 0, "Empty manifest should have no artifacts");
+    }
+}
+
+/// Test 8: Multiple artifacts (kind + name distinction)
+#[test]
+fn a8_multiple_artifacts() {
+    let mut filter = DerivedFilter::new(0.8);
+
+    let skill1 = ArtifactRecord::new(
+        "skill",
+        "deploy-prod",
+        "Deploy to production",
+        "2025-01-26T10:00:00Z",
+    );
+    let skill2 = ArtifactRecord::new(
+        "reference",
+        "kubectl-docs",
+        "kubectl is a command line tool",
+        "2025-01-26T10:00:00Z",
+    );
+
+    filter.add_artifact(skill1);
+    filter.add_artifact(skill2);
+
+    assert_eq!(filter.artifacts.len(), 2);
+
+    // Should match first artifact
+    let result1 = filter.is_derived("Deploy to production");
+    assert!(result1.is_some());
+    if let Some(m) = result1 {
+        assert_eq!(m.artifact_name, "deploy-prod");
+    }
+
+    // Should match second artifact
+    let result2 = filter.is_derived("kubectl is a command line tool");
+    assert!(result2.is_some());
+    if let Some(m) = result2 {
+        assert_eq!(m.artifact_name, "kubectl-docs");
+    }
+}
+
+/// Test 9: Partial overlap below threshold is not excluded
+#[test]
+fn a9_partial_overlap_below_threshold() {
+    let mut filter = DerivedFilter::new(0.9);
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "deploy",
+        "Deploy service to production",
+        "2025-01-26T10:00:00Z",
+    );
+    filter.add_artifact(artifact);
+
+    // Similar but different text (below 0.9 threshold)
+    let record = "Deploy service to staging";
+    let result = filter.is_derived(record);
+
+    assert!(result.is_none(), "Partial overlap below threshold should not be excluded");
+}
+
+/// Test 10: Artifact provenance is retained
+#[test]
+fn a10_artifact_provenance() {
+    let artifact = ArtifactRecord::new(
+        "skill",
+        "my-skill",
+        "some content",
+        "2025-01-26T14:32:00Z",
+    );
+
+    assert_eq!(artifact.kind, "skill");
+    assert_eq!(artifact.name, "my-skill");
+    assert_eq!(artifact.emitted_at, "2025-01-26T14:32:00Z");
+    assert!(!artifact.sha256.is_empty());
+    assert!(!artifact.shingles.is_empty());
+}