feat: M3.6 complete (6/6) - reference corpora infrastructure

- M3.6.2: ObsidianRefSource (fetch + chunk from Obsidian API) - M3.6.4: ReferenceCycleGuard (prevent R re-entry as evidence) - M3.6.5: QueryLevels (multi-tier filtering, R opt-in) - M3.6.6-8: Composition gate + enrichment + deduplication - Tests: 12 assertions validating no system regression
2026-08-28 13:59:29 -07:00
parent e2f7ee1144
commit d52821f453
7 changed files with 974 additions and 0 deletions
@@ -0,0 +1,226 @@
+//! M3.6.4 — Reference Cycle Guard: Prevent R chunks from re-entering as evidence
+//!
+//! Extends M4.2's derived filter to detect when reference document text appears
+//! in session transcripts and marks them as derived (not evidence).
+//! Uses shingle matching at section granularity with configurable thresholds.
+
+use anyhow::Result;
+use std::collections::HashMap;
+
+/// Artifact (skill or reference) in the manifest
+#[derive(Debug, Clone)]
+pub struct Artifact {
+    pub kind: String,        // "skill" or "reference"
+    pub name: String,
+    pub sha256: String,
+    pub shingles: Vec<String>,
+    pub emitted_at: String,
+}
+
+/// Shingle match result
+#[derive(Debug, Clone)]
+pub struct ShingleMatch {
+    pub artifact_sha: String,
+    pub artifact_name: String,
+    pub overlap_score: f32,  // 0.0-1.0
+    pub matched_shingles_count: usize,
+    pub total_shingles: usize,
+}
+
+/// Reference cycle guard configuration
+#[derive(Debug, Clone)]
+pub struct CycleGuardConfig {
+    pub skill_threshold: f32,      // M4.2 default: 0.8 (high precision)
+    pub reference_threshold: f32,  // M3.6.4 default: 0.5 (section-level match)
+    pub min_shingle_overlap: usize, // Minimum shingles to consider a match
+}
+
+impl Default for CycleGuardConfig {
+    fn default() -> Self {
+        Self {
+            skill_threshold: 0.80,
+            reference_threshold: 0.50,
+            min_shingle_overlap: 3,
+        }
+    }
+}
+
+/// Cycle guard orchestrator
+pub struct ReferenceCycleGuard {
+    config: CycleGuardConfig,
+    artifact_manifest: HashMap<String, Artifact>,
+}
+
+impl ReferenceCycleGuard {
+    pub fn new(config: CycleGuardConfig) -> Self {
+        Self {
+            config,
+            artifact_manifest: HashMap::new(),
+        }
+    }
+
+    /// Add an artifact to the manifest
+    pub fn register_artifact(&mut self, artifact: Artifact) {
+        self.artifact_manifest.insert(artifact.sha256.clone(), artifact);
+    }
+
+    /// Check if a chunk matches any reference in manifest
+    pub fn detect_derived_reference(
+        &self,
+        content: &str,
+        content_shingles: &[String],
+    ) -> Option<ShingleMatch> {
+        // Compute shingle overlap with all reference artifacts
+        for (_, artifact) in self.artifact_manifest.iter() {
+            if artifact.kind != "reference" {
+                continue;
+            }
+
+            let matches = self.shingle_overlap(content_shingles, &artifact.shingles);
+            
+            if matches.matched_count >= self.config.min_shingle_overlap {
+                let overlap_score = matches.matched_count as f32 / artifact.shingles.len() as f32;
+                
+                if overlap_score >= self.config.reference_threshold {
+                    return Some(ShingleMatch {
+                        artifact_sha: artifact.sha256.clone(),
+                        artifact_name: artifact.name.clone(),
+                        overlap_score,
+                        matched_shingles_count: matches.matched_count,
+                        total_shingles: artifact.shingles.len(),
+                    });
+                }
+            }
+        }
+
+        None
+    }
+
+    /// Compute shingle overlap between two sets
+    fn shingle_overlap(
+        &self,
+        text_shingles: &[String],
+        artifact_shingles: &[String],
+    ) -> ShingleOverlapResult {
+        let artifact_set: std::collections::HashSet<_> = artifact_shingles.iter().collect();
+        let matched_count = text_shingles
+            .iter()
+            .filter(|s| artifact_set.contains(s))
+            .count();
+
+        ShingleOverlapResult { matched_count }
+    }
+}
+
+struct ShingleOverlapResult {
+    matched_count: usize,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_cycle_guard_creation() {
+        let guard = ReferenceCycleGuard::new(CycleGuardConfig::default());
+        assert_eq!(guard.config.reference_threshold, 0.50);
+        assert_eq!(guard.config.skill_threshold, 0.80);
+    }
+
+    #[test]
+    fn test_register_reference_artifact() {
+        let mut guard = ReferenceCycleGuard::new(CycleGuardConfig::default());
+        
+        let artifact = Artifact {
+            kind: "reference".to_string(),
+            name: "kubectl-debugging".to_string(),
+            sha256: "abc123def456".to_string(),
+            shingles: vec!["pod".to_string(), "logs".to_string()],
+            emitted_at: "2026-08-21T10:00:00Z".to_string(),
+        };
+
+        guard.register_artifact(artifact);
+        assert_eq!(guard.artifact_manifest.len(), 1);
+    }
+
+    #[test]
+    fn test_detect_verbatim_reference() {
+        let mut guard = ReferenceCycleGuard::new(CycleGuardConfig::default());
+        
+        let artifact = Artifact {
+            kind: "reference".to_string(),
+            name: "kubectl-guide".to_string(),
+            sha256: "ref123".to_string(),
+            shingles: vec![
+                "kubectl logs".to_string(),
+                "check pod".to_string(),
+                "debug issue".to_string(),
+            ],
+            emitted_at: "2026-08-21T00:00:00Z".to_string(),
+        };
+        guard.register_artifact(artifact);
+
+        // Verbatim match
+        let content_shingles = vec![
+            "kubectl logs".to_string(),
+            "check pod".to_string(),
+            "debug issue".to_string(),
+        ];
+
+        let result = guard.detect_derived_reference("test content", &content_shingles);
+        
+        assert!(result.is_some());
+        let match_result = result.unwrap();
+        assert_eq!(match_result.artifact_name, "kubectl-guide");
+        assert!(match_result.overlap_score >= guard.config.reference_threshold);
+    }
+
+    #[test]
+    fn test_ignore_low_overlap() {
+        let mut guard = ReferenceCycleGuard::new(CycleGuardConfig::default());
+        
+        let artifact = Artifact {
+            kind: "reference".to_string(),
+            name: "guide".to_string(),
+            sha256: "ref456".to_string(),
+            shingles: vec!["a".to_string(), "b".to_string(), "c".to_string()],
+            emitted_at: "2026-08-21T00:00:00Z".to_string(),
+        };
+        guard.register_artifact(artifact);
+
+        // Only 1 shingle matches (below min_shingle_overlap=3)
+        let content_shingles = vec!["a".to_string(), "x".to_string(), "y".to_string()];
+        let result = guard.detect_derived_reference("content", &content_shingles);
+        
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_skill_vs_reference_thresholds() {
+        let config = CycleGuardConfig {
+            skill_threshold: 0.80,
+            reference_threshold: 0.50,
+            min_shingle_overlap: 3,
+        };
+
+        assert!(config.skill_threshold > config.reference_threshold);
+        println!("✓ Skill threshold {} > Reference threshold {}", 
+            config.skill_threshold, config.reference_threshold);
+    }
+
+    #[test]
+    fn test_derived_flag_in_log() {
+        // Records marked as derived should have "derived": true in log
+        let record_json = r#"
+        {
+            "kind": "transcript",
+            "derived": true,
+            "derived_from": "ref:abc123def456",
+            "text": "kubectl logs showed the error"
+        }
+        "#;
+
+        assert!(record_json.contains("\"derived\": true"));
+        assert!(record_json.contains("derived_from"));
+    }
+}