poimen-memory/tests/it_phase5_phase6.rs

/// Integration Tests: Phase 5 (Metadata) + Phase 6 (Cache) + Full Pipeline
///
/// Tests end-to-end flow with all phases integrated:
/// 1. Query intent inference
/// 2. Wiki-scoped + hybrid retrieval
/// 3. LLM optimization
/// 4. Metadata boost based on intent-category match
/// 5. Cache alignment with wiki distances

use std::collections::BTreeMap;
use std::sync::Arc;

use mem_core::{GlobalTfIdfScorer, SemanticScorer};
use mem_ingest::wiki_link::WikiLinkGraph;
use mem_cli::{
    FullPipeline, PipelineConfig, PipelineBuilder, EnrichedChunk,
    MetadataExtractor, MetadataBooster, ChunkCategory, QueryIntent,
    LruChunkCache, KvCacheAligner, CacheLocalityAnalyzer, CacheMetrics,
};

// ============================================================================
// Test Fixtures
// ============================================================================

fn create_test_vocab() -> Arc<BTreeMap<String, f32>> {
    let mut vocab = BTreeMap::new();
    vocab.insert("kubernetes".to_string(), 0.8);
    vocab.insert("pod".to_string(), 0.7);
    vocab.insert("error".to_string(), 0.95);
    vocab.insert("crashloopbackoff".to_string(), 1.0);
    vocab.insert("fix".to_string(), 0.85);
    vocab.insert("solution".to_string(), 0.8);
    vocab.insert("debug".to_string(), 0.9);
    vocab.insert("explain".to_string(), 0.75);
    vocab.insert("concept".to_string(), 0.7);
    vocab.insert("api".to_string(), 0.6);
    Arc::new(vocab)
}

fn create_test_pipeline() -> FullPipeline {
    let vocab = create_test_vocab();
    let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
    let semantic = Arc::new(SemanticScorer::new());
    FullPipeline::new(tfidf, semantic, PipelineConfig::default())
}

fn create_test_wiki_graph() -> WikiLinkGraph {
    let mut graph = WikiLinkGraph::new("poimen");
    // Build realistic wiki structure
    graph.add_link("index.md", "tools/kubectl.md");
    graph.add_link("index.md", "concepts/pods.md");
    graph.add_link("tools/kubectl.md", "debugging/pod-errors.md");
    graph.add_link("debugging/pod-errors.md", "solutions/restart-pod.md");
    graph.add_link("concepts/pods.md", "concepts/lifecycle.md");
    graph
}

fn create_diverse_candidates() -> Vec<(String, String)> {
    vec![
        // Error category
        ("debugging/pod-errors.md".to_string(), 
         "# Pod Errors\n\nError: CrashLoopBackOff when pod fails to start. Check container logs.".to_string()),
        
        // Solution category  
        ("solutions/restart-pod.md".to_string(),
         "# Restart Pod Solution\n\nTo fix the crashing pod, configure restart policy and check resources.".to_string()),
        
        // Tool category
        ("tools/kubectl.md".to_string(),
         "# Kubectl Tool\n\nUsage: kubectl get pods\n$ kubectl describe pod <name>\nAPI reference for kubernetes CLI.".to_string()),
        
        // Concept category
        ("concepts/pods.md".to_string(),
         "# Pod Concepts\n\nA kubernetes pod is the smallest deployable unit. Explain the design pattern.".to_string()),
        
        // Reference category
        ("concepts/lifecycle.md".to_string(),
         "# Pod Lifecycle Reference\n\nDocumentation and specification for pod states: Pending, Running, Succeeded, Failed.".to_string()),
        
        // Index
        ("index.md".to_string(),
         "# Kubernetes Guide\n\nMain entry point for kubernetes documentation.".to_string()),
         
        // Unrelated (outside wiki scope)
        ("unrelated/docker.md".to_string(),
         "# Docker Guide\n\nDocker container basics unrelated to kubernetes.".to_string()),
    ]
}

// ============================================================================
// Phase 5: Metadata Enhancement Tests
// ============================================================================

#[test]
fn test_query_intent_inference() {
    // FixError intents
    assert_eq!(MetadataExtractor::infer_query_intent("fix pod crash"), QueryIntent::FixError);
    assert_eq!(MetadataExtractor::infer_query_intent("debug kubernetes error"), QueryIntent::FixError);
    assert_eq!(MetadataExtractor::infer_query_intent("troubleshoot deployment"), QueryIntent::FixError);
    
    // LearnConcept intents
    assert_eq!(MetadataExtractor::infer_query_intent("explain kubernetes pods"), QueryIntent::LearnConcept);
    assert_eq!(MetadataExtractor::infer_query_intent("understand deployment patterns"), QueryIntent::LearnConcept);
    assert_eq!(MetadataExtractor::infer_query_intent("what is a service mesh"), QueryIntent::LearnConcept);
    
    // UseTool intents
    assert_eq!(MetadataExtractor::infer_query_intent("use kubectl api"), QueryIntent::UseTool);
    assert_eq!(MetadataExtractor::infer_query_intent("run helm command"), QueryIntent::UseTool);
    assert_eq!(MetadataExtractor::infer_query_intent("call kubernetes api"), QueryIntent::UseTool);
    
    // FindReference intents
    assert_eq!(MetadataExtractor::infer_query_intent("reference for pod spec"), QueryIntent::FindReference);
    assert_eq!(MetadataExtractor::infer_query_intent("definition of deployment"), QueryIntent::FindReference);
}

#[test]
fn test_category_inference() {
    // Error category
    let (cat, conf) = MetadataExtractor::infer_category("Error: CrashLoopBackOff exception");
    assert_eq!(cat, ChunkCategory::Error);
    assert!(conf >= 0.8);
    
    // Solution category
    let (cat, _) = MetadataExtractor::infer_category("Fix this by configuring the solution");
    assert_eq!(cat, ChunkCategory::Solution);
    
    // Tool category
    let (cat, _) = MetadataExtractor::infer_category("Usage: kubectl get pods\n$ kubectl apply");
    assert_eq!(cat, ChunkCategory::Tool);
    
    // Concept category
    let (cat, _) = MetadataExtractor::infer_category("The design pattern explains the principle");
    assert_eq!(cat, ChunkCategory::Concept);
    
    // Reference category
    let (cat, _) = MetadataExtractor::infer_category("Documentation reference and specification");
    assert_eq!(cat, ChunkCategory::Reference);
}

#[test]
fn test_metadata_extraction_full() {
    let text = "# Pod Debugging\n\nError: CrashLoopBackOff when kubernetes pod fails.";
    let metadata = MetadataExtractor::extract("doc1", text);
    
    assert_eq!(metadata.chunk_id, "doc1");
    assert_eq!(metadata.heading, Some("Pod Debugging".to_string()));
    assert!(!metadata.key_terms.is_empty());
    assert_eq!(metadata.category, ChunkCategory::Error);
    assert!(metadata.category_confidence > 0.0);
}

#[test]
fn test_metadata_booster_intent_match() {
    let booster = MetadataBooster::new();
    
    // Error chunk + FixError intent = boost
    let error_metadata = MetadataExtractor::extract("doc1", "Error: pod crash");
    let boost = booster.calculate_boost(QueryIntent::FixError, &error_metadata);
    assert!(boost > 0.0, "Error chunk should boost for FixError intent");
    
    // Solution chunk + FixError intent = boost
    let solution_metadata = MetadataExtractor::extract("doc2", "Fix the issue by configuring solution");
    let boost = booster.calculate_boost(QueryIntent::FixError, &solution_metadata);
    assert!(boost > 0.0, "Solution chunk should boost for FixError intent");
    
    // Concept chunk + LearnConcept intent = boost
    let concept_metadata = MetadataExtractor::extract("doc3", "Explain the design pattern principle");
    let boost = booster.calculate_boost(QueryIntent::LearnConcept, &concept_metadata);
    assert!(boost > 0.0, "Concept chunk should boost for LearnConcept intent");
    
    // Tool chunk + UseTool intent = boost
    let tool_metadata = MetadataExtractor::extract("doc4", "Usage: kubectl get pods\n$ kubectl apply");
    let boost = booster.calculate_boost(QueryIntent::UseTool, &tool_metadata);
    assert!(boost > 0.0, "Tool chunk should boost for UseTool intent");
}

#[test]
fn test_metadata_booster_intent_mismatch() {
    let booster = MetadataBooster::new();
    
    // Reference chunk + FixError intent = no boost
    let ref_metadata = MetadataExtractor::extract("doc1", "Documentation reference specification");
    let boost = booster.calculate_boost(QueryIntent::FixError, &ref_metadata);
    assert_eq!(boost, 0.0, "Reference chunk should not boost for FixError intent");
}

#[test]
fn test_boost_application() {
    let booster = MetadataBooster::new();
    
    // Normal boost
    let score = booster.apply_boost(0.7, 0.15);
    assert_eq!(score, 0.85);
    
    // Capped at 1.0
    let score = booster.apply_boost(0.95, 0.2);
    assert_eq!(score, 1.0);
    
    // Zero boost
    let score = booster.apply_boost(0.7, 0.0);
    assert_eq!(score, 0.7);
}

// ============================================================================
// Phase 6: Cache Alignment Tests
// ============================================================================

#[test]
fn test_lru_cache_basic() {
    let cache = LruChunkCache::new(3);
    
    cache.put("chunk1", "content1");
    cache.put("chunk2", "content2");
    
    assert_eq!(cache.get("chunk1"), Some("content1".to_string()));
    assert_eq!(cache.get("chunk2"), Some("content2".to_string()));
    assert_eq!(cache.get("nonexistent"), None);
    
    let metrics = cache.metrics();
    assert_eq!(metrics.hits, 2);
    assert_eq!(metrics.misses, 1);
}

#[test]
fn test_lru_cache_eviction() {
    let cache = LruChunkCache::new(2);
    
    cache.put("chunk1", "content1");
    cache.put("chunk2", "content2");
    cache.put("chunk3", "content3"); // Should evict chunk1
    
    assert_eq!(cache.get("chunk1"), None); // Evicted
    assert_eq!(cache.get("chunk2"), Some("content2".to_string()));
    assert_eq!(cache.get("chunk3"), Some("content3".to_string()));
    
    assert_eq!(cache.metrics().evictions, 1);
}

#[test]
fn test_cache_locality_distance() {
    let mut graph = std::collections::HashMap::new();
    graph.insert("root".to_string(), vec!["level1".to_string()]);
    graph.insert("level1".to_string(), vec!["level2".to_string()]);
    graph.insert("level2".to_string(), vec!["level3".to_string()]);
    
    assert_eq!(CacheLocalityAnalyzer::calculate_distance("root", "root", &graph), 0);
    assert_eq!(CacheLocalityAnalyzer::calculate_distance("level1", "root", &graph), 1);
    assert_eq!(CacheLocalityAnalyzer::calculate_distance("level2", "root", &graph), 2);
    assert_eq!(CacheLocalityAnalyzer::calculate_distance("level3", "root", &graph), 3);
    assert_eq!(CacheLocalityAnalyzer::calculate_distance("nonexistent", "root", &graph), u32::MAX);
}

#[test]
fn test_kv_cache_aligner_slot_assignment() {
    let aligner = KvCacheAligner::new(4096, 100, 10);
    
    let chunks = vec![
        mem_cli::cache_alignment::CachedChunk {
            chunk_id: "chunk1".to_string(),
            text: "content1".to_string(),
            score: 0.9,
            cache_distance: 1,
            access_count: 5,
            last_accessed_slot: 0,
        },
        mem_cli::cache_alignment::CachedChunk {
            chunk_id: "chunk2".to_string(),
            text: "content2".to_string(),
            score: 0.8,
            cache_distance: 2,
            access_count: 3,
            last_accessed_slot: 1,
        },
    ];
    
    let slots = aligner.assign_slots(&chunks);
    assert_eq!(slots.len(), 2);
    assert_eq!(slots[0], ("chunk1".to_string(), 0));
    assert_eq!(slots[1], ("chunk2".to_string(), 1));
}

#[test]
fn test_kv_cache_will_fit() {
    let aligner = KvCacheAligner::new(1000, 100, 10);
    
    assert!(aligner.will_fit(5));   // 500 tokens < 1000
    assert!(aligner.will_fit(10));  // 1000 tokens = 1000 (fits)
    assert!(!aligner.will_fit(15)); // 1500 tokens > 1000
}

// ============================================================================
// Full Pipeline Integration Tests
// ============================================================================

#[tokio::test]
async fn test_full_pipeline_with_wiki() {
    let pipeline = create_test_pipeline();
    let graph = create_test_wiki_graph();
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_with_wiki("fix kubernetes pod error", &graph, candidates)
        .await
        .unwrap();
    
    // Query should be classified as FixError
    assert_eq!(result.query_intent, QueryIntent::FixError);
    
    // Should have processed candidates
    assert!(result.metrics.wiki_scope_docs > 0);
    
    // Phase timings should be recorded
    assert!(!result.metrics.phase_timings.is_empty());
    
    // Total latency should be recorded (may be 0 for fast operations)
    assert!(result.metrics.total_latency_ms >= 0);
}

#[tokio::test]
async fn test_full_pipeline_direct() {
    let pipeline = create_test_pipeline();
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_direct("explain kubernetes concepts", candidates)
        .await
        .unwrap();
    
    // Query should be classified as LearnConcept
    assert_eq!(result.query_intent, QueryIntent::LearnConcept);
    
    // All candidates should be in scope (no wiki filtering)
    assert!(result.metrics.wiki_scope_docs >= 5);
}

#[tokio::test]
async fn test_metadata_boost_integration() {
    let vocab = create_test_vocab();
    let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
    let semantic = Arc::new(SemanticScorer::new());
    
    let config = PipelineConfig {
        enable_metadata_boost: true,
        ..PipelineConfig::default()
    };
    
    let pipeline = FullPipeline::new(tfidf, semantic, config);
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_direct("fix pod crash error", candidates)
        .await
        .unwrap();
    
    // Should have applied metadata boosts
    // Error/Solution chunks should get boosted for FixError query
    assert!(result.metrics.metadata_boosts_applied >= 0);
    
    // Chunks with boost should have query_intent_match = true
    for chunk in &result.chunks {
        if chunk.metadata_boost > 0.0 {
            assert!(chunk.query_intent_match);
        }
    }
}

#[tokio::test]
async fn test_metadata_boost_disabled() {
    let vocab = create_test_vocab();
    let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
    let semantic = Arc::new(SemanticScorer::new());
    
    let config = PipelineConfig {
        enable_metadata_boost: false,
        ..PipelineConfig::default()
    };
    
    let pipeline = FullPipeline::new(tfidf, semantic, config);
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_direct("fix pod error", candidates)
        .await
        .unwrap();
    
    // No metadata boosts should be applied
    assert_eq!(result.metrics.metadata_boosts_applied, 0);
    assert_eq!(result.metrics.avg_boost, 0.0);
    
    // All chunks should have metadata_boost = 0
    for chunk in &result.chunks {
        assert_eq!(chunk.metadata_boost, 0.0);
    }
}

#[tokio::test]
async fn test_cache_preload_integration() {
    let vocab = create_test_vocab();
    let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
    let semantic = Arc::new(SemanticScorer::new());
    
    let config = PipelineConfig {
        preload_top_k: 3,
        ..PipelineConfig::default()
    };
    
    let pipeline = FullPipeline::new(tfidf, semantic, config);
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_direct("kubernetes", candidates)
        .await
        .unwrap();
    
    // Should have preloaded up to preload_top_k chunks
    assert!(result.metrics.preloaded_chunks <= 3);
}

#[tokio::test]
async fn test_wiki_distance_in_enriched_chunks() {
    let pipeline = create_test_pipeline();
    let graph = create_test_wiki_graph();
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_with_wiki("kubernetes", &graph, candidates)
        .await
        .unwrap();
    
    // Chunks should have wiki_distance populated (within max_hops)
    for chunk in &result.chunks {
        if let Some(dist) = chunk.wiki_distance {
            assert!(dist <= pipeline.config().max_wiki_hops);
        }
    }
}

#[tokio::test]
async fn test_cache_priority_ordering() {
    let pipeline = create_test_pipeline();
    let graph = create_test_wiki_graph();
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_with_wiki("kubernetes", &graph, candidates)
        .await
        .unwrap();
    
    // Cache priority should be positive for all chunks
    for chunk in &result.chunks {
        assert!(chunk.cache_priority >= 0.0);
    }
    
    // Higher scoring chunks closer in wiki-graph should have higher priority
    if result.chunks.len() >= 2 {
        let priorities: Vec<f32> = result.chunks.iter().map(|c| c.cache_priority).collect();
        // Just verify priorities are computed, not necessarily ordered
        // (ordering depends on score * distance factor)
        assert!(priorities.iter().all(|&p| p >= 0.0));
    }
}

#[tokio::test]
async fn test_pipeline_builder_full() {
    let vocab = create_test_vocab();
    let tfidf = Arc::new(GlobalTfIdfScorer::new(vocab));
    let semantic = Arc::new(SemanticScorer::new());
    
    let pipeline = PipelineBuilder::new()
        .with_scorers(tfidf, semantic)
        .with_project("test-project")
        .with_wiki_root("docs/index.md")
        .with_budget(4096)
        .with_score_threshold(0.7)
        .with_metadata_boost(true)
        .with_cache_capacity(500)
        .build()
        .unwrap();
    
    assert_eq!(pipeline.config().project, "test-project");
    assert_eq!(pipeline.config().wiki_root_doc, "docs/index.md");
    assert_eq!(pipeline.config().budget_bytes, 4096);
    assert_eq!(pipeline.config().score_threshold, 0.7);
    assert!(pipeline.config().enable_metadata_boost);
    assert_eq!(pipeline.config().cache_capacity, 500);
}

#[tokio::test]
async fn test_enriched_chunk_all_fields() {
    let pipeline = create_test_pipeline();
    let candidates = vec![
        ("doc1.md".to_string(), "# Error Handling\n\nError: CrashLoopBackOff fix solution.".to_string()),
    ];
    
    let result = pipeline
        .execute_direct("fix error", candidates)
        .await
        .unwrap();
    
    if !result.chunks.is_empty() {
        let chunk = &result.chunks[0];
        
        // All fields should be populated
        assert!(!chunk.id.is_empty());
        assert!(!chunk.text.is_empty());
        assert!(chunk.final_score >= 0.0);
        assert!(chunk.final_score <= 1.0);
        
        // Heading should be extracted if present
        // Category should be inferred
        assert!(matches!(chunk.category, 
            ChunkCategory::Error | ChunkCategory::Solution | 
            ChunkCategory::Tool | ChunkCategory::Concept | 
            ChunkCategory::Reference | ChunkCategory::Unknown
        ));
    }
}

#[tokio::test]
async fn test_phase_timing_coverage() {
    let pipeline = create_test_pipeline();
    let candidates = create_diverse_candidates();
    
    let result = pipeline
        .execute_direct("test query", candidates)
        .await
        .unwrap();
    
    let phase_names: Vec<&str> = result.metrics.phase_timings
        .iter()
        .map(|(name, _)| name.as_str())
        .collect();
    
    // All phases should be timed
    assert!(phase_names.contains(&"intent_inference"), "Missing intent_inference timing");
    assert!(phase_names.contains(&"routing_retrieval"), "Missing routing_retrieval timing");
    assert!(phase_names.contains(&"metadata_boost"), "Missing metadata_boost timing");
    assert!(phase_names.contains(&"cache_alignment"), "Missing cache_alignment timing");
}

// ============================================================================
// Edge Cases
// ============================================================================

#[tokio::test]
async fn test_empty_candidates() {
    let pipeline = create_test_pipeline();
    let graph = create_test_wiki_graph();
    
    let result = pipeline
        .execute_with_wiki("query", &graph, vec![])
        .await
        .unwrap();
    
    assert!(result.chunks.is_empty());
    assert_eq!(result.metrics.post_optimization_count, 0);
    assert_eq!(result.metrics.preloaded_chunks, 0);
}

#[tokio::test]
async fn test_no_wiki_matches() {
    let pipeline = create_test_pipeline();
    let graph = create_test_wiki_graph();
    
    // Candidates that don't match wiki graph
    let candidates = vec![
        ("orphan1.md".to_string(), "orphan content".to_string()),
        ("orphan2.md".to_string(), "more orphan content".to_string()),
    ];
    
    let result = pipeline
        .execute_with_wiki("query", &graph, candidates)
        .await
        .unwrap();
    
    // Should still complete (graceful handling)
    assert!(result.metrics.total_latency_ms >= 0);
    assert!(!result.metrics.phase_timings.is_empty());
}

#[tokio::test]
async fn test_unknown_query_intent() {
    let pipeline = create_test_pipeline();
    let candidates = create_diverse_candidates();
    
    // Ambiguous query
    let result = pipeline
        .execute_direct("something random here", candidates)
        .await
        .unwrap();
    
    // Should default to Unknown intent
    assert_eq!(result.query_intent, QueryIntent::Unknown);
}