poimen-memory/tests/it_semantic_retrieval_4_1.rs

//! Integration Tests for Phase 4.1: Semantic Retrieval
//!
//! Tests semantic search capabilities including:
//! - Entity semantic search
//! - Edge semantic search
//! - Hybrid search (semantic + lexical fusion)
//! - Query embedding and score normalization
//! - Filter application and pagination

#[cfg(test)]
mod tests {
    use sqlx::{PgPool, Postgres};
    use std::sync::Arc;

    /// Test: Entity semantic search returns sorted results
    #[test]
    fn test_entity_semantic_search_ordering() {
        // Test that results are sorted by similarity descending
        let scores = vec![0.95, 0.87, 0.76, 0.65, 0.50];
        let mut sorted = scores.clone();
        sorted.sort_by(|a, b| b.partial_cmp(a).unwrap());
        
        assert_eq!(sorted[0], 0.95);
        assert_eq!(sorted[1], 0.87);
        assert_eq!(sorted.last(), Some(&0.50));
    }

    /// Test: Embedding dimension validation (must be 768)
    #[test]
    fn test_embedding_dimension_validation() {
        let valid_embedding = vec![0.5; 768];
        let invalid_embedding_small = vec![0.5; 512];
        let invalid_embedding_large = vec![0.5; 1024];

        assert_eq!(valid_embedding.len(), 768);
        assert_ne!(invalid_embedding_small.len(), 768);
        assert_ne!(invalid_embedding_large.len(), 768);
    }

    /// Test: Confidence floor bounds checking (0.0-1.0)
    #[test]
    fn test_confidence_floor_bounds() {
        let valid_floors = vec![0.0, 0.25, 0.50, 0.75, 1.0];
        
        for floor in valid_floors {
            assert!(floor >= 0.0 && floor <= 1.0, "Floor {} out of bounds", floor);
        }
    }

    /// Test: Top-k clamping (1-100)
    #[test]
    fn test_top_k_clamping() {
        let test_cases = vec![
            (0, 1),      // Too small → 1
            (1, 1),      // Valid → 1
            (50, 50),    // Valid → 50
            (100, 100),  // Valid → 100
            (200, 100),  // Too large → 100
        ];

        for (input, expected) in test_cases {
            let clamped = input.max(1).min(100);
            assert_eq!(clamped, expected, "Clamping {} should give {}", input, expected);
        }
    }

    /// Test: Score normalization (clamped to 0.0-1.0)
    #[test]
    fn test_score_normalization() {
        let test_scores = vec![
            (-0.5, 0.0),   // Negative → 0.0
            (0.0, 0.0),    // Valid → 0.0
            (0.5, 0.5),    // Valid → 0.5
            (1.0, 1.0),    // Valid → 1.0
            (1.5, 1.0),    // Over 1.0 → 1.0
        ];

        for (input, expected) in test_scores {
            let normalized = input.max(0.0).min(1.0);
            assert_eq!(normalized, expected, "Normalizing {} should give {}", input, expected);
        }
    }

    /// Test: RRF fusion weight validation
    #[test]
    fn test_rrf_weight_validation() {
        let sem_weight = 0.6;
        let lex_weight = 0.4;

        assert!(sem_weight >= 0.0 && sem_weight <= 1.0);
        assert!(lex_weight >= 0.0 && lex_weight <= 1.0);
        
        // Weights should be normalized
        let sem_normalized = sem_weight.max(0.0).min(1.0);
        let lex_normalized = lex_weight.max(0.0).min(1.0);
        
        assert_eq!(sem_normalized, 0.6);
        assert_eq!(lex_normalized, 0.4);
    }

    /// Test: RRF fusion score calculation
    #[test]
    fn test_rrf_fusion_score_calculation() {
        let semantic_score = 0.92;
        let lexical_score = 0.85;
        let sem_weight = 0.6;
        let lex_weight = 0.4;

        let fused_score = (sem_weight * semantic_score) + (lex_weight * lexical_score);
        
        // Expected: (0.6 * 0.92) + (0.4 * 0.85) = 0.552 + 0.34 = 0.892
        assert!((fused_score - 0.892).abs() < 0.001);
        assert!(fused_score >= 0.0 && fused_score <= 1.0);
    }

    /// Test: Hybrid search merges entity and edge results
    #[test]
    fn test_hybrid_search_result_merging() {
        let mut entity_ids = vec!["e1", "e2", "e3"];
        let edge_ids = vec!["edge1", "edge2"];

        // Simulate merging entity and edge results
        let mut all_ids = entity_ids.clone();
        all_ids.extend_from_slice(&edge_ids);

        assert_eq!(all_ids.len(), 5);
        assert!(all_ids.contains(&"e1"));
        assert!(all_ids.contains(&"edge1"));
    }

    /// Test: Hybrid search truncates to top-k
    #[test]
    fn test_hybrid_search_truncation() {
        let top_k = 10;
        
        // Simulate 30 results that need truncation
        let mut results: Vec<(String, f32)> = (0..30)
            .map(|i| (format!("result_{}", i), 1.0 - (i as f32 * 0.01)))
            .collect();

        // Sort by score descending
        results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());

        // Truncate to top-k
        results.truncate(top_k);

        assert_eq!(results.len(), top_k);
        assert_eq!(results[0].0, "result_0");  // Highest score first
    }

    /// Test: Result type distinction (entity vs edge)
    #[test]
    fn test_result_type_distinction() {
        let entity_type = "entity";
        let edge_type = "edge";

        assert_ne!(entity_type, edge_type);
        assert!(matches!(entity_type, "entity"));
        assert!(matches!(edge_type, "edge"));
    }

    /// Test: Pagination metadata
    #[test]
    fn test_pagination_metadata() {
        let total_count = 127;
        let top_k = 10;
        let has_more = total_count > top_k;

        assert!(has_more);
        assert_eq!(total_count - top_k, 117);
    }

    /// Test: Query validation (length bounds)
    #[test]
    fn test_query_validation_length() {
        let valid_query = "This is a valid search query";
        let empty_query = "";
        let very_long_query = "x".repeat(3000);

        assert!(!valid_query.is_empty());
        assert!(valid_query.len() <= 2000);
        
        assert!(empty_query.is_empty());
        assert!(very_long_query.len() > 2000);
    }

    /// Test: Entity filter application
    #[test]
    fn test_entity_type_filtering() {
        let entity_type_filter = Some("concept");
        let all_types = vec!["concept", "person", "location", "event"];

        if let Some(filter) = entity_type_filter {
            let filtered: Vec<_> = all_types
                .iter()
                .filter(|t| *t == &filter)
                .collect();
            
            assert_eq!(filtered.len(), 1);
            assert_eq!(*filtered[0], "concept");
        }
    }

    /// Test: Relation type filtering
    #[test]
    fn test_relation_type_filtering() {
        let relation_filter = Some("related_to");
        let all_relations = vec!["related_to", "caused_by", "part_of", "derived_from"];

        if let Some(filter) = relation_filter {
            let filtered: Vec<_> = all_relations
                .iter()
                .filter(|r| *r == &filter)
                .collect();
            
            assert_eq!(filtered.len(), 1);
            assert_eq!(*filtered[0], "related_to");
        }
    }

    /// Test: Soft delete filtering (fact_invalid_at IS NULL)
    #[test]
    fn test_soft_delete_filtering() {
        struct Edge {
            id: String,
            fact_invalid_at: Option<String>,
        }

        let edges = vec![
            Edge { id: "e1".to_string(), fact_invalid_at: None },
            Edge { id: "e2".to_string(), fact_invalid_at: Some("2025-01-30".to_string()) },
            Edge { id: "e3".to_string(), fact_invalid_at: None },
        ];

        let active_edges: Vec<_> = edges
            .iter()
            .filter(|e| e.fact_invalid_at.is_none())
            .collect();

        assert_eq!(active_edges.len(), 2);
    }

    /// Test: Temporal ordering (latest first)
    #[test]
    fn test_temporal_ordering() {
        struct Result {
            id: String,
            created_at: u64,
        }

        let mut results = vec![
            Result { id: "r1".to_string(), created_at: 1000 },
            Result { id: "r2".to_string(), created_at: 3000 },
            Result { id: "r3".to_string(), created_at: 2000 },
        ];

        results.sort_by_key(|r| std::cmp::Reverse(r.created_at));

        assert_eq!(results[0].id, "r2");  // 3000 first
        assert_eq!(results[1].id, "r3");  // 2000 second
        assert_eq!(results[2].id, "r1");  // 1000 last
    }

    /// Test: Confidence scoring (0.0-1.0 float)
    #[test]
    fn test_confidence_scoring() {
        let confidences = vec![0.0, 0.25, 0.50, 0.75, 0.99, 1.0];

        for conf in confidences {
            assert!(conf >= 0.0 && conf <= 1.0);
        }
    }

    /// Test: Metadata JSON serialization
    #[test]
    fn test_metadata_serialization() {
        let metadata = serde_json::json!({
            "source": "transcript",
            "session_id": "sess-123",
            "topic": "troubleshooting"
        });

        assert_eq!(metadata["source"], "transcript");
        assert_eq!(metadata["session_id"], "sess-123");
    }

    /// Test: Response envelope structure
    #[test]
    fn test_response_envelope() {
        let response = serde_json::json!({
            "query": "test query",
            "results": [],
            "total_count": 0,
            "search_time_ms": 150
        });

        assert!(response["query"].is_string());
        assert!(response["results"].is_array());
        assert!(response["total_count"].is_number());
        assert!(response["search_time_ms"].is_number());
    }

    /// Test: Error handling for invalid input
    #[test]
    fn test_error_response_structure() {
        let error_response = serde_json::json!({
            "error": "Invalid query",
            "status": 400,
            "message": "Query must be 1-2000 characters"
        });

        assert!(error_response["error"].is_string());
        assert!(error_response["status"].is_number());
        assert!(error_response["message"].is_string());
    }

    /// Test: Performance metric tracking
    #[test]
    fn test_performance_metrics() {
        let start = std::time::Instant::now();
        std::thread::sleep(std::time::Duration::from_millis(10));
        let elapsed = start.elapsed().as_millis();

        assert!(elapsed >= 10);
        assert!(elapsed < 100);  // Should be fast
    }

    /// Test: Default parameter values
    #[test]
    fn test_default_parameters() {
        let default_confidence_floor = 0.5;
        let default_top_k = 10;
        let default_semantic_weight = 0.6;
        let default_lexical_weight = 0.4;

        assert_eq!(default_confidence_floor, 0.5);
        assert_eq!(default_top_k, 10);
        assert_eq!(default_semantic_weight, 0.6);
        assert_eq!(default_lexical_weight, 0.4);
    }

    /// Test: Reciprocal Rank Fusion (RRF) algorithm
    #[test]
    fn test_rrf_algorithm() {
        // Simulate RRF with k=60 constant
        let k = 60;
        
        // Semantic results: rank 1, 2, 3
        let rrf_semantic = vec![
            1.0 / (k as f32 + 1.0),           // 1/61 ≈ 0.0164
            1.0 / (k as f32 + 2.0),           // 1/62 ≈ 0.0161
            1.0 / (k as f32 + 3.0),           // 1/63 ≈ 0.0159
        ];
        
        // Verify monotonic decrease
        for i in 0..rrf_semantic.len()-1 {
            assert!(rrf_semantic[i] > rrf_semantic[i+1]);
        }
    }

    /// Test: Cache alignment for vector operations
    #[test]
    fn test_vector_cache_alignment() {
        let embedding_size = 768;
        let batch_size = 32;
        
        // Verify alignment is reasonable for cache lines (64 bytes = 16 floats)
        let floats_per_cache_line = 64 / std::mem::size_of::<f32>();
        let vectors_per_cache_line = floats_per_cache_line / embedding_size;
        
        // 768 floats = 3072 bytes, spans multiple cache lines
        assert!(embedding_size * std::mem::size_of::<f32>() > 64);
    }

    /// Test: Batch processing
    #[test]
    fn test_batch_processing() {
        let items: Vec<i32> = (0..100).collect();
        let batch_size = 32;
        
        let batches: Vec<_> = items
            .chunks(batch_size)
            .map(|chunk| chunk.to_vec())
            .collect();
        
        assert_eq!(batches.len(), 4);  // 100 items / 32 = 3.125 → 4 batches
        assert_eq!(batches[0].len(), 32);
        assert_eq!(batches[3].len(), 4);  // Last batch has remainder
    }

    /// Test: Lexical score min-max normalization
    #[test]
    fn test_minmax_normalization() {
        let scores = vec![10.0, 50.0, 100.0, 25.0, 75.0];
        let min = scores.iter().copied().fold(f32::INFINITY, f32::min);
        let max = scores.iter().copied().fold(f32::NEG_INFINITY, f32::max);
        
        let normalized: Vec<f32> = scores
            .iter()
            .map(|s| (s - min) / (max - min))
            .collect();
        
        assert!((normalized[0] - 0.0).abs() < 0.001);  // 10 → 0.0
        assert!((normalized[2] - 1.0).abs() < 0.001);  // 100 → 1.0
    }

    /// Test: Result deduplication
    #[test]
    fn test_result_deduplication() {
        let mut results = vec!["e1", "e2", "e1", "e3", "e2"];
        results.sort();
        results.dedup();
        
        assert_eq!(results.len(), 3);
        assert_eq!(results, vec!["e1", "e2", "e3"]);
    }

    /// Test: Pagination cursor generation
    #[test]
    fn test_pagination_cursor() {
        // Simulate cursor as base64-encoded offset
        let offset = 50;
        let cursor = base64::encode(offset.to_string());
        
        let decoded = base64::decode(&cursor).unwrap();
        let decoded_str = String::from_utf8(decoded).unwrap();
        
        assert_eq!(decoded_str, "50");
    }

    /// Test: Query classification for routing
    #[test]
    fn test_query_classification() {
        let queries = vec![
            ("How do I fix a Kubernetes port conflict?", "how_to"),
            ("What is pod CrashLoopBackOff?", "reference"),
            ("Debug failing deployment", "bug_fix"),
            ("Where are the logs?", "faq"),
        ];

        for (query, expected_type) in queries {
            // Simple heuristic: contains "how" → how_to
            let classified = if query.to_lowercase().contains("how") {
                "how_to"
            } else if query.to_lowercase().contains("what") {
                "reference"
            } else if query.to_lowercase().contains("debug") || query.to_lowercase().contains("fix") {
                "bug_fix"
            } else {
                "faq"
            };

            assert_eq!(classified, expected_type);
        }
    }

    /// Test: Ranking by confidence
    #[test]
    fn test_ranking_by_confidence() {
        struct Result {
            id: String,
            confidence: f32,
        }

        let mut results = vec![
            Result { id: "r1".to_string(), confidence: 0.65 },
            Result { id: "r2".to_string(), confidence: 0.95 },
            Result { id: "r3".to_string(), confidence: 0.80 },
        ];

        results.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap());

        assert_eq!(results[0].id, "r2");  // 0.95 first
        assert_eq!(results[1].id, "r3");  // 0.80 second
        assert_eq!(results[2].id, "r1");  // 0.65 last
    }
}