poimen-memory/tests/it_calibration.rs

use mem_llm::{CalibrationResults, CalibrationSample, stratified_sample};

/// M5.2 Integration Tests — Labeler Calibration
/// 
/// Verifies calibration measurement before training:
/// - Worksheet is blind (hides labeler answers)
/// - Stratified sampling (50/50 positive/negative)
/// - Cohen's kappa computed correctly
/// - Precision/recall separated
/// - Gate checks kappa >= 0.6

#[test]
fn a1_worksheet_is_blind() {
    let sample = CalibrationSample::new(
        "abc123".to_string(),
        "What is the issue?".to_string(),
        "The service timed out.".to_string(),
        true,
        "Contains evidence of timeout".to_string(),
    );
    
    let blind_json = sample.to_blind_json();
    
    // Labeler's answer should NOT be visible
    let serialized = blind_json.to_string();
    assert!(
        !serialized.contains("labeler"),
        "Blind worksheet should not contain labeler answers"
    );
    
    // But question and chunk should be
    assert!(serialized.contains("What is the issue?"));
    assert!(serialized.contains("timed out"));
}

#[test]
fn a2_stratified_sampling() {
    // 95 negative, 5 positive (realistic class imbalance)
    let mut labels = Vec::new();
    for i in 0..95 {
        labels.push((format!("chunk_{}", i), false));
    }
    for i in 0..5 {
        labels.push((format!("positive_{}", i), true));
    }
    
    let sample_indices = stratified_sample(&labels, 100, 0);
    
    // Count positive and negative in sample
    let positive = sample_indices
        .iter()
        .filter(|&&i| labels[i].1)
        .count();
    let negative = sample_indices.len() - positive;
    
    // With only 5 positives in corpus, can't reach 50/50 on 100 samples
    // But should include most positives
    assert!(positive >= 4, "Should include available positive examples");
    assert!(negative >= 50, "Should include substantial negatives");
}

#[test]
fn a3_kappa_perfect_agreement() {
    let results = CalibrationResults::new(50, 50, 0, 0);
    
    // Perfect agreement should give kappa = 1.0
    assert!((results.kappa - 1.0).abs() < 0.01, "Kappa should be 1.0 for perfect agreement");
}

#[test]
fn a4_kappa_vs_accuracy() {
    // Synthetic all-negative labeler on 95/5 class imbalance
    let results = CalibrationResults::new(0, 95, 0, 5);
    
    // Accuracy is high (95%)
    assert!(results.accuracy > 0.9, "Accuracy misleadingly high");
    
    // Kappa should be low despite high accuracy
    assert!(results.kappa < 0.6, "Kappa correctly penalizes class imbalance: {}", results.kappa);
    assert!(results.kappa > 0.0, "Kappa should still be positive (some structure)");
}

#[test]
fn a5_confusion_matrix() {
    let results = CalibrationResults::new(70, 20, 10, 0);
    
    // Check all four cells are recorded
    assert_eq!(results.tp, 70);
    assert_eq!(results.tn, 20);
    assert_eq!(results.fp, 10);
    assert_eq!(results.fn_, 0);
    
    // Total should sum
    assert_eq!(results.total, 100);
}

#[test]
fn a6_precision_recall_separate() {
    // Case 1: High precision, low recall
    let high_prec = CalibrationResults::new(50, 40, 5, 5);
    assert!(high_prec.precision > 0.8, "High precision");
    assert!(high_prec.recall > 0.8, "Decent recall");
    
    // Case 2: Low precision, high recall
    let low_prec = CalibrationResults::new(50, 20, 30, 0);
    assert!(low_prec.precision < 0.7, "Low precision (many false positives)");
    assert_eq!(low_prec.recall, 1.0, "Perfect recall (no false negatives)");
}

#[test]
fn a7_gate_threshold_kappa_06() {
    let pass_065 = CalibrationResults::new(65, 30, 3, 2);
    let fail_059 = CalibrationResults::new(59, 35, 4, 2);
    
    // Kappa >= 0.6 passes
    if pass_065.kappa >= 0.6 {
        assert!(pass_065.passes_gate());
    }
    
    // Kappa < 0.6 fails
    if fail_059.kappa < 0.6 {
        assert!(!fail_059.passes_gate());
    }
}

#[test]
fn a8_f1_score_computed() {
    let results = CalibrationResults::new(70, 20, 10, 0);
    
    // F1 should be harmonic mean of precision and recall
    // Precision = 70/(70+10) = 0.875
    // Recall = 70/70 = 1.0
    // F1 = 2 * (0.875 * 1.0) / (0.875 + 1.0) ≈ 0.933
    
    assert!(results.f1 > 0.9, "F1 score should be high: {}", results.f1);
}

#[test]
fn a9_calibration_sample_roundtrip() {
    let sample = CalibrationSample::new(
        "sha256_abc".to_string(),
        "What causes failure?".to_string(),
        "Missing database index on query".to_string(),
        true,
        "Identifies root cause".to_string(),
    );
    
    // Should serialize/deserialize with human fields optional
    let json = serde_json::to_string(&sample).expect("Should serialize");
    let deserialized: CalibrationSample = serde_json::from_str(&json)
        .expect("Should deserialize");
    
    assert_eq!(deserialized.chunk_sha, sample.chunk_sha);
    assert_eq!(deserialized.question, sample.question);
    assert!(deserialized.human_label.is_none(), "Human fields should be None initially");
}

#[test]
fn a10_disagreement_analysis() {
    // Three types of disagreements
    let disagreements = vec![
        ("FP", 10, "Labeler says yes, human says no"),
        ("FN", 5, "Labeler says no, human says yes"),
    ];
    
    let mut total_disagreement = 0;
    for (dtype, count, _desc) in disagreements {
        total_disagreement += count;
        assert!(count > 0, "Disagreement count should be tracked");
    }
    
    assert_eq!(total_disagreement, 15, "All disagreements should be counted");
}

#[test]
fn a11_sample_size_sufficient() {
    // With 100 samples:
    // - ~50 positives (for precision on minority class)
    // - ~50 negatives
    // Distinguishes 0.7 kappa from 0.9 kappa
    
    let sample_size = 100;
    assert!(sample_size >= 100, "Sample size should be sufficient");
}

#[test]
fn a12_kappa_formula_correct() {
    // Hand-computed example:
    // 60 agree yes, 30 agree no, 5 FP, 5 FN = 100 total
    // Po (observed) = 90/100 = 0.9
    // Pe (chance) = (65/100)² + (35/100)² = 0.5525
    // Kappa = (0.9 - 0.5525) / (1 - 0.5525) ≈ 0.789
    
    let results = CalibrationResults::new(60, 30, 5, 5);
    assert!(results.kappa > 0.70 && results.kappa < 0.90, 
        "Kappa should be ~0.79, got {}", results.kappa);
}