poimen-memory/tests/it_m3_8_optimizer_benchmarks.rs

//! M3.8.5 — Compression Benchmarks & Search Quality Validation
//!
//! Validates that M3.8 optimization improves search quality without sacrificing performance.

use mem_core::optimizer::{ContextOptimizer, ContextOptimizerConfig, ContentType};
use std::time::Instant;

#[test]
fn test_optimizer_compression_ratio_log() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig {
        enabled: true,
        use_magika: false,
        magika_threshold: 0.8,
        compress_json: true,
        compress_diff: true,
        compress_log: true,
        compress_text: true,
        ccr_enabled: true,
        ccr_size_mb: 100,
    }).expect("failed to create optimizer");

    let log_content = "ERROR: Connection timeout at line 42\nSTACK TRACE:\n  at func1:10\n  at func2:20\nERROR: Retry 3/5\nWARN: Performance degradation";
    
    let result = optimizer.optimize(log_content).expect("optimize failed");
    
    let input_size = log_content.len();
    let output_size = result.compressed.len();
    let ratio = output_size as f32 / input_size as f32;
    
    // Log compression should achieve 80-95% ratio (20-80% reduction)
    assert!(ratio < 0.95, "log compression ratio {} should be < 0.95", ratio);
    assert!(ratio > 0.05, "log compression ratio {} should be > 0.05", ratio);
    
    assert_eq!(result.content_type, ContentType::Log);
}

#[test]
fn test_optimizer_compression_ratio_json() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let json_content = r#"{"user": "alice", "id": 12345, "timestamp": "2024-01-01T00:00:00Z", "data": {"nested": true, "values": [1,2,3]}, "metadata": {"source": "api", "version": "1.0"}}"#;
    
    let result = optimizer.optimize(json_content).expect("optimize failed");
    
    let input_size = json_content.len();
    let output_size = result.compressed.len();
    let ratio = output_size as f32 / input_size as f32;
    
    // JSON compression should achieve 70-90% ratio
    assert!(ratio < 0.95, "json compression ratio should be < 0.95");
}

#[test]
fn test_optimizer_compression_ratio_text() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let text_content = "The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog.";
    
    let result = optimizer.optimize(text_content).expect("optimize failed");
    
    let input_size = text_content.len();
    let output_size = result.compressed.len();
    
    // Text compression should be modest
    assert!(output_size <= input_size, "compressed should not exceed input");
}

#[test]
fn test_optimizer_performance_single_chunk() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let content = "ERROR: error 1\nERROR: error 2\nINFO: info\n".repeat(10);
    
    let start = Instant::now();
    let _result = optimizer.optimize(&content).expect("optimize failed");
    let elapsed = start.elapsed();
    
    // Should complete in < 100ms for a typical log chunk
    assert!(elapsed.as_millis() < 100, "optimization took {}ms, should be < 100ms", elapsed.as_millis());
}

#[test]
fn test_optimizer_preserves_signals() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let content = "ERROR: database connection failed at line 42\nStack: connect.rs:100";
    
    let result = optimizer.optimize(content).expect("optimize failed");
    
    // Optimized text should still contain key signals
    assert!(result.compressed.to_lowercase().contains("error"), "should preserve ERROR signal");
    assert!(result.compressed.to_lowercase().contains("database"), "should preserve domain term");
}

#[test]
fn test_optimizer_handles_empty() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let result = optimizer.optimize("").expect("optimize failed");
    assert_eq!(result.compressed, "");
}

#[test]
fn test_optimizer_config_from_env() {
    let config = ContextOptimizerConfig::from_env();
    assert!(config.enabled);  // Should be enabled by default
}

#[test]
fn test_optimizer_compression_summary() {
    // Test typical compression ratios across different content types
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let test_cases = vec![
        (
            "ERROR: failed\nERROR: retry\nWARN: slow",
            "log",
            0.9,  // Max ratio for log
        ),
        (
            r#"{"a":1,"b":2,"c":{"d":3}}"#,
            "json",
            0.9,  // Max ratio for JSON
        ),
        (
            "The quick brown fox jumps over the lazy dog.",
            "text",
            1.0,  // Max ratio for plain text (may not compress)
        ),
    ];

    for (content, name, max_ratio) in test_cases {
        let result = optimizer.optimize(content).expect("optimize failed");
        let ratio = result.compressed.len() as f32 / content.len() as f32;
        assert!(ratio <= max_ratio, "{}: ratio {} should be <= {}", name, ratio, max_ratio);
    }
}

#[test]
fn test_optimizer_batch_compression() {
    let optimizer = ContextOptimizer::new(ContextOptimizerConfig::from_env())
        .expect("failed to create optimizer");

    let chunks = vec![
        "ERROR: connection timeout",
        "INFO: starting request",
        "ERROR: failed to connect",
        "DEBUG: retry attempt 1",
    ];

    let mut total_input = 0;
    let mut total_output = 0;

    for content in chunks {
        total_input += content.len();
        let result = optimizer.optimize(content).expect("optimize failed");
        total_output += result.compressed.len();
    }

    // Overall batch should compress
    let ratio = total_output as f32 / total_input as f32;
    assert!(ratio < 0.99, "batch compression ratio {} should be < 0.99", ratio);
}