diff --git a/crates/mem-core/tests/it_m3_8_benchmarks.rs b/crates/mem-core/tests/it_m3_8_benchmarks.rs new file mode 100644 index 0000000..2f42f99 --- /dev/null +++ b/crates/mem-core/tests/it_m3_8_benchmarks.rs @@ -0,0 +1,386 @@ +//! M3.8.5 Compression & Search Quality Benchmarks +//! +//! Validates: +//! - Compression reduces size meaningfully +//! - Semantic meaning is preserved +//! - Performance baseline (latency, throughput) + +use mem_core::ContextOptimizer; +use std::time::Instant; + +fn measure_compression(input: &str, optimizer: &ContextOptimizer) -> (usize, usize, f32) { + let input_bytes = input.len(); + + let optimized = match optimizer.optimize(input) { + Ok(chunk) => chunk.compressed, + Err(_) => { + return (input_bytes, input_bytes, 100.0); + } + }; + + let output_bytes = optimized.len(); + let ratio = (output_bytes as f32 / input_bytes as f32) * 100.0; + + (input_bytes, output_bytes, ratio) +} + +// ============================================================================ +// M3.8.5.1 — Compression Ratio Benchmarks (5 tests) +// ============================================================================ + +#[test] +fn benchmark_mixed_logs_compression() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let (input, output, ratio) = measure_compression(content, &optimizer); + + println!( + "Mixed logs: {} bytes → {} bytes ({:.1}% remaining)", + input, output, ratio + ); + + // Logs should compress significantly (lots of timestamps, debug noise) + assert!(ratio < 50.0, "logs should compress to <50% (got {:.1}%)", ratio); + assert!(output > 0, "should produce some output"); +} + +#[test] +fn benchmark_json_output_compression() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/json-output.json"); + + let (input, output, ratio) = measure_compression(content, &optimizer); + + println!( + "JSON output: {} bytes → {} bytes ({:.1}% remaining)", + input, output, ratio + ); + + // JSON may not compress as much (structured data) + assert!( + ratio > 0.0 && ratio < 100.0, + "json should compress somewhat (got {:.1}%)", + ratio + ); +} + +#[test] +fn benchmark_markdown_docs_compression() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/markdown-docs.txt"); + + let (input, output, ratio) = measure_compression(content, &optimizer); + + println!( + "Markdown docs: {} bytes → {} bytes ({:.1}% remaining)", + input, output, ratio + ); + + // Prose text compresses differently depending on content + // Should at least not expand + assert!( + ratio <= 100.0, + "compression should not expand text (got {:.1}%)", + ratio + ); +} + +#[test] +fn benchmark_aggregate_compression_all_sources() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + + let fixtures = vec![ + (include_str!("../../../fixtures/benchmarks/mixed-logs.txt"), "logs"), + (include_str!("../../../fixtures/benchmarks/json-output.json"), "json"), + (include_str!("../../../fixtures/benchmarks/markdown-docs.txt"), "text"), + ]; + + let mut total_input = 0; + let mut total_output = 0; + + for (content, name) in fixtures { + let (input, output, ratio) = measure_compression(content, &optimizer); + + println!(" {}: {:.1}%", name, ratio); + + total_input += input; + total_output += output; + } + + let overall_ratio = (total_output as f32 / total_input as f32) * 100.0; + + println!( + "Overall aggregate: {} bytes → {} bytes ({:.1}% remaining)", + total_input, total_output, overall_ratio + ); + + // Mixed content should compress overall + assert!( + overall_ratio < 100.0, + "aggregate should compress (got {:.1}%)", + overall_ratio + ); +} + +#[test] +fn benchmark_compression_meaningful() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let (input_bytes, output_bytes, _ratio) = measure_compression(content, &optimizer); + + let savings = input_bytes - output_bytes; + let savings_percent = (savings as f32 / input_bytes as f32) * 100.0; + + println!( + "Compression savings: {} bytes ({:.1}%) out of {} bytes", + savings, savings_percent, input_bytes + ); + + // Verify meaningful compression happened on logs + assert!(savings_percent > 1.0, "logs should compress >1%"); +} + +// ============================================================================ +// M3.8.5.2 — Search Quality Validation (8 tests) +// ============================================================================ + +#[test] +fn test_optimization_preserves_semantic_meaning() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + + // Log entry with noise and signal + let original = "ERROR: failed to connect\nDEBUG: thread id=1234\nTRACE: entering method\nERROR: connection refused"; + + let optimized = optimizer.optimize(original).expect("optimize"); + + // Key semantic: "ERROR" and "connection" should still be there + assert!( + !optimized.compressed.is_empty(), + "should produce some output" + ); + assert!( + optimized.compressed.contains("ERROR") || optimized.compressed.contains("error"), + "should preserve error keyword" + ); +} + +#[test] +fn test_compression_deterministic() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let result1 = optimizer.optimize(content).expect("first optimization"); + let result2 = optimizer.optimize(content).expect("second optimization"); + + assert_eq!( + result1.compressed, result2.compressed, + "compression should be deterministic" + ); +} + +#[test] +fn test_optimization_idempotent() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let once = optimizer.optimize(content).expect("first optimization"); + let twice = optimizer.optimize(&once.compressed).expect("re-optimize"); + + // Re-optimizing compressed content should not change it significantly + let ratio = (twice.compressed.len() as f32 / once.compressed.len() as f32) * 100.0; + + // Should be >95% identical + assert!( + ratio > 95.0, + "re-optimization should be nearly idempotent (got {:.1}%)", + ratio + ); +} + +#[test] +fn test_compression_no_information_loss_on_json() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/json-output.json"); + + let optimized = optimizer.optimize(content).expect("optimize"); + + // Should still be valid (non-empty) JSON + assert!(!optimized.compressed.is_empty(), "should produce output"); + + // Try to parse - if original is JSON, optimized should be too or at least valid + let parsed: Result = serde_json::from_str(&optimized.compressed); + + if parsed.is_err() { + // If it's not JSON, at least it should be shorter + assert!( + optimized.compressed.len() < content.len(), + "if not valid JSON, should at least compress" + ); + } +} + +#[test] +fn test_compression_preserves_critical_content() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let optimized = optimizer.optimize(content).expect("optimize"); + + // Should not be empty + assert!(!optimized.compressed.is_empty(), "should preserve some content"); + + // Should have significantly different size (some compression happened) + let ratio = (optimized.compressed.len() as f32 / content.len() as f32) * 100.0; + + // Logs should compress meaningfully + assert!(ratio < 50.0, "logs should compress to <50% (got {:.1}%)", ratio); +} + +#[test] +fn test_compression_handles_large_content() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + + // Create large content (10KB) + let large_content = (0..100) + .map(|i| { + format!( + "ERROR: failed at line {}\nDEBUG: context info\nTRACE: stack depth", + i + ) + }) + .collect::>() + .join("\n"); + + let optimized = optimizer.optimize(&large_content).expect("optimize"); + + // Should produce non-empty result + assert!(!optimized.compressed.is_empty(), "should handle large content"); + + // Should compress logs significantly + let ratio = (optimized.compressed.len() as f32 / large_content.len() as f32) * 100.0; + assert!(ratio < 50.0, "large log content should compress well (got {:.1}%)", ratio); +} + +#[test] +fn test_multi_chunk_search_consistency() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + + let chunks = vec![ + "ERROR: connection failed\nDEBUG: thread id=100", + "ERROR: timeout after 5000ms\nTRACE: stack unwinding", + "ERROR: retry attempt 2\nDEBUG: backoff delay=200ms", + ]; + + let optimized_chunks: Vec<_> = chunks + .iter() + .map(|chunk| optimizer.optimize(chunk).expect("optimize").compressed) + .collect(); + + // All chunks should produce non-empty output + for chunk in &optimized_chunks { + assert!(!chunk.is_empty(), "all chunks should produce output"); + } + + // Chunks should compress to reasonable sizes + for (orig, opt) in chunks.iter().zip(&optimized_chunks) { + let ratio = (opt.len() as f32 / orig.len() as f32) * 100.0; + assert!(ratio < 100.0, "chunks should not expand (got {:.1}%)", ratio); + } +} + +// ============================================================================ +// M3.8.5.3 — Performance Baseline (3 tests) +// ============================================================================ + +#[test] +fn test_optimization_latency_reasonable() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let mut latencies = Vec::new(); + + for _ in 0..50 { + let start = Instant::now(); + let _ = optimizer.optimize(content); + latencies.push(start.elapsed()); + } + + latencies.sort(); + let p95_ms = latencies[47].as_secs_f64() * 1000.0; // P95 + + println!("Optimization latency P95: {:.2}ms", p95_ms); + + // Should complete reasonably quickly + assert!( + p95_ms < 50.0, + "optimization latency P95 {:.2}ms should be <50ms", + p95_ms + ); +} + +#[test] +fn test_throughput_reasonable() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + let content = include_str!("../../../fixtures/benchmarks/mixed-logs.txt"); + + let start = Instant::now(); + let mut count = 0; + + // Process for up to 1 second + while start.elapsed().as_secs_f64() < 0.5 && count < 5000 { + let _ = optimizer.optimize(content); + count += 1; + } + + let elapsed_secs = start.elapsed().as_secs_f64(); + let throughput = count as f64 / elapsed_secs; + + println!( + "Throughput: {:.0} records/sec (processed {} in {:.2}s)", + throughput, count, elapsed_secs + ); + + // Should maintain reasonable throughput + assert!( + throughput >= 100.0, + "throughput {:.0} records/sec should be >=100", + throughput + ); +} + +#[test] +fn test_no_performance_regression_on_large_content() { + let optimizer = ContextOptimizer::new().expect("optimizer init"); + + // Create large content (50KB) + let large_content = (0..500) + .map(|i| { + format!( + "ERROR: failed at line {}\nDEBUG: context info\nTRACE: stack depth", + i + ) + }) + .collect::>() + .join("\n"); + + let start = Instant::now(); + let result = optimizer.optimize(&large_content); + let elapsed = start.elapsed(); + + assert!(result.is_ok(), "should handle large content"); + + // Should complete in reasonable time + assert!( + elapsed.as_millis() < 100, + "optimization of 50KB content should complete in <100ms (got {}ms)", + elapsed.as_millis() + ); + + println!( + "Large content ({}KB) optimized in {:.2}ms", + large_content.len() / 1024, + elapsed.as_secs_f64() * 1000.0 + ); +} diff --git a/fixtures/benchmarks/json-output.json b/fixtures/benchmarks/json-output.json new file mode 100644 index 0000000..6e933e5 --- /dev/null +++ b/fixtures/benchmarks/json-output.json @@ -0,0 +1,109 @@ +{ + "events": [ + { + "timestamp": "2024-08-20T12:00:00Z", + "level": "ERROR", + "message": "failed to connect to database", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "error_code": "CONNECTION_TIMEOUT", + "error_message": "connection refused after 5000ms", + "stack_trace": "at Database.connect (src/db.rs:45)\nat Server.init (src/main.rs:123)", + "attempt": 1, + "max_attempts": 3 + } + }, + { + "timestamp": "2024-08-20T12:00:01Z", + "level": "INFO", + "message": "attempting reconnection strategy exponential_backoff", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "strategy": "exponential_backoff", + "initial_delay_ms": 100, + "max_delay_ms": 30000, + "backoff_multiplier": 2.0 + } + }, + { + "timestamp": "2024-08-20T12:00:02Z", + "level": "DEBUG", + "message": "opening new connection pool", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "pool_size": 10, + "min_idle": 2, + "max_lifetime_seconds": 3600, + "idle_timeout_seconds": 600 + } + }, + { + "timestamp": "2024-08-20T12:00:03Z", + "level": "TRACE", + "message": "acquiring connection from pool", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "available_connections": 8, + "waiting_requests": 0, + "pool_stats": { + "created": 10, + "reused": 1234, + "destroyed": 0 + } + } + }, + { + "timestamp": "2024-08-20T12:00:04Z", + "level": "DEBUG", + "message": "connection timeout after 5000ms", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "timeout_ms": 5000, + "elapsed_ms": 5023, + "reason": "no available connections" + } + }, + { + "timestamp": "2024-08-20T12:00:05Z", + "level": "ERROR", + "message": "failed to connect to database", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "error": "connection timeout", + "details": { + "host": "memory-db.poimen.svc.cluster.local", + "port": 5432, + "database": "memory", + "username": "app_user" + } + } + }, + { + "timestamp": "2024-08-20T12:00:06Z", + "level": "INFO", + "message": "retrying with exponential backoff", + "context": { + "service": "api-server", + "instance": "pod-abc123", + "attempt": 1, + "max_attempts": 3, + "delay_ms": 100, + "next_retry": "2024-08-20T12:00:06.100Z" + } + } + ], + "summary": { + "total_events": 7, + "errors": 2, + "warnings": 0, + "info": 2, + "debug": 2, + "trace": 1 + } +} diff --git a/fixtures/benchmarks/markdown-docs.txt b/fixtures/benchmarks/markdown-docs.txt new file mode 100644 index 0000000..80d3a61 --- /dev/null +++ b/fixtures/benchmarks/markdown-docs.txt @@ -0,0 +1,146 @@ +# Poimen Memory System Architecture + +## Overview + +The Poimen Memory system is a distributed, multi-tier memory management platform designed for AI applications. It provides persistent storage, semantic search, and intelligent caching for conversations, logs, and structured data. + +## Core Components + +### 1. PostgreSQL with pgvector + +PostgreSQL serves as our primary data store with pgvector extension for semantic search. The system uses 768-dimensional embeddings generated by the nomic-embed-text-v2-moe model. + +Features: +- HNSW indexes for fast approximate nearest neighbor search +- Full ACID compliance with 2-node HA cluster +- Automatic failover with 10-minute RTO +- 10GB persistent volumes with daily backups + +### 2. OpenSearch Cluster + +OpenSearch provides full-text search and BM25 ranking. Documents are indexed with both raw text and preprocessed fields. + +Configuration: +- 2-node cluster (1 master, 1 data) +- 8GB heap per node +- 20GB storage per node +- Refresh interval: 10s +- Index shards: 3, replicas: 1 + +### 3. Memory Ingest Pipeline + +Records flow through a 4-stage pipeline: +1. Source extraction (Pi sessions, Claude transcripts, doc corpus) +2. Content routing (Magika ML classification) +3. Type-specific compression (log, json, diff, text) +4. Embedding generation and indexing + +### 4. Query Path (Hybrid Search) + +Queries use dual retrieval: +- 60% pgvector semantic search (top-k nearest neighbors) +- 40% OpenSearch BM25 ranking +- Fusion via Reciprocal Rank Weighting (RRW) + +Results are re-ranked and deduplicated before LLM context window. + +## M3.8 Context Optimization + +The context optimizer runs at ingest time, improving data quality before embedding: + +### Compression Targets +- Logs: 85-95% (remove timestamps, debug lines) +- JSON: 70-90% (minify, remove verbose keys) +- Text: 30-50% (remove markdown artifacts) +- Diffs: 60-80% (remove context lines) + +### Benefits +- Better pgvector embeddings (clean input = better semantic quality) +- Better BM25 ranking (signal-rich text = stronger matches) +- Reduced storage (lower bandwidth, faster queries) +- All queries benefit (optimization happens once) + +## Performance Targets + +- Ingest latency: <1ms per record +- Query latency: <100ms P95 (hybrid search) +- Embedding generation: <500ms for 50-record batch +- Indexing throughput: 1000+ records/sec +- Search throughput: 100+ queries/sec + +## Monitoring & Observability + +### Metrics Exported + +Via Prometheus `/metrics` endpoint: +- `m3_8_optimization_records_total` - records processed +- `m3_8_optimization_compression_ratio` - overall compression % +- `m3_8_optimization_compressor_ratio` - per-type compression +- Query latency distribution (P50, P95, P99) +- Embedding cache hit ratio + +### Logging + +Structured logs via tracing: +- INFO: ingest completion, query execution, errors +- DEBUG: compression stats, cache hits, routing decisions +- TRACE: individual record processing + +## Deployment + +### Kubernetes + +Resources deployed in `poimen` namespace: +- Deployment: memory-api (2 replicas) +- StatefulSet: memory-db (PostgreSQL) +- Deployment: opensearch (2 replicas) +- ConfigMap: optimization settings +- Secret: database credentials, API keys + +### Environment Variables + +- `MEM_CONTEXT_OPTIMIZER` - optimizer mode (on|off) +- `MEM_COMPRESSION_TARGETS` - JSON targets per type +- `MEM_CACHE_SIZE_MB` - compression cache size +- `MEM_PROMETHEUS_ENABLED` - metrics export + +## Testing Strategy + +### Unit Tests (62 tests) +- Individual compressor algorithms +- Content routing accuracy +- Cache behavior + +### Integration Tests (37 tests) +- End-to-end ingest pipeline +- Search quality on compressed content +- Metrics collection accuracy + +### Benchmark Tests (16 tests) +- Compression ratio validation +- Query performance with/without optimization +- Throughput and latency targets + +### Gate Tests (13 tests) +- Safety assertions (no data loss) +- Performance assertions (latency <3ms) +- Quality assertions (compression targets met) + +## Roadmap + +### Current (M3.8) +✅ Core optimizer (62 tests) +✅ Ingest integration (5 tests) +✅ Metrics & monitoring (7 tests) +⏳ Benchmarks (16 tests) +⏳ Gate verification (13 tests) + +### Next (M3.7.4-6) +- Context endpoint (semantic + reference tiers) +- Dual-write indexer (pgvector + OpenSearch) +- Composition gate + +### Future (M4-M7) +- Skill management +- Source connectors (Obsidian, git) +- Frontend React app diff --git a/fixtures/benchmarks/mixed-logs.txt b/fixtures/benchmarks/mixed-logs.txt new file mode 100644 index 0000000..81d8292 --- /dev/null +++ b/fixtures/benchmarks/mixed-logs.txt @@ -0,0 +1,45 @@ +2024-08-20T12:00:00Z ERROR failed to connect to database +2024-08-20T12:00:01Z INFO attempting reconnection strategy exponential_backoff +2024-08-20T12:00:02Z DEBUG opening new connection pool size=10 +2024-08-20T12:00:03Z TRACE acquiring connection from pool +2024-08-20T12:00:04Z DEBUG connection timeout after 5000ms +2024-08-20T12:00:05Z ERROR failed to connect to database: connection timeout +2024-08-20T12:00:06Z INFO retrying with exponential backoff attempt=1 delay=100ms +2024-08-20T12:00:07Z DEBUG creating new TCP socket +2024-08-20T12:00:08Z TRACE establishing TLS handshake +2024-08-20T12:00:09Z DEBUG TLS version: TLSv1.3 cipher: TLS_AES_256_GCM_SHA384 +2024-08-20T12:00:10Z INFO connection established successfully +2024-08-20T12:00:11Z DEBUG setting connection parameters max_connections=50 +2024-08-20T12:00:12Z TRACE executing connection setup queries +2024-08-20T12:00:13Z DEBUG query: SELECT version() -> PostgreSQL 15.3 +2024-08-20T12:00:14Z INFO database initialization complete version=15.3 +2024-08-20T12:00:15Z DEBUG running schema migrations +2024-08-20T12:00:16Z TRACE loading migration 001_init_schema.sql +2024-08-20T12:00:17Z INFO applied migration 001_init_schema +2024-08-20T12:00:18Z TRACE loading migration 002_add_indices.sql +2024-08-20T12:00:19Z INFO applied migration 002_add_indices +2024-08-20T12:00:20Z DEBUG creating index on chunks(embedding_id) +2024-08-20T12:00:21Z TRACE index creation started +2024-08-20T12:00:22Z DEBUG index chunks_embedding_idx created in 1234ms +2024-08-20T12:00:23Z INFO all migrations complete +2024-08-20T12:00:24Z DEBUG starting http server on 0.0.0.0:8080 +2024-08-20T12:00:25Z INFO listening on 0.0.0.0:8080 +2024-08-20T12:00:26Z TRACE handler registered: GET /health +2024-08-20T12:00:27Z DEBUG handler registered: POST /memory/ingest +2024-08-20T12:00:28Z TRACE handler registered: GET /memory/query +2024-08-20T12:00:29Z INFO http server ready +2024-08-20T12:00:30Z TRACE incoming request GET /health from 127.0.0.1:54321 +2024-08-20T12:00:31Z DEBUG request id=abc123 +2024-08-20T12:00:32Z TRACE processing request +2024-08-20T12:00:33Z DEBUG cache hit for /health +2024-08-20T12:00:34Z INFO request completed in 1ms status=200 +2024-08-20T12:00:35Z TRACE response sent to 127.0.0.1:54321 +2024-08-20T12:00:36Z DEBUG connection kept-alive +2024-08-20T12:00:37Z INFO active connections: 1 +2024-08-20T12:00:38Z DEBUG monitoring metrics every 60s +2024-08-20T12:00:39Z TRACE collecting metrics +2024-08-20T12:00:40Z DEBUG requests_total=1234 errors=0 latency_p99=45ms +2024-08-20T12:00:41Z INFO metrics: requests=1234 errors=0 uptime=41s +2024-08-20T12:00:42Z TRACE finalizing metrics snapshot +2024-08-20T12:00:43Z DEBUG memory usage: heap=24.5MB resident=32MB +2024-08-20T12:00:44Z INFO health check passed