#[test] #[ignore] fn m1_gate_update_rate_under_30percent() { // LIVE TEST: Runs against real LLM gateway // run with: MEM_API_KEY="" cargo test --test it_m1_gate -- --ignored --nocapture use mem_core::{QuerySet, gated_loop::{run_loop, LoopConfig}, Level, Chunk, Record, Provenance, Role}; use mem_llm::ChatClient; use std::env; use time::OffsetDateTime; let api_key = env::var("MEM_API_KEY").unwrap_or_default(); // Load query set let query_set = match QuerySet::load("queries/poimen.yaml") { Ok(qs) => qs, Err(e) => { println!("SKIP: Could not load poimen query set: {}", e); return; } }; let llm = match ChatClient::new("https://api.riotpiao.com/v1", api_key, "qwen2.5:3b-instruct") { Ok(llm) => llm, Err(e) => { println!("SKIP: Could not create LLM client: {}", e); return; } }; // Load chunks from fixtures let chunks = load_test_chunks(); if chunks.is_empty() { println!("SKIP: No chunks to test"); return; } println!("Loaded {} chunks for testing", chunks.len()); let mut total_seen = 0; let mut total_used = 0; for query in &query_set.queries { let config = LoopConfig { level: Level::L1, query: query.clone(), memory_budget: 1024, use_exit_gate: false, }; match run_loop(config, chunks.clone(), &llm) { Ok(outcome) => { total_seen += outcome.chunks_seen; total_used += outcome.chunks_used; let update_rate = if outcome.chunks_seen > 0 { (outcome.chunks_used as f32) / (outcome.chunks_seen as f32) } else { 0.0 }; println!("Query '{}': {}/{} chunks used ({:.1}%)", query.id, outcome.chunks_used, outcome.chunks_seen, update_rate * 100.0 ); } Err(e) => println!("Error running loop for {}: {}", query.id, e), } } let overall_rate = if total_seen > 0 { (total_used as f32) / (total_seen as f32) } else { 0.0 }; println!("\n=== M1.8 GATE RESULT ==="); println!("Total: {}/{} chunks used ({:.1}%)", total_used, total_seen, overall_rate * 100.0); println!("Target: < 30%"); println!("Status: {}", if overall_rate < 0.3 { "✅ PASS" } else { "❌ FAIL" }); assert!(overall_rate < 0.3, "Update rate {:.1}% exceeds 30% threshold", overall_rate * 100.0 ); } /// Load test chunks from fixture files. fn load_test_chunks() -> Vec { use mem_core::{Chunk, Record, Provenance, Role}; use std::fs; use time::OffsetDateTime; let mut chunks = Vec::new(); let mut turn = 1u32; // Load from Pi session fixture if let Ok(content) = fs::read_to_string("fixtures/pi-session-small.jsonl") { for (i, line) in content.lines().enumerate() { if let Ok(value) = serde_json::from_str::(line) { if let Some(msg) = value.get("message") { if let Some(text) = msg.get("content").and_then(|c| c.as_str()) { let role = msg.get("role") .and_then(|r| r.as_str()) .map(|r| if r == "user" { Role::User } else { Role::Assistant }) .unwrap_or(Role::User); let record = Record { role, text: text.to_string(), timestamp: OffsetDateTime::now_utc(), provenance: Provenance { source_id: "pi-fixture".to_string(), offset: i as u64, }, }; let tokens = text.len() / 4; chunks.push(Chunk::new(turn, vec![record], tokens)); turn += 1; } } } } } // Load from Claude transcript fixture if let Ok(content) = fs::read_to_string("fixtures/claude-transcript-small.jsonl") { for (i, line) in content.lines().enumerate() { if let Ok(value) = serde_json::from_str::(line) { if let Some(text) = value.get("content").and_then(|c| c.as_str()) { let role = value.get("role") .and_then(|r| r.as_str()) .map(|r| if r == "user" { Role::User } else { Role::Assistant }) .unwrap_or(Role::User); let record = Record { role, text: text.to_string(), timestamp: OffsetDateTime::now_utc(), provenance: Provenance { source_id: "claude-fixture".to_string(), offset: i as u64, }, }; let tokens = text.len() / 4; chunks.push(Chunk::new(turn, vec![record], tokens)); turn += 1; } } } } chunks } #[test] fn m1_gate_framework_compiles() { // Verifies all components work together without live gateway use mem_core::gated_loop::{LlmClient, LoopConfig, run_loop}; use mem_core::{Level, Query}; use anyhow::Result; struct FakeLlm; impl LlmClient for FakeLlm { fn complete_blocking(&self, _s: &str, _u: &str, _m: usize) -> Result { Ok("nonoxcontinue".to_string()) } } let config = LoopConfig { level: Level::L1, query: Query { id: "test".to_string(), question: "Test?".to_string(), exit_gate: false, }, memory_budget: 1024, use_exit_gate: false, }; let outcome = run_loop(config, vec![], &FakeLlm).unwrap(); assert_eq!(outcome.chunks_seen, 0); assert_eq!(outcome.chunks_used, 0); }