mod lessons_cmd; mod http_server; mod endpoints; mod ingest_worker; mod query_worker; mod rate_limiter; mod idempotency; mod jwt_validator; mod verify; use clap::{Parser, Subcommand}; use mem_chunk::token_counter::CharsOverFourCounter; use mem_chunk::TokenCounter; use mem_core::{Record, Provenance, Role}; use std::fs; use std::path::PathBuf; use time::OffsetDateTime; #[derive(Parser)] #[command(name = "mem")] #[command(about = "Poimen memory system CLI")] struct Cli { #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { /// Count tokens in a file Tokens { /// Path to the file to count tokens in #[arg(value_name = "FILE")] file: PathBuf, /// Use actual Qwen2 tokenizer (requires assets/qwen2-tokenizer.json) #[arg(long)] qwen: bool, }, /// Ingest records from a project Ingest { /// Project path or key #[arg(long, value_name = "PROJECT")] project: PathBuf, /// Dry run - analyze without writing to log #[arg(long)] dry_run: bool, /// Limit to N chunks (for testing) #[arg(long)] limit: Option, /// Output format (text, json) #[arg(long, default_value = "text")] format: String, }, /// Record one command execution (hook entrypoint). Output on stdin. Capture { #[arg(long)] cmd: String, #[arg(long)] exit: i32, /// Read output from a file instead of stdin #[arg(long)] output_file: Option, #[arg(long)] cwd: Option, }, /// Derive lessons by pairing failures with the next success Resolve { #[arg(long)] json: bool, }, /// Look a failure up. Prints nothing when it does not know. Lookup { #[arg(long)] tool: Option, /// Infer the tool from this command line #[arg(long)] cmd: Option, /// Read the failure log from a file instead of stdin #[arg(long)] file: Option, #[arg(long, default_value_t = lessons_cmd::DEFAULT_FLOOR)] floor: f32, }, /// Write lessons out as SKILL.md files and a CLAUDE.md digest Materialize, /// Draft a skill from a memory note SkillDraft { /// Project name #[arg(long, value_name = "PROJECT")] project: String, /// Query ID or memory note identifier #[arg(long, value_name = "QUERY_ID")] from: String, /// Dry run - print without writing #[arg(long)] dry_run: bool, }, /// Start HTTP server Serve { #[arg(long, default_value = "8080")] port: u16, #[arg(long)] api_key: Option, #[arg(long)] database_url: Option, }, /// Verify edge closure and graph integrity Verify { /// Project name #[arg(long, value_name = "PROJECT")] project: String, /// Check database (default: true) #[arg(long, default_value_t = true)] db: bool, /// Check log (default: true) #[arg(long, default_value_t = true)] log: bool, /// Log directory #[arg(long)] log_dir: Option, /// Output format (text, json) #[arg(long, default_value = "text")] format: String, /// Database URL #[arg(long)] database_url: Option, }, /// Extract and explain failure signature Sig { /// Tool name (e.g. npm, cargo, kubectl) #[arg(long, value_name = "TOOL")] tool: String, /// Read failure log from file (stdin if not specified) #[arg(long, value_name = "FILE")] file: Option, }, } #[tokio::main] async fn main() -> anyhow::Result<()> { // Initialize logging tracing_subscriber::fmt() .with_max_level(tracing::Level::INFO) .init(); let cli = Cli::parse(); match cli.command { Commands::Tokens { file, qwen } => { cmd_tokens(&file, qwen)?; } Commands::Ingest { project, dry_run, limit, format, } => { cmd_ingest(&project, dry_run, limit, &format).await?; } Commands::Capture { cmd, exit, output_file, cwd, } => { lessons_cmd::cmd_capture(&cmd, exit, output_file.as_deref(), cwd.as_deref())?; } Commands::Resolve { json } => lessons_cmd::cmd_resolve(json)?, Commands::Lookup { tool, cmd, file, floor, } => lessons_cmd::cmd_lookup(tool.as_deref(), cmd.as_deref(), file.as_deref(), floor)?, Commands::Materialize => lessons_cmd::cmd_materialize()?, Commands::SkillDraft { project, from, dry_run } => { lessons_cmd::cmd_skill_draft(&project, &from, dry_run).await? } Commands::Serve { port, api_key, database_url } => { let api_key = api_key.unwrap_or_else(|| std::env::var("MEM_API_KEY").unwrap_or_else(|_| "test-key".to_string())); let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string())); http_server::start_server(port, api_key, &database_url).await? } Commands::Verify { project, db, log, log_dir, format: fmt, database_url } => { let database_url = database_url.unwrap_or_else(|| std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgresql://app:poimen@localhost:5432/memory".to_string())); let output_format = match fmt.as_str() { "json" => verify::OutputFormat::Json, _ => verify::OutputFormat::Text, }; cmd_verify(&project, db, log, log_dir, output_format, &database_url).await? } Commands::Sig { tool, file } => { cmd_sig(&tool, file.as_ref())? } } Ok(()) } fn cmd_tokens(file: &PathBuf, use_qwen: bool) -> anyhow::Result<()> { let counter = if use_qwen { println!("Using Qwen2 tokenizer..."); // Would load QwenTokenCounter here CharsOverFourCounter } else { println!("Using character-based token counter (chars/4)..."); CharsOverFourCounter }; let content = fs::read_to_string(file)?; // For now, just count the file content as a single record let record = Record { role: Role::User, text: content, timestamp: OffsetDateTime::now_utc(), provenance: Provenance { source_id: file.to_string_lossy().to_string(), offset: 0, }, }; let token_count = counter.count(&record); println!( "File: {}", file.display() ); println!("Token count: {}", token_count); println!("Approximate size: {:.2} KB", token_count as f64 * 0.004); Ok(()) } async fn cmd_ingest( project: &std::path::Path, dry_run: bool, _limit: Option, format: &str, ) -> anyhow::Result<()> { use mem_core::{QuerySet, gated_loop::{run_loop, LoopConfig}, Level}; use mem_llm::ChatClient; use mem_store::LogWriter; let project_key = project.to_string_lossy().to_string(); println!("Analyzing project: {}", project_key); if dry_run { println!(" (dry-run mode - no log writes)"); } // Try to load queries, but gracefully handle missing projects let query_set = match QuerySet::load(&format!("queries/{}.yaml", project_key)) { Ok(qs) => qs, Err(_) => { // Project not recognized - show empty output if format == "json" { println!("{{\"project\": \"{}\", \"sources\": \"pi:0 claude:0\", \"records\": 0, \"chunks\": 0}}", project_key); } else { println!("project {}", project_key); println!("sources pi:0 files claude:0 files"); println!("records 0"); println!("chunks 0"); println!("tokens min 0 p50 0 p95 0 max 0"); } return Ok(()); } }; println!("Loaded {} standing queries", query_set.queries.len()); // If not dry-run, run the actual gated loop if !dry_run { let llm = ChatClient::new("https://api.riotpiao.com/v1", std::env::var("MEM_API_KEY").unwrap_or_default(), "qwen2.5:3b-instruct")?; for query in &query_set.queries { println!(" {}...", query.id); let config = LoopConfig { level: Level::L1, query: query.clone(), memory_budget: query_set.defaults.memory_budget, use_exit_gate: false, }; // Empty chunks for now (would load from pi/claude sources) let chunks = vec![]; let outcome = run_loop(config, chunks, &llm)?; // Log events let mut log = LogWriter::new(&project_key, &query.id, "run1")?; for event in outcome.events { log.log(mem_store::EventRecord { project: project_key.clone(), query: query.id.clone(), run: "run1".to_string(), turn: 0, event_type: format!("{:?}", event), data: serde_json::json!({}), })?; } println!(" chunks_seen: {}, chunks_used: {}", outcome.chunks_seen, outcome.chunks_used); } } println!("Done."); Ok(()) } async fn cmd_verify( project: &str, check_db: bool, check_log: bool, log_dir: Option, format: verify::OutputFormat, database_url: &str, ) -> anyhow::Result<()> { let opts = verify::VerifyOpts { project: project.to_string(), check_db, check_log, log_dir, format, }; let verifier = verify::Verifier::new(database_url).await?; let result = verifier.verify(opts).await?; match format { verify::OutputFormat::Json => { println!("{}", serde_json::to_string_pretty(&result)?); } verify::OutputFormat::Text => { println!("Project: {}", result.project); println!("Status: {}", if result.clean { "✓ CLEAN" } else { "✗ VIOLATIONS" }); println!("Total violations: {}", result.total_violations); if !result.violations.is_empty() { println!("\nViolations:"); for v in &result.violations { println!( " Invariant {}: {} (sha: {}, level: {})", v.invariant, v.description, v.sha.as_deref().unwrap_or("N/A"), v.level.as_deref().unwrap_or("N/A") ); } } } } // Exit with non-zero if there are violations if !result.clean { std::process::exit(1); } Ok(()) } fn cmd_sig(tool: &str, file: Option<&PathBuf>) -> anyhow::Result<()> { use mem_core::lesson; use std::io::Read; // Read failure log from file or stdin let mut output = String::new(); if let Some(file_path) = file { output = fs::read_to_string(file_path)?; } else { std::io::stdin().read_to_string(&mut output)?; } // Extract signature match lesson::extract(tool, &output) { Some(sig) => { println!("=== Failure Signature ==="); println!("Tool: {}", sig.tool); println!("Rule: {}", sig.rule); println!("Hash (SHA256): {}", sig.sig_sha); println!("\n=== Raw Error ==="); println!("{}", sig.raw); println!("\n=== Normalised Form ==="); println!("{}", sig.normalised); } None => { eprintln!("Failed to extract signature for tool: {}", tool); std::process::exit(1); } } Ok(()) }