use clap::{Parser, Subcommand}; use mem_chunk::token_counter::CharsOverFourCounter; use mem_chunk::TokenCounter; use mem_core::{Record, Provenance, Role}; use std::fs; use std::path::PathBuf; use time::OffsetDateTime; #[derive(Parser)] #[command(name = "mem")] #[command(about = "Poimen memory system CLI")] struct Cli { #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { /// Count tokens in a file Tokens { /// Path to the file to count tokens in #[arg(value_name = "FILE")] file: PathBuf, /// Use actual Qwen2 tokenizer (requires assets/qwen2-tokenizer.json) #[arg(long)] qwen: bool, }, /// Ingest records from a project Ingest { /// Project path or key #[arg(long, value_name = "PROJECT")] project: PathBuf, /// Dry run - analyze without writing to log #[arg(long)] dry_run: bool, /// Limit to N chunks (for testing) #[arg(long)] limit: Option, /// Output format (text, json) #[arg(long, default_value = "text")] format: String, }, } #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); match cli.command { Commands::Tokens { file, qwen } => { cmd_tokens(&file, qwen)?; } Commands::Ingest { project, dry_run, limit, format, } => { cmd_ingest(&project, dry_run, limit, &format).await?; } } Ok(()) } fn cmd_tokens(file: &PathBuf, use_qwen: bool) -> anyhow::Result<()> { let counter = if use_qwen { println!("Using Qwen2 tokenizer..."); // Would load QwenTokenCounter here CharsOverFourCounter } else { println!("Using character-based token counter (chars/4)..."); CharsOverFourCounter }; let content = fs::read_to_string(file)?; // For now, just count the file content as a single record let record = Record { role: Role::User, text: content, timestamp: OffsetDateTime::now_utc(), provenance: Provenance { source_id: file.to_string_lossy().to_string(), offset: 0, }, }; let token_count = counter.count(&record); println!( "File: {}", file.display() ); println!("Token count: {}", token_count); println!("Approximate size: {:.2} KB", token_count as f64 * 0.004); Ok(()) } async fn cmd_ingest( project: &std::path::Path, dry_run: bool, _limit: Option, format: &str, ) -> anyhow::Result<()> { let project_key = project.to_string_lossy().to_string(); println!("Analyzing project: {}", project_key); if dry_run { println!(" (dry-run mode - no log writes)"); } // For now, just print a summary if format == "json" { println!("{{\"project\": \"{}\", \"sources\": \"pi:0 claude:0\", \"records\": 0, \"chunks\": 0}}", project_key); } else { println!("project {}", project_key); println!("sources pi:0 files claude:0 files"); println!("records 0"); println!("chunks 0"); println!("tokens min 0 p50 0 p95 0 max 0"); } Ok(()) }