poimen-memory/crates/mem-cli/src/main.rs

use clap::{Parser, Subcommand};
use mem_chunk::token_counter::CharsOverFourCounter;
use mem_chunk::TokenCounter;
use mem_core::{Record, Provenance, Role};
use std::fs;
use std::path::PathBuf;
use time::OffsetDateTime;

#[derive(Parser)]
#[command(name = "mem")]
#[command(about = "Poimen memory system CLI")]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Count tokens in a file
    Tokens {
        /// Path to the file to count tokens in
        #[arg(value_name = "FILE")]
        file: PathBuf,

        /// Use actual Qwen2 tokenizer (requires assets/qwen2-tokenizer.json)
        #[arg(long)]
        qwen: bool,
    },

    /// Ingest records from a project
    Ingest {
        /// Project path or key
        #[arg(long, value_name = "PROJECT")]
        project: PathBuf,

        /// Dry run - analyze without writing to log
        #[arg(long)]
        dry_run: bool,

        /// Limit to N chunks (for testing)
        #[arg(long)]
        limit: Option<usize>,

        /// Output format (text, json)
        #[arg(long, default_value = "text")]
        format: String,
    },
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let cli = Cli::parse();

    match cli.command {
        Commands::Tokens { file, qwen } => {
            cmd_tokens(&file, qwen)?;
        }
        Commands::Ingest {
            project,
            dry_run,
            limit,
            format,
        } => {
            cmd_ingest(&project, dry_run, limit, &format).await?;
        }
    }

    Ok(())
}

fn cmd_tokens(file: &PathBuf, use_qwen: bool) -> anyhow::Result<()> {
    let counter = if use_qwen {
        println!("Using Qwen2 tokenizer...");
        // Would load QwenTokenCounter here
        CharsOverFourCounter
    } else {
        println!("Using character-based token counter (chars/4)...");
        CharsOverFourCounter
    };

    let content = fs::read_to_string(file)?;
    
    // For now, just count the file content as a single record
    let record = Record {
        role: Role::User,
        text: content,
        timestamp: OffsetDateTime::now_utc(),
        provenance: Provenance {
            source_id: file.to_string_lossy().to_string(),
            offset: 0,
        },
    };

    let token_count = counter.count(&record);
    println!(
        "File: {}",
        file.display()
    );
    println!("Token count: {}", token_count);
    println!("Approximate size: {:.2} KB", token_count as f64 * 0.004);

    Ok(())
}

async fn cmd_ingest(
    project: &std::path::Path,
    dry_run: bool,
    _limit: Option<usize>,
    format: &str,
) -> anyhow::Result<()> {
    let project_key = project.to_string_lossy().to_string();
    
    println!("Analyzing project: {}", project_key);
    if dry_run {
        println!("  (dry-run mode - no log writes)");
    }

    // For now, just print a summary
    if format == "json" {
        println!("{{\"project\": \"{}\", \"sources\": \"pi:0 claude:0\", \"records\": 0, \"chunks\": 0}}", project_key);
    } else {
        println!("project  {}", project_key);
        println!("sources  pi:0 files  claude:0 files");
        println!("records  0");
        println!("chunks   0");
        println!("tokens   min 0  p50 0  p95 0  max 0");
    }

    Ok(())
}