poimen-memory/crates/mem-cli/src/main.rs

use clap::{Parser, Subcommand};
use mem_chunk::token_counter::CharsOverFourCounter;
use mem_chunk::TokenCounter;
use mem_core::{Record, Provenance, Role};
use std::fs;
use std::path::PathBuf;
use time::OffsetDateTime;

#[derive(Parser)]
#[command(name = "mem")]
#[command(about = "Poimen memory system CLI")]
struct Cli {
    #[command(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Count tokens in a file
    Tokens {
        /// Path to the file to count tokens in
        #[arg(value_name = "FILE")]
        file: PathBuf,

        /// Use actual Qwen2 tokenizer (requires assets/qwen2-tokenizer.json)
        #[arg(long)]
        qwen: bool,
    },

    /// Ingest records from a source
    Ingest {
        /// Source type (pi-session, claude-transcript)
        #[arg(value_name = "SOURCE_TYPE")]
        source_type: String,

        /// Path to source file
        #[arg(value_name = "FILE")]
        file: PathBuf,

        /// Dry run - don't write to log
        #[arg(long)]
        dry_run: bool,
    },
}

fn main() -> anyhow::Result<()> {
    let cli = Cli::parse();

    match cli.command {
        Commands::Tokens { file, qwen } => {
            cmd_tokens(&file, qwen)?;
        }
        Commands::Ingest {
            source_type,
            file,
            dry_run,
        } => {
            cmd_ingest(&source_type, &file, dry_run)?;
        }
    }

    Ok(())
}

fn cmd_tokens(file: &PathBuf, use_qwen: bool) -> anyhow::Result<()> {
    let counter = if use_qwen {
        println!("Using Qwen2 tokenizer...");
        // Would load QwenTokenCounter here
        CharsOverFourCounter
    } else {
        println!("Using character-based token counter (chars/4)...");
        CharsOverFourCounter
    };

    let content = fs::read_to_string(file)?;
    
    // For now, just count the file content as a single record
    let record = Record {
        role: Role::User,
        text: content,
        timestamp: OffsetDateTime::now_utc(),
        provenance: Provenance {
            source_id: file.to_string_lossy().to_string(),
            offset: 0,
        },
    };

    let token_count = counter.count(&record);
    println!(
        "File: {}",
        file.display()
    );
    println!("Token count: {}", token_count);
    println!("Approximate size: {:.2} KB", token_count as f64 * 0.004);

    Ok(())
}

fn cmd_ingest(source_type: &str, file: &PathBuf, dry_run: bool) -> anyhow::Result<()> {
    println!("Ingesting from {} source: {}", source_type, file.display());
    if dry_run {
        println!("  (dry-run mode - no log writes)");
    }
    
    // Placeholder for actual ingest logic
    println!("Ingest not yet implemented");

    Ok(())
}