feat: add 'mem learn' CLI for markdown knowledge ingestion

6 knowledge files: rust, SOLID/DRY, ast-grep, karpathy, golang, caveman
65 chunks ingested to log/knowledge/learn/latest.jsonl
Chunks on ## headings, SHA256 dedup, configurable chunk size
This commit is contained in:
2026-08-29 22:04:14 -07:00
parent d6b6c763b6
commit a5ff20c9f7
10 changed files with 621 additions and 0 deletions
+166
View File
@@ -155,6 +155,25 @@ enum Commands {
#[arg(long, value_name = "FILE")]
file: Option<PathBuf>,
},
/// Ingest markdown knowledge files into memory
Learn {
/// Markdown files or directories to ingest
#[arg(value_name = "PATH")]
paths: Vec<PathBuf>,
/// Project to file under
#[arg(long, default_value = "knowledge")]
project: String,
/// Dry run — show chunks without writing
#[arg(long)]
dry_run: bool,
/// Maximum chunk size in characters (splits on headings)
#[arg(long, default_value_t = 2000)]
chunk_size: usize,
},
}
#[tokio::main]
@@ -213,6 +232,9 @@ async fn main() -> anyhow::Result<()> {
Commands::Sig { tool, file } => {
cmd_sig(&tool, file.as_ref())?
}
Commands::Learn { paths, project, dry_run, chunk_size } => {
cmd_learn(&paths, &project, dry_run, chunk_size)?;
}
}
Ok(())
@@ -379,6 +401,150 @@ async fn cmd_verify(
Ok(())
}
fn cmd_learn(
paths: &[PathBuf],
project: &str,
dry_run: bool,
max_chunk: usize,
) -> anyhow::Result<()> {
use sha2::{Digest, Sha256};
let mut all_files: Vec<PathBuf> = Vec::new();
for p in paths {
if p.is_dir() {
for entry in walkdir::WalkDir::new(p)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| {
e.path()
.extension()
.map(|ext| ext == "md")
.unwrap_or(false)
})
{
all_files.push(entry.into_path());
}
} else if p.extension().map(|e| e == "md").unwrap_or(false) {
all_files.push(p.clone());
} else {
eprintln!("Skipping non-markdown file: {}", p.display());
}
}
if all_files.is_empty() {
eprintln!("No markdown files found.");
return Ok(());
}
all_files.sort();
println!("Found {} markdown files", all_files.len());
let mut total_chunks = 0usize;
let mut total_bytes = 0usize;
let mut log = if !dry_run {
Some(mem_store::LogWriter::new(project, "learn", "latest")?)
} else {
None
};
for file in &all_files {
let content = fs::read_to_string(file)?;
let filename = file.file_stem().unwrap().to_string_lossy();
let chunks = chunk_markdown(&content, max_chunk);
println!("\n📄 {}{} chunks", file.display(), chunks.len());
for (i, chunk) in chunks.iter().enumerate() {
let mut hasher = Sha256::new();
hasher.update(chunk.as_bytes());
let hash = format!("{:x}", hasher.finalize());
let short_hash = &hash[..12];
total_chunks += 1;
total_bytes += chunk.len();
if dry_run {
let preview: String = chunk.chars().take(80).collect();
println!(
" [{}/{}] {} ({} bytes) {}",
i + 1,
chunks.len(),
short_hash,
chunk.len(),
preview.replace('\n', " ")
);
} else {
let record = mem_store::EventRecord {
project: project.to_string(),
query: format!("{}:{}", filename, i),
run: "latest".to_string(),
turn: i as u32,
event_type: "learn".to_string(),
data: serde_json::json!({
"source": file.to_string_lossy(),
"chunk_index": i,
"total_chunks": chunks.len(),
"sha256": hash,
"level": "L1",
"text": chunk,
}),
};
log.as_mut().unwrap().log(record)?;
println!(" ✓ [{}/{}] {} ({} bytes)", i + 1, chunks.len(), short_hash, chunk.len());
}
}
}
println!("\n{}", "".repeat(50));
println!(
"{} files → {} chunks ({:.1} KB)",
all_files.len(),
total_chunks,
total_bytes as f64 / 1024.0
);
if dry_run {
println!("(dry run — nothing written)");
} else {
println!("Written to log/{}/learn/latest.jsonl", project);
}
Ok(())
}
/// Split markdown on ## headings, respecting max_chunk size.
fn chunk_markdown(content: &str, max_chunk: usize) -> Vec<String> {
let mut chunks = Vec::new();
let mut current = String::new();
for line in content.lines() {
// Split on ## headings (keep # title in first chunk)
if line.starts_with("## ") && !current.is_empty() {
let trimmed = current.trim().to_string();
if !trimmed.is_empty() {
chunks.push(trimmed);
}
current = String::new();
}
current.push_str(line);
current.push('\n');
// Hard split if chunk too large
if current.len() > max_chunk {
let trimmed = current.trim().to_string();
if !trimmed.is_empty() {
chunks.push(trimmed);
}
current = String::new();
}
}
let trimmed = current.trim().to_string();
if !trimmed.is_empty() {
chunks.push(trimmed);
}
chunks
}
fn cmd_sig(tool: &str, file: Option<&PathBuf>) -> anyhow::Result<()> {
use mem_core::lesson;
use std::io::Read;