fix: gitignore log/ dir, remove tracked JSONL from repo

Event logs are runtime data, not source code.
Also adds mem compact command and browser-use + memory-service knowledge.

crates/mem-cli/src/main.rs

@@ -156,6 +156,21 @@ enum Commands {
         file: Option<PathBuf>,
     },
 
+    /// Compact knowledge: deduplicate and merge similar chunks via embeddings + LLM
+    Compact {
+        /// Project name
+        #[arg(long, default_value = "knowledge")]
+        project: String,
+
+        /// Cosine similarity threshold for grouping
+        #[arg(long, default_value_t = 0.82)]
+        threshold: f32,
+
+        /// Dry run — show groups without merging
+        #[arg(long)]
+        dry_run: bool,
+    },
+
     /// Ingest markdown knowledge files into memory
     Learn {
         /// Markdown files or directories to ingest
@@ -235,6 +250,9 @@ async fn main() -> anyhow::Result<()> {
         Commands::Learn { paths, project, dry_run, chunk_size } => {
             cmd_learn(&paths, &project, dry_run, chunk_size)?;
         }
+        Commands::Compact { project, threshold, dry_run } => {
+            cmd_compact(&project, threshold, dry_run).await?;
+        }
     }
 
     Ok(())
@@ -401,6 +419,207 @@ async fn cmd_verify(
     Ok(())
 }
 
+async fn cmd_compact(project: &str, threshold: f32, dry_run: bool) -> anyhow::Result<()> {
+    use mem_llm::{EmbeddingsClient, ChatClient};
+    use sha2::{Digest, Sha256};
+
+    let log_path = format!("log/{}/learn/latest.jsonl", project);
+    let content = fs::read_to_string(&log_path)
+        .map_err(|_| anyhow::anyhow!("No log at {}", log_path))?;
+
+    let mut records: Vec<serde_json::Value> = content
+        .lines()
+        .filter(|l| !l.is_empty())
+        .map(|l| serde_json::from_str(l).unwrap())
+        .collect();
+
+    let texts: Vec<String> = records
+        .iter()
+        .map(|r| r["data"]["text"].as_str().unwrap_or("").to_string())
+        .collect();
+
+    println!("Loaded {} chunks from {}", records.len(), log_path);
+
+    // Embed all chunks
+    println!("Embedding {} chunks...", texts.len());
+    let embedder = EmbeddingsClient::from_env()?;
+    let mut vectors = Vec::new();
+    for batch in texts.chunks(8) {
+        let batch_strs: Vec<String> = batch.to_vec();
+        match embedder.embed(&batch_strs).await {
+            Ok(v) => vectors.extend(v),
+            Err(e) => {
+                eprintln!("Embedding batch failed: {}. Retrying in 5s...", e);
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+                let v = embedder.embed(&batch_strs).await?;
+                vectors.extend(v);
+            }
+        }
+        eprint!(".");
+    }
+    eprintln!();
+    println!("Embedded {} vectors (768-dim)", vectors.len());
+
+    // Compute cosine similarity and group
+    let n = vectors.len();
+    let raw_vecs: Vec<Vec<f32>> = vectors.iter().map(|v| v.to_vec()).collect();
+
+    // Normalize vectors
+    let norms: Vec<f32> = raw_vecs
+        .iter()
+        .map(|v| {
+            let s: f32 = v.iter().map(|x| x * x).sum();
+            s.sqrt().max(1e-10)
+        })
+        .collect();
+    let normed: Vec<Vec<f32>> = raw_vecs
+        .iter()
+        .zip(norms.iter())
+        .map(|(v, n)| v.iter().map(|x| x / n).collect())
+        .collect();
+
+    // Find similar groups
+    let mut visited = vec![false; n];
+    let mut groups: Vec<Vec<usize>> = Vec::new();
+
+    for i in 0..n {
+        if visited[i] { continue; }
+        let mut group = vec![i];
+        visited[i] = true;
+        for j in (i + 1)..n {
+            if visited[j] { continue; }
+            let sim: f32 = normed[i].iter().zip(normed[j].iter()).map(|(a, b)| a * b).sum();
+            if sim > threshold {
+                group.push(j);
+                visited[j] = true;
+            }
+        }
+        if group.len() > 1 {
+            groups.push(group);
+        }
+    }
+
+    if groups.is_empty() {
+        println!("\n\u{2713} No similar chunks found. Knowledge is already compact.");
+        return Ok(());
+    }
+
+    let total_mergeable: usize = groups.iter().map(|g| g.len()).sum();
+    let savings = total_mergeable - groups.len();
+    println!("\nFound {} groups ({} chunks \u{2192} {} merged, saving {})",
+        groups.len(), total_mergeable, groups.len(), savings);
+
+    let llm = ChatClient::new(
+        std::env::var("LLM_API_BASE").unwrap_or_else(|_| "https://api.riotpiao.com".to_string()),
+        std::env::var("LLM_API_KEY").unwrap_or_default(),
+        "reasoning",
+    )?;
+
+    let mut to_remove: Vec<usize> = Vec::new();
+
+    for (gi, group) in groups.iter().enumerate() {
+        let group_texts: Vec<&str> = group.iter().map(|&i| texts[i].as_str()).collect();
+        let group_sources: Vec<&str> = group
+            .iter()
+            .map(|&i| records[i]["data"]["source"].as_str().unwrap_or("?"))
+            .collect();
+
+        // Compute max similarity in group
+        let mut max_sim: f32 = 0.0;
+        for a in 0..group.len() {
+            for b in (a + 1)..group.len() {
+                let sim: f32 = normed[group[a]].iter().zip(normed[group[b]].iter()).map(|(x, y)| x * y).sum();
+                max_sim = max_sim.max(sim);
+            }
+        }
+
+        println!("\nGroup {} (sim={:.3}, {} chunks):", gi + 1, max_sim, group.len());
+        for &idx in group {
+            let preview: String = texts[idx].chars().take(80).collect();
+            let src = std::path::Path::new(group_sources[group.iter().position(|&i| i == idx).unwrap()])
+                .file_stem()
+                .map(|s| s.to_string_lossy().to_string())
+                .unwrap_or_else(|| "?".to_string());
+            println!("  [{}] {}...", src, preview.replace('\n', " "));
+        }
+
+        if dry_run {
+            continue;
+        }
+
+        // Merge via LLM
+        println!("  \u{2192} Merging with reasoning model...");
+        let numbered: String = group_texts
+            .iter()
+            .zip(group_sources.iter())
+            .enumerate()
+            .map(|(i, (t, s))| format!("[Chunk {} from {}]:\n{}", i + 1, s, t))
+            .collect::<Vec<_>>()
+            .join("\n\n");
+
+        let merged = llm.complete(
+            "Merge these similar knowledge chunks into ONE concise chunk. Keep ALL unique facts. \
+             Remove redundancy. Keep markdown formatting. Output ONLY the merged text.",
+            &format!("Merge these {} chunks:\n\n{}", group.len(), numbered),
+            1500,
+        ).await?;
+
+        // Strip <think> tags
+        let merged_text = merged.text.split("</think>").last().unwrap_or(&merged.text).trim().to_string();
+        let old_size: usize = group_texts.iter().map(|t| t.len()).sum();
+        println!("  \u{2192} Merged: {} chars (was {} chars, {:.0}% reduction)",
+            merged_text.len(), old_size, (1.0 - merged_text.len() as f64 / old_size as f64) * 100.0);
+
+        // Update first chunk with merged content
+        let mut hasher = Sha256::new();
+        hasher.update(merged_text.as_bytes());
+        let new_hash = format!("{:x}", hasher.finalize());
+
+        records[group[0]]["data"]["text"] = serde_json::Value::String(merged_text);
+        records[group[0]]["data"]["sha256"] = serde_json::Value::String(new_hash);
+        records[group[0]]["data"]["merged_from"] = serde_json::json!(group.len());
+
+        // Mark rest for removal
+        for &idx in &group[1..] {
+            to_remove.push(idx);
+        }
+    }
+
+    if dry_run {
+        println!("\n(dry run \u{2014} no changes written)");
+        return Ok(());
+    }
+
+    // Write compacted log
+    let to_remove_set: std::collections::HashSet<usize> = to_remove.into_iter().collect();
+    let compacted: Vec<&serde_json::Value> = records
+        .iter()
+        .enumerate()
+        .filter(|(i, _)| !to_remove_set.contains(i))
+        .map(|(_, r)| r)
+        .collect();
+
+    // Backup
+    let backup = format!("{}.bak", log_path);
+    fs::copy(&log_path, &backup)?;
+
+    // Write
+    let mut f = fs::File::create(&log_path)?;
+    use std::io::Write;
+    for r in &compacted {
+        serde_json::to_writer(&mut f, r)?;
+        f.write_all(b"\n")?;
+    }
+
+    println!("\n{}", "\u{2500}".repeat(50));
+    println!("Before: {} chunks", records.len());
+    println!("After:  {} chunks (-{})", compacted.len(), records.len() - compacted.len());
+    println!("Backup: {}", backup);
+    println!("Written: {}", log_path);
+
+    Ok(())
+}
+
 fn cmd_learn(
     paths: &[PathBuf],
     project: &str,