diff --git a/.gitignore b/.gitignore index cbc435a..28426cd 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,6 @@ target/ vault/ # Do NOT ignore these - they are authoritative: -# log/ - JSONL event log (authoritative record) +log/ # tasks/ - Task board and acceptance criteria CLAUDE.md diff --git a/crates/mem-cli/src/main.rs b/crates/mem-cli/src/main.rs index 8563158..db38dff 100644 --- a/crates/mem-cli/src/main.rs +++ b/crates/mem-cli/src/main.rs @@ -156,6 +156,21 @@ enum Commands { file: Option, }, + /// Compact knowledge: deduplicate and merge similar chunks via embeddings + LLM + Compact { + /// Project name + #[arg(long, default_value = "knowledge")] + project: String, + + /// Cosine similarity threshold for grouping + #[arg(long, default_value_t = 0.82)] + threshold: f32, + + /// Dry run — show groups without merging + #[arg(long)] + dry_run: bool, + }, + /// Ingest markdown knowledge files into memory Learn { /// Markdown files or directories to ingest @@ -235,6 +250,9 @@ async fn main() -> anyhow::Result<()> { Commands::Learn { paths, project, dry_run, chunk_size } => { cmd_learn(&paths, &project, dry_run, chunk_size)?; } + Commands::Compact { project, threshold, dry_run } => { + cmd_compact(&project, threshold, dry_run).await?; + } } Ok(()) @@ -401,6 +419,207 @@ async fn cmd_verify( Ok(()) } +async fn cmd_compact(project: &str, threshold: f32, dry_run: bool) -> anyhow::Result<()> { + use mem_llm::{EmbeddingsClient, ChatClient}; + use sha2::{Digest, Sha256}; + + let log_path = format!("log/{}/learn/latest.jsonl", project); + let content = fs::read_to_string(&log_path) + .map_err(|_| anyhow::anyhow!("No log at {}", log_path))?; + + let mut records: Vec = content + .lines() + .filter(|l| !l.is_empty()) + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + + let texts: Vec = records + .iter() + .map(|r| r["data"]["text"].as_str().unwrap_or("").to_string()) + .collect(); + + println!("Loaded {} chunks from {}", records.len(), log_path); + + // Embed all chunks + println!("Embedding {} chunks...", texts.len()); + let embedder = EmbeddingsClient::from_env()?; + let mut vectors = Vec::new(); + for batch in texts.chunks(8) { + let batch_strs: Vec = batch.to_vec(); + match embedder.embed(&batch_strs).await { + Ok(v) => vectors.extend(v), + Err(e) => { + eprintln!("Embedding batch failed: {}. Retrying in 5s...", e); + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + let v = embedder.embed(&batch_strs).await?; + vectors.extend(v); + } + } + eprint!("."); + } + eprintln!(); + println!("Embedded {} vectors (768-dim)", vectors.len()); + + // Compute cosine similarity and group + let n = vectors.len(); + let raw_vecs: Vec> = vectors.iter().map(|v| v.to_vec()).collect(); + + // Normalize vectors + let norms: Vec = raw_vecs + .iter() + .map(|v| { + let s: f32 = v.iter().map(|x| x * x).sum(); + s.sqrt().max(1e-10) + }) + .collect(); + let normed: Vec> = raw_vecs + .iter() + .zip(norms.iter()) + .map(|(v, n)| v.iter().map(|x| x / n).collect()) + .collect(); + + // Find similar groups + let mut visited = vec![false; n]; + let mut groups: Vec> = Vec::new(); + + for i in 0..n { + if visited[i] { continue; } + let mut group = vec![i]; + visited[i] = true; + for j in (i + 1)..n { + if visited[j] { continue; } + let sim: f32 = normed[i].iter().zip(normed[j].iter()).map(|(a, b)| a * b).sum(); + if sim > threshold { + group.push(j); + visited[j] = true; + } + } + if group.len() > 1 { + groups.push(group); + } + } + + if groups.is_empty() { + println!("\n\u{2713} No similar chunks found. Knowledge is already compact."); + return Ok(()); + } + + let total_mergeable: usize = groups.iter().map(|g| g.len()).sum(); + let savings = total_mergeable - groups.len(); + println!("\nFound {} groups ({} chunks \u{2192} {} merged, saving {})", + groups.len(), total_mergeable, groups.len(), savings); + + let llm = ChatClient::new( + std::env::var("LLM_API_BASE").unwrap_or_else(|_| "https://api.riotpiao.com".to_string()), + std::env::var("LLM_API_KEY").unwrap_or_default(), + "reasoning", + )?; + + let mut to_remove: Vec = Vec::new(); + + for (gi, group) in groups.iter().enumerate() { + let group_texts: Vec<&str> = group.iter().map(|&i| texts[i].as_str()).collect(); + let group_sources: Vec<&str> = group + .iter() + .map(|&i| records[i]["data"]["source"].as_str().unwrap_or("?")) + .collect(); + + // Compute max similarity in group + let mut max_sim: f32 = 0.0; + for a in 0..group.len() { + for b in (a + 1)..group.len() { + let sim: f32 = normed[group[a]].iter().zip(normed[group[b]].iter()).map(|(x, y)| x * y).sum(); + max_sim = max_sim.max(sim); + } + } + + println!("\nGroup {} (sim={:.3}, {} chunks):", gi + 1, max_sim, group.len()); + for &idx in group { + let preview: String = texts[idx].chars().take(80).collect(); + let src = std::path::Path::new(group_sources[group.iter().position(|&i| i == idx).unwrap()]) + .file_stem() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| "?".to_string()); + println!(" [{}] {}...", src, preview.replace('\n', " ")); + } + + if dry_run { + continue; + } + + // Merge via LLM + println!(" \u{2192} Merging with reasoning model..."); + let numbered: String = group_texts + .iter() + .zip(group_sources.iter()) + .enumerate() + .map(|(i, (t, s))| format!("[Chunk {} from {}]:\n{}", i + 1, s, t)) + .collect::>() + .join("\n\n"); + + let merged = llm.complete( + "Merge these similar knowledge chunks into ONE concise chunk. Keep ALL unique facts. \ + Remove redundancy. Keep markdown formatting. Output ONLY the merged text.", + &format!("Merge these {} chunks:\n\n{}", group.len(), numbered), + 1500, + ).await?; + + // Strip tags + let merged_text = merged.text.split("").last().unwrap_or(&merged.text).trim().to_string(); + let old_size: usize = group_texts.iter().map(|t| t.len()).sum(); + println!(" \u{2192} Merged: {} chars (was {} chars, {:.0}% reduction)", + merged_text.len(), old_size, (1.0 - merged_text.len() as f64 / old_size as f64) * 100.0); + + // Update first chunk with merged content + let mut hasher = Sha256::new(); + hasher.update(merged_text.as_bytes()); + let new_hash = format!("{:x}", hasher.finalize()); + + records[group[0]]["data"]["text"] = serde_json::Value::String(merged_text); + records[group[0]]["data"]["sha256"] = serde_json::Value::String(new_hash); + records[group[0]]["data"]["merged_from"] = serde_json::json!(group.len()); + + // Mark rest for removal + for &idx in &group[1..] { + to_remove.push(idx); + } + } + + if dry_run { + println!("\n(dry run \u{2014} no changes written)"); + return Ok(()); + } + + // Write compacted log + let to_remove_set: std::collections::HashSet = to_remove.into_iter().collect(); + let compacted: Vec<&serde_json::Value> = records + .iter() + .enumerate() + .filter(|(i, _)| !to_remove_set.contains(i)) + .map(|(_, r)| r) + .collect(); + + // Backup + let backup = format!("{}.bak", log_path); + fs::copy(&log_path, &backup)?; + + // Write + let mut f = fs::File::create(&log_path)?; + use std::io::Write; + for r in &compacted { + serde_json::to_writer(&mut f, r)?; + f.write_all(b"\n")?; + } + + println!("\n{}", "\u{2500}".repeat(50)); + println!("Before: {} chunks", records.len()); + println!("After: {} chunks (-{})", compacted.len(), records.len() - compacted.len()); + println!("Backup: {}", backup); + println!("Written: {}", log_path); + + Ok(()) +} + fn cmd_learn( paths: &[PathBuf], project: &str, diff --git a/knowledge/browser-use.md b/knowledge/browser-use.md new file mode 100644 index 0000000..0f978dd --- /dev/null +++ b/knowledge/browser-use.md @@ -0,0 +1,186 @@ +# browser-use — AI Browser Automation + +## What It Is +- AI agent that controls a real browser via CDP (Chrome DevTools Protocol). +- Describe task in natural language → agent clicks, types, navigates, extracts data. +- Python library for code-level automation OR CLI for agent integration (Claude Code, Cursor, etc). +- Open source, self-hosted. Optional cloud for stealth/scaling. + +## Installation +```bash +# Python library +uv add browser-use +# or: pip install browser-use + +# CLI (via uvx, no install needed) +uvx browser-use + +# Install Chromium + dependencies +browser-use install + +# Register skill for coding agents +browser-use skill install +``` + +## CLI Usage +```bash +# Run inline Python with pre-imported helpers +browser-use <<'PY' +new_tab("https://example.com") +print(page_info()) +PY + +# Check local Chrome connection +browser-use --doctor + +# macOS: approve remote debugging permission +browser-use mac-approve + +# Auth for cloud browsers +browser-use auth login +``` + +## Python Library — Basic Agent +```python +import asyncio +from browser_use import Agent, ChatBrowserUse + +async def main(): + agent = Agent( + task="Find the price of iPhone 16 on Amazon", + llm=ChatBrowserUse(model='openai/gpt-5.5'), + ) + result = await agent.run() + print(result) + +asyncio.run(main()) +``` + +## Structured Output +```python +from pydantic import BaseModel, Field +from browser_use import Agent, ChatBrowserUse + +class ProductInfo(BaseModel): + title: str = Field(..., description='Product name') + price: float = Field(..., description='Price as number') + url: str = Field(..., description='Product URL') + +agent = Agent( + task="Find the top 3 results for 'mechanical keyboard' on Amazon", + llm=ChatBrowserUse(model='bu-2-0-mini-preview'), + output_model_schema=ProductInfo, +) +result = asyncio.run(agent.run()) +if result and result.structured_output: + print(result.structured_output) +``` + +## Connect to Existing Browser +```python +from browser_use import Agent, Browser, ChatBrowserUse + +# Connect to local Chrome with CDP +browser = Browser(cdp_url='http://localhost:9222') +agent = Agent(browser=browser, llm=ChatBrowserUse(), task="...") +``` + +## CLI Helper Functions +Pre-imported in `browser-use` CLI context: +- `new_tab(url)` — open URL in new tab (first navigation per task) +- `goto_url(url)` — navigate current tab +- `page_info()` — get current page info +- `current_tab()` — get current tab info +- `list_tabs()` — list all open tabs +- `switch_tab(target)` — switch to existing tab +- `activate_tab(target)` — bring tab to foreground +- `click_at_xy(x, y)` — click at viewport coordinates +- `js(code)` — execute JavaScript in page +- `wait_for_load()` — wait for page load after navigation +- `ensure_real_tab()` — skip internal/stale tabs +- `cdp("Domain.method", ...)` — raw CDP command +- `start_recording(name)` / `stop_recording()` — record session + +## Remote/Cloud Browsers +```bash +# Start remote browser (for parallel tasks or stealth) +browser-use <<'PY' +start_remote_daemon("my-task") +PY + +# Use remote browser +BU_NAME=my-task browser-use <<'PY' +new_tab("https://example.com") +print(page_info()) +PY + +# Stop when done (billed until stopped) +browser-use <<'PY' +stop_remote_daemon("my-task") +PY +``` + +## Page Interaction Workflow +1. Find elements via accessibility tree: `cdp("Accessibility.getFullAXTree")["nodes"]` +2. Get coordinates: `cdp("DOM.getBoxModel", backendNodeId=n)` → compute center +3. Click: `click_at_xy(x, y)` +4. Verify: `js(...)` or `page_info()` to confirm action +5. Fallback to raw HTML via `js(...)` only when AX tree lacks element + +## When to Use browser-use vs curl +- **Use curl**: public pages, APIs, static content, docs +- **Use browser-use**: login-required pages, JS-rendered content, form filling, clicking, bot-protected sites, interactive workflows + +## Environment Variables +- `BROWSER_USE_API_KEY` — API key for ChatBrowserUse or cloud +- `BU_NAME` — name for remote daemon +- `BU_CDP_URL` — custom CDP endpoint +- `BH_DOMAIN_SKILLS=1` — enable domain-specific skills +- `BH_AGENT_WORKSPACE` — path to agent workspace with helpers + +## Recordings +```bash +# Enable/disable background recording +browser-use recordings enable +browser-use recordings disable + +# View latest recording +browser-use recordings --latest +``` + +## Common Use Cases +- Fill job applications with resume data +- Extract structured data from websites (followers, products, prices) +- Compare prices across multiple sites +- Automate checkout flows +- Screenshot pages for verification +- QA testing of web applications +- Monitor website changes + +## Integration with Verification Workflow +```bash +# After deploying an API, verify it works via browser +browser-use <<'PY' +new_tab("https://api.riotpiao.com/health") +info = page_info() +print(info) # Should show {"status": "ok"} +PY + +# Verify UI renders correctly after deployment +browser-use <<'PY' +new_tab("https://myapp.riotpiao.com") +wait_for_load() +# Check page title +title = js("document.title") +print(f"Page title: {title}") +PY +``` + +## Gotchas +- First navigation per task must use `new_tab(url)`, not `goto_url(url)` +- Don't open duplicate tabs — check `list_tabs()` first +- Login walls: stop and ask user. Exception: SSO if already signed in +- `chrome://inspect/#remote-debugging` must be enabled for local Chrome +- CDP target order ≠ Chrome's visible tab-strip order +- Always call `wait_for_load()` after navigation +- Cloud browsers bill until stopped — always `stop_remote_daemon()` when done diff --git a/knowledge/poimen-memory-service.md b/knowledge/poimen-memory-service.md new file mode 100644 index 0000000..bb24fe5 --- /dev/null +++ b/knowledge/poimen-memory-service.md @@ -0,0 +1,237 @@ +# Poimen Memory Service — Agent Integration Guide + +## What Is This +Poimen Memory is a knowledge management system that stores, retrieves, and reasons over learned facts. Agents use it to remember solutions, patterns, and skills across sessions. It prevents re-learning the same things and surfaces relevant knowledge when needed. + +## Architecture +- **Event Log** (JSONL): Immutable source of truth. All knowledge written here first. +- **pgvector** (Postgres): Vector store for semantic search (768-dim embeddings). +- **OpenSearch**: Lexical index for BM25 keyword search. +- **Hybrid Search**: 60% semantic + 40% lexical via RRF fusion. +- **Three-Tier Retrieval**: Tier 1 (exact signature) → Tier 2 (hybrid search) → Tier 3 (reference docs). + +## When to Memorize (What's Worth Storing) + +### DO Memorize +- **Bug fixes**: Error message → root cause → solution. Next time the same error appears, instant recall. +- **Configuration patterns**: Port conflicts, env var requirements, service dependencies. +- **Architecture decisions**: Why we chose X over Y, with trade-offs documented. +- **Tool usage patterns**: Commands that work, flags that matter, common gotchas. +- **Deployment procedures**: Steps to deploy, verify, rollback. Infra-specific knowledge. +- **API contracts**: Request/response formats, auth requirements, rate limits. +- **Design patterns**: Patterns that worked well in this codebase, anti-patterns to avoid. +- **Failure signatures**: Specific error outputs mapped to known fixes. + +### DO NOT Memorize +- Temporary debugging output or scratch work. +- Information already in official docs (link to docs instead). +- Personal preferences or style opinions (put in CLAUDE.md instead). +- Secrets, tokens, passwords, API keys — NEVER. +- Exact code implementations (too large, changes too often — store the pattern, not the code). +- Trivial facts that any LLM already knows (e.g., "Rust uses cargo for builds"). + +### Quality Bar +Ask: "If I hit this problem again in 3 months, would this knowledge save me 30+ minutes?" If yes, memorize it. If not, skip it. + +## CLI Usage — `mem learn` + +### Ingest Markdown Knowledge +```bash +# Ingest a single file +mem learn knowledge/rust-fundamentals.md + +# Ingest entire directory +mem learn knowledge/ + +# Preview chunks without writing (dry run) +mem learn knowledge/ --dry-run + +# Custom project namespace +mem learn docs/ --project my-project + +# Smaller chunks for fine-grained retrieval +mem learn knowledge/ --chunk-size 500 +``` + +### How Chunking Works +- Splits on `## ` headings — each section becomes a chunk. +- If a section exceeds `--chunk-size`, it hard-splits at the boundary. +- Each chunk gets a SHA256 hash for deduplication. +- Chunks are stored as JSONL events in `log//learn/latest.jsonl`. + +### Writing Good Knowledge Files +```markdown +# Topic Title + +## Subtopic A +- Concise bullet points with actionable information. +- Include the WHY, not just the WHAT. +- Code examples with context, not bare snippets. + +## Subtopic B +- Each ## section becomes one retrievable chunk. +- Keep sections focused — one concept per section. +- 200-500 chars per section is ideal for retrieval precision. +``` + +## CLI Usage — Other Commands + +### Capture Failures (Auto-Learn from Errors) +```bash +# Record a command execution and its output +mem capture --cmd "cargo build" --exit 1 --output-file /tmp/build-error.log + +# Extract failure signature +mem sig --tool cargo --file /tmp/build-error.log +``` + +### Resolve Lessons (Pair Failures with Fixes) +```bash +# After fixing a bug, derive the lesson +mem resolve --json +``` + +### Lookup Known Fixes +```bash +# Search memory for known fix +echo "error[E0308]: mismatched types" | mem lookup --tool cargo + +# With minimum confidence threshold +mem lookup --tool kubectl --file /tmp/error.log --floor 0.7 +``` + +### Materialize Skills +```bash +# Generate SKILL.md files from memory +mem materialize +``` + +## HTTP API — Programmatic Access + +### Health Check +```bash +curl -s http://localhost:8080/health | jq . +# {"status": "ok", "timestamp": "..."} +``` + +### Ingest Records +```bash +curl -s -X POST http://localhost:8080/memory/ingest \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "project": "poimen", + "source": "transcript://session-123", + "kind": "L1", + "text": "Port 8080 conflict fixed by changing to 8081 in deployment.yaml", + "metadata": {"topic": "kubernetes", "session_id": "sess-123"} + }' | jq . +# Returns: 201 with chunk ID and SHA256 +``` + +### Query Memory (Hybrid Search) +```bash +curl -s -X POST http://localhost:8080/memory/query \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "project": "poimen", + "query": "how to fix kubernetes port conflict", + "limit": 5, + "floor": 0.6 + }' | jq '.results[] | {score, text}' +# Returns: ranked results with semantic + lexical scores +``` + +### Context Lookup (Three-Tier) +```bash +curl -s -X POST http://localhost:8080/memory/context \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "project": "poimen", + "tool": "cargo", + "task": "build", + "budget": 4096 + }' | jq . +# Returns: tier-1 signature matches, tier-2 search results, tier-3 reference docs +``` + +### Browse Vault +```bash +curl -s "http://localhost:8080/memory/vault?project=poimen" \ + -H "Authorization: Bearer $TOKEN" | jq '.files[] | .path' +``` + +### Rebuild from Log +```bash +curl -s -X POST http://localhost:8080/memory/rebuild \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"project": "poimen", "dry_run": true, "verify_parity": true}' | jq . +``` + +## Authentication +```bash +# Get JWT token from Authentik +TOKEN=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \ + -d "grant_type=client_credentials" \ + -d "client_id=$CLIENT_ID" \ + -d "client_secret=$CLIENT_SECRET" | jq -r .access_token) + +# Use token +curl -s -H "Authorization: Bearer $TOKEN" http://localhost:8080/memory/query ... +``` + +## Rate Limits +- Ingest: 100 requests/hour +- Query: 1000 requests/hour +- Context: 100 requests/hour +- Idempotency: 24-hour TTL (same ingest key → 409 Conflict) + +## Agent Workflow — When to Query vs Ingest + +### Before Starting a Task +``` +1. Extract key terms from the task description +2. Query memory: mem lookup --tool or POST /memory/query +3. If results found with score > 0.7 → apply known solution +4. If no results → proceed with normal problem-solving +``` + +### After Completing a Task +``` +1. Did I learn something new? (bug fix, config pattern, deployment step) +2. Would this save 30+ minutes if encountered again? +3. If yes → write a markdown note → mem learn note.md +4. If it was a failure→fix pair → mem capture + mem resolve +``` + +### During Debugging +``` +1. Hit an error → mem sig --tool --file error.log +2. Check if signature matches known fix → mem lookup +3. If known → apply fix +4. If unknown → debug normally, then memorize the solution +``` + +## Knowledge Levels +- **L0**: Raw evidence (transcripts, logs). Automatically extracted. +- **L1**: Per-query memory (specific facts, fixes, patterns). Most useful for agents. +- **L2**: Project-level synthesis (cross-cutting themes, architecture summaries). +- **R**: Reference documents (Obsidian vault, external docs). Read-only fallback. + +## Infrastructure +- **Cluster**: Kubernetes (poimen namespace) +- **Database**: CNPG pgvector (2 replicas) +- **Search**: OpenSearch (2-node HA) +- **Auth**: Authentik (JWT/OIDC) +- **Deployment**: ArgoCD from `forgejo.riotpiao.com/rock/poimen-memory` +- **Obsidian**: REST API at `obsidian-server.poimen.svc` (reference corpus) + +## Gotchas +- Event log is immutable — you can't delete knowledge, only supersede it with newer facts. +- Rebuild from log is deterministic — same log → same vector store state. +- OpenSearch is fail-soft — if it's down, semantic search still works via pgvector alone. +- Embedding model is `sentence-transformers/all-MiniLM-L6-v2` (768-dim). Don't mix with other embedding models. +- `mem learn` appends to `latest.jsonl` — running it twice on the same files creates duplicate chunks (use SHA256 to detect). diff --git a/log/knowledge/learn/latest.jsonl b/log/knowledge/learn/latest.jsonl deleted file mode 100644 index 41a8917..0000000 --- a/log/knowledge/learn/latest.jsonl +++ /dev/null @@ -1,96 +0,0 @@ -{"project":"knowledge","query":"andrej-karpathy:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"96fc747c0bcda29f58e27bd541fc24402bca94fcf59038cd9822e66b2e0ac3ce","source":"knowledge/andrej-karpathy.md","text":"# Andrej Karpathy — Key Insights & Practices","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"bc2e7975fd968bb3323a4109d0236bf8c41e742abaf3bfebbb96cc325c77ec19","source":"knowledge/andrej-karpathy.md","text":"## Software 2.0\n- Traditional software (1.0): explicit rules written by programmers.\n- Software 2.0: behavior learned from data via neural networks. Code = weights.\n- Implication: datasets are the new source code. Data curation > clever algorithms.\n- Debug by inspecting data, not stepping through logic.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"b5acf1cd6a76fbeaf16d760d18e2b57113a037a94f6ef2a53d029b4d06699d3b","source":"knowledge/andrej-karpathy.md","text":"## Training Neural Networks — A Recipe\n1. **Become one with the data** — visualize, understand distributions, find patterns and anomalies before writing any model code.\n2. **Set up end-to-end training/eval skeleton** — simplest possible model first. Get the pipeline working.\n3. **Overfit first** — if model can't memorize a single batch, architecture is wrong.\n4. **Regularize** — only add dropout, weight decay, augmentation after overfitting confirmed.\n5. **Tune** — learning rate is the most important hyperparameter. Use LR finder.\n6. **Squeeze** — ensembles, larger models, more data. Diminishing returns here.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"8768a071d887b0c38a0a822edf92be2604e029a7ed97634390ce288597a669ee","source":"knowledge/andrej-karpathy.md","text":"## Most Common Neural Net Mistakes\n- Not looking at data first.\n- Forgetting to set model to eval mode (BatchNorm, Dropout change behavior).\n- Forgetting to zero gradients.\n- Using softmax with cross-entropy (use logits directly).\n- Not normalizing inputs.\n- Applying augmentation to validation set.\n- Silent shape broadcasting bugs — always assert tensor shapes.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"f8b49eeeb82c80afd8b3e24904abe4a1505fe4af4a3708f5795c307f3aba2396","source":"knowledge/andrej-karpathy.md","text":"## LLM Insights (Post-GPT Era)\n- LLMs are \"operating systems\" — CPU is the transformer, context window is RAM, training data is disk.\n- Tokenization is a key bottleneck — BPE artifacts cause many failure modes.\n- Temperature controls creativity vs precision. T=0 for factual, T>0 for creative.\n- Chain-of-thought works because it gives the model \"working memory\" in the output tokens.\n- Prompt engineering is programming in natural language. Be explicit, give examples.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"5bdeb96ce1d96a7a340e955e756044e43a6f65f706dfb27549d5305b7a0edbaa","source":"knowledge/andrej-karpathy.md","text":"## Build Nanograd / Micrograd Philosophy\n- Understand backpropagation by implementing it from scratch.\n- A neural net is just: forward pass → compute loss → backward pass → update weights.\n- Autograd: track operations, build computation graph, reverse-mode differentiation.\n- Every complex framework (PyTorch, JAX) is built on these same primitives.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"965a6782e73ba162337b00c2f37d6eecd7ada8ec43a91c39b24c2369e165304b","source":"knowledge/andrej-karpathy.md","text":"## Practical ML Engineering\n- Start simple: logistic regression baseline before deep learning.\n- Measure everything: loss curves, gradient norms, weight distributions.\n- Reproducibility: fix seeds, log hyperparameters, version datasets.\n- Don't trust your code — trust your loss curve. If loss isn't going down, something is wrong.\n- Data quality > model complexity. 10x data often beats 10x model size.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"296e684f1b57af077482c6cf83697f36f01b22cf3d32555bde95a001760c5ee4","source":"knowledge/andrej-karpathy.md","text":"## Scaling Laws\n- Performance scales predictably with compute, data, and parameters (Chinchilla scaling).\n- Compute-optimal training: balance model size and training tokens.\n- Emergent abilities appear at scale — capabilities that don't exist in smaller models.","total_chunks":9}} -{"project":"knowledge","query":"andrej-karpathy:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"aa6a971d54fafc4ba53aaaf1ba36afe8caf8454ab39698bdd9a20136111335a4","source":"knowledge/andrej-karpathy.md","text":"## On AI Engineering\n- The best AI engineers understand both ML and systems engineering.\n- Inference optimization matters as much as training — quantization, batching, KV-cache.\n- Eval is everything. If you can't measure it, you can't improve it.\n- Build evaluation suites before building features.","total_chunks":9}} -{"project":"knowledge","query":"ast-grep:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"938aaf1d005021f300ecda8246052070b35257dc205e1def9e18dd3917c92206","source":"knowledge/ast-grep.md","text":"# AST-Grep (sg) — Structural Code Search & Transform","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"926ae186e84b30d159dfc3104253b9e204acaf17740833b3a918982931c3ca16","source":"knowledge/ast-grep.md","text":"## Core Concept\n- AST-grep searches/transforms code using Abstract Syntax Tree patterns, not regex.\n- Pattern matches structural meaning, ignoring whitespace, comments, formatting.\n- Works across: Rust, Go, Python, JS/TS, Java, C, C++, Ruby, Kotlin, Lua, CSS, HTML.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"3932d5e5acd8bfaf85b0e92fa587b1231f1e9a28b3a58027d7f170daae97bfc9","source":"knowledge/ast-grep.md","text":"## CLI Usage\n- `sg --pattern 'unwrap()' -l rust` — find all `.unwrap()` calls in Rust files.\n- `sg --pattern 'println!($$$ARGS)' -l rust` — find all println macros with any args.\n- `sg --pattern '$A.unwrap()' --rewrite '$A.expect(\"TODO\")' -l rust` — rewrite unwrap to expect.\n- `sg scan` — run lint rules from `sgconfig.yml`.\n- `sg test` — test rules against fixtures.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"e3524a773966ec62343402bd8aa0349610889f725fd193e84e9cb9cd90a796a7","source":"knowledge/ast-grep.md","text":"## Pattern Syntax\n- `$VAR` matches single AST node (identifier, expression, etc).\n- `$$$VARS` matches zero or more nodes (variadic).\n- `$$VAR` matches zero or one node (optional).\n- Literal code matches itself: `if true { $$$BODY }` matches any `if true` block.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"405b19c2dcb3b59a88faeef7d8ab1eb47d6bf5f4e7c090de9b496b24ee9bb93b","source":"knowledge/ast-grep.md","text":"## Meta Variables\n- `$A` in pattern captures node, available in `--rewrite` as `$A`.\n- Named captures: same name must match same content. `$A == $A` matches `x == x` but not `x == y`.\n- `$_` is anonymous — matches anything without capturing.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"2c7d1bb19a23f90fbdb48206839d9c4c13dfe742df28ed4de6cfcaf65cd6414b","source":"knowledge/ast-grep.md","text":"## Rule YAML Format\n```yaml\nid: no-unwrap\nlanguage: rust\nrule:\n pattern: $A.unwrap()\n not:\n inside:\n kind: test_function\nfix: $A.expect(\"handle error\")\nmessage: \"Use .expect() instead of .unwrap() in production code\"\nseverity: warning\n```","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"7d9376b357ff5d81343cbef5adbd6c3946bf381c673c8c052308c0b1a0d52790","source":"knowledge/ast-grep.md","text":"## Composite Rules\n- `all: [rule1, rule2]` — both must match.\n- `any: [rule1, rule2]` — either matches.\n- `not: rule` — negation.\n- `matches: rule-id` — reference another rule.\n- `inside: { kind: function_item }` — must be inside a function.\n- `has: { pattern: $EXPR }` — must contain sub-pattern.\n- `follows: { pattern: ... }` — must follow another pattern.\n- `precedes: { pattern: ... }` — must precede another pattern.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"149ef392299445e1d63e719c0650746b6ea6cfd4f4267414ccf7ca01da2bdf9c","source":"knowledge/ast-grep.md","text":"## Kind Selectors\n- `kind: function_item` — match AST node type directly.\n- `kind: call_expression` — match function calls.\n- Use `sg --debug-query='println!(\"hello\")'` to see AST node kinds.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"949924afcbaa21f05d1de58c291c6c05335b647b6ac58a2ae46ba1595400968c","source":"knowledge/ast-grep.md","text":"## Configuration (sgconfig.yml)\n```yaml\nruleDirs:\n - rules/\ntestConfigs:\n - rules/tests/\n```","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"9931dea657038e2bcf246db03cec83142d0c86acc2a4c1eaf7e0f19c87e7bdf8","source":"knowledge/ast-grep.md","text":"## Advanced Patterns\n- Find unused variables: `let $VAR = $EXPR;` where `$VAR` not referenced later.\n- Find API migrations: `old_function($$$ARGS)` → `new_function($$$ARGS)`.\n- Enforce patterns: ensure all error handling uses `?` not `.unwrap()`.\n- Security: find `eval($EXPR)`, SQL injection patterns, hardcoded secrets.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:10","run":"latest","turn":10,"event_type":"learn","data":{"chunk_index":10,"level":"L1","sha256":"8fa60c107e2ae5f2a943e2b9106e185cd9641b6ad783fbcfe21a874bc5eb4bd4","source":"knowledge/ast-grep.md","text":"## Integration\n- CI/CD: `sg scan --json` for machine-readable output.\n- Pre-commit hooks: `sg scan --rule rules/` on staged files.\n- Editor: VSCode extension, LSP support.\n- Programmatic: `@ast-grep/napi` Node.js binding for custom tools.","total_chunks":12}} -{"project":"knowledge","query":"ast-grep:11","run":"latest","turn":11,"event_type":"learn","data":{"chunk_index":11,"level":"L1","sha256":"736f7a348fc3d929d781b5fbc13b687de0bfe3b506364f033df4637bb2a8bd28","source":"knowledge/ast-grep.md","text":"## vs Regex\n- Regex: `unwrap\\(\\)` matches in comments, strings, docs. AST-grep: only actual code.\n- Regex can't match nested structures. AST-grep handles `if { if { unwrap() } }`.\n- AST-grep understands scope, types, structure. Regex is text-level.","total_chunks":12}} -{"project":"knowledge","query":"caveman-communication:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"69035d3903b4172138b5fddfb4cee9270f8bb1fc5019aa8b812dc279b170abe8","source":"knowledge/caveman-communication.md","text":"# Caveman Communication — Ultra-Compressed Output","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"a7996bc7fca36cf342cdc29c4c3fb7914e951ea1041f2b94be69ed085d96021a","source":"knowledge/caveman-communication.md","text":"## Core Principle\n- Cut output tokens 60-70% while keeping 100% technical accuracy.\n- Speak like caveman: drop articles, filler, hedging. Keep nouns, verbs, code.\n- Every word must carry information. If removing a word doesn't lose meaning, remove it.","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"b41c9d147a56faac608dadd281e3f8f83f98de679991cad1625f66f305d60f70","source":"knowledge/caveman-communication.md","text":"## Rules\n- No articles (a, an, the) unless ambiguous without them.\n- No filler phrases: \"I think\", \"it seems like\", \"you might want to\", \"basically\".\n- No hedging: \"perhaps\", \"maybe\", \"it's possible that\".\n- No repetition of what user said. They know what they asked.\n- No \"Let me\" or \"I'll\" — just do it.\n- Code > prose. Show the fix, not a paragraph about the fix.\n- Use symbols: → (leads to), ✓ (done/correct), ✗ (wrong), = (equals/means).","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"28b85d8ab18ba4c526c3f7fdf67b350b772858b9c0b62f454b1d93bb16fc6ce4","source":"knowledge/caveman-communication.md","text":"## Intensity Levels\n- **Lite**: Drop filler, keep sentence structure. \"Fix the port conflict by changing 8080 to 8081.\"\n- **Full**: Drop articles, compress sentences. \"Fix port conflict — change 8080→8081.\"\n- **Ultra**: Telegraphic. Maximum compression. \"port 8080→8081. done.\"","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"28444219d8111bbb9fae3391dc30f2f9f96be99c39b92e81e39036f5c238f0e9","source":"knowledge/caveman-communication.md","text":"## Examples\nBad (verbose):\n\"I think the issue you're experiencing is that the Kubernetes service is trying to bind to port 8080, which is already in use by another process. You might want to try changing the port to 8081 in your deployment manifest.\"\n\nGood (caveman):\n\"Port 8080 conflict. Change in deployment.yaml: containerPort: 8081. Restart pod.\"","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"b39e55cff4b18a8a1ade0156d2d56c223e44d0e4e0bee64dd39b9b75da25cd78","source":"knowledge/caveman-communication.md","text":"## When NOT to Caveman\n- User explicitly asks for detailed explanation.\n- Teaching a new concept — clarity > brevity.\n- Error messages and warnings — be precise.\n- Documentation writing — full prose expected.","total_chunks":7}} -{"project":"knowledge","query":"caveman-communication:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"2642fc49d0e044e16ff37e990a8a1a1b6d9c96cf937cec715278a61dc10d852f","source":"knowledge/caveman-communication.md","text":"## Token Savings\n- Average English: 1.3 tokens per word. 100 words ≈ 130 tokens.\n- Caveman same content: ~35 words ≈ 46 tokens. 65% reduction.\n- Over a session: saves thousands of output tokens → more context for actual work.","total_chunks":7}} -{"project":"knowledge","query":"golang-skills:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"307bfa816f5657b69cfcd94a2582713263adc3a5c32532f6d47fd3be3c535e16","source":"knowledge/golang-skills.md","text":"# Go (Golang) Skills","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"a1fe77aaa004f914a14b51e920ad0eb0992e6ea3b5f1e3b912389f6a6821f3f0","source":"knowledge/golang-skills.md","text":"## Core Idioms\n- Accept interfaces, return structs.\n- Errors are values — check them explicitly. `if err != nil { return err }`.\n- Don't panic in library code. Reserve panic for truly unrecoverable situations.\n- Zero values are useful — `var m map[string]int` is nil but `var s []int` is usable.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"e5476fdaa6443f6a5d61dcfd780a2468c933ea93240f7413f02ceccac5a38647","source":"knowledge/golang-skills.md","text":"## Error Handling\n- Wrap errors with context: `fmt.Errorf(\"failed to open %s: %w\", path, err)`.\n- Sentinel errors: `var ErrNotFound = errors.New(\"not found\")`. Check with `errors.Is(err, ErrNotFound)`.\n- Custom error types: `type ValidationError struct { Field, Message string }`. Check with `errors.As()`.\n- Never ignore errors: `_ = doSomething()` is a code smell. At minimum, log it.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"d31c0300bb450c928105c5393a8267f4b5995559dd1355c4c0ec0c88c7e4850c","source":"knowledge/golang-skills.md","text":"## Concurrency\n- \"Don't communicate by sharing memory; share memory by communicating.\" — use channels.\n- `go func()` launches goroutine. Always ensure goroutines terminate (context, done channel).\n- `sync.WaitGroup` to wait for goroutine completion.\n- `sync.Mutex` when channels are overkill (protecting a counter, map).\n- `context.Context` for cancellation, timeouts, and request-scoped values. Always first parameter.\n- `errgroup.Group` for parallel tasks with error propagation.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"4b9d554419821786d90683057cc49429c9dbb4e4bda9e2637caa9829bf32710d","source":"knowledge/golang-skills.md","text":"## Channel Patterns\n- `ch := make(chan T)` unbuffered (synchronous). `make(chan T, n)` buffered.\n- Fan-out: multiple goroutines read from one channel.\n- Fan-in: multiple channels merged into one via select.\n- Pipeline: chain of stages connected by channels.\n- `select` with `case <-ctx.Done():` for cancellation.\n- Close channels from sender side only. Never close from receiver.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"5bc4131050a7e4a04dfcb611651d0549d7fd0b6897c732434fc4fb52992b5ba1","source":"knowledge/golang-skills.md","text":"## Interfaces\n- Interfaces are satisfied implicitly — no `implements` keyword.\n- Keep interfaces small: `io.Reader` has one method. `io.ReadWriteCloser` composes three.\n- Define interfaces where they're used, not where they're implemented.\n- `interface{}` (or `any`) is a code smell — prefer generics or specific interfaces.\n- Type assertions: `v, ok := i.(ConcreteType)`. Type switch: `switch v := i.(type) { ... }`.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"51d99bf523bdf2d615cc9081f81ca5699268b0658ea4c8143406df14c32f1bb6","source":"knowledge/golang-skills.md","text":"## Generics (Go 1.18+)\n- `func Map[T, U any](s []T, f func(T) U) []U` — generic function.\n- Constraints: `comparable`, `~int | ~float64`, custom interface constraints.\n- Use generics for data structures and utility functions, not business logic.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"21bf4ec3471d9712b9d62cc8cee1fa27261c6ddfa74d65c0e65f58ee400b8c4c","source":"knowledge/golang-skills.md","text":"## Project Structure\n```\ncmd/\n myapp/main.go # entrypoint\ninternal/ # private packages\n domain/ # business logic, no external deps\n repository/ # data access\n handler/ # HTTP handlers\npkg/ # public library code\n```\n- `internal/` enforced by Go compiler — cannot be imported outside module.\n- One package per directory. Package name = directory name.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"c1210d8d99a94ce28e581ff93beedbcf1fd6a236b3a314bc73b0e9784bbd05ef","source":"knowledge/golang-skills.md","text":"## Testing\n- `func TestFoo(t *testing.T)` — test functions.\n- Table-driven tests: `tests := []struct{ name string; input int; want int }{ ... }`.\n- `t.Run(name, func(t *testing.T) { ... })` for subtests.\n- `t.Parallel()` for concurrent test execution.\n- `testify/assert` for cleaner assertions. `testify/mock` for mocking.\n- `httptest.NewServer()` for HTTP integration tests.\n- Benchmarks: `func BenchmarkFoo(b *testing.B) { for i := 0; i < b.N; i++ { ... } }`.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"925cdfafae3fcfa87c5a792565582a8f6170e6cdd3a35f3b5861f4c8a09895b9","source":"knowledge/golang-skills.md","text":"## HTTP Server\n- `http.HandlerFunc` wraps functions as handlers.\n- Middleware pattern: `func Logging(next http.Handler) http.Handler`.\n- Use `chi` or `echo` for routing. Stdlib `http.ServeMux` improved in Go 1.22.\n- Always set timeouts: `srv := &http.Server{ReadTimeout: 5*time.Second, WriteTimeout: 10*time.Second}`.\n- Graceful shutdown: `signal.Notify` + `srv.Shutdown(ctx)`.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:10","run":"latest","turn":10,"event_type":"learn","data":{"chunk_index":10,"level":"L1","sha256":"19d6e7a82d2c7bb00f492fcb68f51f961467ec78227ba42a9ee0ec4844d7cb2d","source":"knowledge/golang-skills.md","text":"## Performance\n- `pprof` for CPU/memory profiling: `go tool pprof http://localhost:6060/debug/pprof/profile`.\n- `sync.Pool` for reducing GC pressure on frequently allocated objects.\n- Pre-allocate slices: `make([]T, 0, expectedLen)`.\n- String building: `strings.Builder` not `+` concatenation.\n- Avoid interface boxing in hot paths.","total_chunks":12}} -{"project":"knowledge","query":"golang-skills:11","run":"latest","turn":11,"event_type":"learn","data":{"chunk_index":11,"level":"L1","sha256":"5a872b66f1a89f540f08ece84d9c040d68e3236ffd71cb981cc5d371b991d20f","source":"knowledge/golang-skills.md","text":"## Common Gotchas\n- Loop variable capture in goroutines (fixed in Go 1.22, but still common in older code).\n- Nil interface vs nil pointer: `var p *MyType = nil; var i MyInterface = p; i != nil` is TRUE.\n- Maps are not safe for concurrent access — use `sync.Map` or `sync.RWMutex`.\n- Slice append may or may not create a new backing array — never hold stale slice references.\n- `defer` evaluates arguments immediately, runs function at return.\n- `init()` runs before `main()` — avoid side effects, prefer explicit initialization.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"938a5b95800adb2f3bddc13bc3b793170e678c308e4d4661dfc994fb38692659","source":"knowledge/rust-fundamentals.md","text":"# Rust Fundamentals","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"40f33225751c58fd80b9960c6daf90cf2bfb005f03f2db4b1761ec739a7e44cb","source":"knowledge/rust-fundamentals.md","text":"## Ownership & Borrowing\n- Every value has exactly one owner. When owner goes out of scope, value is dropped.\n- `&T` immutable borrow, `&mut T` mutable borrow. Cannot have `&mut` while `&` exists.\n- Move semantics by default for non-Copy types. Clone for explicit deep copy.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"4c3838041a71cf0d3b9a3203eb1733c698bbc87d6dd201c159344f56a231b07c","source":"knowledge/rust-fundamentals.md","text":"## Lifetimes\n- `'a` annotations tell compiler how long references live.\n- Elision rules: single input lifetime → applied to all outputs. `&self` → output gets `'self` lifetime.\n- `'static` means reference lives for entire program. String literals are `&'static str`.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"6f51e7f86ccef640864780050e660757cc868d20faad5642fa4f1e6e52fcbd54","source":"knowledge/rust-fundamentals.md","text":"## Error Handling\n- `Result` for recoverable errors, `panic!` for unrecoverable.\n- `?` operator propagates errors. Use `anyhow::Result` for application code, `thiserror` for library errors.\n- Never use `.unwrap()` in production — use `.expect(\"reason\")` or proper error handling.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"7f2ecd5e8344f9aacc2310acd018f2367bd8b3b5b165484e43e08088eb621a81","source":"knowledge/rust-fundamentals.md","text":"## Traits & Generics\n- Traits define shared behavior: `trait Summary { fn summarize(&self) -> String; }`\n- Trait bounds: `fn notify(item: &impl Summary)` or `fn notify(item: &T)`\n- `dyn Trait` for trait objects (dynamic dispatch), `impl Trait` for static dispatch.\n- Blanket implementations: `impl ToString for T`","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"e64dd65ddf651cb176b7507bfd92d97ecc4263e02574741ef62e813cfff3551d","source":"knowledge/rust-fundamentals.md","text":"## Smart Pointers\n- `Box` heap allocation with single ownership.\n- `Rc` reference-counted shared ownership (single-threaded).\n- `Arc` atomic reference-counted (thread-safe). Use with `Mutex` or `RwLock`.\n- `Cow<'a, T>` clone-on-write — borrows when possible, clones when mutation needed.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"3f69f64923035ec5cd075ec2e15b3ed2f035949ad052285da103ef6a90d180a0","source":"knowledge/rust-fundamentals.md","text":"## Concurrency\n- `Send` — type can be transferred across threads. `Sync` — type can be shared between threads.\n- `tokio::spawn` for async tasks. `rayon` for data parallelism.\n- Channels: `mpsc::channel()` for multi-producer single-consumer. `crossbeam` for advanced patterns.\n- `async/await` — futures are lazy, must be `.await`ed or spawned.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"b8f8fc26834fe910d6434a529cddd11339ade3dc2ae32dd378c2875b7d72e149","source":"knowledge/rust-fundamentals.md","text":"## Pattern Matching\n- `match` is exhaustive — must cover all variants.\n- `if let Some(x) = option` for single-pattern matching.\n- Destructuring: `let (a, b) = tuple;` and `let Point { x, y } = point;`\n- Guards: `match x { n if n > 0 => ..., _ => ... }`","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"bccdf62a0b7c9be157961e5c36f2d962fc53ec24009e0686ecae0691aa933f3c","source":"knowledge/rust-fundamentals.md","text":"## Module System\n- `mod foo;` loads from `foo.rs` or `foo/mod.rs`.\n- `pub(crate)` visible within crate only. `pub(super)` visible to parent module.\n- `use crate::module::Type` absolute path. `use super::Type` relative path.\n- Re-exports: `pub use inner::Type;` to flatten module hierarchy.","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"746c2db8686ce1f93e7ccd39e7617ee9db7708f0cf2b8dcf5f93cd61a1d2fc87","source":"knowledge/rust-fundamentals.md","text":"## Iterators\n- `.iter()` borrows, `.into_iter()` consumes, `.iter_mut()` mutable borrow.\n- Lazy — nothing happens until consumed (`.collect()`, `.for_each()`, `.count()`).\n- Chaining: `.filter().map().flat_map().take().collect::>()`\n- `impl Iterator for MyType { type Item = T; fn next(&mut self) -> Option }`","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:10","run":"latest","turn":10,"event_type":"learn","data":{"chunk_index":10,"level":"L1","sha256":"4981e3e218a0c6f17546dc9b7385351115066ae01cde02588feb82a567085495","source":"knowledge/rust-fundamentals.md","text":"## Macros\n- `macro_rules!` for declarative macros. `#[derive(...)]` for derive macros.\n- `proc_macro` for procedural macros (attribute, derive, function-like).\n- `vec![1, 2, 3]` expands to `{ let mut v = Vec::new(); v.push(1); ... v }`","total_chunks":12}} -{"project":"knowledge","query":"rust-fundamentals:11","run":"latest","turn":11,"event_type":"learn","data":{"chunk_index":11,"level":"L1","sha256":"4c137d96c638eaa8922a353b4ff70e20c4edcad6a3995b5524bdd04ec6439586","source":"knowledge/rust-fundamentals.md","text":"## Common Patterns\n- Builder pattern: `MyStruct::new().with_field(val).build()`\n- Newtype pattern: `struct UserId(u64);` for type safety without runtime cost.\n- Type state pattern: use generics to encode state in the type system.\n- Interior mutability: `Cell`, `RefCell` for single-threaded, `Mutex` for multi-threaded.","total_chunks":12}} -{"project":"knowledge","query":"solid-dry-principles:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"7721d79c72c3b3c2dfa2b0acf902bfe1a2da7d622f9472c15a3e08fc51c76cb3","source":"knowledge/solid-dry-principles.md","text":"# SOLID & DRY Design Principles","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"3fd2840a348033491348006884d34089184f072f6b71a43afc0331ba84184d77","source":"knowledge/solid-dry-principles.md","text":"## Single Responsibility Principle (SRP)\n- A class/module should have one and only one reason to change.\n- Each module owns exactly one actor's requirements.\n- Bad: `UserService` that handles auth, email, and database. Good: separate `AuthService`, `EmailService`, `UserRepository`.\n- In Rust: one struct per concern. `ChunkProcessor` doesn't also handle HTTP routing.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"927592c6c68dbd42ab38b31d39c7b81f4160a180c24b8d71b2d1e6197168c1eb","source":"knowledge/solid-dry-principles.md","text":"## Open/Closed Principle (OCP)\n- Software entities should be open for extension, closed for modification.\n- Use traits/interfaces to allow new behavior without changing existing code.\n- Strategy pattern: `trait Scorer { fn score(&self, doc: &Doc) -> f64; }` — add new scorers without modifying search.\n- In Rust: trait objects or generics. `fn process(s: &S)` — new strategies don't touch `process`.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"1c6d960c61ff7a8b5b6dd7b26c70ac88884da0bcc3cc568fb05b2c74687cec0e","source":"knowledge/solid-dry-principles.md","text":"## Liskov Substitution Principle (LSP)\n- Subtypes must be substitutable for their base types without breaking correctness.\n- If `fn accept(animal: &dyn Animal)` works with `Dog`, it must work with `Cat` too.\n- Violated when: subtype throws unexpected errors, ignores base contract, strengthens preconditions.\n- In Rust: trait implementations must honor the trait's documented contract.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"972c4a715547bc6028099526d21dd2c30c2942c8ccc1910a0dfd58af0c40125e","source":"knowledge/solid-dry-principles.md","text":"## Interface Segregation Principle (ISP)\n- Clients should not be forced to depend on interfaces they don't use.\n- Many small traits > one fat trait.\n- Bad: `trait Repository { fn read(); fn write(); fn delete(); fn audit(); }` — read-only clients forced to see write methods.\n- Good: `trait Readable`, `trait Writable`, `trait Auditable` — compose as needed.\n- In Rust: supertraits for composition: `trait FullRepo: Readable + Writable + Auditable {}`","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"e52391adbdeceb264716cda68857f5bee8563183cc9692681d215305e6454ea5","source":"knowledge/solid-dry-principles.md","text":"## Dependency Inversion Principle (DIP)\n- High-level modules should not depend on low-level modules. Both should depend on abstractions.\n- In Rust: accept `impl Trait` or `&dyn Trait`, not concrete types.\n- `fn search(store: &dyn VectorStore)` — works with Postgres, OpenSearch, or in-memory mock.\n- Constructor injection: `struct SearchEngine { store: Box }`","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"edcdeee3a6c405e2adad092d78b8f5f6f7beed778649c36780bc677418a21e7c","source":"knowledge/solid-dry-principles.md","text":"## DRY (Don't Repeat Yourself)\n- Every piece of knowledge should have a single, unambiguous, authoritative representation.\n- DRY is about knowledge, not code. Two functions with same code but different reasons to change are NOT duplication.\n- Extract when: same logic appears 3+ times AND changes for the same reason.\n- Wrong DRY: coupling unrelated code just because it looks similar. Right DRY: shared business rules in one place.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"774b41e79a231bcd65412b85d255252736f202e3c85c35167ceac005499e74d0","source":"knowledge/solid-dry-principles.md","text":"## WET (Write Everything Twice) — When DRY Goes Wrong\n- Premature DRY creates coupling worse than duplication.\n- Rule of three: duplicate is fine, triplicate means extract.\n- Tests should be WET — readability > DRYness in test code.\n- Configuration can be WET — explicit is better than magic shared config.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"dc611d933234e9bf4cd2f7f1b077237564c21305ecaa4b9bf1e772b8830e1936","source":"knowledge/solid-dry-principles.md","text":"## KISS (Keep It Simple, Stupid)\n- Simplest solution that works is usually the best.\n- Avoid: premature abstraction, speculative generality, framework-itis.\n- Measure complexity: if a new team member can't understand it in 15 minutes, simplify.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"1fca71e8e16d245085dbe72d12ad950fa10fc431ae018d0fc196dec11cbfa195","source":"knowledge/solid-dry-principles.md","text":"## YAGNI (You Aren't Gonna Need It)\n- Don't build features until you actually need them.\n- Speculative code rots — it's untested, unmaintained, and misleading.\n- Exception: known architectural boundaries (API versioning, database migrations).","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:10","run":"latest","turn":10,"event_type":"learn","data":{"chunk_index":10,"level":"L1","sha256":"e5fb640cbedc9f409f3fd73978f0578f2af62c89df104ec3054d1f19579179f7","source":"knowledge/solid-dry-principles.md","text":"## Composition Over Inheritance\n- Prefer composing objects over class hierarchies.\n- In Rust: no inheritance. Composition is the default via struct fields + trait delegation.\n- `struct HttpServer { router: Router, auth: AuthMiddleware, rate_limiter: RateLimiter }`","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:11","run":"latest","turn":11,"event_type":"learn","data":{"chunk_index":11,"level":"L1","sha256":"0606d00ead93bf2f15c0e75447791e02ada10a1eef34e8ea5de20f7d705642e8","source":"knowledge/solid-dry-principles.md","text":"## Law of Demeter\n- Only talk to your immediate friends. Don't chain: `a.b().c().d()`.\n- Tell, don't ask: `order.ship()` not `order.get_warehouse().get_shipping().create_label()`.\n- In Rust: expose methods that encapsulate internal structure.","total_chunks":13}} -{"project":"knowledge","query":"solid-dry-principles:12","run":"latest","turn":12,"event_type":"learn","data":{"chunk_index":12,"level":"L1","sha256":"4eaad9bdaf561bc037d4bf306d8838bc7341f117ef3cd28cd9be5571d4262548","source":"knowledge/solid-dry-principles.md","text":"## Practical Application\n- Start concrete, extract abstractions when patterns emerge.\n- Refactor in small steps with tests as safety net.\n- Code review checklist: SRP violated? Unnecessary coupling? Duplicated knowledge? Over-engineered?","total_chunks":13}} -{"project":"knowledge","query":"curl-api-testing:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"9ecbcde1289b24e760f605561c56abebcfef2ca9f4b3c6416eff7cc1934b0f78","source":"knowledge/curl-api-testing.md","text":"# curl — API Testing & Verification","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"8591f6f335f1533290505b772f7867d7ee2ad5d0089f665947d5730c3669d39b","source":"knowledge/curl-api-testing.md","text":"## Why Test With curl\n- Every API endpoint must be testable with curl before marking a task done.\n- curl is the ground truth — if curl can't hit it, the API doesn't work.\n- Test locally first (`localhost`), then staging, then production.\n- Save working curl commands as documentation for future reference.","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"45ad8d80df104d0a775eb6463796a833550047103ccae29b1a5a2077241fc351","source":"knowledge/curl-api-testing.md","text":"## Basic Patterns\n```bash\n# GET request\ncurl -s http://localhost:8080/health | jq .\n\n# POST with JSON body\ncurl -s -X POST http://localhost:8080/memory/ingest \\\n -H \"Content-Type: application/json\" \\\n -d '{\"project\": \"poimen\", \"text\": \"test record\", \"kind\": \"L1\"}'\n\n# PUT update\ncurl -s -X PUT http://localhost:8080/resource/123 \\\n -H \"Content-Type: application/json\" \\\n -d '{\"field\": \"new_value\"}'\n\n# DELETE\ncurl -s -X DELETE http://localhost:8080/resource/123\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"d62b7acda370c2dac68e079a45a5d3e53a4740893dff238315e85cc0ff51e0b7","source":"knowledge/curl-api-testing.md","text":"## Authentication\n```bash\n# Bearer token (JWT)\ncurl -s -H \"Authorization: Bearer $TOKEN\" http://localhost:8080/memory/query\n\n# Get JWT from Authentik\nTOKEN=$(curl -s -X POST https://authentik.riotpiao.com/application/o/token/ \\\n -d \"grant_type=client_credentials\" \\\n -d \"client_id=$CLIENT_ID\" \\\n -d \"client_secret=$CLIENT_SECRET\" | jq -r .access_token)\n\n# API key header\ncurl -s -H \"X-API-Key: $API_KEY\" http://localhost:8080/endpoint\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"b4aa92ecf23be98e8b03fbbf14520a3914493b1bdbef4b6df3ee96232ee1cf2e","source":"knowledge/curl-api-testing.md","text":"## Response Inspection\n```bash\n# Status code only\ncurl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080/health\n\n# Headers + body\ncurl -sv http://localhost:8080/health 2>&1\n\n# Response time\ncurl -s -o /dev/null -w \"%{time_total}s\" http://localhost:8080/health\n\n# Follow redirects\ncurl -sL http://localhost:8080/old-path\n\n# Pretty print JSON\ncurl -s http://localhost:8080/health | python3 -m json.tool\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"00c2d2ecaedf760754881e0c937b59f8b951f3fb2290eceed5a64115912bda4c","source":"knowledge/curl-api-testing.md","text":"## Error Testing\n```bash\n# Expect 401 — missing auth\ncurl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080/memory/query\n# Should return: 401\n\n# Expect 400 — bad request body\ncurl -s -X POST http://localhost:8080/memory/ingest \\\n -H \"Content-Type: application/json\" \\\n -d '{\"invalid\": true}'\n# Should return: 400\n\n# Expect 429 — rate limit\nfor i in $(seq 1 200); do\n curl -s -o /dev/null -w \"%{http_code}\\n\" -H \"Authorization: Bearer $TOKEN\" \\\n http://localhost:8080/memory/query -d '{\"query\":\"test\"}'\ndone | sort | uniq -c\n# Should see 429 after limit exceeded\n\n# Expect 404 — not found\ncurl -s -o /dev/null -w \"%{http_code}\" http://localhost:8080/nonexistent\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"c5c5274b426bee99520b53a494c1fc7bb7eb74bda278118307699d5721d779a3","source":"knowledge/curl-api-testing.md","text":"## Kubernetes Service Testing\n```bash\n# Port-forward to test in-cluster service\nkubectl port-forward -n poimen svc/poimen-memory 8080:8080 &\ncurl -s http://localhost:8080/health\n\n# Direct pod exec curl\nkubectl exec -n poimen deploy/poimen-memory -- curl -s http://localhost:8080/health\n\n# Test from inside cluster (debug pod)\nkubectl run -it --rm curl-test --image=curlimages/curl -- \\\n curl -s http://poimen-memory.poimen.svc.cluster.local:8080/health\n\n# Test ingress from outside\ncurl -sk https://api.riotpiao.com/health\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"892aa36e1cfad7cee79eda7df75a25df9d22ff6b9eb1637760ec92982e6bf038","source":"knowledge/curl-api-testing.md","text":"## Verification Checklist\nAfter completing any API work, verify with curl:\n1. **Happy path** — correct input returns expected output and status code\n2. **Auth required** — missing token returns 401, bad token returns 403\n3. **Validation** — bad input returns 400 with error message\n4. **Not found** — missing resource returns 404\n5. **Idempotency** — same request twice returns same result (for POST with idempotency key)\n6. **Rate limiting** — excessive requests return 429\n7. **Content type** — response has correct Content-Type header\n8. **Latency** — response time within SLA (< 500ms for queries, < 50ms for health)","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"c51a5781f8c34d9194d4ac5e6cb6de13bfbc9b84d0e06f524e9a8d479c8ee614","source":"knowledge/curl-api-testing.md","text":"## Scripted Verification\n```bash\n#!/bin/bash\n# verify-api.sh — run after any API change\nBASE=\"http://localhost:8080\"\nPASS=0; FAIL=0\n\ncheck() {\n local desc=\"$1\" expected=\"$2\" actual=\"$3\"\n if [ \"$expected\" = \"$actual\" ]; then\n echo \"✓ $desc\"; ((PASS++))\n else\n echo \"✗ $desc (expected $expected, got $actual)\"; ((FAIL++))\n fi\n}\n\ncheck \"health returns 200\" \"200\" \\\n \"$(curl -s -o /dev/null -w '%{http_code}' $BASE/health)\"\n\ncheck \"query without auth returns 401\" \"401\" \\\n \"$(curl -s -o /dev/null -w '%{http_code}' -X POST $BASE/memory/query)\"\n\ncheck \"ingest with bad body returns 400\" \"400\" \\\n \"$(curl -s -o /dev/null -w '%{http_code}' -X POST $BASE/memory/ingest \\\n -H 'Content-Type: application/json' -d '{}')\"\n\necho \"---\"\necho \"$PASS passed, $FAIL failed\"\n[ $FAIL -eq 0 ] && exit 0 || exit 1\n```","total_chunks":10}} -{"project":"knowledge","query":"curl-api-testing:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"e8e32cac4614b04e7df4cf0cf923c2225155c361060c5f49648bda79239fa301","source":"knowledge/curl-api-testing.md","text":"## Data Piping\n```bash\n# Ingest from file\ncurl -s -X POST http://localhost:8080/memory/ingest \\\n -H \"Content-Type: application/json\" \\\n -H \"Authorization: Bearer $TOKEN\" \\\n -d @payload.json\n\n# Query and filter with jq\ncurl -s -H \"Authorization: Bearer $TOKEN\" \\\n http://localhost:8080/memory/query \\\n -d '{\"project\":\"poimen\",\"query\":\"rust error handling\"}' | \\\n jq '.results[] | {score, text}'\n\n# Chain: query → pipe to next call\nCHUNK_ID=$(curl -s ... | jq -r '.results[0].id')\ncurl -s http://localhost:8080/memory/chunk/$CHUNK_ID\n```","total_chunks":10}} -{"project":"knowledge","query":"tea-cli:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"f6422b9218ba12e9826a237d4ae4fc36da9643b750fa7bcd02f29fa91667fb8a","source":"knowledge/tea-cli.md","text":"# tea CLI — Gitea/Forgejo Command Line","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"25dfbeb95e7841646e833e0de0ad6592838236436c495aaf2e547f7b9f8c8956","source":"knowledge/tea-cli.md","text":"## Setup\n```bash\n# Login to Forgejo instance\ntea login add --name riotpiao \\\n --url https://git.riotpiao.com \\\n --token \n\n# Verify\ntea whoami\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"fb250f6093342be0709daa5b8d2368cef007f91ee55490dda5388465dae5ce66","source":"knowledge/tea-cli.md","text":"## Repository Operations\n```bash\n# List repos\ntea repos ls\n\n# Clone\ntea clone rock/poimen-memory\n\n# Open in browser\ntea open\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"a5c77ed2128b946d903e503abeed5129f743ea6c7d0a1e02e5a6fae32d74b99d","source":"knowledge/tea-cli.md","text":"## Issues & Tickets\n```bash\n# List issues\ntea issues ls --repo rock/poimen-memory\ntea issues ls --state open\ntea issues ls --labels bug\n\n# Create issue\ntea issues create --title \"Fix port conflict\" --body \"Port 8080 is in use\"\n\n# Close issue\ntea issues close 42\n\n# Comment on issue\ntea comments create 42 --body \"Fixed in commit abc123\"\n\n# Assign\ntea issues edit 42 --assignees rock\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"9ec3fe3fe3d53241f09991f62cce4fe5abff9e99a3f7778b20189f04f49b6cb1","source":"knowledge/tea-cli.md","text":"## Pull Requests\n```bash\n# List PRs\ntea pr ls --repo rock/poimen-memory\ntea pr ls --state open\n\n# Create PR from current branch\ntea pr create --title \"feat: add learn command\" --base main\n\n# Checkout a PR locally\ntea pr checkout 15\n\n# Merge PR\ntea pr merge 15 --style squash\n\n# Review PR\ntea pr review 15 --approve\ntea pr review 15 --request-changes --body \"Fix the unwrap()\"\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"4847500431acbf2641a6ef5396166cdb316999f0d12a97856d9e9e39cfa3c5fe","source":"knowledge/tea-cli.md","text":"## CI/CD — Actions & Workflows\n```bash\n# List workflow runs\ntea actions runs list --repo rock/poimen-memory\ntea actions runs list --repo rock/poimen-memory --limit 5\n\n# View specific run\ntea actions runs view --repo rock/poimen-memory\n\n# List workflows\ntea actions workflows list --repo rock/poimen-memory\n\n# Secrets management\ntea actions secrets list --repo rock/poimen-memory\ntea actions secrets create --repo rock/poimen-memory --name MY_SECRET --value \"secret123\"\n\n# Variables\ntea actions variables list --repo rock/poimen-memory\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"a1b2e7855fbec4d603f343695f43f88ce2c226dd64c79ffdde74d8ef5aed7086","source":"knowledge/tea-cli.md","text":"## Releases\n```bash\n# List releases\ntea releases ls --repo rock/poimen-memory\n\n# Create release\ntea releases create --repo rock/poimen-memory \\\n --tag v1.0.0 \\\n --title \"v1.0.0 — Production Release\" \\\n --note \"First production release\"\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"a1dfa3250a846495a4c18eaffbc79056dd18df89f80525324f37acb9d799fd69","source":"knowledge/tea-cli.md","text":"## Wiki\n```bash\n# List wiki pages\ntea wiki ls --repo rock/poimen-memory\n\n# Create wiki page\ntea wiki create --repo rock/poimen-memory \\\n --title \"Setup Guide\" \\\n --content \"# Setup\\n...\"\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"21e4a3407188a0ef9d952c67a94adda8124744d34108e2bacce0d3e80e2fa7c6","source":"knowledge/tea-cli.md","text":"## API Direct Access\n```bash\n# Raw API call (authenticated)\ntea api /repos/rock/poimen-memory\ntea api /repos/rock/poimen-memory/issues?state=open\n\n# POST via API\ntea api --method POST /repos/rock/poimen-memory/issues \\\n --body '{\"title\":\"test\",\"body\":\"test issue\"}'\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"d93eebb42cfc4bb536b23d7e5bf7091c5213a59bad764ac1e32c2e3475049ead","source":"knowledge/tea-cli.md","text":"## Ticket Verification Workflow\nAfter completing a task, verify the ticket is done:\n```bash\n# 1. Check CI passed\ntea actions runs list --repo rock/poimen-memory --limit 1\n# Should show: ✓ completed / success\n\n# 2. Check issue is closed or PR merged\ntea issues ls --repo rock/poimen-memory --state closed --limit 5\n\n# 3. Verify the API endpoint works (curl the actual service)\ncurl -s http://localhost:8080/health | jq .status\n# Should return: \"ok\"\n\n# 4. Tag release if milestone complete\ntea releases create --tag v1.x.x --title \"Milestone X complete\"\n```","total_chunks":11}} -{"project":"knowledge","query":"tea-cli:10","run":"latest","turn":10,"event_type":"learn","data":{"chunk_index":10,"level":"L1","sha256":"23974031e10bd55fb41c2f4f5b3ab8a1b1b431d038d9d241e5ec7648af2ab54b","source":"knowledge/tea-cli.md","text":"## Useful Flags\n- `--repo owner/name` — specify repo (or use current git context)\n- `--output simple` — machine-readable output\n- `--output yaml` — YAML format\n- `--output json` — JSON format for piping to jq\n- `--limit N` — limit results\n- `--state open|closed|all` — filter by state\n- `--fields name,status` — select columns","total_chunks":11}} -{"project":"knowledge","query":"verify-done:0","run":"latest","turn":0,"event_type":"learn","data":{"chunk_index":0,"level":"L1","sha256":"f2350e4f9651681fadc58444daaf29d716ee1a4e43e284b4e42e3c122cfcb2fe","source":"knowledge/verify-done.md","text":"# Verify Done — Confirming Task Completion","total_chunks":10}} -{"project":"knowledge","query":"verify-done:1","run":"latest","turn":1,"event_type":"learn","data":{"chunk_index":1,"level":"L1","sha256":"605504294c3771888d3e1d8d66dca9e8fba9a101d51f9b0a0905ac354c68db00","source":"knowledge/verify-done.md","text":"## Principle\nA task is NOT done until it's verified with real data against the real service. Code review + CI green is necessary but not sufficient.","total_chunks":10}} -{"project":"knowledge","query":"verify-done:2","run":"latest","turn":2,"event_type":"learn","data":{"chunk_index":2,"level":"L1","sha256":"fe04a7472414971a4823f2dd5c3eda05ad3493955a7cc7f4c2a49f22d22e7865","source":"knowledge/verify-done.md","text":"## Definition of Done Checklist\n1. **Code compiles** — `cargo build` / `go build` passes with no errors\n2. **Tests pass** — `cargo test` / `go test ./...` all green\n3. **CI green** — `tea actions runs list` shows latest run succeeded\n4. **API accessible** — `curl` the endpoint, get expected response\n5. **Auth works** — requests without token return 401, with token return 200\n6. **Error paths tested** — bad input returns proper error codes\n7. **Deployed** — ArgoCD synced, pod running, ingress reachable\n8. **Documented** — endpoint added to API docs or CLAUDE.md","total_chunks":10}} -{"project":"knowledge","query":"verify-done:3","run":"latest","turn":3,"event_type":"learn","data":{"chunk_index":3,"level":"L1","sha256":"c382f438306e367545fea933072e8804f95d8b8ec8cb5511a9210a0081a7fff7","source":"knowledge/verify-done.md","text":"## Verification Flow\n```\nCode change → Push → CI passes → ArgoCD deploys → curl test → Done\n ↓\n If fails → fix → repeat\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:4","run":"latest","turn":4,"event_type":"learn","data":{"chunk_index":4,"level":"L1","sha256":"a2c594aefc35b343153a14945c7c42a0cd8a86018c17bc7e854cd540a7d551fe","source":"knowledge/verify-done.md","text":"## API Endpoint Verification Template\nFor every new or changed endpoint, run:\n```bash\nENDPOINT=\"https://api.riotpiao.com\"\n\n# 1. Is it alive?\ncurl -s -o /dev/null -w \"%{http_code}\" $ENDPOINT/health\n# Expect: 200\n\n# 2. Does the new endpoint exist?\ncurl -s -o /dev/null -w \"%{http_code}\" $ENDPOINT/new-endpoint\n# Expect: NOT 404\n\n# 3. Does auth gate work?\ncurl -s -o /dev/null -w \"%{http_code}\" $ENDPOINT/new-endpoint\n# Expect: 401 (no token)\n\n# 4. Does it return correct data?\ncurl -s -H \"Authorization: Bearer $TOKEN\" $ENDPOINT/new-endpoint | jq .\n# Expect: meaningful JSON response\n\n# 5. Does it handle bad input?\ncurl -s -o /dev/null -w \"%{http_code}\" -X POST $ENDPOINT/new-endpoint \\\n -H \"Content-Type: application/json\" -d '{}'\n# Expect: 400\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:5","run":"latest","turn":5,"event_type":"learn","data":{"chunk_index":5,"level":"L1","sha256":"04fc352fdec5bbd2998a83672301c81524725ed898f148f6347f0a1e93c69521","source":"knowledge/verify-done.md","text":"## CI Verification\n```bash\n# Check last CI run status\ntea actions runs list --repo rock/poimen-memory --limit 1\n\n# If failed, check logs\ntea actions runs view --repo rock/poimen-memory\n\n# Check if ArgoCD synced\nkubectl get application -n argocd poimen-memory-app -o jsonpath='{.status.sync.status}'\n# Expect: Synced\n\n# Check pod health\nkubectl get pods -n poimen | grep poimen-memory\n# Expect: Running, no restarts\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:6","run":"latest","turn":6,"event_type":"learn","data":{"chunk_index":6,"level":"L1","sha256":"28fdf5743cfe78a0abdc83dc496fbc9eb49c5a066e2d8ef2340e45fe79872587","source":"knowledge/verify-done.md","text":"## Deployment Verification\n```bash\n# Pod running and ready\nkubectl get pods -n poimen -l app.kubernetes.io/name=poimen-memory\n# Expect: 1/1 Running\n\n# Service reachable internally\nkubectl exec -n poimen deploy/poimen-memory -- curl -s http://localhost:8080/health\n\n# Ingress reachable externally\ncurl -sk https://api.riotpiao.com/health\n\n# Logs clean (no panics, no errors on startup)\nkubectl logs -n poimen deploy/poimen-memory --tail=20\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:7","run":"latest","turn":7,"event_type":"learn","data":{"chunk_index":7,"level":"L1","sha256":"dc3c71bd7455fddc2d271561e4d1e7f2f157c44b9e7156dd964117d863a7d8c0","source":"knowledge/verify-done.md","text":"## Git Commit Verification\nBefore pushing, confirm:\n```bash\n# Commit message follows conventional commits\ngit log -1 --oneline\n# Expect: feat: / fix: / docs: / refactor: prefix\n\n# No secrets in diff\ngit diff --cached | grep -iE \"password|secret|token|api_key\"\n# Expect: empty (no matches)\n\n# No .env or key files staged\ngit diff --cached --name-only | grep -iE \"\\.env|\\.key|\\.pem\"\n# Expect: empty\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:8","run":"latest","turn":8,"event_type":"learn","data":{"chunk_index":8,"level":"L1","sha256":"3337996473ce8d78166fed6ef051b3af9d305be1510d249715772db3c5ad8430","source":"knowledge/verify-done.md","text":"## Memory System Specific Verification\n```bash\n# After ingest changes\ncurl -s -X POST https://api.riotpiao.com/memory/ingest \\\n -H \"Authorization: Bearer $TOKEN\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"project\":\"test\",\"text\":\"verify ingest\",\"kind\":\"L1\"}' | jq .\n# Expect: 201 with chunk ID\n\n# After query changes\ncurl -s -X POST https://api.riotpiao.com/memory/query \\\n -H \"Authorization: Bearer $TOKEN\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"project\":\"test\",\"query\":\"verify\"}' | jq .\n# Expect: 200 with results array\n\n# After context endpoint changes\ncurl -s -X POST https://api.riotpiao.com/memory/context \\\n -H \"Authorization: Bearer $TOKEN\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"project\":\"test\",\"tool\":\"cargo\",\"task\":\"build\"}' | jq .\n# Expect: 200 with tier, lessons, budget\n\n# After rebuild changes\ncurl -s -X POST https://api.riotpiao.com/memory/rebuild \\\n -H \"Authorization: Bearer $TOKEN\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"project\":\"test\",\"dry_run\":true}' | jq .\n# Expect: 200 with records_processed count\n```","total_chunks":10}} -{"project":"knowledge","query":"verify-done:9","run":"latest","turn":9,"event_type":"learn","data":{"chunk_index":9,"level":"L1","sha256":"2c19a95eba6e0f454324431f1ca416ddc6ce37b47e0227df14c718b3e5513fb8","source":"knowledge/verify-done.md","text":"## Anti-Patterns\n- ❌ \"CI passed so it's done\" — CI doesn't test the real deployment\n- ❌ \"It works on my machine\" — must work in-cluster\n- ❌ Marking done without curl-testing the endpoint\n- ❌ Skipping error path testing (400, 401, 404, 429)\n- ❌ Not checking ArgoCD sync status after push\n- ❌ Trusting `kubectl apply` over ArgoCD (let ArgoCD manage state)","total_chunks":10}} diff --git a/scripts/compact_knowledge.py b/scripts/compact_knowledge.py new file mode 100644 index 0000000..2b2d3d2 --- /dev/null +++ b/scripts/compact_knowledge.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +""" +mem compact — Deduplicate and merge similar knowledge chunks. + +Flow: +1. Read all chunks from JSONL event log +2. Embed each chunk via embedding API +3. Compute cosine similarity matrix +4. Group chunks with similarity > threshold +5. Send each group to reasoning model to merge into one +6. Write compacted JSONL + update markdown files + +Usage: + python3 scripts/compact_knowledge.py [--threshold 0.80] [--dry-run] [--project knowledge] +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +import requests +import numpy as np + +API_BASE = os.environ.get("MEM_API_BASE", "https://api.riotpiao.com/v1") +EMBED_MODEL = "nomic-ai/nomic-embed-text-v2-moe" +REASON_MODEL = "reasoning" + + +def embed_batch(texts: list[str], batch_size: int = 32) -> list[list[float]]: + """Embed texts in batches.""" + all_embeddings = [] + for i in range(0, len(texts), batch_size): + batch = texts[i:i + batch_size] + resp = requests.post( + f"{API_BASE}/embeddings", + json={"model": EMBED_MODEL, "input": batch}, + ) + resp.raise_for_status() + data = resp.json()["data"] + # Sort by index to maintain order + data.sort(key=lambda x: x["index"]) + all_embeddings.extend([d["embedding"] for d in data]) + if i + batch_size < len(texts): + sys.stderr.write(f" Embedded {i + batch_size}/{len(texts)}...\n") + return all_embeddings + + +def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float: + """Cosine similarity between two vectors.""" + return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-10)) + + +def find_similar_groups( + chunks: list[dict], embeddings: list[list[float]], threshold: float +) -> list[list[int]]: + """Group chunk indices by cosine similarity > threshold.""" + n = len(chunks) + vecs = np.array(embeddings) + # Normalize + norms = np.linalg.norm(vecs, axis=1, keepdims=True) + vecs_norm = vecs / (norms + 1e-10) + # Similarity matrix + sim_matrix = vecs_norm @ vecs_norm.T + + visited = set() + groups = [] + + for i in range(n): + if i in visited: + continue + group = [i] + visited.add(i) + for j in range(i + 1, n): + if j in visited: + continue + if sim_matrix[i, j] > threshold: + group.append(j) + visited.add(j) + if len(group) > 1: + groups.append(group) + + return groups + + +def merge_with_llm(chunks_text: list[str], source_files: list[str]) -> str: + """Send similar chunks to reasoning model for merging.""" + numbered = "\n\n".join( + f"[Chunk {i+1} from {src}]:\n{text}" + for i, (text, src) in enumerate(zip(chunks_text, source_files)) + ) + + resp = requests.post( + f"{API_BASE}/chat/completions", + json={ + "model": REASON_MODEL, + "messages": [ + { + "role": "system", + "content": ( + "You are a knowledge compactor. Merge the following similar chunks " + "into ONE concise chunk. Keep ALL unique facts. Remove redundancy. " + "Keep markdown formatting. Output ONLY the merged text, no explanation. " + "No meta-commentary. No 'Here is the merged version'. Just the content." + ), + }, + { + "role": "user", + "content": f"Merge these {len(chunks_text)} similar chunks:\n\n{numbered}", + }, + ], + "temperature": 0.1, + "max_tokens": 1500, + }, + ) + resp.raise_for_status() + import re + result = resp.json()["choices"][0]["message"]["content"] + # Strip tags from reasoning model + result = re.sub(r".*?", "", result, flags=re.DOTALL).strip() + return result + + +def main(): + parser = argparse.ArgumentParser(description="Compact knowledge chunks") + parser.add_argument( + "--threshold", + type=float, + default=0.82, + help="Cosine similarity threshold for grouping (default: 0.82)", + ) + parser.add_argument("--dry-run", action="store_true", help="Show groups without merging") + parser.add_argument("--project", default="knowledge", help="Project name") + parser.add_argument( + "--log-file", + default="log/knowledge/learn/latest.jsonl", + help="Path to JSONL log", + ) + args = parser.parse_args() + + # 1. Read chunks + log_path = Path(args.log_file) + if not log_path.exists(): + print(f"No log file at {log_path}") + sys.exit(1) + + records = [] + with open(log_path) as f: + for line in f: + records.append(json.loads(line)) + + texts = [r["data"]["text"] for r in records] + sources = [r["data"].get("source", "unknown") for r in records] + print(f"Loaded {len(records)} chunks from {log_path}") + + # 2. Embed all chunks + print("Embedding chunks...") + embeddings = embed_batch(texts) + print(f"Embedded {len(embeddings)} chunks ({len(embeddings[0])} dims)") + + # 3. Find similar groups + print(f"Finding groups with similarity > {args.threshold}...") + groups = find_similar_groups(records, embeddings, args.threshold) + + if not groups: + print("✓ No similar chunks found. Knowledge is already compact.") + sys.exit(0) + + # 4. Report + total_mergeable = sum(len(g) for g in groups) + savings = total_mergeable - len(groups) + print(f"\nFound {len(groups)} groups ({total_mergeable} chunks → {len(groups)} merged)") + print(f"Estimated savings: {savings} chunks removed\n") + + for gi, group in enumerate(groups): + group_texts = [texts[i] for i in group] + group_sources = [sources[i] for i in group] + max_sim = 0 + for a in range(len(group)): + for b in range(a + 1, len(group)): + s = cosine_similarity( + np.array(embeddings[group[a]]), np.array(embeddings[group[b]]) + ) + max_sim = max(max_sim, s) + + print(f"Group {gi+1} (sim={max_sim:.3f}, {len(group)} chunks):") + for idx in group: + preview = texts[idx][:80].replace("\n", " ") + src = Path(sources[idx]).stem + print(f" [{src}] {preview}...") + + if not args.dry_run: + print(f" → Merging with {REASON_MODEL}...") + merged = merge_with_llm(group_texts, group_sources) + print(f" → Merged: {len(merged)} chars (was {sum(len(t) for t in group_texts)} chars)") + # Replace first chunk with merged, mark others for removal + records[group[0]]["data"]["text"] = merged + records[group[0]]["data"]["merged_from"] = len(group) + for idx in group[1:]: + records[idx] = None # Mark for removal + print() + + if args.dry_run: + print("(dry run — no changes written)") + sys.exit(0) + + # 5. Write compacted log + compacted = [r for r in records if r is not None] + backup = log_path.with_suffix(".jsonl.bak") + log_path.rename(backup) + + with open(log_path, "w") as f: + for r in compacted: + f.write(json.dumps(r) + "\n") + + print(f"{'─' * 50}") + print(f"Before: {len(records)} chunks") + print(f"After: {len(compacted)} chunks (-{len(records) - len(compacted)})") + print(f"Backup: {backup}") + print(f"Written: {log_path}") + + +if __name__ == "__main__": + main()