feat: POST /memory/learn endpoint + refactor mem learn CLI
Learning flow now goes through the service, not local JSONL: - POST /memory/learn: accepts markdown, chunks it, runs gated loop (LLM evaluates + compacts), stores in pgvector. OpenAI-style API. - mem learn CLI: reads files, calls POST /memory/learn per file - Removed cmd_compact (gated loop IS the compaction) - Updated README with new commands and API docs Memory never grows unbounded — every update is a rewrite, not append. The gated loop LLM acts as evaluator + compactor in one pass.
This commit is contained in:
@@ -357,6 +357,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
||||
.route("/memory/context", web::post().to(context_handler))
|
||||
.route("/memory/projects", web::get().to(projects_handler))
|
||||
.route("/memory/skills", web::get().to(skills_handler))
|
||||
.route("/memory/learn", web::post().to(learn_handler))
|
||||
.route("/memory/vault/generate", web::post().to(vault_generate_handler))
|
||||
.route("/memory/vault", web::get().to(vault_browser_handler))
|
||||
.route("/memory/vault/{project}", web::get().to(vault_project_handler))
|
||||
@@ -562,6 +563,193 @@ async fn optimize_search_results(
|
||||
optimized
|
||||
}
|
||||
|
||||
/// POST /memory/learn — Ingest knowledge via gated loop (LLM evaluates + compacts)
|
||||
///
|
||||
/// OpenAI-compatible style endpoint. Accepts markdown text, chunks it,
|
||||
/// runs each chunk through the gated loop where the LLM decides whether
|
||||
/// to accept/reject and rewrites memory to stay compact.
|
||||
///
|
||||
/// Request:
|
||||
/// POST /memory/learn
|
||||
/// { "project": "knowledge", "text": "## Rust\n- ownership...", "query": "What are key Rust patterns?" }
|
||||
///
|
||||
/// Response:
|
||||
/// { "project": "knowledge", "chunks_seen": 5, "chunks_used": 3,
|
||||
/// "memory": "compacted memory text...", "status": "completed" }
|
||||
pub async fn learn_handler(
|
||||
req: HttpRequest,
|
||||
body: web::Json<serde_json::Value>,
|
||||
state: web::Data<AppState>,
|
||||
) -> HttpResponse {
|
||||
let (claims, _token) = match validate_auth(&req, &state).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => return e,
|
||||
};
|
||||
|
||||
if !has_capability(&claims, "memory:write") {
|
||||
return HttpResponse::Forbidden().json(json!({
|
||||
"error": "forbidden",
|
||||
"reason": "missing capability: memory:write"
|
||||
}));
|
||||
}
|
||||
|
||||
if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
|
||||
return e;
|
||||
}
|
||||
|
||||
let project = body["project"].as_str().unwrap_or("knowledge").to_string();
|
||||
let text = match body["text"].as_str() {
|
||||
Some(t) => t.to_string(),
|
||||
None => return HttpResponse::BadRequest().json(json!({
|
||||
"error": "bad_request",
|
||||
"reason": "missing required field: text"
|
||||
})),
|
||||
};
|
||||
let question = body["query"].as_str()
|
||||
.unwrap_or("What are the key facts, patterns, and practices in this knowledge?")
|
||||
.to_string();
|
||||
let memory_budget = body["memory_budget"].as_u64().unwrap_or(4096) as u32;
|
||||
let chunk_size = body["chunk_size"].as_u64().unwrap_or(2000) as usize;
|
||||
let model = body["model"].as_str().unwrap_or("qwen2.5:3b-instruct").to_string();
|
||||
|
||||
// Chunk the markdown
|
||||
let chunks = chunk_markdown_text(&text, chunk_size);
|
||||
if chunks.is_empty() {
|
||||
return HttpResponse::BadRequest().json(json!({
|
||||
"error": "bad_request",
|
||||
"reason": "text produced no chunks"
|
||||
}));
|
||||
}
|
||||
|
||||
// Build domain chunks
|
||||
let domain_chunks: Vec<mem_core::domain::Chunk> = chunks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, text)| {
|
||||
let record = mem_core::Record {
|
||||
role: mem_core::Role::User,
|
||||
text: text.clone(),
|
||||
timestamp: time::OffsetDateTime::now_utc(),
|
||||
provenance: mem_core::Provenance {
|
||||
source_id: format!("learn://{}:{}", project, i),
|
||||
offset: i as u64,
|
||||
},
|
||||
};
|
||||
mem_core::domain::Chunk::new((i + 1) as u32, vec![record], text.len() / 4)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Build query for gated loop
|
||||
let query = mem_core::query::Query {
|
||||
id: format!("learn-{}", uuid::Uuid::new_v4()),
|
||||
question,
|
||||
exit_gate: false,
|
||||
};
|
||||
|
||||
let config = mem_core::gated_loop::LoopConfig {
|
||||
level: mem_core::Level::L1,
|
||||
query,
|
||||
memory_budget,
|
||||
use_exit_gate: false,
|
||||
};
|
||||
|
||||
// Create LLM client
|
||||
let llm_base = std::env::var("LLM_API_BASE")
|
||||
.unwrap_or_else(|_| "https://api.riotpiao.com".to_string());
|
||||
let llm_key = std::env::var("LLM_API_KEY")
|
||||
.or_else(|_| std::env::var("MEM_API_KEY"))
|
||||
.unwrap_or_default();
|
||||
let llm = match mem_llm::ChatClient::new(&llm_base, &llm_key, &model) {
|
||||
Ok(c) => c,
|
||||
Err(e) => return HttpResponse::InternalServerError().json(json!({
|
||||
"error": "llm_init_failed",
|
||||
"reason": e.to_string()
|
||||
})),
|
||||
};
|
||||
|
||||
// Run gated loop
|
||||
let outcome = match mem_core::gated_loop::run_loop(config, domain_chunks, &llm) {
|
||||
Ok(o) => o,
|
||||
Err(e) => return HttpResponse::InternalServerError().json(json!({
|
||||
"error": "gated_loop_failed",
|
||||
"reason": e.to_string()
|
||||
})),
|
||||
};
|
||||
|
||||
// Store compacted memory in pgvector if non-empty
|
||||
let mut stored = false;
|
||||
if !outcome.final_memory.is_empty() {
|
||||
match state.embeddings.embed_one(&outcome.final_memory).await {
|
||||
Ok(embedding) => {
|
||||
let chunk_id = uuid::Uuid::new_v4();
|
||||
let sha = {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut h = Sha256::new();
|
||||
h.update(outcome.final_memory.as_bytes());
|
||||
format!("{:x}", h.finalize())
|
||||
};
|
||||
let result = sqlx::query(
|
||||
"INSERT INTO memory_chunks (id, project, level, text, embedding, sha256, source, created_at)
|
||||
VALUES ($1, $2, 'L1', $3, $4, $5, $6, NOW())
|
||||
ON CONFLICT (sha256) DO UPDATE SET text = $3, embedding = $4",
|
||||
)
|
||||
.bind(chunk_id)
|
||||
.bind(&project)
|
||||
.bind(&outcome.final_memory)
|
||||
.bind(embedding.to_vec())
|
||||
.bind(&sha)
|
||||
.bind(format!("learn://{}", project))
|
||||
.fetch_optional(&state.pool)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(_) => { stored = true; }
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to store compacted memory: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to embed compacted memory: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
HttpResponse::Ok().json(json!({
|
||||
"project": project,
|
||||
"status": "completed",
|
||||
"chunks_seen": outcome.chunks_seen,
|
||||
"chunks_used": outcome.chunks_used,
|
||||
"memory": outcome.final_memory,
|
||||
"memory_tokens": outcome.final_memory.len() / 4,
|
||||
"stored": stored,
|
||||
"model": model,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Split markdown on ## headings for learn endpoint.
|
||||
fn chunk_markdown_text(content: &str, max_chunk: usize) -> Vec<String> {
|
||||
let mut chunks = Vec::new();
|
||||
let mut current = String::new();
|
||||
for line in content.lines() {
|
||||
if line.starts_with("## ") && !current.is_empty() {
|
||||
let trimmed = current.trim().to_string();
|
||||
if !trimmed.is_empty() { chunks.push(trimmed); }
|
||||
current = String::new();
|
||||
}
|
||||
current.push_str(line);
|
||||
current.push('\n');
|
||||
if current.len() > max_chunk {
|
||||
let trimmed = current.trim().to_string();
|
||||
if !trimmed.is_empty() { chunks.push(trimmed); }
|
||||
current = String::new();
|
||||
}
|
||||
}
|
||||
let trimmed = current.trim().to_string();
|
||||
if !trimmed.is_empty() { chunks.push(trimmed); }
|
||||
chunks
|
||||
}
|
||||
|
||||
/// GET /memory/query — semantic search across memories
|
||||
pub async fn query_handler(
|
||||
req: HttpRequest,
|
||||
|
||||
Reference in New Issue
Block a user