feat: add Obsidian vault projection with Longhorn storage #13
@@ -1,5 +1,6 @@
|
||||
use actix_web::{web, App, HttpServer, HttpResponse, HttpRequest, middleware::Logger};
|
||||
use anyhow::Result;
|
||||
use chrono;
|
||||
use mem_llm::{EmbeddingsClient, RerankClient};
|
||||
use mem_store::{init_schema, VectorStore};
|
||||
use serde_json::json;
|
||||
@@ -74,6 +75,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
|
||||
.route("/memory/query", web::get().to(query_handler))
|
||||
.route("/memory/projects", web::get().to(projects_handler))
|
||||
.route("/memory/skills", web::get().to(skills_handler))
|
||||
.route("/memory/vault", web::post().to(vault_handler))
|
||||
})
|
||||
.bind(("0.0.0.0", port))?
|
||||
.run()
|
||||
@@ -298,3 +300,101 @@ pub async fn skills_handler(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// POST /memory/vault — generate Obsidian vault from memories
|
||||
pub async fn vault_handler(
|
||||
req: HttpRequest,
|
||||
state: web::Data<AppState>,
|
||||
) -> HttpResponse {
|
||||
if let Err(e) = check_auth(&req, &state) {
|
||||
return e;
|
||||
}
|
||||
|
||||
let vault_dir = std::env::var("MEM_HOME").unwrap_or_else(|_| "/data".to_string());
|
||||
|
||||
// Get all projects from database
|
||||
let projects_result = sqlx::query_as::<_, (String,)>(
|
||||
"SELECT DISTINCT project FROM memories_l1 ORDER BY project",
|
||||
)
|
||||
.fetch_all(&state.pool)
|
||||
.await;
|
||||
|
||||
match projects_result {
|
||||
Ok(projects) => {
|
||||
let mut generated = 0;
|
||||
let mut errors = Vec::new();
|
||||
|
||||
for (project,) in projects {
|
||||
let project_vault_dir = format!("{}/vault/{}", vault_dir, project);
|
||||
|
||||
// Create project directory
|
||||
if let Err(e) = std::fs::create_dir_all(&project_vault_dir) {
|
||||
errors.push(format!("Failed to create {}: {}", project_vault_dir, e));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get all L1 memories for this project
|
||||
let l1s_result = sqlx::query_as::<_, (String, String, String)>(
|
||||
"SELECT id, query_id, content FROM memories_l1 WHERE project = $1 ORDER BY updated_at DESC",
|
||||
)
|
||||
.bind(&project)
|
||||
.fetch_all(&state.pool)
|
||||
.await;
|
||||
|
||||
match l1s_result {
|
||||
Ok(l1s) => {
|
||||
for (id, query_id, content) in l1s {
|
||||
let filename = format!("{}/{}.md", project_vault_dir, query_id);
|
||||
let note = format!(
|
||||
"---\nproject: {}\nlevel: L1\nquery_id: {}\nid: {}\nupdated: {}\n---\n\n{}",
|
||||
project,
|
||||
query_id,
|
||||
id,
|
||||
chrono::Utc::now().to_rfc3339(),
|
||||
content
|
||||
);
|
||||
if std::fs::write(&filename, note).is_ok() {
|
||||
generated += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
errors.push(format!("Failed to fetch L1s for {}: {}", project, e));
|
||||
}
|
||||
}
|
||||
|
||||
// Get L2 synthesis
|
||||
let l2_result = sqlx::query_as::<_, (String,)>(
|
||||
"SELECT content FROM memories_l2 WHERE project = $1",
|
||||
)
|
||||
.bind(&project)
|
||||
.fetch_optional(&state.pool)
|
||||
.await;
|
||||
|
||||
if let Ok(Some((content,))) = l2_result {
|
||||
let filename = format!("{}/index.md", project_vault_dir);
|
||||
let note = format!(
|
||||
"---\nproject: {}\nlevel: L2\ntitle: {} Synthesis\nupdated: {}\n---\n\n{}",
|
||||
project,
|
||||
project,
|
||||
chrono::Utc::now().to_rfc3339(),
|
||||
content
|
||||
);
|
||||
if std::fs::write(&filename, note).is_ok() {
|
||||
generated += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
HttpResponse::Ok().json(json!({
|
||||
"status": "generated",
|
||||
"vault_dir": format!("{}/vault", vault_dir),
|
||||
"notes_created": generated,
|
||||
"errors": errors
|
||||
}))
|
||||
}
|
||||
Err(_) => {
|
||||
HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
mod lessons_cmd;
|
||||
mod http_server;
|
||||
mod endpoints;
|
||||
mod ingest_worker;
|
||||
mod query_worker;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use mem_chunk::token_counter::CharsOverFourCounter;
|
||||
|
||||
@@ -87,7 +87,8 @@ spec:
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
persistentVolumeClaim:
|
||||
claimName: poimen-memory-vault
|
||||
# Tolerate control-plane nodes
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
|
||||
@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
namespace: poimen
|
||||
resources:
|
||||
- vault-pvc.yaml
|
||||
- deployment.yaml
|
||||
- service.yaml
|
||||
# Secret managed separately (SealedSecret in homelab)
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: poimen-memory-vault
|
||||
namespace: poimen
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: longhorn
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
+20
-14
@@ -7,11 +7,13 @@ async fn a1_bare_array_parsed() {
|
||||
let mock_server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(vec![
|
||||
serde_json::json!({"index": 0, "score": 0.98}),
|
||||
serde_json::json!({"index": 1, "score": 0.01}),
|
||||
]))
|
||||
.and(path("/v1/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"results": [
|
||||
{"index": 0, "score": 0.98},
|
||||
{"index": 1, "score": 0.01},
|
||||
]
|
||||
})))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
@@ -32,11 +34,13 @@ async fn a2_index_mapping() {
|
||||
|
||||
// Return out-of-order: index 1 first, then index 0
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(vec![
|
||||
serde_json::json!({"index": 1, "score": 0.99}),
|
||||
serde_json::json!({"index": 0, "score": 0.01}),
|
||||
]))
|
||||
.and(path("/v1/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"results": [
|
||||
{"index": 1, "score": 0.99},
|
||||
{"index": 0, "score": 0.01},
|
||||
]
|
||||
})))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
@@ -68,10 +72,12 @@ async fn a4_apikey_sent() {
|
||||
let mock_server = MockServer::start().await;
|
||||
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(vec![
|
||||
serde_json::json!({"index": 0, "score": 0.95}),
|
||||
]))
|
||||
.and(path("/v1/rerank"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"results": [
|
||||
{"index": 0, "score": 0.95},
|
||||
]
|
||||
})))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user