rock/poimen-memory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: replace Obsidian projector with standalone service (ppatlabs/obsidian)

Browse Source
This commit is contained in:
Story Crater Bot
2026-08-27 21:35:07 -07:00
parent 83b9dcf5f9
commit 0eecca815b
11 changed files with 1050 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/mem-store/src/lib.rs
-2
View File
@@ -2,12 +2,10 @@ pub mod event_log;
pub mod pgvector;
pub mod rebuild;
pub mod pg_repo;
pub mod obsidian;
pub mod schema;
pub use event_log::{EventRecord, LogWriter};
pub use pgvector::{VectorRecord, VectorStore, ChunkL0, MemoryL1, MemoryL2};
pub use rebuild::{RebuildEngine, RebuildOpts, RebuildStats};
pub use pg_repo::{PgRepo, MemoryNode, VectorKind, Level, ScoredNode, Scope, SignatureHit};
pub use obsidian::{ObsidianProjector, ProjectorOpts, MemoryRecord, MemoryParent};
pub use schema::init_schema;
crates/mem-store/src/obsidian.rs
-471
View File
@@ -1,471 +0,0 @@
use anyhow::{anyhow, Result};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
use std::fs;
use std::path::{Path, PathBuf};
/// Obsidian projector options
#[derive(Debug, Clone)]
pub struct ProjectorOpts {
pub emit_evidence_notes: bool,
}
impl Default for ProjectorOpts {
fn default() -> Self {
Self {
emit_evidence_notes: false,
}
}
}
/// Memory record from the event log
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryRecord {
pub level: String, // "L0", "L1", "L2"
pub project: String,
pub query_id: Option<String>, // Some for L0/L1, None for L2
pub text: String, // Final memory text
pub updated: String, // ISO8601 timestamp from run
pub run_id: String,
pub t: i32, // Timestamp/sequence
pub source: Option<String>, // "pi", "claude", "transcript" for L0
pub chunks_seen: Option<i32>,
pub chunks_used: Option<i32>,
pub parents: Vec<MemoryParent>, // Provenance (L0 chunks, L1 references)
}
/// Parent reference for provenance
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryParent {
pub source: String, // "pi", "claude", etc.
pub t: i32, // Timestamp/sequence
pub description: Option<String>, // e.g., "chunk 66 — Kong body buffer"
}
/// Obsidian vault projector
pub struct ObsidianProjector;
impl ObsidianProjector {
/// Project memories from log into vault tree
///
/// Steps:
/// 1. Read log directory (multiple runs)
/// 2. Extract final memory per query (L1) and L2
/// 3. Generate deterministic frontmatter
/// 4. Write markdown with provenance
/// 5. Optionally write L0 evidence notes
pub async fn project<P: AsRef<Path>>(
log_dir: P,
vault_dir: P,
opts: ProjectorOpts,
) -> Result<ProjectorStats> {
let log_path = log_dir.as_ref();
let vault_path = vault_dir.as_ref();
// Create vault directory if needed
fs::create_dir_all(&vault_path)?;
let mut stats = ProjectorStats::default();
let mut l1_by_query: HashMap<String, MemoryRecord> = HashMap::new();
let mut l2_record: Option<MemoryRecord> = None;
let mut l0_records: Vec<MemoryRecord> = Vec::new();
// Read all memory records from log (simplified: assume JSON array for now)
// In real implementation, parse JSONL log files
let all_memories = Self::read_log_memories(log_path).await?;
for memory in all_memories {
match memory.level.as_str() {
"L0" => {
l0_records.push(memory);
}
"L1" => {
if let Some(qid) = &memory.query_id {
l1_by_query.insert(qid.clone(), memory);
}
}
"L2" => {
l2_record = Some(memory);
}
_ => {}
}
}
// Write index.md (L2 synthesis)
if let Some(l2) = l2_record.clone() {
let index_path = vault_path.join(format!("{}", l2.project)).join("index.md");
fs::create_dir_all(index_path.parent().unwrap())?;
let content = Self::render_l2(&l2, &l1_by_query)?;
Self::write_deterministic(&index_path, &content)?;
stats.files_written += 1;
}
// Write L1 notes (one per query)
for (query_id, l1) in l1_by_query.iter() {
let note_path = vault_path
.join(format!("{}", l1.project))
.join(format!("{}.md", query_id));
fs::create_dir_all(note_path.parent().unwrap())?;
let content = Self::render_l1(l1)?;
Self::write_deterministic(&note_path, &content)?;
stats.files_written += 1;
}
// Write L0 evidence notes (if enabled)
if opts.emit_evidence_notes {
for l0 in l0_records.iter() {
if let Some(source) = &l0.source {
let evidence_dir = vault_path
.join(format!("{}", l0.project))
.join("evidence");
fs::create_dir_all(&evidence_dir)?;
let note_name = format!("{}-{}", source, l0.t);
let note_path = evidence_dir.join(format!("{}.md", note_name));
let content = Self::render_l0(l0)?;
Self::write_deterministic(&note_path, &content)?;
stats.files_written += 1;
}
}
}
Ok(stats)
}
/// Render L2 (index.md) — links every L1 note
fn render_l2(
l2: &MemoryRecord,
l1_notes: &HashMap<String, MemoryRecord>,
) -> Result<String> {
let mut fm = BTreeMap::new();
fm.insert("project", l2.project.clone());
fm.insert("level", "L2".to_string());
fm.insert("updated", l2.updated.clone());
fm.insert("run_id", l2.run_id.clone());
let frontmatter = Self::render_frontmatter(&fm)?;
let mut body = format!("# {} — Memory\n\n", l2.project);
body.push_str(&l2.text);
body.push_str("\n\n");
// Links to all L1 notes (sorted by query_id for determinism)
if !l1_notes.is_empty() {
body.push_str("## Standing Queries\n\n");
let mut query_ids: Vec<_> = l1_notes.keys().collect();
query_ids.sort();
for qid in query_ids {
body.push_str(&format!("- [[{}]]\n", qid));
}
}
Ok(format!("{}{}", frontmatter, body))
}
/// Render L1 (query note) — memory + provenance + backlink to L2
fn render_l1(l1: &MemoryRecord) -> Result<String> {
let mut fm = BTreeMap::new();
if let Some(qid) = &l1.query_id {
fm.insert("project".to_string(), l1.project.clone());
fm.insert("level".to_string(), "L1".to_string());
fm.insert("query_id".to_string(), qid.clone());
fm.insert("updated".to_string(), l1.updated.clone());
fm.insert("run_id".to_string(), l1.run_id.clone());
if let Some(cs) = l1.chunks_seen {
fm.insert("chunks_seen".to_string(), cs.to_string());
}
if let Some(cu) = l1.chunks_used {
fm.insert("chunks_used".to_string(), cu.to_string());
}
} else {
return Err(anyhow!("L1 memory must have query_id"));
}
let frontmatter = Self::render_frontmatter_ordered(&fm)?;
let mut body = String::new();
// Title
if let Some(qid) = &l1.query_id {
body.push_str(&format!("# {} — {}\n\n", qid, l1.project));
}
// Memory text (empty memory shows explicit message)
if l1.text.trim().is_empty() {
body.push_str("_No evidence found for this query._\n\n");
} else {
body.push_str(&l1.text);
body.push_str("\n\n");
}
// Provenance section (sorted by source, then t)
if !l1.parents.is_empty() {
body.push_str("## Provenance\n\n");
let mut sorted_parents = l1.parents.clone();
sorted_parents.sort_by(|a, b| {
a.source
.cmp(&b.source)
.then(a.t.cmp(&b.t))
});
for parent in sorted_parents {
let desc = parent
.description
.as_ref()
.map(|d| format!(" — {}", d))
.unwrap_or_default();
body.push_str(&format!(
"- [[{}-{}]]{}\n",
parent.source, parent.t, desc
));
}
body.push_str("\n");
}
// Backlink to index.md
body.push_str("[[index]]\n");
Ok(format!("{}{}", frontmatter, body))
}
/// Render L0 (evidence note) — raw chunk
fn render_l0(l0: &MemoryRecord) -> Result<String> {
let mut fm = BTreeMap::new();
fm.insert("project", l0.project.clone());
fm.insert("level", "L0".to_string());
if let Some(src) = &l0.source {
fm.insert("source", src.clone());
}
fm.insert("updated", l0.updated.clone());
let frontmatter = Self::render_frontmatter(&fm)?;
let title = format!(
"# {} — {}\n\n",
l0.source
.as_ref()
.map(|s| s.as_str())
.unwrap_or("unknown"),
l0.project
);
Ok(format!("{}{}{}", frontmatter, title, l0.text))
}
/// Render YAML frontmatter with stable key order
fn render_frontmatter(fm: &BTreeMap<&str, String>) -> Result<String> {
let mut result = "---\n".to_string();
for (k, v) in fm {
result.push_str(&format!("{}: {}\n", k, v));
}
result.push_str("---\n");
Ok(result)
}
/// Render YAML frontmatter from String keys (for ordered insertion)
fn render_frontmatter_ordered(fm: &BTreeMap<String, String>) -> Result<String> {
let mut result = "---\n".to_string();
for (k, v) in fm {
result.push_str(&format!("{}: {}\n", k, v));
}
result.push_str("---\n");
Ok(result)
}
/// Write file with deterministic formatting:
/// - \n line endings (not \r\n)
/// - No trailing whitespace on lines
/// - Exactly one trailing newline
fn write_deterministic(path: &Path, content: &str) -> Result<()> {
let lines: Vec<&str> = content.lines().collect();
let mut output = String::new();
for line in lines {
output.push_str(line.trim_end());
output.push('\n');
}
// Ensure exactly one trailing newline (last line already has one)
let final_content = output.trim_end_matches('\n').to_string() + "\n";
fs::write(path, final_content)?;
Ok(())
}
/// Read memory records from log directory (placeholder)
/// In real implementation, parse JSONL or other event format
async fn read_log_memories(_log_dir: &Path) -> Result<Vec<MemoryRecord>> {
// TODO: Implement actual log parsing
// For now, return empty (tests will mock this)
Ok(Vec::new())
}
}
/// Projector statistics
#[derive(Debug, Clone, Default)]
pub struct ProjectorStats {
pub files_written: usize,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_render_l1_frontmatter_order() {
let l1 = MemoryRecord {
level: "L1".to_string(),
project: "test".to_string(),
query_id: Some("q1".to_string()),
text: "memory".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 0,
source: None,
chunks_seen: Some(10),
chunks_used: Some(5),
parents: vec![],
};
let content = ObsidianProjector::render_l1(&l1).expect("render");
// Verify frontmatter key order
assert!(content.starts_with("---\n"));
let lines: Vec<&str> = content.lines().collect();
let fm_lines: Vec<&str> = lines
.iter()
.skip(1)
.take_while(|l| !l.is_empty() && l != &&"---")
.copied()
.collect();
// Expected order: project, level, query_id, updated, run_id, chunks_seen, chunks_used
assert_eq!(fm_lines[0], "chunks_seen: 10"); // BTreeMap sorts alphabetically
assert_eq!(fm_lines[1], "chunks_used: 5");
assert_eq!(fm_lines[2], "level: L1");
assert_eq!(fm_lines[3], "project: test");
assert_eq!(fm_lines[4], "query_id: q1");
}
#[test]
fn test_write_deterministic_line_endings() {
let content = "line1\r\nline2\nline3";
let cleaned = content.lines().collect::<Vec<_>>().join("\n") + "\n";
assert!(!cleaned.contains("\r\n"));
assert!(cleaned.ends_with("\n"));
assert!(!cleaned.ends_with("\n\n"));
}
#[test]
fn test_l2_links_all_queries() {
let mut l1_notes = HashMap::new();
l1_notes.insert(
"query-a".to_string(),
MemoryRecord {
level: "L1".to_string(),
project: "p1".to_string(),
query_id: Some("query-a".to_string()),
text: "memory a".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 0,
source: None,
chunks_seen: None,
chunks_used: None,
parents: vec![],
},
);
l1_notes.insert(
"query-b".to_string(),
MemoryRecord {
level: "L1".to_string(),
project: "p1".to_string(),
query_id: Some("query-b".to_string()),
text: "memory b".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 1,
source: None,
chunks_seen: None,
chunks_used: None,
parents: vec![],
},
);
let l2 = MemoryRecord {
level: "L2".to_string(),
project: "p1".to_string(),
query_id: None,
text: "synthesis".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 2,
source: None,
chunks_seen: None,
chunks_used: None,
parents: vec![],
};
let content = ObsidianProjector::render_l2(&l2, &l1_notes).expect("render");
assert!(content.contains("[[query-a]]"));
assert!(content.contains("[[query-b]]"));
}
#[test]
fn test_empty_memory_text() {
let l1 = MemoryRecord {
level: "L1".to_string(),
project: "test".to_string(),
query_id: Some("q1".to_string()),
text: "".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 0,
source: None,
chunks_seen: None,
chunks_used: None,
parents: vec![],
};
let content = ObsidianProjector::render_l1(&l1).expect("render");
assert!(content.contains("_No evidence found"));
}
#[test]
fn test_provenance_sorting() {
let parents = vec![
MemoryParent {
source: "claude".to_string(),
t: 2,
description: None,
},
MemoryParent {
source: "pi".to_string(),
t: 1,
description: Some("chunk 1".to_string()),
},
MemoryParent {
source: "claude".to_string(),
t: 1,
description: None,
},
];
let l1 = MemoryRecord {
level: "L1".to_string(),
project: "test".to_string(),
query_id: Some("q1".to_string()),
text: "memory".to_string(),
updated: "2025-01-27".to_string(),
run_id: "r1".to_string(),
t: 0,
source: None,
chunks_seen: None,
chunks_used: None,
parents,
};
let content = ObsidianProjector::render_l1(&l1).expect("render");
// Should be sorted by source, then t: claude-1, claude-2, pi-1
let provenance_section = content.split("## Provenance").nth(1).unwrap();
let lines: Vec<&str> = provenance_section.lines().collect();
assert!(lines[1].contains("claude-1"));
assert!(lines[2].contains("claude-2"));
assert!(lines[3].contains("pi-1"));
}
}
crates/mem-store/src/rebuild.rs
+31 -22
View File
@@ -1,23 +1,39 @@
use anyhow::{anyhow, Result};
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use sha2::{Digest, Sha256};
use serde::{Deserialize, Serialize};
use crate::{
MemoryNode, MemoryRecord, MemoryParent, Level, VectorKind, PgRepo, ObsidianProjector,
ProjectorOpts,
};
use crate::{MemoryNode, Level, PgRepo};
/// Memory record from log (local copy for rebuild purposes)
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MemoryRecord {
pub level: String,
pub project: String,
pub query_id: Option<String>,
pub text: String,
pub run_id: String,
pub t: i32,
pub source: Option<String>,
pub parents: Vec<MemoryParent>,
}
/// Parent reference for provenance
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MemoryParent {
pub source: String,
pub t: i32,
pub description: Option<String>,
}
/// Rebuild options
#[derive(Debug, Clone)]
pub struct RebuildOpts {
pub project: String,
pub vault_only: bool, // Only rebuild vault, not database
pub db_only: bool, // Only rebuild database, not vault
pub allow_partial: bool, // Allow rebuilding from incomplete logs
pub embedding_cache_dir: Option<PathBuf>,
pub vault_dir: Option<PathBuf>,
pub log_dir: Option<PathBuf>,
}
@@ -32,7 +48,7 @@ pub struct RebuildStats {
pub embeddings_cached: i64,
}
/// Rebuild orchestrator: drop projections, rebuild from log
/// Rebuild orchestrator: rebuild database from log (vault projection delegated to Obsidian service)
pub struct RebuildEngine {
repo: PgRepo,
}
@@ -44,12 +60,12 @@ impl RebuildEngine {
Ok(Self { repo })
}
/// Execute full rebuild: clear → insert nodes → insert edges → project vault
/// Execute database rebuild: clear → insert nodes → insert edges
///
/// Order matters: nodes first (foreign key constraint), then edges, then vault projection
/// Order matters: nodes first (foreign key constraint), then edges.
/// Vault projection delegated to Obsidian service.
pub async fn rebuild(&self, opts: RebuildOpts) -> Result<RebuildStats> {
let log_dir = opts.log_dir.unwrap_or_else(|| PathBuf::from("log"));
let vault_dir = opts.vault_dir.unwrap_or_else(|| PathBuf::from("vault"));
let cache_dir = opts
.embedding_cache_dir
.clone()
@@ -63,13 +79,11 @@ impl RebuildEngine {
let mut stats = RebuildStats::default();
// PASS 1: Clear project (if not vault-only)
if !opts.vault_only {
self.repo.clear_project(&opts.project).await?;
}
// PASS 1: Clear project
self.repo.clear_project(&opts.project).await?;
// PASS 2: Insert all nodes (convert memories to nodes, batch embeddings)
if !opts.vault_only {
{
let mut nodes_by_sha: HashMap<String, MemoryNode> = HashMap::new();
let mut sha_to_level: HashMap<String, Level> = HashMap::new();
let mut sha_to_parents: HashMap<String, Vec<String>> = HashMap::new();
@@ -130,11 +144,6 @@ impl RebuildEngine {
stats.embeddings_computed = 0;
}
// PASS 4: Project vault (if not db-only)
if !opts.db_only {
ObsidianProjector::project(&log_dir, &vault_dir, ProjectorOpts::default()).await?;
}
Ok(stats)
}