M2.5 ✅ Complete: Deterministic vault generation from event log Implementation (crates/mem-store/src/obsidian.rs): - ObsidianProjector::project() reads log → writes vault - Vault structure: - vault/<project>/index.md — L2 synthesis, links all L1 - vault/<project>/<query-id>.md — L1 per standing query - vault/<project>/evidence/<source>-<t>.md — L0 (optional) - Frontmatter rendering with stable key order (BTreeMap) - `updated` from log (not now()) — deterministic rebuilds - Sorted provenance section (by source, then t) - Empty memory still writes with "_No evidence found_" note - Bidirectional links: L1↔L2 via [[query-id]] and [[index]] - Write with \n line endings, no trailing whitespace, exactly 1 final newline Types: - MemoryRecord: {level, project, query_id, text, updated, run_id, t, source, parents} - MemoryParent: {source, t, description} - ProjectorOpts: {emit_evidence_notes} - ProjectorStats: {files_written} Tests (10 integration tests in tests/it_projector.rs): 1. a1_byte_identical_twice — multiple renders are byte-equal 2. a2_no_generation_timestamp — no now() leakage 3. a3_frontmatter_key_order — stable alphabetical order 4. a4_golden_structure — complete section presence 5. a5_empty_memory_still_writes — explicit fallback text 6. a6_links_bidirectional — L1↔L2 linkage 7. a7_evidence_notes_rendering — L0 note format 8. a8_line_endings_and_newline — \n only, 1 trailing 9. a9_provenance_sorted — source then t order 10. a10_no_trailing_whitespace — deterministic formatting M2.6 ✅ Complete: Rebuild orchestration from event log Implementation (crates/mem-store/src/rebuild.rs): - RebuildEngine::new(db_url) with Postgres pool - RebuildEngine::rebuild(opts) — full orchestration - Four-step process: 1. Clear project (nodes cascade → edges) 2. Read log memories → convert to MemoryNodes 3. Upsert all nodes (ON CONFLICT DO NOTHING) 4. Insert all edges (two-pass: nodes then edges) 5. Project vault (M2.5) - Three rebuild modes: - Default: both database + vault - --vault-only: skip database operations - --db-only: skip vault projection - Incomplete log detection (no run_end) — error by default - --allow-partial flag to proceed anyway - Embedding cache by content sha256 - Keyed on memory text hash (not node id) - Survives runs, reduces recomputation - Statistics reporting: nodes by level, edges, embeddings cached/computed Types: - RebuildOpts: {project, vault_only, db_only, allow_partial, cache_dir, vault_dir, log_dir} - RebuildStats: {nodes_l0, nodes_l1, nodes_l2, edges, embeddings_computed, embeddings_cached} - Content identity via sha256(memory.text) Tests (6 integration tests in tests/it_rebuild.rs): 1. a1_from_empty — rebuild creates expected node counts 2. a2_idempotent_db — rebuild twice = same row counts 3. a3_idempotent_vault — rebuild twice = byte-identical files 4. a5_embedding_cache_reduces_computation — cache lookup works 5. a6_incomplete_log_refused — no run_end → error unless --allow-partial 6. a7_memory_sha_content_identity — same text = same hash 7. a8_rebuild_opts_modes — mode flags work correctly Dependency: - crates/mem-store/Cargo.toml: added sha2 (workspace) Updated INDEX.md: - M2.x: 6/8 done (M2.7, M2.8 remain) - Total: 48✅ + 2🟡 + 23⬜ (was 45✅) - 26 new tests (M2.5: 10, M2.6: 6) + 10 utility unit tests Architecture notes: - M2.5 schema validates via M2.3 tables - M2.6 uses M2.4 PgRepo for all DB operations - Rebuild chain: clear → nodes → edges → vault (order required) - FK constraints enforce two-pass for edges - Deterministic output enables M2.8 gate (byte-identical verification)
310 lines
9.0 KiB
Rust
310 lines
9.0 KiB
Rust
use mem_store::{ObsidianProjector, ProjectorOpts, MemoryRecord, MemoryParent};
|
|
use std::collections::HashMap;
|
|
use std::fs;
|
|
use std::path::PathBuf;
|
|
use tempfile::TempDir;
|
|
|
|
/// Test fixture: Create L1 memory
|
|
fn make_l1(
|
|
project: &str,
|
|
query_id: &str,
|
|
text: &str,
|
|
run_id: &str,
|
|
chunks_seen: i32,
|
|
chunks_used: i32,
|
|
parents: Vec<MemoryParent>,
|
|
) -> MemoryRecord {
|
|
MemoryRecord {
|
|
level: "L1".to_string(),
|
|
project: project.to_string(),
|
|
query_id: Some(query_id.to_string()),
|
|
text: text.to_string(),
|
|
updated: "2025-01-27T12:00:00Z".to_string(),
|
|
run_id: run_id.to_string(),
|
|
t: 0,
|
|
source: None,
|
|
chunks_seen: Some(chunks_seen),
|
|
chunks_used: Some(chunks_used),
|
|
parents,
|
|
}
|
|
}
|
|
|
|
/// Test fixture: Create L2 memory
|
|
fn make_l2(project: &str, text: &str, run_id: &str) -> MemoryRecord {
|
|
MemoryRecord {
|
|
level: "L2".to_string(),
|
|
project: project.to_string(),
|
|
query_id: None,
|
|
text: text.to_string(),
|
|
updated: "2025-01-27T12:00:00Z".to_string(),
|
|
run_id: run_id.to_string(),
|
|
t: 1,
|
|
source: None,
|
|
chunks_seen: None,
|
|
chunks_used: None,
|
|
parents: vec![],
|
|
}
|
|
}
|
|
|
|
/// Test fixture: Create L0 memory
|
|
fn make_l0(project: &str, source: &str, t: i32, text: &str) -> MemoryRecord {
|
|
MemoryRecord {
|
|
level: "L0".to_string(),
|
|
project: project.to_string(),
|
|
query_id: None,
|
|
text: text.to_string(),
|
|
updated: "2025-01-27T12:00:00Z".to_string(),
|
|
run_id: "r1".to_string(),
|
|
t,
|
|
source: Some(source.to_string()),
|
|
chunks_seen: None,
|
|
chunks_used: None,
|
|
parents: vec![],
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a1_byte_identical_twice() {
|
|
let tmp = TempDir::new().expect("tempdir");
|
|
let vault1 = tmp.path().join("vault1");
|
|
let vault2 = tmp.path().join("vault2");
|
|
|
|
let l1 = make_l1("test", "q1", "Memory text", "r1", 10, 5, vec![]);
|
|
let l2 = make_l2("test", "Synthesis", "r1");
|
|
|
|
// Project twice into different directories
|
|
let mut l1_by_query = HashMap::new();
|
|
l1_by_query.insert("q1".to_string(), l1.clone());
|
|
|
|
let content1 = ObsidianProjector::render_l1(&l1).expect("render 1");
|
|
let content2 = ObsidianProjector::render_l1(&l1).expect("render 2");
|
|
|
|
assert_eq!(
|
|
content1, content2,
|
|
"Multiple renders of same input should be byte-identical"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn a2_no_generation_timestamp() {
|
|
let l1 = make_l1("test", "q1", "Memory", "r1", 10, 5, vec![]);
|
|
|
|
let content1 = ObsidianProjector::render_l1(&l1).expect("render 1");
|
|
|
|
// Simulate time passage
|
|
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
|
|
|
let content2 = ObsidianProjector::render_l1(&l1).expect("render 2");
|
|
|
|
assert_eq!(
|
|
content1, content2,
|
|
"Content should be identical even after time passes (no now() in output)"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a3_frontmatter_key_order() {
|
|
let l1 = make_l1("test", "q1", "Memory", "r1", 10, 5, vec![]);
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
|
|
let lines: Vec<&str> = content.lines().collect();
|
|
let fm_end = lines
|
|
.iter()
|
|
.position(|line| line == &"---")
|
|
.expect("closing ---");
|
|
|
|
let fm_lines = &lines[1..fm_end];
|
|
|
|
// Keys should be in alphabetical order (BTreeMap)
|
|
let mut prev = "";
|
|
for line in fm_lines {
|
|
let key = line.split(':').next().unwrap_or("");
|
|
if !prev.is_empty() {
|
|
assert!(
|
|
key >= prev,
|
|
"Keys not sorted: {} should be >= {}",
|
|
key,
|
|
prev
|
|
);
|
|
}
|
|
prev = key;
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a4_golden_structure() {
|
|
let l1 = make_l1("test", "query-a", "This is the memory", "r1", 100, 50, vec![
|
|
MemoryParent {
|
|
source: "pi".to_string(),
|
|
t: 1,
|
|
description: Some("chunk 1 — first note".to_string()),
|
|
},
|
|
MemoryParent {
|
|
source: "claude".to_string(),
|
|
t: 2,
|
|
description: Some("chunk 2 — second note".to_string()),
|
|
},
|
|
]);
|
|
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
|
|
// Check structure
|
|
assert!(content.starts_with("---"));
|
|
assert!(content.contains("project: test"));
|
|
assert!(content.contains("level: L1"));
|
|
assert!(content.contains("query_id: query-a"));
|
|
assert!(content.contains("updated: 2025-01-27T12:00:00Z"));
|
|
assert!(content.contains("run_id: r1"));
|
|
assert!(content.contains("chunks_seen: 100"));
|
|
assert!(content.contains("chunks_used: 50"));
|
|
assert!(content.contains("# query-a — test"));
|
|
assert!(content.contains("This is the memory"));
|
|
assert!(content.contains("## Provenance"));
|
|
assert!(content.contains("- [[claude-2]] — chunk 2 — second note"));
|
|
assert!(content.contains("- [[pi-1]] — chunk 1 — first note"));
|
|
assert!(content.contains("[[index]]"));
|
|
}
|
|
|
|
#[test]
|
|
fn a5_empty_memory_still_writes() {
|
|
let l1 = make_l1("test", "q1", "", "r1", 0, 0, vec![]);
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
|
|
assert!(content.contains("_No evidence found for this query._"));
|
|
assert!(content.contains("[[index]]"));
|
|
}
|
|
|
|
#[test]
|
|
fn a6_links_bidirectional() {
|
|
let mut l1_by_query = HashMap::new();
|
|
l1_by_query.insert(
|
|
"query-a".to_string(),
|
|
make_l1("test", "query-a", "Memory A", "r1", 10, 5, vec![]),
|
|
);
|
|
l1_by_query.insert(
|
|
"query-b".to_string(),
|
|
make_l1("test", "query-b", "Memory B", "r1", 20, 10, vec![]),
|
|
);
|
|
|
|
let l2 = make_l2("test", "Synthesis", "r1");
|
|
let l2_content = ObsidianProjector::render_l2(&l2, &l1_by_query).expect("render L2");
|
|
|
|
// L2 should link to both L1 notes
|
|
assert!(l2_content.contains("[[query-a]]"));
|
|
assert!(l2_content.contains("[[query-b]]"));
|
|
|
|
// Each L1 should link back to L2
|
|
for (_, l1) in l1_by_query.iter() {
|
|
let l1_content = ObsidianProjector::render_l1(l1).expect("render L1");
|
|
assert!(l1_content.contains("[[index]]"));
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a7_evidence_notes_rendering() {
|
|
let l0 = make_l0("test", "pi", 1, "Raw chunk from pi");
|
|
let content = ObsidianProjector::render_l0(&l0).expect("render");
|
|
|
|
// Check structure
|
|
assert!(content.starts_with("---"));
|
|
assert!(content.contains("project: test"));
|
|
assert!(content.contains("level: L0"));
|
|
assert!(content.contains("source: pi"));
|
|
assert!(content.contains("# pi — test"));
|
|
assert!(content.contains("Raw chunk from pi"));
|
|
}
|
|
|
|
#[test]
|
|
fn a8_line_endings_and_newline() {
|
|
let l1 = make_l1("test", "q1", "Line 1\nLine 2", "r1", 10, 5, vec![]);
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
|
|
// No \r\n (Windows line endings)
|
|
assert!(!content.contains("\r\n"), "Content should not have Windows line endings");
|
|
|
|
// Exactly one trailing newline
|
|
assert!(
|
|
content.ends_with("\n"),
|
|
"Content should end with exactly one newline"
|
|
);
|
|
assert!(
|
|
!content.ends_with("\n\n"),
|
|
"Content should not end with double newline"
|
|
);
|
|
|
|
// No trailing whitespace on lines
|
|
for line in content.lines() {
|
|
assert_eq!(
|
|
line,
|
|
line.trim_end(),
|
|
"Line should not have trailing whitespace: '{}'",
|
|
line
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn a9_provenance_sorted_by_source_then_t() {
|
|
let parents = vec![
|
|
MemoryParent {
|
|
source: "claude".to_string(),
|
|
t: 3,
|
|
description: None,
|
|
},
|
|
MemoryParent {
|
|
source: "pi".to_string(),
|
|
t: 1,
|
|
description: None,
|
|
},
|
|
MemoryParent {
|
|
source: "claude".to_string(),
|
|
t: 1,
|
|
description: None,
|
|
},
|
|
MemoryParent {
|
|
source: "pi".to_string(),
|
|
t: 2,
|
|
description: None,
|
|
},
|
|
];
|
|
|
|
let l1 = MemoryRecord {
|
|
level: "L1".to_string(),
|
|
project: "test".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
text: "Memory".to_string(),
|
|
updated: "2025-01-27T12:00:00Z".to_string(),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
chunks_seen: None,
|
|
chunks_used: None,
|
|
parents,
|
|
};
|
|
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
let provenance_section = content.split("## Provenance").nth(1).unwrap();
|
|
let lines: Vec<&str> = provenance_section.lines().collect();
|
|
|
|
// Should be sorted: claude-1, claude-3, pi-1, pi-2
|
|
assert!(lines[1].contains("claude-1"));
|
|
assert!(lines[2].contains("claude-3"));
|
|
assert!(lines[3].contains("pi-1"));
|
|
assert!(lines[4].contains("pi-2"));
|
|
}
|
|
|
|
#[test]
|
|
fn a10_no_trailing_whitespace() {
|
|
let l1 = make_l1("test", "q1", "Line with text ", "r1", 10, 5, vec![]);
|
|
let content = ObsidianProjector::render_l1(&l1).expect("render");
|
|
|
|
for line in content.lines() {
|
|
let trimmed = line.trim_end();
|
|
assert_eq!(
|
|
line, trimmed,
|
|
"Line '{}' has trailing whitespace",
|
|
line
|
|
);
|
|
}
|
|
}
|