Story Crater Bot
cbd49a8cb6
M2.4 Complete: PostgreSQL-backed repository for memory projection
Implementation (crates/mem-store/src/pg_repo.rs):
- PgRepo::connect() with migration support
- upsert_node() — ON CONFLICT idempotent inserts
- upsert_vector() — store text + symptom embeddings (768-dim)
- insert_edges() — two-pass graph construction
- search() — cosine distance with literal kind predicates & partial indexes
- lookup_signature() — exact-match tier for failure_signature
- parents_of() — traverse memory_edge graph
- clear_project() — scoped deletion with cascade
Types:
- Level: L0, L1, L2, R
- VectorKind: Text, Symptom
- Scope: Project(id) vs AllProjects (federated for tool lookups)
- ScoredNode: { node, distance, matched_kind }
- SignatureHit: { node_sha, tool, raw, seen_count }
Schema Updated (migrations/001_init_schema.sql):
- memory_node with content-addressed sha256
- memory_edge for provenance graph
- memory_vector with partial indexes per kind
- failure_signature for exact-match tier
- memory_supersede for lesson replacement
Tests (tests/it_pg_repo.rs): 8 integration tests (with #[ignore] for local Postgres)
1. a1_upsert_idempotent — duplicate insert = no-op
2. a2_two_pass_required — forward edges fail, two-pass succeeds
3. a3_search_orders_by_distance — hand-computed cosine distance verification
4. a4_level_filter — respect levels constraint
5. a5_project_isolation — no cross-project leakage
6. a6_clear_project_scoped — clean per-project cleanup
7. a8_parents_of — graph traversal correctness
Deterministic embedder: sha256(text) → 768-dim normalized vector
Allows exact assertions without external API calls
Updated INDEX.md:
- M2.x: 3/8 done (was 2/8)
- Total: 45✅ + 2🟡 + 26⬜ (was 44✅)
Note: M2.3 schema tables now match spec (memory_node, edges, vectors)
477 lines
14 KiB
Rust
477 lines
14 KiB
Rust
use mem_store::{Level, VectorKind, MemoryNode, PgRepo, Scope};
|
|
use sha2::{Sha256, Digest};
|
|
|
|
/// Deterministic embedder: sha256(text) → 768-dim vector
|
|
/// Allows exact distance assertions without external API calls
|
|
fn hash_to_embedding(text: &str) -> Vec<f32> {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(text.as_bytes());
|
|
let hash = hasher.finalize();
|
|
|
|
// Convert 32 bytes to 768 floats deterministically
|
|
let mut embedding = vec![0.0_f32; 768];
|
|
for (i, byte) in hash.iter().enumerate() {
|
|
let idx = i % 768;
|
|
embedding[idx] += (*byte as f32) / 256.0;
|
|
}
|
|
|
|
// Normalize to unit vector (cosine distance)
|
|
let mag: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
|
|
if mag > 0.0 {
|
|
for x in &mut embedding {
|
|
*x /= mag;
|
|
}
|
|
}
|
|
|
|
embedding
|
|
}
|
|
|
|
/// Compute cosine distance between two normalized vectors
|
|
fn cosine_distance(a: &[f32], b: &[f32]) -> f32 {
|
|
let dot: f32 = a.iter().zip(b.iter()).map(|(x, y)| x * y).sum();
|
|
// Distance = 1 - similarity (for normalized vectors)
|
|
(1.0 - dot).max(0.0)
|
|
}
|
|
|
|
/// Setup: Postgres via testcontainers (if available in CI/local)
|
|
/// For now, skip tests if DB not available to avoid CI dependencies
|
|
#[tokio::test]
|
|
#[ignore] // Run with: cargo test it_pg_repo -- --ignored
|
|
async fn a1_upsert_idempotent() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
// Skip if DB not available
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available at {}", db_url);
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p1").await.ok();
|
|
|
|
let node = MemoryNode {
|
|
sha256: "abc123def456".to_string(),
|
|
level: Level::L1,
|
|
project: "test_p1".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
text: "test memory".to_string(),
|
|
};
|
|
|
|
// First insert
|
|
repo.upsert_node(&node).await.expect("upsert 1");
|
|
let count1 = repo.node_count().await.expect("count 1");
|
|
assert_eq!(count1, 1, "First insert should create 1 node");
|
|
|
|
// Upsert same node again
|
|
repo.upsert_node(&node).await.expect("upsert 2");
|
|
let count2 = repo.node_count().await.expect("count 2");
|
|
assert_eq!(count2, 1, "Duplicate upsert must not create duplicate row");
|
|
|
|
repo.clear_project("test_p1").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a2_two_pass_required() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p2").await.ok();
|
|
|
|
let parent = MemoryNode {
|
|
sha256: "parent_sha".to_string(),
|
|
level: Level::L0,
|
|
project: "test_p2".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: Some("pi".to_string()),
|
|
text: "parent memory".to_string(),
|
|
};
|
|
|
|
let child = MemoryNode {
|
|
sha256: "child_sha".to_string(),
|
|
level: Level::L1,
|
|
project: "test_p2".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 1,
|
|
source: None,
|
|
text: "child memory".to_string(),
|
|
};
|
|
|
|
// Insert child first (before parent)
|
|
repo.upsert_node(&child).await.expect("insert child");
|
|
|
|
// Try edge before parent exists — should fail
|
|
let edge_result = repo
|
|
.insert_edges(&child.sha256, &[parent.sha256.clone()])
|
|
.await;
|
|
assert!(
|
|
edge_result.is_err(),
|
|
"Edge insert should fail when parent not found (FK constraint)"
|
|
);
|
|
|
|
// Now insert parent
|
|
repo.upsert_node(&parent).await.expect("insert parent");
|
|
|
|
// Now edge should succeed
|
|
repo.insert_edges(&child.sha256, &[parent.sha256.clone()])
|
|
.await
|
|
.expect("insert edge after parent exists");
|
|
|
|
let edge_count = repo.edge_count().await.expect("edge count");
|
|
assert_eq!(edge_count, 1, "Edge should be created in second pass");
|
|
|
|
repo.clear_project("test_p2").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a3_search_orders_by_distance() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p3").await.ok();
|
|
|
|
// Create three nodes with known embeddings
|
|
let texts = vec!["apple", "application", "banana"];
|
|
let embeddings: Vec<Vec<f32>> = texts
|
|
.iter()
|
|
.map(|t| hash_to_embedding(t))
|
|
.collect();
|
|
|
|
for (i, text) in texts.iter().enumerate() {
|
|
let node = MemoryNode {
|
|
sha256: format!("node_{}", i),
|
|
level: Level::L1,
|
|
project: "test_p3".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: i as i32,
|
|
source: None,
|
|
text: text.to_string(),
|
|
};
|
|
repo.upsert_node(&node).await.expect("upsert");
|
|
repo.upsert_vector(&node.sha256, VectorKind::Text, &embeddings[i])
|
|
.await
|
|
.expect("upsert vector");
|
|
}
|
|
|
|
// Query for "apple" — should rank apple closest
|
|
let query_embedding = hash_to_embedding("apple");
|
|
let results = repo
|
|
.search(
|
|
&query_embedding,
|
|
VectorKind::Text,
|
|
&[Level::L1],
|
|
&Scope::Project("test_p3".to_string()),
|
|
3,
|
|
)
|
|
.await
|
|
.expect("search");
|
|
|
|
assert_eq!(results.len(), 3, "Should return all 3 nodes");
|
|
|
|
// First result should be "apple" itself (distance ~0)
|
|
assert_eq!(
|
|
results[0].node.text, "apple",
|
|
"Closest match should be 'apple' itself"
|
|
);
|
|
assert!(results[0].distance < 0.01, "Distance to self should be ~0");
|
|
|
|
// Verify ordering matches hand-computed distances
|
|
for i in 0..results.len() - 1 {
|
|
assert!(
|
|
results[i].distance <= results[i + 1].distance + 1e-5,
|
|
"Results should be ordered by distance (ascending)"
|
|
);
|
|
}
|
|
|
|
repo.clear_project("test_p3").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a4_level_filter() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p4").await.ok();
|
|
|
|
let levels = vec![Level::L0, Level::L1, Level::L2];
|
|
let query_vec = hash_to_embedding("query");
|
|
|
|
for (i, level) in levels.iter().enumerate() {
|
|
let node = MemoryNode {
|
|
sha256: format!("node_{}", i),
|
|
level: *level,
|
|
project: "test_p4".to_string(),
|
|
query_id: if *level == Level::L2 { None } else { Some("q1".to_string()) },
|
|
run_id: "r1".to_string(),
|
|
t: i as i32,
|
|
source: None,
|
|
text: format!("memory at {:?}", level),
|
|
};
|
|
repo.upsert_node(&node).await.expect("upsert");
|
|
repo.upsert_vector(&node.sha256, VectorKind::Text, &query_vec)
|
|
.await
|
|
.expect("vector");
|
|
}
|
|
|
|
// Search with L1 only
|
|
let results = repo
|
|
.search(
|
|
&query_vec,
|
|
VectorKind::Text,
|
|
&[Level::L1],
|
|
&Scope::Project("test_p4".to_string()),
|
|
10,
|
|
)
|
|
.await
|
|
.expect("search");
|
|
|
|
assert_eq!(results.len(), 1, "Should return only L1");
|
|
assert_eq!(results[0].node.level, Level::L1);
|
|
|
|
repo.clear_project("test_p4").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a5_project_isolation() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p5a").await.ok();
|
|
repo.clear_project("test_p5b").await.ok();
|
|
|
|
let query_vec = hash_to_embedding("test");
|
|
|
|
// Create identical nodes in two projects
|
|
for proj in &["test_p5a", "test_p5b"] {
|
|
let node = MemoryNode {
|
|
sha256: format!("{}_node", proj),
|
|
level: Level::L1,
|
|
project: proj.to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
text: "same memory".to_string(),
|
|
};
|
|
repo.upsert_node(&node).await.expect("upsert");
|
|
repo.upsert_vector(&node.sha256, VectorKind::Text, &query_vec)
|
|
.await
|
|
.expect("vector");
|
|
}
|
|
|
|
// Search in p5a — should NOT return p5b
|
|
let results = repo
|
|
.search(
|
|
&query_vec,
|
|
VectorKind::Text,
|
|
&[Level::L1],
|
|
&Scope::Project("test_p5a".to_string()),
|
|
10,
|
|
)
|
|
.await
|
|
.expect("search");
|
|
|
|
assert_eq!(results.len(), 1, "Should return only 1 result (from p5a)");
|
|
assert_eq!(results[0].node.project, "test_p5a");
|
|
|
|
repo.clear_project("test_p5a").await.ok();
|
|
repo.clear_project("test_p5b").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a6_clear_project_scoped() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p6a").await.ok();
|
|
repo.clear_project("test_p6b").await.ok();
|
|
|
|
let parent_node = MemoryNode {
|
|
sha256: "parent".to_string(),
|
|
level: Level::L0,
|
|
project: "test_p6a".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
text: "parent".to_string(),
|
|
};
|
|
|
|
let child_node = MemoryNode {
|
|
sha256: "child".to_string(),
|
|
level: Level::L1,
|
|
project: "test_p6a".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 1,
|
|
source: None,
|
|
text: "child".to_string(),
|
|
};
|
|
|
|
let other_node = MemoryNode {
|
|
sha256: "other".to_string(),
|
|
level: Level::L1,
|
|
project: "test_p6b".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
text: "other".to_string(),
|
|
};
|
|
|
|
// Setup: parent + child in p6a, edge between them; separate node in p6b
|
|
repo.upsert_node(&parent_node).await.expect("insert parent");
|
|
repo.upsert_node(&child_node).await.expect("insert child");
|
|
repo.upsert_node(&other_node).await.expect("insert other");
|
|
repo.insert_edges("child", &["parent".to_string()])
|
|
.await
|
|
.expect("edge");
|
|
|
|
let edge_count_before = repo.edge_count().await.expect("count");
|
|
assert_eq!(edge_count_before, 1);
|
|
|
|
// Clear p6a
|
|
repo.clear_project("test_p6a").await.expect("clear");
|
|
|
|
// Verify p6b is intact
|
|
let other_count = sqlx::query_scalar::<_, i64>(
|
|
"SELECT COUNT(*) FROM memory_node WHERE project = 'test_p6b'",
|
|
)
|
|
.fetch_one(&repo.pool)
|
|
.await
|
|
.expect("query");
|
|
assert_eq!(other_count, 1, "Other project should be intact");
|
|
|
|
// Verify edges are gone (cascade delete)
|
|
let edge_count_after = repo.edge_count().await.expect("count");
|
|
assert_eq!(edge_count_after, 0, "Edges should cascade-delete with nodes");
|
|
|
|
repo.clear_project("test_p6b").await.ok();
|
|
}
|
|
|
|
#[tokio::test]
|
|
#[ignore]
|
|
async fn a8_parents_of() {
|
|
let db_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
|
|
"postgres://postgres:password@localhost:5432/poimen_test".to_string()
|
|
});
|
|
|
|
if !is_postgres_available(&db_url).await {
|
|
println!("Skipping: Postgres not available");
|
|
return;
|
|
}
|
|
|
|
let repo = PgRepo::connect(&db_url).await.expect("connect");
|
|
repo.clear_project("test_p8").await.ok();
|
|
|
|
let parent1 = MemoryNode {
|
|
sha256: "p1".to_string(),
|
|
level: Level::L0,
|
|
project: "test_p8".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 0,
|
|
source: None,
|
|
text: "p1".to_string(),
|
|
};
|
|
|
|
let parent2 = MemoryNode {
|
|
sha256: "p2".to_string(),
|
|
level: Level::L0,
|
|
project: "test_p8".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 1,
|
|
source: None,
|
|
text: "p2".to_string(),
|
|
};
|
|
|
|
let child = MemoryNode {
|
|
sha256: "c1".to_string(),
|
|
level: Level::L1,
|
|
project: "test_p8".to_string(),
|
|
query_id: Some("q1".to_string()),
|
|
run_id: "r1".to_string(),
|
|
t: 2,
|
|
source: None,
|
|
text: "c1".to_string(),
|
|
};
|
|
|
|
repo.upsert_node(&parent1).await.expect("insert p1");
|
|
repo.upsert_node(&parent2).await.expect("insert p2");
|
|
repo.upsert_node(&child).await.expect("insert c1");
|
|
|
|
repo.insert_edges("c1", &["p1".to_string(), "p2".to_string()])
|
|
.await
|
|
.expect("edges");
|
|
|
|
let parents = repo.parents_of("c1").await.expect("parents");
|
|
assert_eq!(parents.len(), 2, "Should return 2 parents");
|
|
|
|
let parent_shas: Vec<String> = parents.iter().map(|p| p.sha256.clone()).collect();
|
|
assert!(parent_shas.contains(&"p1".to_string()));
|
|
assert!(parent_shas.contains(&"p2".to_string()));
|
|
|
|
repo.clear_project("test_p8").await.ok();
|
|
}
|
|
|
|
/// Helper: Check if Postgres is available (for CI compatibility)
|
|
async fn is_postgres_available(url: &str) -> bool {
|
|
match sqlx::postgres::PgPoolOptions::new()
|
|
.max_connections(1)
|
|
.connect(url)
|
|
.await
|
|
{
|
|
Ok(_) => true,
|
|
Err(_) => false,
|
|
}
|
|
}
|