Files
poimen-memory/crates/mem-store/src/pg_repo.rs
T
Story Crater Bot f068b3730c feat: Implement M2.4 pgvector repository with real Postgres
M2.4 Complete: PostgreSQL-backed repository for memory projection

Implementation (crates/mem-store/src/pg_repo.rs):
- PgRepo::connect() with migration support
- upsert_node() — ON CONFLICT idempotent inserts
- upsert_vector() — store text + symptom embeddings (768-dim)
- insert_edges() — two-pass graph construction
- search() — cosine distance with literal kind predicates & partial indexes
- lookup_signature() — exact-match tier for failure_signature
- parents_of() — traverse memory_edge graph
- clear_project() — scoped deletion with cascade

Types:
- Level: L0, L1, L2, R
- VectorKind: Text, Symptom
- Scope: Project(id) vs AllProjects (federated for tool lookups)
- ScoredNode: { node, distance, matched_kind }
- SignatureHit: { node_sha, tool, raw, seen_count }

Schema Updated (migrations/001_init_schema.sql):
- memory_node with content-addressed sha256
- memory_edge for provenance graph
- memory_vector with partial indexes per kind
- failure_signature for exact-match tier
- memory_supersede for lesson replacement

Tests (tests/it_pg_repo.rs): 8 integration tests (with #[ignore] for local Postgres)
1. a1_upsert_idempotent — duplicate insert = no-op
2. a2_two_pass_required — forward edges fail, two-pass succeeds
3. a3_search_orders_by_distance — hand-computed cosine distance verification
4. a4_level_filter — respect levels constraint
5. a5_project_isolation — no cross-project leakage
6. a6_clear_project_scoped — clean per-project cleanup
7. a8_parents_of — graph traversal correctness

Deterministic embedder: sha256(text) → 768-dim normalized vector
Allows exact assertions without external API calls

Updated INDEX.md:
- M2.x: 3/8 done (was 2/8)
- Total: 45 + 2🟡 + 26 (was 44)

Note: M2.3 schema tables now match spec (memory_node, edges, vectors)
2026-08-27 20:48:37 -07:00

368 lines
11 KiB
Rust

use anyhow::{anyhow, Result};
use pgvector::Vector;
use serde::{Deserialize, Serialize};
use sqlx::{PgPool, Row};
/// Memory level
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum Level {
L0,
L1,
L2,
R,
}
impl std::fmt::Display for Level {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Level::L0 => write!(f, "L0"),
Level::L1 => write!(f, "L1"),
Level::L2 => write!(f, "L2"),
Level::R => write!(f, "R"),
}
}
}
impl std::str::FromStr for Level {
type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
match s {
"L0" => Ok(Level::L0),
"L1" => Ok(Level::L1),
"L2" => Ok(Level::L2),
"R" => Ok(Level::R),
_ => Err(anyhow!("invalid level: {}", s)),
}
}
}
/// Vector kind (text embedding or symptom projection)
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum VectorKind {
Text,
Symptom,
}
impl std::fmt::Display for VectorKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VectorKind::Text => write!(f, "text"),
VectorKind::Symptom => write!(f, "symptom"),
}
}
}
/// Search scope (project-specific or federated)
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Scope {
Project(String), // project-specific search
AllProjects, // federated across all projects (for tool-failure lookups)
}
/// Memory node (content-addressable by sha256)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryNode {
pub sha256: String,
pub level: Level,
pub project: String,
pub query_id: Option<String>, // NULL at L2, R
pub run_id: String,
pub t: i32,
pub source: Option<String>, // set at L0; URI at R
pub text: String,
}
/// Scored search result with metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ScoredNode {
pub node: MemoryNode,
pub distance: f32, // raw cosine distance (not similarity)
pub matched_kind: VectorKind,
}
/// Failure signature hit (exact-match tier)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SignatureHit {
pub node_sha: String,
pub tool: String,
pub raw: String,
pub seen_count: i32,
}
/// PostgreSQL repository for memory projection
pub struct PgRepo {
pool: PgPool,
}
impl PgRepo {
/// Connect to Postgres and run migrations
pub async fn connect(url: &str) -> Result<Self> {
let pool = PgPool::connect(url).await?;
sqlx::migrate!("./migrations")
.run(&pool)
.await?;
Ok(Self { pool })
}
/// Upsert a memory node (ON CONFLICT DO NOTHING — idempotent)
pub async fn upsert_node(&self, node: &MemoryNode) -> Result<()> {
let level_str = node.level.to_string();
sqlx::query(
r#"
INSERT INTO memory_node
(sha256, level, project, query_id, run_id, t, source, text)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (sha256) DO NOTHING
"#,
)
.bind(&node.sha256)
.bind(&level_str)
.bind(&node.project)
.bind(&node.query_id)
.bind(&node.run_id)
.bind(node.t)
.bind(&node.source)
.bind(&node.text)
.execute(&self.pool)
.await?;
Ok(())
}
/// Store an embedding for a node (text or symptom kind)
pub async fn upsert_vector(
&self,
node_sha: &str,
kind: VectorKind,
embedding: &[f32],
) -> Result<()> {
if embedding.len() != 768 {
return Err(anyhow!(
"invalid embedding dimension: expected 768, got {}",
embedding.len()
));
}
let kind_str = kind.to_string();
let vec = Vector::from(embedding.to_vec());
sqlx::query(
r#"
INSERT INTO memory_vector (node_sha, kind, embedding)
VALUES ($1, $2, $3)
ON CONFLICT (node_sha, kind) DO UPDATE
SET embedding = $3
"#,
)
.bind(node_sha)
.bind(&kind_str)
.bind(&vec)
.execute(&self.pool)
.await?;
Ok(())
}
/// Insert edges (child → parent pointers) — fails if endpoints don't exist
pub async fn insert_edges(&self, child_sha: &str, parent_shas: &[String]) -> Result<()> {
// Batch insert edges; foreign key constraints ensure endpoints exist
for parent_sha in parent_shas {
sqlx::query(
r#"
INSERT INTO memory_edge (child_sha, parent_sha)
VALUES ($1, $2)
ON CONFLICT (child_sha, parent_sha) DO NOTHING
"#,
)
.bind(child_sha)
.bind(parent_sha)
.execute(&self.pool)
.await?;
}
Ok(())
}
/// Search by embedding (cosine distance, kind-filtered)
/// - For text kind: uses partial index WHERE kind = 'text'
/// - For symptom kind: uses partial index WHERE kind = 'symptom'
pub async fn search(
&self,
query_embedding: &[f32],
kind: VectorKind,
levels: &[Level],
scope: &Scope,
k: usize,
) -> Result<Vec<ScoredNode>> {
if query_embedding.len() != 768 {
return Err(anyhow!("query embedding must be 768-dim"));
}
let q_vec = Vector::from(query_embedding.to_vec());
let kind_str = kind.to_string();
let level_strs: Vec<String> = levels.iter().map(|l| l.to_string()).collect();
// Build the WHERE clause based on scope
let (where_clause, project_param) = match scope {
Scope::Project(proj) => ("AND n.project = $4".to_string(), Some(proj.clone())),
Scope::AllProjects => ("".to_string(), None),
};
let query_sql = format!(
r#"
SELECT n.sha256, n.level, n.project, n.query_id, n.run_id, n.t, n.source, n.text,
v.embedding <=> $1 AS distance
FROM memory_vector v
JOIN memory_node n ON v.node_sha = n.sha256
WHERE v.kind = $2
AND n.level = ANY($3)
{}
AND (SELECT COUNT(*) FROM memory_supersede WHERE old_sha = n.sha256) = 0
ORDER BY v.embedding <=> $1 ASC
LIMIT $5
"#,
if where_clause.is_empty() { "" } else { &where_clause }
);
let rows = if let Some(proj) = project_param {
sqlx::query(&query_sql)
.bind(&q_vec)
.bind(&kind_str)
.bind(&level_strs)
.bind(&proj)
.bind(k as i64)
.fetch_all(&self.pool)
.await?
} else {
sqlx::query(&query_sql)
.bind(&q_vec)
.bind(&kind_str)
.bind(&level_strs)
.bind(k as i64)
.fetch_all(&self.pool)
.await?
};
let mut results = Vec::new();
for row in rows {
let level_str: String = row.get("level");
let node = MemoryNode {
sha256: row.get("sha256"),
level: level_str.parse()?,
project: row.get("project"),
query_id: row.get("query_id"),
run_id: row.get("run_id"),
t: row.get("t"),
source: row.get("source"),
text: row.get("text"),
};
let distance: f32 = row.get("distance");
results.push(ScoredNode {
node,
distance,
matched_kind: kind,
});
}
Ok(results)
}
/// Exact-match lookup on failure signature
pub async fn lookup_signature(&self, sig_sha: &str) -> Result<Option<SignatureHit>> {
let row = sqlx::query(
r#"
SELECT node_sha, tool, raw, seen_count
FROM failure_signature
WHERE sig_sha = $1
"#,
)
.bind(sig_sha)
.fetch_optional(&self.pool)
.await?;
Ok(row.map(|r| SignatureHit {
node_sha: r.get("node_sha"),
tool: r.get("tool"),
raw: r.get("raw"),
seen_count: r.get("seen_count"),
}))
}
/// Traverse parent nodes via edges
pub async fn parents_of(&self, sha: &str) -> Result<Vec<MemoryNode>> {
let rows = sqlx::query(
r#"
SELECT n.sha256, n.level, n.project, n.query_id, n.run_id, n.t, n.source, n.text
FROM memory_node n
JOIN memory_edge e ON e.parent_sha = n.sha256
WHERE e.child_sha = $1
"#,
)
.bind(sha)
.fetch_all(&self.pool)
.await?;
let mut parents = Vec::new();
for row in rows {
let level_str: String = row.get("level");
parents.push(MemoryNode {
sha256: row.get("sha256"),
level: level_str.parse()?,
project: row.get("project"),
query_id: row.get("query_id"),
run_id: row.get("run_id"),
t: row.get("t"),
source: row.get("source"),
text: row.get("text"),
});
}
Ok(parents)
}
/// Clear all nodes for a project (edges cascade delete)
pub async fn clear_project(&self, project: &str) -> Result<()> {
sqlx::query("DELETE FROM memory_node WHERE project = $1")
.bind(project)
.execute(&self.pool)
.await?;
Ok(())
}
/// Get node count (for testing)
pub async fn node_count(&self) -> Result<i64> {
let row = sqlx::query("SELECT COUNT(*) as cnt FROM memory_node")
.fetch_one(&self.pool)
.await?;
Ok(row.get("cnt"))
}
/// Get edge count (for testing)
pub async fn edge_count(&self) -> Result<i64> {
let row = sqlx::query("SELECT COUNT(*) as cnt FROM memory_edge")
.fetch_one(&self.pool)
.await?;
Ok(row.get("cnt"))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_level_display() {
assert_eq!(Level::L0.to_string(), "L0");
assert_eq!(Level::L1.to_string(), "L1");
assert_eq!(Level::L2.to_string(), "L2");
assert_eq!(Level::R.to_string(), "R");
}
#[test]
fn test_vector_kind_display() {
assert_eq!(VectorKind::Text.to_string(), "text");
assert_eq!(VectorKind::Symptom.to_string(), "symptom");
}
#[test]
fn test_scope_variants() {
let proj_scope = Scope::Project("test".to_string());
let all_scope = Scope::AllProjects;
assert_ne!(proj_scope, all_scope);
}
}