feat: Implement M2.4 pgvector repository with real Postgres
M2.4 Complete: PostgreSQL-backed repository for memory projection
Implementation (crates/mem-store/src/pg_repo.rs):
- PgRepo::connect() with migration support
- upsert_node() — ON CONFLICT idempotent inserts
- upsert_vector() — store text + symptom embeddings (768-dim)
- insert_edges() — two-pass graph construction
- search() — cosine distance with literal kind predicates & partial indexes
- lookup_signature() — exact-match tier for failure_signature
- parents_of() — traverse memory_edge graph
- clear_project() — scoped deletion with cascade
Types:
- Level: L0, L1, L2, R
- VectorKind: Text, Symptom
- Scope: Project(id) vs AllProjects (federated for tool lookups)
- ScoredNode: { node, distance, matched_kind }
- SignatureHit: { node_sha, tool, raw, seen_count }
Schema Updated (migrations/001_init_schema.sql):
- memory_node with content-addressed sha256
- memory_edge for provenance graph
- memory_vector with partial indexes per kind
- failure_signature for exact-match tier
- memory_supersede for lesson replacement
Tests (tests/it_pg_repo.rs): 8 integration tests (with #[ignore] for local Postgres)
1. a1_upsert_idempotent — duplicate insert = no-op
2. a2_two_pass_required — forward edges fail, two-pass succeeds
3. a3_search_orders_by_distance — hand-computed cosine distance verification
4. a4_level_filter — respect levels constraint
5. a5_project_isolation — no cross-project leakage
6. a6_clear_project_scoped — clean per-project cleanup
7. a8_parents_of — graph traversal correctness
Deterministic embedder: sha256(text) → 768-dim normalized vector
Allows exact assertions without external API calls
Updated INDEX.md:
- M2.x: 3/8 done (was 2/8)
- Total: 45✅ + 2🟡 + 26⬜ (was 44✅)
Note: M2.3 schema tables now match spec (memory_node, edges, vectors)
This commit is contained in:
Story Crater Bot
+53
-106
@@ -1,123 +1,70 @@
|
||||
-- Poimen Memory schema (M2.3, M2.4 spec)
|
||||
-- Tables: memory_node, memory_edge, memory_vector, failure_signature, memory_supersede
|
||||
|
||||
-- Enable pgvector extension
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- Event log — source of truth for all memory
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
-- Core memory node (sha256 = content identity, idempotent upsert key)
|
||||
CREATE TABLE IF NOT EXISTS memory_node (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
project VARCHAR NOT NULL,
|
||||
query_id VARCHAR NOT NULL,
|
||||
run_id VARCHAR NOT NULL,
|
||||
turn INT NOT NULL,
|
||||
event_type VARCHAR NOT NULL, -- "ingest", "gate_update", "gate_exit", "synthesis"
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
data JSONB NOT NULL,
|
||||
UNIQUE(project, query_id, run_id, turn)
|
||||
sha256 TEXT NOT NULL UNIQUE,
|
||||
level TEXT NOT NULL CHECK (level IN ('L0', 'L1', 'L2', 'R')),
|
||||
project TEXT NOT NULL,
|
||||
query_id TEXT, -- NULL at L2 and R; CHECK enforced below
|
||||
run_id TEXT NOT NULL,
|
||||
t INT NOT NULL,
|
||||
source TEXT, -- set at L0; source URI at R
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
CONSTRAINT level_query_id_check CHECK (
|
||||
(level IN ('L2', 'R') AND query_id IS NULL) OR
|
||||
(level NOT IN ('L2', 'R') AND query_id IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_events_project_query ON events(project, query_id);
|
||||
CREATE INDEX idx_events_run ON events(run_id);
|
||||
CREATE INDEX idx_events_type ON events(event_type);
|
||||
CREATE INDEX idx_memory_node_project_level ON memory_node(project, level);
|
||||
CREATE INDEX idx_memory_node_sha256 ON memory_node(sha256);
|
||||
|
||||
-- L0: Evidence chunks (raw, with source reference)
|
||||
CREATE TABLE IF NOT EXISTS chunks_l0 (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL,
|
||||
query_id VARCHAR NOT NULL,
|
||||
source VARCHAR NOT NULL, -- "pi", "claude", "transcript"
|
||||
content TEXT NOT NULL,
|
||||
tokens INT NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
-- Memory edges (provenance graph: child → parent)
|
||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
||||
child_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
parent_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
PRIMARY KEY (child_sha, parent_sha)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_chunks_l0_project_query ON chunks_l0(project, query_id);
|
||||
CREATE INDEX idx_memory_edge_parent ON memory_edge(parent_sha);
|
||||
|
||||
-- L1: Per-query memories (one per standing query, up to 1024 tokens)
|
||||
CREATE TABLE IF NOT EXISTS memories_l1 (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL,
|
||||
query_id VARCHAR NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
tokens INT NOT NULL,
|
||||
embedding vector(768), -- nomic-embed-text-v2-moe
|
||||
chunks_seen INT NOT NULL,
|
||||
chunks_used INT NOT NULL,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
run_id VARCHAR NOT NULL,
|
||||
UNIQUE(project, query_id)
|
||||
-- Vector storage (one node can have multiple kinds: text + symptom)
|
||||
CREATE TABLE IF NOT EXISTS memory_vector (
|
||||
node_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('text', 'symptom')),
|
||||
embedding vector(768) NOT NULL,
|
||||
PRIMARY KEY (node_sha, kind)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_memories_l1_project ON memories_l1(project);
|
||||
CREATE INDEX idx_memories_l1_embedding ON memories_l1 USING ivfflat (embedding vector_cosine_ops);
|
||||
-- Partial indexes per kind (literal predicate required for planner to use them)
|
||||
CREATE INDEX idx_memory_vector_text ON memory_vector USING hnsw (embedding vector_cosine_ops) WHERE kind = 'text';
|
||||
CREATE INDEX idx_memory_vector_symptom ON memory_vector USING hnsw (embedding vector_cosine_ops) WHERE kind = 'symptom';
|
||||
|
||||
-- L1 -> L0 provenance (which evidence chunks produced this memory)
|
||||
CREATE TABLE IF NOT EXISTS l1_l0_edges (
|
||||
l1_id UUID REFERENCES memories_l1(id) ON DELETE CASCADE,
|
||||
l0_id UUID REFERENCES chunks_l0(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (l1_id, l0_id)
|
||||
-- Exact-match failure signature tier (M3.7.7)
|
||||
CREATE TABLE IF NOT EXISTS failure_signature (
|
||||
sig_sha TEXT PRIMARY KEY, -- hash of normalized signature
|
||||
node_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
tool TEXT NOT NULL, -- 'github-actions' | 'kubectl' | 'npm' | ...
|
||||
raw TEXT NOT NULL, -- pre-normalisation, for display
|
||||
seen_count INT NOT NULL DEFAULT 1,
|
||||
last_seen TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
-- L2: Project synthesis (one per project, up to 1024 tokens)
|
||||
CREATE TABLE IF NOT EXISTS memories_l2 (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL UNIQUE,
|
||||
content TEXT NOT NULL,
|
||||
tokens INT NOT NULL,
|
||||
embedding vector(768),
|
||||
l1_count INT NOT NULL, -- how many L1 memories were used
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
run_id VARCHAR NOT NULL
|
||||
CREATE INDEX idx_failure_signature_tool ON failure_signature(tool);
|
||||
CREATE INDEX idx_failure_signature_node ON failure_signature(node_sha);
|
||||
|
||||
-- Memory supersession (old lesson replaced by new one)
|
||||
CREATE TABLE IF NOT EXISTS memory_supersede (
|
||||
old_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
new_sha TEXT NOT NULL REFERENCES memory_node(sha256) ON DELETE CASCADE,
|
||||
reason TEXT,
|
||||
PRIMARY KEY (old_sha, new_sha)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_memories_l2_project ON memories_l2(project);
|
||||
CREATE INDEX idx_memories_l2_embedding ON memories_l2 USING ivfflat (embedding vector_cosine_ops);
|
||||
|
||||
-- L2 -> L1 provenance (which L1 memories produced this synthesis)
|
||||
CREATE TABLE IF NOT EXISTS l2_l1_edges (
|
||||
l2_id UUID REFERENCES memories_l2(id) ON DELETE CASCADE,
|
||||
l1_id UUID REFERENCES memories_l1(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (l2_id, l1_id)
|
||||
);
|
||||
|
||||
-- Reference corpus (not gated, used in queries)
|
||||
CREATE TABLE IF NOT EXISTS reference_corpus (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL, -- doc name or skill name
|
||||
content TEXT NOT NULL,
|
||||
embedding vector(768),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(project, name)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_corpus_project ON reference_corpus(project);
|
||||
CREATE INDEX idx_corpus_embedding ON reference_corpus USING ivfflat (embedding vector_cosine_ops);
|
||||
|
||||
-- Ingest jobs (async queue)
|
||||
CREATE TABLE IF NOT EXISTS ingest_jobs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL,
|
||||
ingest_id VARCHAR NOT NULL UNIQUE,
|
||||
status VARCHAR NOT NULL DEFAULT 'pending', -- pending, processing, done, failed
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
error TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX idx_ingest_jobs_project ON ingest_jobs(project);
|
||||
CREATE INDEX idx_ingest_jobs_status ON ingest_jobs(status);
|
||||
|
||||
-- Skills extracted from memories
|
||||
CREATE TABLE IF NOT EXISTS skills (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project VARCHAR NOT NULL,
|
||||
name VARCHAR NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
when_to_use TEXT,
|
||||
examples TEXT,
|
||||
l1_source UUID NOT NULL REFERENCES memories_l1(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(project, name)
|
||||
);
|
||||
|
||||
CREATE INDEX idx_skills_project ON skills(project);
|
||||
CREATE INDEX idx_memory_supersede_new ON memory_supersede(new_sha);
|
||||
|
||||
Reference in New Issue
Block a user