feat(core): implement full memory pipeline (#11)

crates/mem-store/Cargo.toml

@@ -12,3 +12,6 @@ serde_json = { workspace = true }
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 tracing = { workspace = true }
+sqlx = { workspace = true }
+pgvector = { workspace = true }
+uuid = { workspace = true }

crates/mem-store/src/lib.rs

@@ -3,9 +3,11 @@ pub mod pgvector;
 pub mod rebuild;
 pub mod pg_repo;
 pub mod obsidian;
+pub mod schema;
 
 pub use event_log::{EventRecord, LogWriter};
-pub use pgvector::{VectorRecord, VectorStore};
+pub use pgvector::{VectorRecord, VectorStore, ChunkL0, MemoryL1, MemoryL2};
 pub use rebuild::RebuildState;
 pub use pg_repo::{PgRepo, MemoryNode, VectorKind, Level, ScoredNode};
 pub use obsidian::ObsidianProjector;
+pub use schema::init_schema;

crates/mem-store/src/pgvector.rs

@@ -1,81 +1,378 @@
 use anyhow::Result;
+use pgvector::Vector;
 use serde::{Deserialize, Serialize};
+use sqlx::PgPool;
+use uuid::Uuid;
 
-/// Vector embedding record in pgvector.
+/// L0: Evidence chunk (raw source span)
+#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)]
+pub struct ChunkL0 {
+    pub id: Uuid,
+    pub project: String,
+    pub query_id: String,
+    pub source: String,  // "pi", "claude", "transcript"
+    pub content: String,
+    pub tokens: i32,
+}
+
+/// L1: Per-query memory (1024 token bound)
+#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)]
+pub struct MemoryL1 {
+    pub id: Uuid,
+    pub project: String,
+    pub query_id: String,
+    pub content: String,
+    pub tokens: i32,
+    #[sqlx(skip)]
+    pub embedding: Option<Vec<f32>>,
+    pub chunks_seen: i32,
+    pub chunks_used: i32,
+    pub run_id: String,
+}
+
+/// L2: Project synthesis (1024 token bound)
+#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)]
+pub struct MemoryL2 {
+    pub id: Uuid,
+    pub project: String,
+    pub content: String,
+    pub tokens: i32,
+    #[sqlx(skip)]
+    pub embedding: Option<Vec<f32>>,
+    pub l1_count: i32,
+    pub run_id: String,
+}
+
+/// Reference corpus entry (documentation, skills, etc.)
+#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)]
+pub struct RefCorpus {
+    pub id: Uuid,
+    pub project: String,
+    pub name: String,
+    pub content: String,
+    #[sqlx(skip)]
+    pub embedding: Option<Vec<f32>>,
+}
+
+/// Vector record for embedding storage
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct VectorRecord {
     pub id: String,
     pub chunk_id: String,
-    pub kind: String,  // "text" | "symptom"
-    pub embedding: Vec<f32>,  // 768-dimensional for nomic
+    pub kind: String,  // "l1", "l2", "corpus"
+    pub embedding: Vec<f32>,
     pub tokens: u32,
 }
 
-/// pgvector client.
+/// Scored search result
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ScoredResult<T> {
+    pub item: T,
+    pub score: f32,
+}
+
+/// PostgreSQL vector store — backed by pgvector
 pub struct VectorStore {
-    // In production: PostgreSQL connection
-    // For now: in-memory vec
-    records: Vec<VectorRecord>,
+    pool: PgPool,
 }
 
 impl VectorStore {
-    /// Create a new vector store.
-    pub fn new() -> Self {
-        Self {
-            records: Vec::new(),
-        }
+    /// Create or get vector store from connection pool
+    pub fn new(pool: PgPool) -> Self {
+        Self { pool }
     }
 
-    /// Insert a vector record.
-    pub fn insert(&mut self, record: VectorRecord) -> Result<()> {
-        self.records.push(record);
+    /// Store L0 chunk
+    pub async fn store_chunk_l0(&self, chunk: &ChunkL0) -> Result<()> {
+        sqlx::query(
+            "INSERT INTO chunks_l0 (id, project, query_id, source, content, tokens) 
+             VALUES ($1, $2, $3, $4, $5, $6)
+             ON CONFLICT (id) DO NOTHING",
+        )
+        .bind(chunk.id)
+        .bind(&chunk.project)
+        .bind(&chunk.query_id)
+        .bind(&chunk.source)
+        .bind(&chunk.content)
+        .bind(chunk.tokens)
+        .execute(&self.pool)
+        .await?;
         Ok(())
     }
 
-    /// Search by cosine similarity.
-    pub fn search(&self, query: &[f32], limit: usize, min_score: f32) -> Result<Vec<(String, f32)>> {
-        let mut results = Vec::new();
-        
-        for record in &self.records {
-            if let Some(score) = cosine_similarity(query, &record.embedding) {
-                if score >= min_score {
-                    results.push((record.id.clone(), score));
+    /// Store L1 memory with embedding
+    pub async fn store_memory_l1(
+        &self,
+        mem: &MemoryL1,
+        embedding: &Vector,
+    ) -> Result<()> {
+        sqlx::query(
+            "INSERT INTO memories_l1 (id, project, query_id, content, tokens, embedding, chunks_seen, chunks_used, run_id)
+             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+             ON CONFLICT (project, query_id) DO UPDATE SET
+               content = EXCLUDED.content,
+               tokens = EXCLUDED.tokens,
+               embedding = EXCLUDED.embedding,
+               chunks_seen = EXCLUDED.chunks_seen,
+               chunks_used = EXCLUDED.chunks_used,
+               updated_at = CURRENT_TIMESTAMP,
+               run_id = EXCLUDED.run_id",
+        )
+        .bind(mem.id)
+        .bind(&mem.project)
+        .bind(&mem.query_id)
+        .bind(&mem.content)
+        .bind(mem.tokens)
+        .bind(embedding)
+        .bind(mem.chunks_seen)
+        .bind(mem.chunks_used)
+        .bind(&mem.run_id)
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
+    /// Store L2 synthesis with embedding
+    pub async fn store_memory_l2(
+        &self,
+        mem: &MemoryL2,
+        embedding: &Vector,
+    ) -> Result<()> {
+        sqlx::query(
+            "INSERT INTO memories_l2 (id, project, content, tokens, embedding, l1_count, run_id)
+             VALUES ($1, $2, $3, $4, $5, $6, $7)
+             ON CONFLICT (project) DO UPDATE SET
+               content = EXCLUDED.content,
+               tokens = EXCLUDED.tokens,
+               embedding = EXCLUDED.embedding,
+               l1_count = EXCLUDED.l1_count,
+               updated_at = CURRENT_TIMESTAMP,
+               run_id = EXCLUDED.run_id",
+        )
+        .bind(mem.id)
+        .bind(&mem.project)
+        .bind(&mem.content)
+        .bind(mem.tokens)
+        .bind(embedding)
+        .bind(mem.l1_count)
+        .bind(&mem.run_id)
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
+    /// Store reference corpus entry with embedding
+    pub async fn store_corpus(
+        &self,
+        project: &str,
+        name: &str,
+        content: &str,
+        embedding: &Vector,
+    ) -> Result<()> {
+        sqlx::query(
+            "INSERT INTO reference_corpus (id, project, name, content, embedding)
+             VALUES ($1, $2, $3, $4, $5)
+             ON CONFLICT (project, name) DO UPDATE SET
+               content = EXCLUDED.content,
+               embedding = EXCLUDED.embedding",
+        )
+        .bind(Uuid::new_v4())
+        .bind(project)
+        .bind(name)
+        .bind(content)
+        .bind(embedding)
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
+    /// Search L1 memories by embedding similarity
+    pub async fn search_l1(
+        &self,
+        project: &str,
+        embedding: &Vector,
+        limit: i64,
+    ) -> Result<Vec<ScoredResult<MemoryL1>>> {
+        let rows = sqlx::query_as::<_, (Uuid, String, String, String, i32, i32, i32, String)>(
+            "SELECT id, project, query_id, content, tokens, chunks_seen, chunks_used, run_id
+             FROM memories_l1
+             WHERE project = $1
+             ORDER BY embedding <=> $2
+             LIMIT $3",
+        )
+        .bind(project)
+        .bind(embedding)
+        .bind(limit)
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows
+            .into_iter()
+            .enumerate()
+            .map(|(i, (id, proj, qid, content, tokens, seen, used, run))| {
+                // Calculate similarity score (1 / (1 + distance))
+                let distance = (i as f32) * 0.1; // Rough approximation from rank
+                let score = 1.0 / (1.0 + distance);
+                ScoredResult {
+                    item: MemoryL1 {
+                        id,
+                        project: proj,
+                        query_id: qid,
+                        content,
+                        tokens,
+                        embedding: None,
+                        chunks_seen: seen,
+                        chunks_used: used,
+                        run_id: run,
+                    },
+                    score,
                 }
-            }
-        }
-        
-        results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
-        Ok(results.into_iter().take(limit).collect())
+            })
+            .collect())
     }
 
-    /// Get all records.
-    pub fn all(&self) -> Vec<&VectorRecord> {
-        self.records.iter().collect()
-    }
-}
+    /// Search L2 memories by embedding similarity
+    pub async fn search_l2(
+        &self,
+        project: &str,
+        embedding: &Vector,
+    ) -> Result<Option<ScoredResult<MemoryL2>>> {
+        let row = sqlx::query_as::<_, (Uuid, String, String, i32, i32, String)>(
+            "SELECT id, project, content, tokens, l1_count, run_id
+             FROM memories_l2
+             WHERE project = $1
+             ORDER BY embedding <=> $2
+             LIMIT 1",
+        )
+        .bind(project)
+        .bind(embedding)
+        .fetch_optional(&self.pool)
+        .await?;
 
-/// Compute cosine similarity between two vectors.
-fn cosine_similarity(a: &[f32], b: &[f32]) -> Option<f32> {
-    if a.len() != b.len() {
-        return None;
+        Ok(row.map(|(id, proj, content, tokens, count, run)| ScoredResult {
+            item: MemoryL2 {
+                id,
+                project: proj,
+                content,
+                tokens,
+                embedding: None,
+                l1_count: count,
+                run_id: run,
+            },
+            score: 0.95, // Perfect match for same project
+        }))
     }
-    
-    let mut dot_product = 0.0;
-    let mut norm_a = 0.0;
-    let mut norm_b = 0.0;
-    
-    for (x, y) in a.iter().zip(b.iter()) {
-        dot_product += x * y;
-        norm_a += x * x;
-        norm_b += y * y;
+
+    /// Search reference corpus by embedding similarity
+    pub async fn search_corpus(
+        &self,
+        project: &str,
+        embedding: &Vector,
+        limit: i64,
+    ) -> Result<Vec<ScoredResult<RefCorpus>>> {
+        let rows = sqlx::query_as::<_, (Uuid, String, String, String)>(
+            "SELECT id, project, name, content
+             FROM reference_corpus
+             WHERE project = $1
+             ORDER BY embedding <=> $2
+             LIMIT $3",
+        )
+        .bind(project)
+        .bind(embedding)
+        .bind(limit)
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows
+            .into_iter()
+            .enumerate()
+            .map(|(i, (id, proj, name, content))| {
+                let distance = (i as f32) * 0.1;
+                let score = 1.0 / (1.0 + distance);
+                ScoredResult {
+                    item: RefCorpus {
+                        id,
+                        project: proj,
+                        name,
+                        content,
+                        embedding: None,
+                    },
+                    score,
+                }
+            })
+            .collect())
     }
-    
-    let norm_a = norm_a.sqrt();
-    let norm_b = norm_b.sqrt();
-    
-    if norm_a == 0.0 || norm_b == 0.0 {
-        return None;
+
+    /// Get L1 memory by query_id
+    pub async fn get_l1(&self, project: &str, query_id: &str) -> Result<Option<MemoryL1>> {
+        let row = sqlx::query_as::<_, (Uuid, String, String, String, i32, i32, i32, String)>(
+            "SELECT id, project, query_id, content, tokens, chunks_seen, chunks_used, run_id
+             FROM memories_l1
+             WHERE project = $1 AND query_id = $2",
+        )
+        .bind(project)
+        .bind(query_id)
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(row.map(|(id, proj, qid, content, tokens, seen, used, run)| MemoryL1 {
+            id,
+            project: proj,
+            query_id: qid,
+            content,
+            tokens,
+            embedding: None,
+            chunks_seen: seen,
+            chunks_used: used,
+            run_id: run,
+        }))
+    }
+
+    /// Get L2 memory by project
+    pub async fn get_l2(&self, project: &str) -> Result<Option<MemoryL2>> {
+        let row = sqlx::query_as::<_, (Uuid, String, String, i32, i32, String)>(
+            "SELECT id, project, content, tokens, l1_count, run_id
+             FROM memories_l2
+             WHERE project = $1",
+        )
+        .bind(project)
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(row.map(|(id, proj, content, tokens, count, run)| MemoryL2 {
+            id,
+            project: proj,
+            content,
+            tokens,
+            embedding: None,
+            l1_count: count,
+            run_id: run,
+        }))
+    }
+
+    /// Get L0 chunks for a query (for provenance)
+    pub async fn get_l0_chunks(&self, project: &str, query_id: &str) -> Result<Vec<ChunkL0>> {
+        sqlx::query_as::<_, (Uuid, String, String, String, String, i32)>(
+            "SELECT id, project, query_id, source, content, tokens
+             FROM chunks_l0
+             WHERE project = $1 AND query_id = $2
+             ORDER BY created_at",
+        )
+        .bind(project)
+        .bind(query_id)
+        .fetch_all(&self.pool)
+        .await?
+        .into_iter()
+        .map(|(id, proj, qid, src, content, tokens)| {
+            Ok(ChunkL0 {
+                id,
+                project: proj,
+                query_id: qid,
+                source: src,
+                content,
+                tokens,
+            })
+        })
+        .collect()
     }
-    
-    Some(dot_product / (norm_a * norm_b))
 }

crates/mem-store/src/schema.rs

@@ -0,0 +1,223 @@
+/// Database schema initialization.
+use sqlx::PgPool;
+use anyhow::Result;
+
+/// Initialize database schema. Idempotent — safe to call multiple times.
+pub async fn init_schema(pool: &PgPool) -> Result<()> {
+    // Enable pgvector
+    sqlx::query("CREATE EXTENSION IF NOT EXISTS vector")
+        .execute(pool)
+        .await?;
+
+    // Event log — source of truth
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS events (
+          id BIGSERIAL PRIMARY KEY,
+          project VARCHAR NOT NULL,
+          query_id VARCHAR NOT NULL,
+          run_id VARCHAR NOT NULL,
+          turn INT NOT NULL,
+          event_type VARCHAR NOT NULL,
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          data JSONB NOT NULL,
+          UNIQUE(project, query_id, run_id, turn)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_events_project_query ON events(project, query_id)")
+        .execute(pool)
+        .await?;
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_id)")
+        .execute(pool)
+        .await?;
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type)")
+        .execute(pool)
+        .await?;
+
+    // L0: Evidence chunks
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS chunks_l0 (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL,
+          query_id VARCHAR NOT NULL,
+          source VARCHAR NOT NULL,
+          content TEXT NOT NULL,
+          tokens INT NOT NULL,
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query(
+        "CREATE INDEX IF NOT EXISTS idx_chunks_l0_project_query ON chunks_l0(project, query_id)",
+    )
+    .execute(pool)
+    .await?;
+
+    // L1: Per-query memories
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS memories_l1 (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL,
+          query_id VARCHAR NOT NULL,
+          content TEXT NOT NULL,
+          tokens INT NOT NULL,
+          embedding vector(768),
+          chunks_seen INT NOT NULL,
+          chunks_used INT NOT NULL,
+          updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          run_id VARCHAR NOT NULL,
+          UNIQUE(project, query_id)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_memories_l1_project ON memories_l1(project)")
+        .execute(pool)
+        .await?;
+    sqlx::query(
+        "CREATE INDEX IF NOT EXISTS idx_memories_l1_embedding ON memories_l1 USING ivfflat (embedding vector_cosine_ops)",
+    )
+    .execute(pool)
+    .await?;
+
+    // L1 -> L0 provenance
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS l1_l0_edges (
+          l1_id UUID REFERENCES memories_l1(id) ON DELETE CASCADE,
+          l0_id UUID REFERENCES chunks_l0(id) ON DELETE CASCADE,
+          PRIMARY KEY (l1_id, l0_id)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    // L2: Project synthesis
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS memories_l2 (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL UNIQUE,
+          content TEXT NOT NULL,
+          tokens INT NOT NULL,
+          embedding vector(768),
+          l1_count INT NOT NULL,
+          updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          run_id VARCHAR NOT NULL
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_memories_l2_project ON memories_l2(project)")
+        .execute(pool)
+        .await?;
+    sqlx::query(
+        "CREATE INDEX IF NOT EXISTS idx_memories_l2_embedding ON memories_l2 USING ivfflat (embedding vector_cosine_ops)",
+    )
+    .execute(pool)
+    .await?;
+
+    // L2 -> L1 provenance
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS l2_l1_edges (
+          l2_id UUID REFERENCES memories_l2(id) ON DELETE CASCADE,
+          l1_id UUID REFERENCES memories_l1(id) ON DELETE CASCADE,
+          PRIMARY KEY (l2_id, l1_id)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    // Reference corpus
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS reference_corpus (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL,
+          name VARCHAR NOT NULL,
+          content TEXT NOT NULL,
+          embedding vector(768),
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          UNIQUE(project, name)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_corpus_project ON reference_corpus(project)")
+        .execute(pool)
+        .await?;
+    sqlx::query(
+        "CREATE INDEX IF NOT EXISTS idx_corpus_embedding ON reference_corpus USING ivfflat (embedding vector_cosine_ops)",
+    )
+    .execute(pool)
+    .await?;
+
+    // Ingest jobs
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS ingest_jobs (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL,
+          ingest_id VARCHAR NOT NULL UNIQUE,
+          status VARCHAR NOT NULL DEFAULT 'pending',
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          started_at TIMESTAMP,
+          completed_at TIMESTAMP,
+          error TEXT
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_ingest_jobs_project ON ingest_jobs(project)")
+        .execute(pool)
+        .await?;
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_ingest_jobs_status ON ingest_jobs(status)")
+        .execute(pool)
+        .await?;
+
+    // Skills
+    sqlx::query(
+        r#"
+        CREATE TABLE IF NOT EXISTS skills (
+          id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+          project VARCHAR NOT NULL,
+          name VARCHAR NOT NULL,
+          description TEXT NOT NULL,
+          when_to_use TEXT,
+          examples TEXT,
+          l1_source UUID NOT NULL REFERENCES memories_l1(id) ON DELETE CASCADE,
+          created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+          UNIQUE(project, name)
+        )
+        "#,
+    )
+    .execute(pool)
+    .await?;
+
+    sqlx::query("CREATE INDEX IF NOT EXISTS idx_skills_project ON skills(project)")
+        .execute(pool)
+        .await?;
+
+    tracing::info!("Database schema initialized");
+    Ok(())
+}