test: production ingest E2E test suite with enhanced logging (#55)
CI / CI (push) Successful in 25m7s
Deploy / Tag & Push Latest (push) Successful in 3m49s

## Summary
Production testing of ingest + embedding pipeline with api-gw integration.

## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.

Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects

Evidence from logs:
```
WARN: Failed to save entity Docker:
      error returned from database: relation "memory_entity" does not exist
```

## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking

## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely

**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)

## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```

Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #55.
This commit is contained in:
2026-09-16 00:10:58 +00:00
committed by rock
co-authored by rock
parent 4169effd8a
commit a88ea918bf
137 changed files with 4628 additions and 7227 deletions
@@ -1,52 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1\n ORDER BY version_num DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_num",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operation",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "snapshot",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "changed_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "changed_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "fields_changed!",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "1e81bb729531ca33e4cef21623bcfe4fafb0c1bd435353b205f582bfda8873bc"
}
@@ -1,52 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1\n ORDER BY version_num DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_num",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operation",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "snapshot",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "changed_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "changed_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "fields_changed!",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "62d65d4afc4d292b37de8e5cb59fbd51c602bdc1b437988f54e6c7fe268b9816"
}
@@ -1,53 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND changed_at <= $2\n ORDER BY version_num DESC\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_num",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operation",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "snapshot",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "changed_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "changed_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "fields_changed!",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Text",
"Timestamptz"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "aee5900f5e3d7cbba23729bbf2dd033dcc4cb41f6c851bf447a9238810684d18"
}
@@ -1,53 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND version_num = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_num",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operation",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "snapshot",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "changed_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "changed_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "fields_changed!",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Text",
"Int4"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "c045466e1fe037dbdafea1008f262f4e48f104ea77732aa1d32ecb797f70e71d"
}
@@ -1,53 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1 AND version_num = $2\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version_num",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operation",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "snapshot",
"type_info": "Jsonb"
},
{
"ordinal": 3,
"name": "changed_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "changed_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "fields_changed!",
"type_info": "TextArray"
}
],
"parameters": {
"Left": [
"Uuid",
"Int4"
]
},
"nullable": [
false,
false,
false,
false,
false,
null
]
},
"hash": "ca6872495bc04c6a65531279af8c758637c902dda2cc10366662988c6973ca48"
}
@@ -1,20 +1,20 @@
-- Migration 009: Temporal edge schema (Zep paper §2.2.2)
-- Replaces old memory_edge (child_sha/parent_sha node graph)
-- with temporal edge schema supporting relation types, facts, and validity periods.
-- Migration 009: Temporal knowledge graph edge schema (Zep paper §2.2.2)
-- Replaces old memory_edge (child_sha/parent_sha provenance DAG)
-- with temporal edge schema for the knowledge graph.
-- Idempotent: safe to run multiple times.
-- Rename old table if it still exists (skip if already migrated)
-- Rename old provenance DAG table if it still has child_sha columns
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_edge'
AND EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_name = 'memory_edge' AND column_name = 'child_sha'))
THEN
ALTER TABLE memory_edge RENAME TO memory_edge_legacy;
ALTER TABLE memory_edge RENAME TO memory_edge_provenance;
END IF;
END $$;
-- Create temporal edge table
-- Create temporal knowledge graph edge table
CREATE TABLE IF NOT EXISTS memory_edge (
id TEXT PRIMARY KEY,
project_id TEXT NOT NULL DEFAULT 'default',
@@ -64,4 +64,5 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_project_name ON memory_entit
-- ROLLBACK instructions:
-- DROP TABLE IF EXISTS memory_edge;
-- ALTER TABLE IF EXISTS memory_edge_legacy RENAME TO memory_edge;
-- ALTER TABLE IF EXISTS memory_edge_provenance RENAME TO memory_edge;
-- DROP INDEX IF EXISTS idx_memory_entity_project_name;
+358
View File
@@ -0,0 +1,358 @@
use anyhow::Result;
use sqlx::{PgPool, FromRow};
use uuid::Uuid;
use serde::{Deserialize, Serialize};
use chrono::{DateTime, Utc};
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct AgentPrompt {
pub id: Uuid,
pub project_id: String,
pub name: String,
pub template: String,
pub target_model: Option<String>,
pub task_category: String,
pub usage_count: i64,
pub avg_quality: f32,
pub last_used: Option<DateTime<Utc>>,
pub active: bool,
pub version: i32,
pub tags: Vec<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct AgentSkill {
pub id: Uuid,
pub project_id: String,
pub agent_id: String,
pub name: String,
pub description: String,
pub trigger_patterns: Vec<String>,
pub success_rate: f32,
pub invocation_count: i64,
pub avg_latency_ms: i64,
pub linked_prompts: Vec<Uuid>,
pub enabled: bool,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct AgentDecision {
pub id: Uuid,
pub project_id: String,
pub agent_id: String,
pub action: String,
pub reasoning: String,
pub alternatives: Vec<String>,
pub confidence: f32,
pub context_entities: Vec<Uuid>,
pub tool: Option<String>,
pub task: Option<String>,
pub outcome_success: Option<bool>,
pub outcome_quality: Option<f32>,
pub outcome_feedback: Option<String>,
pub outcome_recorded_at: Option<DateTime<Utc>>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct RolePromptMapping {
pub id: Uuid,
pub project_id: String,
pub role_name: String,
pub prompt_id: Uuid,
pub priority: i32,
pub active: bool,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct AgentMetrics {
pub id: Uuid,
pub project_id: String,
pub agent_id: String,
pub requests_total: i64,
pub requests_success: i64,
pub requests_failed: i64,
pub average_latency_ms: f32,
pub p95_latency_ms: f32,
pub p99_latency_ms: f32,
pub error_rate: f32,
pub recorded_at: DateTime<Utc>,
}
pub struct AgentRepository {
pool: PgPool,
}
impl AgentRepository {
pub fn new(pool: PgPool) -> Self {
AgentRepository { pool }
}
pub async fn create_prompt(&self, prompt: AgentPrompt) -> Result<AgentPrompt> {
let result = sqlx::query_as::<_, AgentPrompt>(
r#"
INSERT INTO agent_prompt
(project_id, name, template, target_model, task_category, active, version, tags)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING *
"#,
)
.bind(&prompt.project_id)
.bind(&prompt.name)
.bind(&prompt.template)
.bind(&prompt.target_model)
.bind(&prompt.task_category)
.bind(prompt.active)
.bind(prompt.version)
.bind(&prompt.tags)
.fetch_one(&self.pool)
.await?;
Ok(result)
}
pub async fn get_prompt(&self, id: Uuid) -> Result<Option<AgentPrompt>> {
let result = sqlx::query_as::<_, AgentPrompt>(
"SELECT * FROM agent_prompt WHERE id = $1"
)
.bind(id)
.fetch_optional(&self.pool)
.await?;
Ok(result)
}
pub async fn list_prompts(&self, project_id: &str) -> Result<Vec<AgentPrompt>> {
let results = sqlx::query_as::<_, AgentPrompt>(
"SELECT * FROM agent_prompt WHERE project_id = $1 AND active = true ORDER BY created_at DESC"
)
.bind(project_id)
.fetch_all(&self.pool)
.await?;
Ok(results)
}
pub async fn update_prompt_usage(&self, id: Uuid, quality_score: f32) -> Result<()> {
sqlx::query(
r#"
UPDATE agent_prompt
SET usage_count = usage_count + 1,
avg_quality = (avg_quality * (usage_count) + $2) / (usage_count + 1),
last_used = NOW(),
updated_at = NOW()
WHERE id = $1
"#,
)
.bind(id)
.bind(quality_score)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn create_skill(&self, skill: AgentSkill) -> Result<AgentSkill> {
let result = sqlx::query_as::<_, AgentSkill>(
r#"
INSERT INTO agent_skill
(project_id, agent_id, name, description, enabled)
VALUES ($1, $2, $3, $4, $5)
RETURNING *
"#,
)
.bind(&skill.project_id)
.bind(&skill.agent_id)
.bind(&skill.name)
.bind(&skill.description)
.bind(skill.enabled)
.fetch_one(&self.pool)
.await?;
Ok(result)
}
pub async fn get_skill(&self, id: Uuid) -> Result<Option<AgentSkill>> {
let result = sqlx::query_as::<_, AgentSkill>(
"SELECT * FROM agent_skill WHERE id = $1"
)
.bind(id)
.fetch_optional(&self.pool)
.await?;
Ok(result)
}
pub async fn list_skills(&self, project_id: &str, agent_id: &str) -> Result<Vec<AgentSkill>> {
let results = sqlx::query_as::<_, AgentSkill>(
"SELECT * FROM agent_skill WHERE project_id = $1 AND agent_id = $2 AND enabled = true ORDER BY created_at DESC"
)
.bind(project_id)
.bind(agent_id)
.fetch_all(&self.pool)
.await?;
Ok(results)
}
pub async fn create_decision(&self, decision: AgentDecision) -> Result<AgentDecision> {
let result = sqlx::query_as::<_, AgentDecision>(
r#"
INSERT INTO agent_decision
(project_id, agent_id, action, reasoning, confidence, tool, task)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING *
"#,
)
.bind(&decision.project_id)
.bind(&decision.agent_id)
.bind(&decision.action)
.bind(&decision.reasoning)
.bind(decision.confidence)
.bind(&decision.tool)
.bind(&decision.task)
.fetch_one(&self.pool)
.await?;
Ok(result)
}
pub async fn record_decision_outcome(
&self,
id: Uuid,
success: bool,
quality: f32,
feedback: Option<&str>,
) -> Result<()> {
sqlx::query(
r#"
UPDATE agent_decision
SET outcome_success = $2,
outcome_quality = $3,
outcome_feedback = $4,
outcome_recorded_at = NOW(),
updated_at = NOW()
WHERE id = $1
"#,
)
.bind(id)
.bind(success)
.bind(quality)
.bind(feedback)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn create_role_mapping(&self, mapping: RolePromptMapping) -> Result<RolePromptMapping> {
let result = sqlx::query_as::<_, RolePromptMapping>(
r#"
INSERT INTO role_prompt_mapping
(project_id, role_name, prompt_id, priority, active)
VALUES ($1, $2, $3, $4, $5)
RETURNING *
"#,
)
.bind(&mapping.project_id)
.bind(&mapping.role_name)
.bind(mapping.prompt_id)
.bind(mapping.priority)
.bind(mapping.active)
.fetch_one(&self.pool)
.await?;
Ok(result)
}
pub async fn get_prompts_for_role(&self, project_id: &str, role_name: &str) -> Result<Vec<AgentPrompt>> {
let results = sqlx::query_as::<_, AgentPrompt>(
r#"
SELECT ap.* FROM agent_prompt ap
INNER JOIN role_prompt_mapping rpm ON ap.id = rpm.prompt_id
WHERE rpm.project_id = $1 AND rpm.role_name = $2 AND rpm.active = true
ORDER BY rpm.priority DESC, ap.created_at DESC
"#,
)
.bind(project_id)
.bind(role_name)
.fetch_all(&self.pool)
.await?;
Ok(results)
}
pub async fn save_metrics(&self, metrics: AgentMetrics) -> Result<()> {
sqlx::query(
r#"
INSERT INTO agent_metrics
(project_id, agent_id, requests_total, requests_success, requests_failed,
average_latency_ms, p95_latency_ms, p99_latency_ms, error_rate)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (project_id, agent_id, DATE(recorded_at)) DO UPDATE SET
requests_total = EXCLUDED.requests_total,
requests_success = EXCLUDED.requests_success,
requests_failed = EXCLUDED.requests_failed,
average_latency_ms = EXCLUDED.average_latency_ms,
p95_latency_ms = EXCLUDED.p95_latency_ms,
p99_latency_ms = EXCLUDED.p99_latency_ms,
error_rate = EXCLUDED.error_rate
"#,
)
.bind(&metrics.project_id)
.bind(&metrics.agent_id)
.bind(metrics.requests_total)
.bind(metrics.requests_success)
.bind(metrics.requests_failed)
.bind(metrics.average_latency_ms)
.bind(metrics.p95_latency_ms)
.bind(metrics.p99_latency_ms)
.bind(metrics.error_rate)
.execute(&self.pool)
.await?;
Ok(())
}
pub async fn log_prompt_usage(
&self,
project_id: &str,
prompt_id: Uuid,
agent_id: Option<&str>,
model: Option<&str>,
input_tokens: Option<i32>,
output_tokens: Option<i32>,
quality_score: Option<f32>,
duration_ms: i64,
error_message: Option<&str>,
) -> Result<()> {
sqlx::query(
r#"
INSERT INTO prompt_usage_log
(project_id, prompt_id, agent_id, model_used, input_tokens, output_tokens,
quality_score, duration_ms, error_message)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
"#,
)
.bind(project_id)
.bind(prompt_id)
.bind(agent_id)
.bind(model)
.bind(input_tokens)
.bind(output_tokens)
.bind(quality_score)
.bind(duration_ms)
.bind(error_message)
.execute(&self.pool)
.await?;
Ok(())
}
}
+1 -1
View File
@@ -1,7 +1,6 @@
use chrono::{DateTime, Utc};
use sqlx::PgPool;
use uuid::Uuid;
use serde_json::json;
/// Minimal audit logger - records version snapshots on mutation
#[derive(Clone)]
@@ -178,6 +177,7 @@ pub struct AuditedEntityRepo {
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_diff_fields_modified() {
-543
View File
@@ -1,543 +0,0 @@
/// PostgreSQL repository implementation for Phase 2.6 DB Integration.
///
/// Connects ingest pipeline to persistent storage.
/// Handles transactions, error recovery, and audit logging.
use sqlx::{Pool, Postgres, Row, Transaction, Error as SqlxError};
use serde::{Deserialize, Serialize};
use chrono::{DateTime, Utc};
use mem_core::entity::Entity;
use mem_core::edge::Edge;
use crate::entity_repo::EntityRepoOps;
use crate::edge_repo::EdgeRepoOps;
/// Database connection error types
#[derive(Debug, Clone)]
pub enum DbError {
ConnectionFailed(String),
QueryFailed(String),
TransactionFailed(String),
DuplicateKey(String),
NotFound(String),
InvalidData(String),
}
impl std::fmt::Display for DbError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
DbError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg),
DbError::QueryFailed(msg) => write!(f, "Query failed: {}", msg),
DbError::TransactionFailed(msg) => write!(f, "Transaction failed: {}", msg),
DbError::DuplicateKey(msg) => write!(f, "Duplicate key: {}", msg),
DbError::NotFound(msg) => write!(f, "Not found: {}", msg),
DbError::InvalidData(msg) => write!(f, "Invalid data: {}", msg),
}
}
}
impl std::error::Error for DbError {}
/// PostgreSQL repository pool
pub struct DbPool {
pool: Pool<Postgres>,
}
impl DbPool {
/// Create new DB pool from connection string
pub async fn new(database_url: &str) -> Result<Self, DbError> {
let pool = Pool::<Postgres>::connect(database_url)
.await
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
Ok(DbPool { pool })
}
/// Get pool for queries
pub fn pool(&self) -> &Pool<Postgres> {
&self.pool
}
/// Test connection
pub async fn health_check(&self) -> Result<(), DbError> {
sqlx::query("SELECT 1")
.fetch_one(&self.pool)
.await
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
Ok(())
}
}
/// Persistent entity repository
pub struct PersistentEntityRepo {
pool: Pool<Postgres>,
}
impl PersistentEntityRepo {
pub fn new(pool: Pool<Postgres>) -> Self {
Self { pool }
}
/// Save entity to database (idempotent)
pub async fn save(&self, entity: &Entity) -> Result<String, DbError> {
let query = r#"
INSERT INTO memory_entity (id, entity_type, name, description, embedding, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT(id) DO UPDATE SET
name = EXCLUDED.name,
description = EXCLUDED.description,
updated_at = EXCLUDED.updated_at
RETURNING id;
"#;
let id = sqlx::query_scalar::<_, String>(query)
.bind(&entity.id)
.bind(&entity.entity_type)
.bind(&entity.name)
.bind(&entity.description)
.bind(&entity.embedding)
.bind(Utc::now())
.bind(Utc::now())
.fetch_one(&self.pool)
.await
.map_err(|e| {
if e.to_string().contains("duplicate") {
DbError::DuplicateKey(format!("Entity {} already exists", entity.id))
} else {
DbError::QueryFailed(e.to_string())
}
})?;
Ok(id)
}
/// Get entity by ID
pub async fn get(&self, id: &str) -> Result<Option<Entity>, DbError> {
let query = r#"
SELECT id, entity_type, name, description, embedding, created_at, updated_at
FROM memory_entity
WHERE id = $1 AND deleted_at IS NULL;
"#;
let row = sqlx::query(query)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(row.map(|r| Entity {
id: r.get("id"),
entity_type: r.get("entity_type"),
name: r.get("name"),
description: r.get("description"),
embedding: r.get("embedding"),
created_at: r.get("created_at"),
updated_at: r.get("updated_at"),
}))
}
/// List entities with pagination
pub async fn list(&self, limit: i64, offset: i64) -> Result<Vec<Entity>, DbError> {
let query = r#"
SELECT id, entity_type, name, description, embedding, created_at, updated_at
FROM memory_entity
WHERE deleted_at IS NULL
ORDER BY created_at DESC
LIMIT $1 OFFSET $2;
"#;
let rows = sqlx::query(query)
.bind(limit)
.bind(offset)
.fetch_all(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(rows.iter().map(|r| Entity {
id: r.get("id"),
entity_type: r.get("entity_type"),
name: r.get("name"),
description: r.get("description"),
embedding: r.get("embedding"),
created_at: r.get("created_at"),
updated_at: r.get("updated_at"),
}).collect())
}
/// Soft delete entity
pub async fn delete(&self, id: &str) -> Result<(), DbError> {
let query = r#"
UPDATE memory_entity
SET deleted_at = $1
WHERE id = $2;
"#;
sqlx::query(query)
.bind(Utc::now())
.bind(id)
.execute(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(())
}
}
/// Persistent edge repository
pub struct PersistentEdgeRepo {
pool: Pool<Postgres>,
}
impl PersistentEdgeRepo {
pub fn new(pool: Pool<Postgres>) -> Self {
Self { pool }
}
/// Save edge to database (idempotent)
pub async fn save(&self, edge: &Edge) -> Result<String, DbError> {
let query = r#"
INSERT INTO memory_edge (id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT(id) DO UPDATE SET
strength = EXCLUDED.strength,
t_invalid = EXCLUDED.t_invalid,
t_expired = EXCLUDED.t_expired
RETURNING id;
"#;
let id = sqlx::query_scalar::<_, String>(query)
.bind(&edge.id)
.bind(&edge.source_id)
.bind(&edge.target_id)
.bind(&edge.relation_type)
.bind(&edge.fact)
.bind(edge.strength)
.bind(edge.t_valid)
.bind(edge.t_invalid)
.bind(edge.t_created)
.bind(edge.t_expired)
.fetch_one(&self.pool)
.await
.map_err(|e| {
if e.to_string().contains("duplicate") {
DbError::DuplicateKey(format!("Edge {} already exists", edge.id))
} else {
DbError::QueryFailed(e.to_string())
}
})?;
Ok(id)
}
/// Get edge by ID
pub async fn get(&self, id: &str) -> Result<Option<Edge>, DbError> {
let query = r#"
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
FROM memory_edge
WHERE id = $1 AND t_expired IS NULL;
"#;
let row = sqlx::query(query)
.bind(id)
.fetch_optional(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(row.map(|r| Edge {
id: r.get("id"),
source_id: r.get("source_id"),
target_id: r.get("target_id"),
relation_type: r.get("relation_type"),
fact: r.get("fact"),
strength: r.get("strength"),
t_valid: r.get("t_valid"),
t_invalid: r.get("t_invalid"),
t_created: r.get("t_created"),
t_expired: r.get("t_expired"),
}))
}
/// List edges for a source entity
pub async fn list_from(&self, source_id: &str, limit: i64) -> Result<Vec<Edge>, DbError> {
let query = r#"
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
FROM memory_edge
WHERE source_id = $1 AND t_expired IS NULL AND t_invalid IS NULL
ORDER BY t_created DESC
LIMIT $2;
"#;
let rows = sqlx::query(query)
.bind(source_id)
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(rows.iter().map(|r| Edge {
id: r.get("id"),
source_id: r.get("source_id"),
target_id: r.get("target_id"),
relation_type: r.get("relation_type"),
fact: r.get("fact"),
strength: r.get("strength"),
t_valid: r.get("t_valid"),
t_invalid: r.get("t_invalid"),
t_created: r.get("t_created"),
t_expired: r.get("t_expired"),
}).collect())
}
/// Mark edge as contradicted (soft delete)
pub async fn invalidate(&self, id: &str) -> Result<(), DbError> {
let query = r#"
UPDATE memory_edge
SET t_invalid = $1
WHERE id = $2;
"#;
sqlx::query(query)
.bind(Utc::now())
.bind(id)
.execute(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(())
}
}
/// Review queue entry for human verification
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ReviewQueueEntry {
pub id: String,
pub extraction_type: String, // "entity" | "edge" | "contradiction"
pub content: serde_json::Value, // Full extracted data
pub status: String, // "pending" | "approved" | "rejected"
pub created_at: DateTime<Utc>,
pub reviewed_at: Option<DateTime<Utc>>,
pub reviewed_by: Option<String>, // User ID who reviewed
pub rejection_reason: Option<String>,
}
/// Review queue repository
pub struct ReviewQueueRepo {
pool: Pool<Postgres>,
}
impl ReviewQueueRepo {
pub fn new(pool: Pool<Postgres>) -> Self {
Self { pool }
}
/// Add item to review queue
pub async fn enqueue(&self, entry: &ReviewQueueEntry) -> Result<String, DbError> {
let query = r#"
INSERT INTO review_queue (id, extraction_type, content, status, created_at)
VALUES ($1, $2, $3, $4, $5)
RETURNING id;
"#;
let id = sqlx::query_scalar::<_, String>(query)
.bind(&entry.id)
.bind(&entry.extraction_type)
.bind(&entry.content)
.bind(&entry.status)
.bind(Utc::now())
.fetch_one(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(id)
}
/// Get pending items for review
pub async fn list_pending(&self, limit: i64) -> Result<Vec<ReviewQueueEntry>, DbError> {
let query = r#"
SELECT id, extraction_type, content, status, created_at, reviewed_at, reviewed_by, rejection_reason
FROM review_queue
WHERE status = 'pending'
ORDER BY created_at ASC
LIMIT $1;
"#;
let rows = sqlx::query(query)
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(rows.iter().map(|r| ReviewQueueEntry {
id: r.get("id"),
extraction_type: r.get("extraction_type"),
content: r.get("content"),
status: r.get("status"),
created_at: r.get("created_at"),
reviewed_at: r.get("reviewed_at"),
reviewed_by: r.get("reviewed_by"),
rejection_reason: r.get("rejection_reason"),
}).collect())
}
/// Approve review queue entry
pub async fn approve(&self, id: &str, reviewed_by: &str) -> Result<(), DbError> {
let query = r#"
UPDATE review_queue
SET status = 'approved', reviewed_at = $1, reviewed_by = $2
WHERE id = $3;
"#;
sqlx::query(query)
.bind(Utc::now())
.bind(reviewed_by)
.bind(id)
.execute(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(())
}
/// Reject review queue entry
pub async fn reject(&self, id: &str, reviewed_by: &str, reason: &str) -> Result<(), DbError> {
let query = r#"
UPDATE review_queue
SET status = 'rejected', reviewed_at = $1, reviewed_by = $2, rejection_reason = $3
WHERE id = $4;
"#;
sqlx::query(query)
.bind(Utc::now())
.bind(reviewed_by)
.bind(reason)
.bind(id)
.execute(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(())
}
}
/// Extraction Audit Repository (Immutable log for audit trail)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ExtractionAuditEntry {
pub id: String,
pub extraction_type: String, // "entity" | "edge"
pub extraction_id: String, // ID of extracted entity/edge
pub source_content: String, // Original text
pub extracted_data: serde_json::Value,
pub llm_confidence: Option<f32>,
pub contradiction_score: Option<f32>,
pub status: String, // "extracted" | "approved" | "rejected"
pub extracted_at: DateTime<Utc>,
pub extracted_by: String, // User or "system"
}
pub struct ExtractionAuditRepo {
pool: Pool<Postgres>,
}
impl ExtractionAuditRepo {
pub fn new(pool: Pool<Postgres>) -> Self {
Self { pool }
}
/// Log an extraction attempt (immutable append)
pub async fn log_extraction(&self, entry: &ExtractionAuditEntry) -> Result<String, DbError> {
let query = r#"
INSERT INTO extraction_audit (id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING id;
"#;
let id = sqlx::query_scalar::<_, String>(query)
.bind(&entry.id)
.bind(&entry.extraction_type)
.bind(&entry.extraction_id)
.bind(&entry.source_content)
.bind(&entry.extracted_data)
.bind(entry.llm_confidence)
.bind(entry.contradiction_score)
.bind(&entry.status)
.bind(entry.extracted_at)
.bind(&entry.extracted_by)
.fetch_one(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(id)
}
/// Get audit trail for an extracted item
pub async fn get_history(&self, extraction_id: &str) -> Result<Vec<ExtractionAuditEntry>, DbError> {
let query = r#"
SELECT id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by
FROM extraction_audit
WHERE extraction_id = $1
ORDER BY extracted_at DESC;
"#;
let rows = sqlx::query(query)
.bind(extraction_id)
.fetch_all(&self.pool)
.await
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
Ok(rows.iter().map(|r| ExtractionAuditEntry {
id: r.get("id"),
extraction_type: r.get("extraction_type"),
extraction_id: r.get("extraction_id"),
source_content: r.get("source_content"),
extracted_data: r.get("extracted_data"),
llm_confidence: r.get("llm_confidence"),
contradiction_score: r.get("contradiction_score"),
status: r.get("status"),
extracted_at: r.get("extracted_at"),
extracted_by: r.get("extracted_by"),
}).collect())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_db_error_display() {
let err = DbError::ConnectionFailed("test".to_string());
assert!(err.to_string().contains("Connection failed"));
}
#[test]
fn test_review_queue_entry_creation() {
let entry = ReviewQueueEntry {
id: "test-1".to_string(),
extraction_type: "entity".to_string(),
content: serde_json::json!({"name": "test"}),
status: "pending".to_string(),
created_at: Utc::now(),
reviewed_at: None,
reviewed_by: None,
rejection_reason: None,
};
assert_eq!(entry.extraction_type, "entity");
}
#[test]
fn test_dead_letter_entry_creation() {
let entry = DeadLetterEntry {
id: "dlq-1".to_string(),
original_content: "test content".to_string(),
error_message: "extraction failed".to_string(),
error_type: "extraction_failed".to_string(),
retry_count: 0,
max_retries: 3,
created_at: Utc::now(),
last_retry_at: None,
};
assert_eq!(entry.retry_count, 0);
assert!(entry.retry_count < entry.max_retries);
}
}
+1 -1
View File
@@ -8,7 +8,7 @@ pub mod edge_repo;
pub mod community_repo;
pub mod versioning;
pub mod audit_logger;
// pub mod db_repo; // TODO: Fix Entity schema integration
pub mod agent_repo;
pub use event_log::{EventRecord, LogWriter};
pub use pgvector::{VectorRecord, VectorStore, ChunkL0, MemoryL1, MemoryL2};
+2 -2
View File
@@ -12,7 +12,7 @@ use mem_ingest::OptimizationMetrics;
/// Memory record from log (local copy for rebuild purposes)
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MemoryRecord {
pub struct MemoryRecord {
pub level: String,
pub project: String,
pub query_id: Option<String>,
@@ -25,7 +25,7 @@ struct MemoryRecord {
/// Parent reference for provenance
#[derive(Debug, Clone, Serialize, Deserialize)]
struct MemoryParent {
pub struct MemoryParent {
pub source: String,
pub t: i32,
pub description: Option<String>,
+92 -1
View File
@@ -218,6 +218,97 @@ pub async fn init_schema(pool: &PgPool) -> Result<()> {
.execute(pool)
.await?;
tracing::info!("Database schema initialized");
// Memory entity table (temporal knowledge graph)
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS memory_entity (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
project_id VARCHAR(255) NOT NULL,
name VARCHAR(500) NOT NULL,
name_embedding VECTOR(768),
summary TEXT,
description TEXT,
summary_embedding VECTOR(768),
entity_type VARCHAR(50),
t_created TIMESTAMPTZ DEFAULT NOW(),
t_updated TIMESTAMPTZ DEFAULT NOW(),
t_expired TIMESTAMPTZ,
confidence FLOAT DEFAULT 1.0,
source_count INT DEFAULT 1,
source_episodes UUID[] DEFAULT '{}',
access_count BIGINT DEFAULT 0,
last_accessed TIMESTAMPTZ,
UNIQUE(project_id, name)
)
"#,
)
.execute(pool)
.await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_project ON memory_entity(project_id)")
.execute(pool)
.await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_type ON memory_entity(project_id, entity_type)")
.execute(pool)
.await?;
// Memory edge table (temporal knowledge graph)
sqlx::query(
r#"
CREATE TABLE IF NOT EXISTS memory_edge (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
project_id VARCHAR(255) NOT NULL,
source_id UUID NOT NULL,
target_id UUID NOT NULL,
relation_type VARCHAR(100) NOT NULL,
fact TEXT NOT NULL,
fact_embedding VECTOR(768),
t_valid TIMESTAMPTZ,
t_invalid TIMESTAMPTZ,
t_created TIMESTAMPTZ DEFAULT NOW(),
t_expired TIMESTAMPTZ,
confidence FLOAT DEFAULT 1.0,
contradiction_status VARCHAR(20) DEFAULT 'active',
contradiction_confidence FLOAT
)
"#,
)
.execute(pool)
.await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_project ON memory_edge(project_id)")
.execute(pool)
.await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_source ON memory_edge(source_id)")
.execute(pool)
.await?;
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_target ON memory_edge(target_id)")
.execute(pool)
.await?;
// HNSW vector indexes for semantic search (RAG-001)
// name_embedding: primary entity search vector
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_entity_name_emb ON memory_entity \
USING hnsw (name_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
)
.execute(pool)
.await?;
// summary_embedding: secondary entity search vector
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_entity_summary_emb ON memory_entity \
USING hnsw (summary_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
)
.execute(pool)
.await?;
// fact_embedding: edge/relationship search vector
sqlx::query(
"CREATE INDEX IF NOT EXISTS idx_edge_fact_emb ON memory_edge \
USING hnsw (fact_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
)
.execute(pool)
.await?;
tracing::info!("Database schema initialized (including memory_entity + memory_edge + HNSW indexes)");
Ok(())
}
+26 -103
View File
@@ -1,15 +1,16 @@
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use sqlx::{PgPool, FromRow};
use uuid::Uuid;
use chrono::{DateTime, Utc};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
pub struct VersionSnapshot {
pub version_num: i32,
pub operation: String, // 'create' | 'update' | 'delete'
pub snapshot: serde_json::Value,
pub changed_at: DateTime<Utc>,
pub changed_by: String,
#[sqlx(default)]
pub fields_changed: Vec<String>,
}
@@ -40,8 +41,7 @@ impl EntityVersioningService {
/// Get all versions of an entity in descending order
pub async fn get_versions(&self, entity_id: &str) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
sqlx::query_as!(
VersionSnapshot,
sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -49,13 +49,13 @@ impl EntityVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_entity_version
WHERE entity_id = $1
ORDER BY version_num DESC
"#,
entity_id
)
.bind(entity_id)
.fetch_all(&self.pool)
.await
}
@@ -66,8 +66,7 @@ impl EntityVersioningService {
entity_id: &str,
version_num: i32,
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
sqlx::query_as!(
VersionSnapshot,
sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -75,13 +74,13 @@ impl EntityVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_entity_version
WHERE entity_id = $1 AND version_num = $2
"#,
entity_id,
version_num
)
.bind(entity_id)
.bind(version_num)
.fetch_optional(&self.pool)
.await
}
@@ -95,78 +94,7 @@ impl EntityVersioningService {
) -> Result<DiffResult, sqlx::Error> {
let from_snap = self.get_version(entity_id, from_v).await?;
let to_snap = self.get_version(entity_id, to_v).await?;
let from_obj = from_snap
.as_ref()
.and_then(|s| s.snapshot.as_object())
.map(|o| o.clone());
let to_obj = to_snap
.as_ref()
.and_then(|s| s.snapshot.as_object())
.map(|o| o.clone());
let mut added = Vec::new();
let mut removed = Vec::new();
let mut modified = Vec::new();
// Check removed and modified
if let Some(ref from) = from_obj {
for (key, from_val) in from {
if let Some(to) = &to_obj {
if let Some(to_val) = to.get(key) {
if from_val != to_val {
modified.push(DiffField {
name: key.clone(),
from_value: Some(from_val.clone()),
to_value: Some(to_val.clone()),
});
}
} else {
removed.push(DiffField {
name: key.clone(),
from_value: Some(from_val.clone()),
to_value: None,
});
}
} else {
removed.push(DiffField {
name: key.clone(),
from_value: Some(from_val.clone()),
to_value: None,
});
}
}
}
// Check added
if let Some(to) = to_obj {
for (key, to_val) in to {
if let Some(from) = &from_obj {
if !from.contains_key(&key) {
added.push(DiffField {
name: key,
from_value: None,
to_value: Some(to_val),
});
}
} else {
added.push(DiffField {
name: key,
from_value: None,
to_value: Some(to_val),
});
}
}
}
Ok(DiffResult {
from_version: from_v,
to_version: to_v,
added_fields: added,
removed_fields: removed,
modified_fields: modified,
})
compute_diff(from_snap, to_snap, from_v, to_v)
}
/// Get entity state at a point in time
@@ -175,8 +103,7 @@ impl EntityVersioningService {
entity_id: &str,
as_of: DateTime<Utc>,
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
sqlx::query_as!(
VersionSnapshot,
sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -184,15 +111,15 @@ impl EntityVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_entity_version
WHERE entity_id = $1 AND changed_at <= $2
ORDER BY version_num DESC
LIMIT 1
"#,
entity_id,
as_of
)
.bind(entity_id)
.bind(as_of)
.fetch_optional(&self.pool)
.await
}
@@ -210,8 +137,7 @@ impl EdgeVersioningService {
/// Get all versions of an edge
pub async fn get_versions(&self, edge_id: Uuid) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
sqlx::query_as!(
VersionSnapshot,
sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -219,13 +145,13 @@ impl EdgeVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_edge_version
WHERE edge_id = $1
ORDER BY version_num DESC
"#,
edge_id
)
.bind(edge_id)
.fetch_all(&self.pool)
.await
}
@@ -237,8 +163,7 @@ impl EdgeVersioningService {
from_v: i32,
to_v: i32,
) -> Result<DiffResult, sqlx::Error> {
let from_snap = sqlx::query_as!(
VersionSnapshot,
let from_snap = sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -246,18 +171,17 @@ impl EdgeVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_edge_version
WHERE edge_id = $1 AND version_num = $2
"#,
edge_id,
from_v
)
.bind(edge_id)
.bind(from_v)
.fetch_optional(&self.pool)
.await?;
let to_snap = sqlx::query_as!(
VersionSnapshot,
let to_snap = sqlx::query_as::<_, VersionSnapshot>(
r#"
SELECT
version_num,
@@ -265,17 +189,16 @@ impl EdgeVersioningService {
snapshot,
changed_at,
changed_by,
COALESCE(fields_changed, '{}') as "fields_changed!"
COALESCE(fields_changed, '{}') as fields_changed
FROM memory_edge_version
WHERE edge_id = $1 AND version_num = $2
"#,
edge_id,
to_v
)
.bind(edge_id)
.bind(to_v)
.fetch_optional(&self.pool)
.await?;
// Same diff logic as entities
compute_diff(from_snap, to_snap, from_v, to_v)
}
}