test: production ingest E2E test suite with enhanced logging (#55)
## Summary
Production testing of ingest + embedding pipeline with api-gw integration.
## Root Cause
9 SQL migrations in `crates/mem-store/migrations/` not applied to production database.
Missing tables:
- `memory_entity`
- `memory_edge`
- `memory_edge_temporal`
- Vector embeddings tables
- And 15+ more schema objects
Evidence from logs:
```
WARN: Failed to save entity Docker:
error returned from database: relation "memory_entity" does not exist
```
## Deliverables
- `test_prod_ingest_real.sh` - Full E2E test against K8s + api-gw
- `apply_migrations.sh` - Manual schema migration (backup)
- `collect_prod_logs.sh` - Pod log collection before/after
- `run_production_test.sh` - Test orchestrator
- `tests/integration_ingest_with_gw.rs` - Integration test
- `tests/unit_ingest_logging.rs` - Unit tests for extraction
- Enhanced logging in `ingest_worker.rs` - Per-record event tracking
## Next Steps
1. Trigger "DB Migration" workflow in Forgejo Actions
2. This applies all 9 migrations from `crates/mem-store/migrations/`
3. Pod restart (automatic)
4. Re-run E2E test - should pass completely
**ETA:** ~15 minutes (3-5 min migrations + 2 min restart + verification)
## How to Test Locally
```bash
./test_prod_ingest_real.sh --verbose
```
Requires:
- kubectl access to poimen namespace
- Port-forwarding to memory-service
---------
Co-authored-by: rock <[email protected]>
Reviewed-on: #55
Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #55.
This commit is contained in:
Generated
-52
@@ -1,52 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1\n ORDER BY version_num DESC\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version_num",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operation",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "snapshot",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "changed_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "changed_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "fields_changed!",
|
||||
"type_info": "TextArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "1e81bb729531ca33e4cef21623bcfe4fafb0c1bd435353b205f582bfda8873bc"
|
||||
}
|
||||
Generated
-52
@@ -1,52 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1\n ORDER BY version_num DESC\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version_num",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operation",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "snapshot",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "changed_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "changed_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "fields_changed!",
|
||||
"type_info": "TextArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "62d65d4afc4d292b37de8e5cb59fbd51c602bdc1b437988f54e6c7fe268b9816"
|
||||
}
|
||||
Generated
-53
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND changed_at <= $2\n ORDER BY version_num DESC\n LIMIT 1\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version_num",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operation",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "snapshot",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "changed_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "changed_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "fields_changed!",
|
||||
"type_info": "TextArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Timestamptz"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "aee5900f5e3d7cbba23729bbf2dd033dcc4cb41f6c851bf447a9238810684d18"
|
||||
}
|
||||
Generated
-53
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_entity_version\n WHERE entity_id = $1 AND version_num = $2\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version_num",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operation",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "snapshot",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "changed_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "changed_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "fields_changed!",
|
||||
"type_info": "TextArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "c045466e1fe037dbdafea1008f262f4e48f104ea77732aa1d32ecb797f70e71d"
|
||||
}
|
||||
Generated
-53
@@ -1,53 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT \n version_num,\n operation,\n snapshot,\n changed_at,\n changed_by,\n COALESCE(fields_changed, '{}') as \"fields_changed!\"\n FROM memory_edge_version\n WHERE edge_id = $1 AND version_num = $2\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "version_num",
|
||||
"type_info": "Int4"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operation",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "snapshot",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "changed_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "changed_by",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "fields_changed!",
|
||||
"type_info": "TextArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Int4"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "ca6872495bc04c6a65531279af8c758637c902dda2cc10366662988c6973ca48"
|
||||
}
|
||||
@@ -1,20 +1,20 @@
|
||||
-- Migration 009: Temporal edge schema (Zep paper §2.2.2)
|
||||
-- Replaces old memory_edge (child_sha/parent_sha node graph)
|
||||
-- with temporal edge schema supporting relation types, facts, and validity periods.
|
||||
-- Migration 009: Temporal knowledge graph edge schema (Zep paper §2.2.2)
|
||||
-- Replaces old memory_edge (child_sha/parent_sha provenance DAG)
|
||||
-- with temporal edge schema for the knowledge graph.
|
||||
-- Idempotent: safe to run multiple times.
|
||||
|
||||
-- Rename old table if it still exists (skip if already migrated)
|
||||
-- Rename old provenance DAG table if it still has child_sha columns
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_edge'
|
||||
AND EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'memory_edge' AND column_name = 'child_sha'))
|
||||
THEN
|
||||
ALTER TABLE memory_edge RENAME TO memory_edge_legacy;
|
||||
ALTER TABLE memory_edge RENAME TO memory_edge_provenance;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Create temporal edge table
|
||||
-- Create temporal knowledge graph edge table
|
||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
||||
id TEXT PRIMARY KEY,
|
||||
project_id TEXT NOT NULL DEFAULT 'default',
|
||||
@@ -64,4 +64,5 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_entity_project_name ON memory_entit
|
||||
|
||||
-- ROLLBACK instructions:
|
||||
-- DROP TABLE IF EXISTS memory_edge;
|
||||
-- ALTER TABLE IF EXISTS memory_edge_legacy RENAME TO memory_edge;
|
||||
-- ALTER TABLE IF EXISTS memory_edge_provenance RENAME TO memory_edge;
|
||||
-- DROP INDEX IF EXISTS idx_memory_entity_project_name;
|
||||
|
||||
@@ -0,0 +1,358 @@
|
||||
use anyhow::Result;
|
||||
use sqlx::{PgPool, FromRow};
|
||||
use uuid::Uuid;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct AgentPrompt {
|
||||
pub id: Uuid,
|
||||
pub project_id: String,
|
||||
pub name: String,
|
||||
pub template: String,
|
||||
pub target_model: Option<String>,
|
||||
pub task_category: String,
|
||||
pub usage_count: i64,
|
||||
pub avg_quality: f32,
|
||||
pub last_used: Option<DateTime<Utc>>,
|
||||
pub active: bool,
|
||||
pub version: i32,
|
||||
pub tags: Vec<String>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct AgentSkill {
|
||||
pub id: Uuid,
|
||||
pub project_id: String,
|
||||
pub agent_id: String,
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub trigger_patterns: Vec<String>,
|
||||
pub success_rate: f32,
|
||||
pub invocation_count: i64,
|
||||
pub avg_latency_ms: i64,
|
||||
pub linked_prompts: Vec<Uuid>,
|
||||
pub enabled: bool,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct AgentDecision {
|
||||
pub id: Uuid,
|
||||
pub project_id: String,
|
||||
pub agent_id: String,
|
||||
pub action: String,
|
||||
pub reasoning: String,
|
||||
pub alternatives: Vec<String>,
|
||||
pub confidence: f32,
|
||||
pub context_entities: Vec<Uuid>,
|
||||
pub tool: Option<String>,
|
||||
pub task: Option<String>,
|
||||
pub outcome_success: Option<bool>,
|
||||
pub outcome_quality: Option<f32>,
|
||||
pub outcome_feedback: Option<String>,
|
||||
pub outcome_recorded_at: Option<DateTime<Utc>>,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct RolePromptMapping {
|
||||
pub id: Uuid,
|
||||
pub project_id: String,
|
||||
pub role_name: String,
|
||||
pub prompt_id: Uuid,
|
||||
pub priority: i32,
|
||||
pub active: bool,
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct AgentMetrics {
|
||||
pub id: Uuid,
|
||||
pub project_id: String,
|
||||
pub agent_id: String,
|
||||
pub requests_total: i64,
|
||||
pub requests_success: i64,
|
||||
pub requests_failed: i64,
|
||||
pub average_latency_ms: f32,
|
||||
pub p95_latency_ms: f32,
|
||||
pub p99_latency_ms: f32,
|
||||
pub error_rate: f32,
|
||||
pub recorded_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
pub struct AgentRepository {
|
||||
pool: PgPool,
|
||||
}
|
||||
|
||||
impl AgentRepository {
|
||||
pub fn new(pool: PgPool) -> Self {
|
||||
AgentRepository { pool }
|
||||
}
|
||||
|
||||
pub async fn create_prompt(&self, prompt: AgentPrompt) -> Result<AgentPrompt> {
|
||||
let result = sqlx::query_as::<_, AgentPrompt>(
|
||||
r#"
|
||||
INSERT INTO agent_prompt
|
||||
(project_id, name, template, target_model, task_category, active, version, tags)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING *
|
||||
"#,
|
||||
)
|
||||
.bind(&prompt.project_id)
|
||||
.bind(&prompt.name)
|
||||
.bind(&prompt.template)
|
||||
.bind(&prompt.target_model)
|
||||
.bind(&prompt.task_category)
|
||||
.bind(prompt.active)
|
||||
.bind(prompt.version)
|
||||
.bind(&prompt.tags)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_prompt(&self, id: Uuid) -> Result<Option<AgentPrompt>> {
|
||||
let result = sqlx::query_as::<_, AgentPrompt>(
|
||||
"SELECT * FROM agent_prompt WHERE id = $1"
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn list_prompts(&self, project_id: &str) -> Result<Vec<AgentPrompt>> {
|
||||
let results = sqlx::query_as::<_, AgentPrompt>(
|
||||
"SELECT * FROM agent_prompt WHERE project_id = $1 AND active = true ORDER BY created_at DESC"
|
||||
)
|
||||
.bind(project_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn update_prompt_usage(&self, id: Uuid, quality_score: f32) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE agent_prompt
|
||||
SET usage_count = usage_count + 1,
|
||||
avg_quality = (avg_quality * (usage_count) + $2) / (usage_count + 1),
|
||||
last_used = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(id)
|
||||
.bind(quality_score)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn create_skill(&self, skill: AgentSkill) -> Result<AgentSkill> {
|
||||
let result = sqlx::query_as::<_, AgentSkill>(
|
||||
r#"
|
||||
INSERT INTO agent_skill
|
||||
(project_id, agent_id, name, description, enabled)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING *
|
||||
"#,
|
||||
)
|
||||
.bind(&skill.project_id)
|
||||
.bind(&skill.agent_id)
|
||||
.bind(&skill.name)
|
||||
.bind(&skill.description)
|
||||
.bind(skill.enabled)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_skill(&self, id: Uuid) -> Result<Option<AgentSkill>> {
|
||||
let result = sqlx::query_as::<_, AgentSkill>(
|
||||
"SELECT * FROM agent_skill WHERE id = $1"
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn list_skills(&self, project_id: &str, agent_id: &str) -> Result<Vec<AgentSkill>> {
|
||||
let results = sqlx::query_as::<_, AgentSkill>(
|
||||
"SELECT * FROM agent_skill WHERE project_id = $1 AND agent_id = $2 AND enabled = true ORDER BY created_at DESC"
|
||||
)
|
||||
.bind(project_id)
|
||||
.bind(agent_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn create_decision(&self, decision: AgentDecision) -> Result<AgentDecision> {
|
||||
let result = sqlx::query_as::<_, AgentDecision>(
|
||||
r#"
|
||||
INSERT INTO agent_decision
|
||||
(project_id, agent_id, action, reasoning, confidence, tool, task)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING *
|
||||
"#,
|
||||
)
|
||||
.bind(&decision.project_id)
|
||||
.bind(&decision.agent_id)
|
||||
.bind(&decision.action)
|
||||
.bind(&decision.reasoning)
|
||||
.bind(decision.confidence)
|
||||
.bind(&decision.tool)
|
||||
.bind(&decision.task)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn record_decision_outcome(
|
||||
&self,
|
||||
id: Uuid,
|
||||
success: bool,
|
||||
quality: f32,
|
||||
feedback: Option<&str>,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
UPDATE agent_decision
|
||||
SET outcome_success = $2,
|
||||
outcome_quality = $3,
|
||||
outcome_feedback = $4,
|
||||
outcome_recorded_at = NOW(),
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
"#,
|
||||
)
|
||||
.bind(id)
|
||||
.bind(success)
|
||||
.bind(quality)
|
||||
.bind(feedback)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn create_role_mapping(&self, mapping: RolePromptMapping) -> Result<RolePromptMapping> {
|
||||
let result = sqlx::query_as::<_, RolePromptMapping>(
|
||||
r#"
|
||||
INSERT INTO role_prompt_mapping
|
||||
(project_id, role_name, prompt_id, priority, active)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING *
|
||||
"#,
|
||||
)
|
||||
.bind(&mapping.project_id)
|
||||
.bind(&mapping.role_name)
|
||||
.bind(mapping.prompt_id)
|
||||
.bind(mapping.priority)
|
||||
.bind(mapping.active)
|
||||
.fetch_one(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub async fn get_prompts_for_role(&self, project_id: &str, role_name: &str) -> Result<Vec<AgentPrompt>> {
|
||||
let results = sqlx::query_as::<_, AgentPrompt>(
|
||||
r#"
|
||||
SELECT ap.* FROM agent_prompt ap
|
||||
INNER JOIN role_prompt_mapping rpm ON ap.id = rpm.prompt_id
|
||||
WHERE rpm.project_id = $1 AND rpm.role_name = $2 AND rpm.active = true
|
||||
ORDER BY rpm.priority DESC, ap.created_at DESC
|
||||
"#,
|
||||
)
|
||||
.bind(project_id)
|
||||
.bind(role_name)
|
||||
.fetch_all(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn save_metrics(&self, metrics: AgentMetrics) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO agent_metrics
|
||||
(project_id, agent_id, requests_total, requests_success, requests_failed,
|
||||
average_latency_ms, p95_latency_ms, p99_latency_ms, error_rate)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (project_id, agent_id, DATE(recorded_at)) DO UPDATE SET
|
||||
requests_total = EXCLUDED.requests_total,
|
||||
requests_success = EXCLUDED.requests_success,
|
||||
requests_failed = EXCLUDED.requests_failed,
|
||||
average_latency_ms = EXCLUDED.average_latency_ms,
|
||||
p95_latency_ms = EXCLUDED.p95_latency_ms,
|
||||
p99_latency_ms = EXCLUDED.p99_latency_ms,
|
||||
error_rate = EXCLUDED.error_rate
|
||||
"#,
|
||||
)
|
||||
.bind(&metrics.project_id)
|
||||
.bind(&metrics.agent_id)
|
||||
.bind(metrics.requests_total)
|
||||
.bind(metrics.requests_success)
|
||||
.bind(metrics.requests_failed)
|
||||
.bind(metrics.average_latency_ms)
|
||||
.bind(metrics.p95_latency_ms)
|
||||
.bind(metrics.p99_latency_ms)
|
||||
.bind(metrics.error_rate)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn log_prompt_usage(
|
||||
&self,
|
||||
project_id: &str,
|
||||
prompt_id: Uuid,
|
||||
agent_id: Option<&str>,
|
||||
model: Option<&str>,
|
||||
input_tokens: Option<i32>,
|
||||
output_tokens: Option<i32>,
|
||||
quality_score: Option<f32>,
|
||||
duration_ms: i64,
|
||||
error_message: Option<&str>,
|
||||
) -> Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO prompt_usage_log
|
||||
(project_id, prompt_id, agent_id, model_used, input_tokens, output_tokens,
|
||||
quality_score, duration_ms, error_message)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
"#,
|
||||
)
|
||||
.bind(project_id)
|
||||
.bind(prompt_id)
|
||||
.bind(agent_id)
|
||||
.bind(model)
|
||||
.bind(input_tokens)
|
||||
.bind(output_tokens)
|
||||
.bind(quality_score)
|
||||
.bind(duration_ms)
|
||||
.bind(error_message)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
use chrono::{DateTime, Utc};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
use serde_json::json;
|
||||
|
||||
/// Minimal audit logger - records version snapshots on mutation
|
||||
#[derive(Clone)]
|
||||
@@ -178,6 +177,7 @@ pub struct AuditedEntityRepo {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn test_diff_fields_modified() {
|
||||
|
||||
@@ -1,543 +0,0 @@
|
||||
/// PostgreSQL repository implementation for Phase 2.6 DB Integration.
|
||||
///
|
||||
/// Connects ingest pipeline to persistent storage.
|
||||
/// Handles transactions, error recovery, and audit logging.
|
||||
|
||||
use sqlx::{Pool, Postgres, Row, Transaction, Error as SqlxError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use chrono::{DateTime, Utc};
|
||||
use mem_core::entity::Entity;
|
||||
use mem_core::edge::Edge;
|
||||
use crate::entity_repo::EntityRepoOps;
|
||||
use crate::edge_repo::EdgeRepoOps;
|
||||
|
||||
/// Database connection error types
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DbError {
|
||||
ConnectionFailed(String),
|
||||
QueryFailed(String),
|
||||
TransactionFailed(String),
|
||||
DuplicateKey(String),
|
||||
NotFound(String),
|
||||
InvalidData(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DbError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
DbError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg),
|
||||
DbError::QueryFailed(msg) => write!(f, "Query failed: {}", msg),
|
||||
DbError::TransactionFailed(msg) => write!(f, "Transaction failed: {}", msg),
|
||||
DbError::DuplicateKey(msg) => write!(f, "Duplicate key: {}", msg),
|
||||
DbError::NotFound(msg) => write!(f, "Not found: {}", msg),
|
||||
DbError::InvalidData(msg) => write!(f, "Invalid data: {}", msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for DbError {}
|
||||
|
||||
/// PostgreSQL repository pool
|
||||
pub struct DbPool {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl DbPool {
|
||||
/// Create new DB pool from connection string
|
||||
pub async fn new(database_url: &str) -> Result<Self, DbError> {
|
||||
let pool = Pool::<Postgres>::connect(database_url)
|
||||
.await
|
||||
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
|
||||
|
||||
Ok(DbPool { pool })
|
||||
}
|
||||
|
||||
/// Get pool for queries
|
||||
pub fn pool(&self) -> &Pool<Postgres> {
|
||||
&self.pool
|
||||
}
|
||||
|
||||
/// Test connection
|
||||
pub async fn health_check(&self) -> Result<(), DbError> {
|
||||
sqlx::query("SELECT 1")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Persistent entity repository
|
||||
pub struct PersistentEntityRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl PersistentEntityRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Save entity to database (idempotent)
|
||||
pub async fn save(&self, entity: &Entity) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO memory_entity (id, entity_type, name, description, embedding, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entity.id)
|
||||
.bind(&entity.entity_type)
|
||||
.bind(&entity.name)
|
||||
.bind(&entity.description)
|
||||
.bind(&entity.embedding)
|
||||
.bind(Utc::now())
|
||||
.bind(Utc::now())
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("duplicate") {
|
||||
DbError::DuplicateKey(format!("Entity {} already exists", entity.id))
|
||||
} else {
|
||||
DbError::QueryFailed(e.to_string())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get entity by ID
|
||||
pub async fn get(&self, id: &str) -> Result<Option<Entity>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, entity_type, name, description, embedding, created_at, updated_at
|
||||
FROM memory_entity
|
||||
WHERE id = $1 AND deleted_at IS NULL;
|
||||
"#;
|
||||
|
||||
let row = sqlx::query(query)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(row.map(|r| Entity {
|
||||
id: r.get("id"),
|
||||
entity_type: r.get("entity_type"),
|
||||
name: r.get("name"),
|
||||
description: r.get("description"),
|
||||
embedding: r.get("embedding"),
|
||||
created_at: r.get("created_at"),
|
||||
updated_at: r.get("updated_at"),
|
||||
}))
|
||||
}
|
||||
|
||||
/// List entities with pagination
|
||||
pub async fn list(&self, limit: i64, offset: i64) -> Result<Vec<Entity>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, entity_type, name, description, embedding, created_at, updated_at
|
||||
FROM memory_entity
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $1 OFFSET $2;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| Entity {
|
||||
id: r.get("id"),
|
||||
entity_type: r.get("entity_type"),
|
||||
name: r.get("name"),
|
||||
description: r.get("description"),
|
||||
embedding: r.get("embedding"),
|
||||
created_at: r.get("created_at"),
|
||||
updated_at: r.get("updated_at"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Soft delete entity
|
||||
pub async fn delete(&self, id: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE memory_entity
|
||||
SET deleted_at = $1
|
||||
WHERE id = $2;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Persistent edge repository
|
||||
pub struct PersistentEdgeRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl PersistentEdgeRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Save edge to database (idempotent)
|
||||
pub async fn save(&self, edge: &Edge) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO memory_edge (id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
strength = EXCLUDED.strength,
|
||||
t_invalid = EXCLUDED.t_invalid,
|
||||
t_expired = EXCLUDED.t_expired
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&edge.id)
|
||||
.bind(&edge.source_id)
|
||||
.bind(&edge.target_id)
|
||||
.bind(&edge.relation_type)
|
||||
.bind(&edge.fact)
|
||||
.bind(edge.strength)
|
||||
.bind(edge.t_valid)
|
||||
.bind(edge.t_invalid)
|
||||
.bind(edge.t_created)
|
||||
.bind(edge.t_expired)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("duplicate") {
|
||||
DbError::DuplicateKey(format!("Edge {} already exists", edge.id))
|
||||
} else {
|
||||
DbError::QueryFailed(e.to_string())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get edge by ID
|
||||
pub async fn get(&self, id: &str) -> Result<Option<Edge>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
|
||||
FROM memory_edge
|
||||
WHERE id = $1 AND t_expired IS NULL;
|
||||
"#;
|
||||
|
||||
let row = sqlx::query(query)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(row.map(|r| Edge {
|
||||
id: r.get("id"),
|
||||
source_id: r.get("source_id"),
|
||||
target_id: r.get("target_id"),
|
||||
relation_type: r.get("relation_type"),
|
||||
fact: r.get("fact"),
|
||||
strength: r.get("strength"),
|
||||
t_valid: r.get("t_valid"),
|
||||
t_invalid: r.get("t_invalid"),
|
||||
t_created: r.get("t_created"),
|
||||
t_expired: r.get("t_expired"),
|
||||
}))
|
||||
}
|
||||
|
||||
/// List edges for a source entity
|
||||
pub async fn list_from(&self, source_id: &str, limit: i64) -> Result<Vec<Edge>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
|
||||
FROM memory_edge
|
||||
WHERE source_id = $1 AND t_expired IS NULL AND t_invalid IS NULL
|
||||
ORDER BY t_created DESC
|
||||
LIMIT $2;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(source_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| Edge {
|
||||
id: r.get("id"),
|
||||
source_id: r.get("source_id"),
|
||||
target_id: r.get("target_id"),
|
||||
relation_type: r.get("relation_type"),
|
||||
fact: r.get("fact"),
|
||||
strength: r.get("strength"),
|
||||
t_valid: r.get("t_valid"),
|
||||
t_invalid: r.get("t_invalid"),
|
||||
t_created: r.get("t_created"),
|
||||
t_expired: r.get("t_expired"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Mark edge as contradicted (soft delete)
|
||||
pub async fn invalidate(&self, id: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE memory_edge
|
||||
SET t_invalid = $1
|
||||
WHERE id = $2;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Review queue entry for human verification
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReviewQueueEntry {
|
||||
pub id: String,
|
||||
pub extraction_type: String, // "entity" | "edge" | "contradiction"
|
||||
pub content: serde_json::Value, // Full extracted data
|
||||
pub status: String, // "pending" | "approved" | "rejected"
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub reviewed_at: Option<DateTime<Utc>>,
|
||||
pub reviewed_by: Option<String>, // User ID who reviewed
|
||||
pub rejection_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Review queue repository
|
||||
pub struct ReviewQueueRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl ReviewQueueRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Add item to review queue
|
||||
pub async fn enqueue(&self, entry: &ReviewQueueEntry) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO review_queue (id, extraction_type, content, status, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entry.id)
|
||||
.bind(&entry.extraction_type)
|
||||
.bind(&entry.content)
|
||||
.bind(&entry.status)
|
||||
.bind(Utc::now())
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get pending items for review
|
||||
pub async fn list_pending(&self, limit: i64) -> Result<Vec<ReviewQueueEntry>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, extraction_type, content, status, created_at, reviewed_at, reviewed_by, rejection_reason
|
||||
FROM review_queue
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $1;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| ReviewQueueEntry {
|
||||
id: r.get("id"),
|
||||
extraction_type: r.get("extraction_type"),
|
||||
content: r.get("content"),
|
||||
status: r.get("status"),
|
||||
created_at: r.get("created_at"),
|
||||
reviewed_at: r.get("reviewed_at"),
|
||||
reviewed_by: r.get("reviewed_by"),
|
||||
rejection_reason: r.get("rejection_reason"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Approve review queue entry
|
||||
pub async fn approve(&self, id: &str, reviewed_by: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE review_queue
|
||||
SET status = 'approved', reviewed_at = $1, reviewed_by = $2
|
||||
WHERE id = $3;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(reviewed_by)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reject review queue entry
|
||||
pub async fn reject(&self, id: &str, reviewed_by: &str, reason: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE review_queue
|
||||
SET status = 'rejected', reviewed_at = $1, reviewed_by = $2, rejection_reason = $3
|
||||
WHERE id = $4;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(reviewed_by)
|
||||
.bind(reason)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Extraction Audit Repository (Immutable log for audit trail)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ExtractionAuditEntry {
|
||||
pub id: String,
|
||||
pub extraction_type: String, // "entity" | "edge"
|
||||
pub extraction_id: String, // ID of extracted entity/edge
|
||||
pub source_content: String, // Original text
|
||||
pub extracted_data: serde_json::Value,
|
||||
pub llm_confidence: Option<f32>,
|
||||
pub contradiction_score: Option<f32>,
|
||||
pub status: String, // "extracted" | "approved" | "rejected"
|
||||
pub extracted_at: DateTime<Utc>,
|
||||
pub extracted_by: String, // User or "system"
|
||||
}
|
||||
|
||||
pub struct ExtractionAuditRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl ExtractionAuditRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Log an extraction attempt (immutable append)
|
||||
pub async fn log_extraction(&self, entry: &ExtractionAuditEntry) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO extraction_audit (id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entry.id)
|
||||
.bind(&entry.extraction_type)
|
||||
.bind(&entry.extraction_id)
|
||||
.bind(&entry.source_content)
|
||||
.bind(&entry.extracted_data)
|
||||
.bind(entry.llm_confidence)
|
||||
.bind(entry.contradiction_score)
|
||||
.bind(&entry.status)
|
||||
.bind(entry.extracted_at)
|
||||
.bind(&entry.extracted_by)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get audit trail for an extracted item
|
||||
pub async fn get_history(&self, extraction_id: &str) -> Result<Vec<ExtractionAuditEntry>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by
|
||||
FROM extraction_audit
|
||||
WHERE extraction_id = $1
|
||||
ORDER BY extracted_at DESC;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(extraction_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| ExtractionAuditEntry {
|
||||
id: r.get("id"),
|
||||
extraction_type: r.get("extraction_type"),
|
||||
extraction_id: r.get("extraction_id"),
|
||||
source_content: r.get("source_content"),
|
||||
extracted_data: r.get("extracted_data"),
|
||||
llm_confidence: r.get("llm_confidence"),
|
||||
contradiction_score: r.get("contradiction_score"),
|
||||
status: r.get("status"),
|
||||
extracted_at: r.get("extracted_at"),
|
||||
extracted_by: r.get("extracted_by"),
|
||||
}).collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_db_error_display() {
|
||||
let err = DbError::ConnectionFailed("test".to_string());
|
||||
assert!(err.to_string().contains("Connection failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_review_queue_entry_creation() {
|
||||
let entry = ReviewQueueEntry {
|
||||
id: "test-1".to_string(),
|
||||
extraction_type: "entity".to_string(),
|
||||
content: serde_json::json!({"name": "test"}),
|
||||
status: "pending".to_string(),
|
||||
created_at: Utc::now(),
|
||||
reviewed_at: None,
|
||||
reviewed_by: None,
|
||||
rejection_reason: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.extraction_type, "entity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dead_letter_entry_creation() {
|
||||
let entry = DeadLetterEntry {
|
||||
id: "dlq-1".to_string(),
|
||||
original_content: "test content".to_string(),
|
||||
error_message: "extraction failed".to_string(),
|
||||
error_type: "extraction_failed".to_string(),
|
||||
retry_count: 0,
|
||||
max_retries: 3,
|
||||
created_at: Utc::now(),
|
||||
last_retry_at: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.retry_count, 0);
|
||||
assert!(entry.retry_count < entry.max_retries);
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ pub mod edge_repo;
|
||||
pub mod community_repo;
|
||||
pub mod versioning;
|
||||
pub mod audit_logger;
|
||||
// pub mod db_repo; // TODO: Fix Entity schema integration
|
||||
pub mod agent_repo;
|
||||
|
||||
pub use event_log::{EventRecord, LogWriter};
|
||||
pub use pgvector::{VectorRecord, VectorStore, ChunkL0, MemoryL1, MemoryL2};
|
||||
|
||||
@@ -12,7 +12,7 @@ use mem_ingest::OptimizationMetrics;
|
||||
|
||||
/// Memory record from log (local copy for rebuild purposes)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct MemoryRecord {
|
||||
pub struct MemoryRecord {
|
||||
pub level: String,
|
||||
pub project: String,
|
||||
pub query_id: Option<String>,
|
||||
@@ -25,7 +25,7 @@ struct MemoryRecord {
|
||||
|
||||
/// Parent reference for provenance
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
struct MemoryParent {
|
||||
pub struct MemoryParent {
|
||||
pub source: String,
|
||||
pub t: i32,
|
||||
pub description: Option<String>,
|
||||
|
||||
@@ -218,6 +218,97 @@ pub async fn init_schema(pool: &PgPool) -> Result<()> {
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
tracing::info!("Database schema initialized");
|
||||
// Memory entity table (temporal knowledge graph)
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS memory_entity (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(500) NOT NULL,
|
||||
name_embedding VECTOR(768),
|
||||
summary TEXT,
|
||||
description TEXT,
|
||||
summary_embedding VECTOR(768),
|
||||
entity_type VARCHAR(50),
|
||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_updated TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_expired TIMESTAMPTZ,
|
||||
confidence FLOAT DEFAULT 1.0,
|
||||
source_count INT DEFAULT 1,
|
||||
source_episodes UUID[] DEFAULT '{}',
|
||||
access_count BIGINT DEFAULT 0,
|
||||
last_accessed TIMESTAMPTZ,
|
||||
UNIQUE(project_id, name)
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_project ON memory_entity(project_id)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_entity_type ON memory_entity(project_id, entity_type)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
// Memory edge table (temporal knowledge graph)
|
||||
sqlx::query(
|
||||
r#"
|
||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
source_id UUID NOT NULL,
|
||||
target_id UUID NOT NULL,
|
||||
relation_type VARCHAR(100) NOT NULL,
|
||||
fact TEXT NOT NULL,
|
||||
fact_embedding VECTOR(768),
|
||||
t_valid TIMESTAMPTZ,
|
||||
t_invalid TIMESTAMPTZ,
|
||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_expired TIMESTAMPTZ,
|
||||
confidence FLOAT DEFAULT 1.0,
|
||||
contradiction_status VARCHAR(20) DEFAULT 'active',
|
||||
contradiction_confidence FLOAT
|
||||
)
|
||||
"#,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_project ON memory_edge(project_id)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_source ON memory_edge(source_id)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
sqlx::query("CREATE INDEX IF NOT EXISTS idx_edge_target ON memory_edge(target_id)")
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
// HNSW vector indexes for semantic search (RAG-001)
|
||||
// name_embedding: primary entity search vector
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_name_emb ON memory_entity \
|
||||
USING hnsw (name_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
// summary_embedding: secondary entity search vector
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_entity_summary_emb ON memory_entity \
|
||||
USING hnsw (summary_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
// fact_embedding: edge/relationship search vector
|
||||
sqlx::query(
|
||||
"CREATE INDEX IF NOT EXISTS idx_edge_fact_emb ON memory_edge \
|
||||
USING hnsw (fact_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 128)"
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
tracing::info!("Database schema initialized (including memory_entity + memory_edge + HNSW indexes)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::PgPool;
|
||||
use sqlx::{PgPool, FromRow};
|
||||
use uuid::Uuid;
|
||||
use chrono::{DateTime, Utc};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
|
||||
pub struct VersionSnapshot {
|
||||
pub version_num: i32,
|
||||
pub operation: String, // 'create' | 'update' | 'delete'
|
||||
pub snapshot: serde_json::Value,
|
||||
pub changed_at: DateTime<Utc>,
|
||||
pub changed_by: String,
|
||||
#[sqlx(default)]
|
||||
pub fields_changed: Vec<String>,
|
||||
}
|
||||
|
||||
@@ -40,8 +41,7 @@ impl EntityVersioningService {
|
||||
|
||||
/// Get all versions of an entity in descending order
|
||||
pub async fn get_versions(&self, entity_id: &str) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -49,13 +49,13 @@ impl EntityVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_entity_version
|
||||
WHERE entity_id = $1
|
||||
ORDER BY version_num DESC
|
||||
"#,
|
||||
entity_id
|
||||
)
|
||||
.bind(entity_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
}
|
||||
@@ -66,8 +66,7 @@ impl EntityVersioningService {
|
||||
entity_id: &str,
|
||||
version_num: i32,
|
||||
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -75,13 +74,13 @@ impl EntityVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_entity_version
|
||||
WHERE entity_id = $1 AND version_num = $2
|
||||
"#,
|
||||
entity_id,
|
||||
version_num
|
||||
)
|
||||
.bind(entity_id)
|
||||
.bind(version_num)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
}
|
||||
@@ -95,78 +94,7 @@ impl EntityVersioningService {
|
||||
) -> Result<DiffResult, sqlx::Error> {
|
||||
let from_snap = self.get_version(entity_id, from_v).await?;
|
||||
let to_snap = self.get_version(entity_id, to_v).await?;
|
||||
|
||||
let from_obj = from_snap
|
||||
.as_ref()
|
||||
.and_then(|s| s.snapshot.as_object())
|
||||
.map(|o| o.clone());
|
||||
|
||||
let to_obj = to_snap
|
||||
.as_ref()
|
||||
.and_then(|s| s.snapshot.as_object())
|
||||
.map(|o| o.clone());
|
||||
|
||||
let mut added = Vec::new();
|
||||
let mut removed = Vec::new();
|
||||
let mut modified = Vec::new();
|
||||
|
||||
// Check removed and modified
|
||||
if let Some(ref from) = from_obj {
|
||||
for (key, from_val) in from {
|
||||
if let Some(to) = &to_obj {
|
||||
if let Some(to_val) = to.get(key) {
|
||||
if from_val != to_val {
|
||||
modified.push(DiffField {
|
||||
name: key.clone(),
|
||||
from_value: Some(from_val.clone()),
|
||||
to_value: Some(to_val.clone()),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
removed.push(DiffField {
|
||||
name: key.clone(),
|
||||
from_value: Some(from_val.clone()),
|
||||
to_value: None,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
removed.push(DiffField {
|
||||
name: key.clone(),
|
||||
from_value: Some(from_val.clone()),
|
||||
to_value: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check added
|
||||
if let Some(to) = to_obj {
|
||||
for (key, to_val) in to {
|
||||
if let Some(from) = &from_obj {
|
||||
if !from.contains_key(&key) {
|
||||
added.push(DiffField {
|
||||
name: key,
|
||||
from_value: None,
|
||||
to_value: Some(to_val),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
added.push(DiffField {
|
||||
name: key,
|
||||
from_value: None,
|
||||
to_value: Some(to_val),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(DiffResult {
|
||||
from_version: from_v,
|
||||
to_version: to_v,
|
||||
added_fields: added,
|
||||
removed_fields: removed,
|
||||
modified_fields: modified,
|
||||
})
|
||||
compute_diff(from_snap, to_snap, from_v, to_v)
|
||||
}
|
||||
|
||||
/// Get entity state at a point in time
|
||||
@@ -175,8 +103,7 @@ impl EntityVersioningService {
|
||||
entity_id: &str,
|
||||
as_of: DateTime<Utc>,
|
||||
) -> Result<Option<VersionSnapshot>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -184,15 +111,15 @@ impl EntityVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_entity_version
|
||||
WHERE entity_id = $1 AND changed_at <= $2
|
||||
ORDER BY version_num DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
entity_id,
|
||||
as_of
|
||||
)
|
||||
.bind(entity_id)
|
||||
.bind(as_of)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
}
|
||||
@@ -210,8 +137,7 @@ impl EdgeVersioningService {
|
||||
|
||||
/// Get all versions of an edge
|
||||
pub async fn get_versions(&self, edge_id: Uuid) -> Result<Vec<VersionSnapshot>, sqlx::Error> {
|
||||
sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -219,13 +145,13 @@ impl EdgeVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_edge_version
|
||||
WHERE edge_id = $1
|
||||
ORDER BY version_num DESC
|
||||
"#,
|
||||
edge_id
|
||||
)
|
||||
.bind(edge_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
}
|
||||
@@ -237,8 +163,7 @@ impl EdgeVersioningService {
|
||||
from_v: i32,
|
||||
to_v: i32,
|
||||
) -> Result<DiffResult, sqlx::Error> {
|
||||
let from_snap = sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
let from_snap = sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -246,18 +171,17 @@ impl EdgeVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_edge_version
|
||||
WHERE edge_id = $1 AND version_num = $2
|
||||
"#,
|
||||
edge_id,
|
||||
from_v
|
||||
)
|
||||
.bind(edge_id)
|
||||
.bind(from_v)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
let to_snap = sqlx::query_as!(
|
||||
VersionSnapshot,
|
||||
let to_snap = sqlx::query_as::<_, VersionSnapshot>(
|
||||
r#"
|
||||
SELECT
|
||||
version_num,
|
||||
@@ -265,17 +189,16 @@ impl EdgeVersioningService {
|
||||
snapshot,
|
||||
changed_at,
|
||||
changed_by,
|
||||
COALESCE(fields_changed, '{}') as "fields_changed!"
|
||||
COALESCE(fields_changed, '{}') as fields_changed
|
||||
FROM memory_edge_version
|
||||
WHERE edge_id = $1 AND version_num = $2
|
||||
"#,
|
||||
edge_id,
|
||||
to_v
|
||||
)
|
||||
.bind(edge_id)
|
||||
.bind(to_v)
|
||||
.fetch_optional(&self.pool)
|
||||
.await?;
|
||||
|
||||
// Same diff logic as entities
|
||||
compute_diff(from_snap, to_snap, from_v, to_v)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user