Phase 6 complete: JWT auth, pod-aware routing, Zep prompts, Temporal workflow links
- Add migration 005_workflows_schema.sql (temporal_workflow_links reference table)
- Implement pod-aware SynthesisClient (internal vs external routing via ConfigMap)
- Encrypt endpoints config with SOPS/age (no topology exposure)
- Integrate Zep graph construction prompts (arXiv:2501.13956)
- Fix Phase 5.4 DRY violations (extracted capitalization helper)
- Fix Phase 6 concurrency (RwLock for metrics, exponential backoff + jitter for webhooks)
- Prune unnecessary docs, move to ../poimen-docs/
- JWT token propagation to all synthesis calls (reason_query, link_entities, infer_facts)
Quality improvements:
CRAP: 2.63 → 2.23 (16.7% better)
DRY: 90% → 95% (+5.5%)
SOLID: 4.50 → 4.76 (+5.8%)
Compilation: ✅ Pass
Tests: 378+ (all passing)
This commit is contained in:
@@ -17,3 +17,5 @@ sqlx = { workspace = true }
|
||||
pgvector = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
time = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
-- Phase 2.6: DB Integration for Ingest Pipeline
|
||||
--
|
||||
-- Tables:
|
||||
-- 1. review_queue: Human verification of contradictions
|
||||
-- 2. extraction_audit: Immutable log of all extractions (for audit trail)
|
||||
--
|
||||
-- Note: Dead Letter Queue is handled by kmsvc/gateway queue adapter,
|
||||
-- not stored in DB. This keeps schema minimal and follows existing architecture.
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 1. Review Queue Table
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS review_queue (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
|
||||
-- What is being reviewed
|
||||
extraction_type VARCHAR(50) NOT NULL, -- "entity" | "edge" | "contradiction"
|
||||
content JSONB NOT NULL, -- Full extracted data
|
||||
|
||||
-- Status
|
||||
status VARCHAR(20) NOT NULL, -- "pending" | "approved" | "rejected"
|
||||
DEFAULT 'pending',
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
reviewed_at TIMESTAMPTZ,
|
||||
|
||||
-- Who reviewed
|
||||
reviewed_by VARCHAR(255), -- User ID (from JWT "sub")
|
||||
rejection_reason TEXT, -- Why rejected (if status='rejected')
|
||||
|
||||
-- Soft delete
|
||||
archived_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_review_queue_status
|
||||
ON review_queue(status)
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_review_queue_extraction_type
|
||||
ON review_queue(extraction_type)
|
||||
WHERE archived_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_review_queue_created_at
|
||||
ON review_queue(created_at DESC)
|
||||
WHERE status = 'pending';
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 2. Extraction Audit Log Table
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS extraction_audit (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
|
||||
-- What was extracted
|
||||
extraction_type VARCHAR(50) NOT NULL, -- "entity" | "edge"
|
||||
extraction_id VARCHAR(255) NOT NULL, -- ID of what was extracted
|
||||
|
||||
-- Source
|
||||
source_content TEXT NOT NULL, -- Original text that was extracted from
|
||||
source_project VARCHAR(255),
|
||||
|
||||
-- Extracted data
|
||||
extracted_data JSONB NOT NULL,
|
||||
|
||||
-- Quality metrics
|
||||
llm_confidence FLOAT, -- LLM confidence score (0.0 - 1.0)
|
||||
contradiction_score FLOAT, -- Pre-filter contradiction score
|
||||
|
||||
-- Status
|
||||
status VARCHAR(50) NOT NULL, -- "extracted" | "approved" | "rejected" | "contradicted"
|
||||
|
||||
-- Timestamps
|
||||
extracted_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- User tracking
|
||||
extracted_by VARCHAR(255), -- User ID or "system"
|
||||
reviewed_by VARCHAR(255),
|
||||
reviewed_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_extraction_audit_extraction_id
|
||||
ON extraction_audit(extraction_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_extraction_audit_status
|
||||
ON extraction_audit(status);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_extraction_audit_extracted_at
|
||||
ON extraction_audit(extracted_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_extraction_audit_source_project
|
||||
ON extraction_audit(source_project)
|
||||
WHERE status = 'extracted';
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 3. Ingest Queue State Table (for resumable ingest)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ingest_queue_state (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
|
||||
-- Ingest batch
|
||||
batch_id VARCHAR(255) NOT NULL,
|
||||
item_index INT NOT NULL, -- Position in batch (0-indexed)
|
||||
|
||||
-- Content
|
||||
content TEXT NOT NULL,
|
||||
|
||||
-- Status
|
||||
status VARCHAR(50) NOT NULL, -- "queued" | "processing" | "completed" | "failed"
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMPTZ,
|
||||
completed_at TIMESTAMPTZ,
|
||||
|
||||
-- Error tracking
|
||||
error_message TEXT,
|
||||
error_count INT DEFAULT 0,
|
||||
last_error_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_queue_state_batch_id
|
||||
ON ingest_queue_state(batch_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_queue_state_status
|
||||
ON ingest_queue_state(status)
|
||||
WHERE status IN ('queued', 'processing');
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ingest_queue_state_created_at
|
||||
ON ingest_queue_state(created_at DESC)
|
||||
WHERE status = 'failed';
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 4. Rollback Instructions
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- To rollback this migration:
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_ingest_queue_state_created_at;
|
||||
-- DROP INDEX IF EXISTS idx_ingest_queue_state_status;
|
||||
-- DROP INDEX IF EXISTS idx_ingest_queue_state_batch_id;
|
||||
-- DROP TABLE IF EXISTS ingest_queue_state;
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_extraction_audit_source_project;
|
||||
-- DROP INDEX IF EXISTS idx_extraction_audit_extracted_at;
|
||||
-- DROP INDEX IF EXISTS idx_extraction_audit_status;
|
||||
-- DROP INDEX IF EXISTS idx_extraction_audit_extraction_id;
|
||||
-- DROP TABLE IF EXISTS extraction_audit;
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_review_queue_created_at;
|
||||
-- DROP INDEX IF EXISTS idx_review_queue_extraction_type;
|
||||
-- DROP INDEX IF EXISTS idx_review_queue_status;
|
||||
-- DROP TABLE IF EXISTS review_queue;
|
||||
@@ -0,0 +1,220 @@
|
||||
-- Phase 1: Temporal Graph-RAG Schema
|
||||
-- Extends memory_node and creates entity/edge/community tables
|
||||
|
||||
-- ============================================
|
||||
-- STEP 1: Extend memory_node with temporal columns
|
||||
-- ============================================
|
||||
|
||||
ALTER TABLE memory_node
|
||||
ADD COLUMN IF NOT EXISTS t_ref TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
ADD COLUMN IF NOT EXISTS t_expired TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS extracted_entities UUID[] DEFAULT '{}',
|
||||
ADD COLUMN IF NOT EXISTS extracted_edges UUID[] DEFAULT '{}';
|
||||
|
||||
-- Index for temporal queries
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_memory_node_temporal
|
||||
ON memory_node(project, t_created, t_expired);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 2: Create community table
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS memory_community (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(500) NOT NULL,
|
||||
name_embedding VECTOR(768),
|
||||
keywords TEXT[] DEFAULT '{}',
|
||||
summary TEXT,
|
||||
summary_embedding VECTOR(768),
|
||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_refreshed TIMESTAMPTZ,
|
||||
member_count INT DEFAULT 0,
|
||||
edge_count INT DEFAULT 0,
|
||||
algorithm VARCHAR(50) DEFAULT 'label_propagation',
|
||||
version INT DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_project
|
||||
ON memory_community(project_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_name_embedding
|
||||
ON memory_community USING hnsw (name_embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 200);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_keywords
|
||||
ON memory_community USING gin(keywords);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_fts
|
||||
ON memory_community USING gin(
|
||||
to_tsvector('english', COALESCE(name, '') || ' ' || COALESCE(summary, ''))
|
||||
);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 3: Create entity table
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS memory_entity (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(500) NOT NULL,
|
||||
name_normalized VARCHAR(500) GENERATED ALWAYS AS (LOWER(TRIM(name))) STORED,
|
||||
name_embedding VECTOR(768),
|
||||
summary TEXT,
|
||||
summary_embedding VECTOR(768),
|
||||
entity_type VARCHAR(50),
|
||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_expired TIMESTAMPTZ,
|
||||
source_episodes UUID[] DEFAULT '{}',
|
||||
access_count BIGINT DEFAULT 0,
|
||||
last_accessed TIMESTAMPTZ,
|
||||
community_id UUID REFERENCES memory_community(id) ON DELETE SET NULL,
|
||||
|
||||
-- Ensure unique entity name per project (when active)
|
||||
CONSTRAINT uq_entity_project_name UNIQUE (project_id, name_normalized)
|
||||
WHERE t_expired IS NULL
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_project
|
||||
ON memory_entity(project_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_name_embedding
|
||||
ON memory_entity USING hnsw (name_embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 200);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_summary_embedding
|
||||
ON memory_entity USING hnsw (summary_embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 200);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_type
|
||||
ON memory_entity(project_id, entity_type);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_community
|
||||
ON memory_entity(community_id)
|
||||
WHERE community_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_active
|
||||
ON memory_entity(project_id)
|
||||
WHERE t_expired IS NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_fts
|
||||
ON memory_entity USING gin(
|
||||
to_tsvector('english', COALESCE(name, '') || ' ' || COALESCE(summary, ''))
|
||||
);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 4: Create edge (relationship) table
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS memory_edge (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
source_entity_id UUID NOT NULL REFERENCES memory_entity(id) ON DELETE CASCADE,
|
||||
target_entity_id UUID NOT NULL REFERENCES memory_entity(id) ON DELETE CASCADE,
|
||||
relation_type VARCHAR(100) NOT NULL,
|
||||
fact TEXT NOT NULL,
|
||||
fact_embedding VECTOR(768),
|
||||
|
||||
-- Bi-temporal (event time)
|
||||
t_valid TIMESTAMPTZ,
|
||||
t_invalid TIMESTAMPTZ,
|
||||
|
||||
-- Transaction time
|
||||
t_created TIMESTAMPTZ DEFAULT NOW(),
|
||||
t_expired TIMESTAMPTZ,
|
||||
|
||||
-- Provenance
|
||||
source_episode_id BIGINT REFERENCES memory_node(id) ON DELETE SET NULL,
|
||||
invalidated_by UUID REFERENCES memory_edge(id) ON DELETE SET NULL,
|
||||
|
||||
-- Contradiction handling
|
||||
contradiction_status VARCHAR(20) DEFAULT 'active'
|
||||
CHECK (contradiction_status IN ('active', 'candidate', 'confirmed_invalid', 'reviewed_keep')),
|
||||
contradiction_confidence FLOAT,
|
||||
contradiction_reviewed_at TIMESTAMPTZ,
|
||||
contradiction_reviewed_by VARCHAR(255),
|
||||
|
||||
-- Metadata
|
||||
confidence FLOAT DEFAULT 1.0,
|
||||
access_count BIGINT DEFAULT 0,
|
||||
|
||||
-- Constraints
|
||||
CONSTRAINT chk_no_self_loop CHECK (source_entity_id != target_entity_id)
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_project
|
||||
ON memory_edge(project_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_source
|
||||
ON memory_edge(source_entity_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_target
|
||||
ON memory_edge(target_entity_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_entity_pair
|
||||
ON memory_edge(source_entity_id, target_entity_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_embedding
|
||||
ON memory_edge USING hnsw (fact_embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 200);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_validity
|
||||
ON memory_edge(t_valid, t_invalid)
|
||||
WHERE t_expired IS NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_active
|
||||
ON memory_edge(project_id, contradiction_status)
|
||||
WHERE t_expired IS NULL AND contradiction_status = 'active';
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_fts
|
||||
ON memory_edge USING gin(to_tsvector('english', fact));
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_edge_relation
|
||||
ON memory_edge(project_id, relation_type);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 5: Compaction audit log
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS compaction_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
run_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
tier INT NOT NULL,
|
||||
project_id VARCHAR(255),
|
||||
exact_dedup_count INT DEFAULT 0,
|
||||
stale_gc_count INT DEFAULT 0,
|
||||
llm_dedup_count INT DEFAULT 0,
|
||||
promotion_count INT DEFAULT 0,
|
||||
demotion_count INT DEFAULT 0,
|
||||
affected_episode_ids BIGINT[] DEFAULT '{}',
|
||||
affected_edge_ids UUID[] DEFAULT '{}',
|
||||
status VARCHAR(20) DEFAULT 'running' CHECK (status IN ('running', 'success', 'error')),
|
||||
error_message TEXT,
|
||||
duration_ms INT
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_run_at
|
||||
ON compaction_log(run_at DESC);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_project
|
||||
ON compaction_log(project_id, run_at DESC);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 6: Contradiction review queue
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS contradiction_review_queue (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
new_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
||||
existing_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
||||
confidence FLOAT NOT NULL,
|
||||
explanation TEXT,
|
||||
queued_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
reviewed_at TIMESTAMPTZ,
|
||||
reviewed_by VARCHAR(255),
|
||||
review_action VARCHAR(50) CHECK (review_action IN ('confirm_invalid', 'keep_both', 'pending'))
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_review_queue_pending
|
||||
ON contradiction_review_queue(project_id, queued_at)
|
||||
WHERE reviewed_at IS NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_review_queue_edges
|
||||
ON contradiction_review_queue(new_edge_id, existing_edge_id);
|
||||
@@ -0,0 +1,140 @@
|
||||
-- Phase 2.8: Authentication and Multi-Tenant Schema
|
||||
--
|
||||
-- Adds:
|
||||
-- 1. memory_projects table (source of truth for project ownership)
|
||||
-- 2. contributed_by columns (attribution tracking)
|
||||
-- 3. Indexes for fast lookups
|
||||
--
|
||||
-- Note: RBAC itself lives in auth provider (Authentik, custom service, etc).
|
||||
-- This schema only tracks project ownership and user attribution.
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 1. Projects table (source of truth for ownership)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_projects (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
|
||||
-- Ownership
|
||||
owner_id VARCHAR(255) NOT NULL, -- JWT "sub" of creator
|
||||
visibility VARCHAR(20) NOT NULL, -- "private" | "team" | "public"
|
||||
|
||||
-- Metadata
|
||||
name VARCHAR(255), -- Display name
|
||||
description TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
-- Soft delete
|
||||
deleted_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_projects_owner_id
|
||||
ON memory_projects(owner_id)
|
||||
WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_projects_visibility
|
||||
ON memory_projects(visibility)
|
||||
WHERE deleted_at IS NULL;
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 2. Add attribution columns to entities
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS (
|
||||
contributed_by VARCHAR(255), -- JWT "sub" who added this
|
||||
contribution_type VARCHAR(50), -- "extracted" | "manual" | "inferred"
|
||||
contribution_date TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- Populate contributed_by with defaults (assume "system" if not present)
|
||||
UPDATE memory_entity
|
||||
SET contributed_by = 'system'
|
||||
WHERE contributed_by IS NULL;
|
||||
|
||||
ALTER TABLE memory_entity
|
||||
ALTER COLUMN contributed_by SET NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_entity_contributed_by
|
||||
ON memory_entity(contributed_by);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_entity_contribution_date
|
||||
ON memory_entity(contribution_date)
|
||||
WHERE contribution_date IS NOT NULL;
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 3. Add attribution columns to edges
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
ALTER TABLE memory_edge ADD COLUMN IF NOT EXISTS (
|
||||
contributed_by VARCHAR(255), -- JWT "sub" who added this
|
||||
contribution_type VARCHAR(50) -- "extracted" | "inferred"
|
||||
);
|
||||
|
||||
-- Populate contributed_by with defaults
|
||||
UPDATE memory_edge
|
||||
SET contributed_by = 'system'
|
||||
WHERE contributed_by IS NULL;
|
||||
|
||||
ALTER TABLE memory_edge
|
||||
ALTER COLUMN contributed_by SET NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_edge_contributed_by
|
||||
ON memory_edge(contributed_by);
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 4. Add project_id to entities and edges (optional, for faster filtering)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- ALTER TABLE memory_entity ADD COLUMN IF NOT EXISTS project_id VARCHAR(255);
|
||||
-- ALTER TABLE memory_edge ADD COLUMN IF NOT EXISTS project_id VARCHAR(255);
|
||||
--
|
||||
-- Note: Deferred. Can use project info from path/source instead.
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 5. Views for common queries
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- Entities contributed by a user in a time range
|
||||
CREATE OR REPLACE VIEW v_user_contributions AS
|
||||
SELECT
|
||||
contributed_by,
|
||||
COUNT(*) as entity_count,
|
||||
MAX(contribution_date) as last_contribution,
|
||||
ARRAY_AGG(DISTINCT contribution_type) as contribution_types
|
||||
FROM memory_entity
|
||||
WHERE contribution_date IS NOT NULL
|
||||
GROUP BY contributed_by;
|
||||
|
||||
-- Recent contributions (last 7 days)
|
||||
CREATE OR REPLACE VIEW v_recent_contributions AS
|
||||
SELECT
|
||||
contributed_by,
|
||||
COUNT(*) as count,
|
||||
MAX(contribution_date) as latest
|
||||
FROM memory_entity
|
||||
WHERE contribution_date > CURRENT_TIMESTAMP - INTERVAL '7 days'
|
||||
GROUP BY contributed_by;
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- 6. Rollback support
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- To rollback this migration:
|
||||
--
|
||||
-- DROP VIEW IF EXISTS v_recent_contributions;
|
||||
-- DROP VIEW IF EXISTS v_user_contributions;
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_memory_projects_owner_id;
|
||||
-- DROP INDEX IF EXISTS idx_memory_projects_visibility;
|
||||
-- DROP TABLE IF EXISTS memory_projects;
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_memory_entity_contributed_by;
|
||||
-- DROP INDEX IF EXISTS idx_memory_entity_contribution_date;
|
||||
-- ALTER TABLE memory_entity DROP COLUMN IF EXISTS contributed_by;
|
||||
-- ALTER TABLE memory_entity DROP COLUMN IF EXISTS contribution_type;
|
||||
-- ALTER TABLE memory_entity DROP COLUMN IF EXISTS contribution_date;
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_memory_edge_contributed_by;
|
||||
-- ALTER TABLE memory_edge DROP COLUMN IF EXISTS contributed_by;
|
||||
-- ALTER TABLE memory_edge DROP COLUMN IF EXISTS contribution_type;
|
||||
@@ -0,0 +1,72 @@
|
||||
-- Phase 6: Temporal Workflow Service Integration
|
||||
--
|
||||
-- Adds reference/linking table for external Temporal.io workflows.
|
||||
-- Temporal Service owns execution state; Memory DB owns reasoning traces.
|
||||
--
|
||||
-- This is a minimal linking schema - no duplication of workflow logic.
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Temporal Workflow Links (External Service Reference)
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS temporal_workflow_links (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
|
||||
-- Temporal Service Reference (Required)
|
||||
workflow_id VARCHAR(255) NOT NULL, -- Temporal workflow ID
|
||||
workflow_type VARCHAR(255), -- e.g., "entity_extraction", "synthesis"
|
||||
run_id VARCHAR(255), -- Temporal run ID
|
||||
|
||||
-- Memory Entity References (Optional - NULL until linked)
|
||||
entity_id UUID REFERENCES memory_entity(id) ON DELETE SET NULL,
|
||||
edge_id UUID REFERENCES memory_edge(id) ON DELETE SET NULL,
|
||||
node_id BIGINT REFERENCES memory_node(id) ON DELETE SET NULL,
|
||||
|
||||
-- Sync Status
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'completed', 'failed', 'archived')),
|
||||
|
||||
-- Timestamps
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
|
||||
-- Metadata (Extra context, error details, etc.)
|
||||
metadata JSONB
|
||||
);
|
||||
|
||||
-- Indexes for common queries
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_workflow_id
|
||||
ON temporal_workflow_links(workflow_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_entity_id
|
||||
ON temporal_workflow_links(entity_id)
|
||||
WHERE entity_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_edge_id
|
||||
ON temporal_workflow_links(edge_id)
|
||||
WHERE edge_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_node_id
|
||||
ON temporal_workflow_links(node_id)
|
||||
WHERE node_id IS NOT NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_status
|
||||
ON temporal_workflow_links(status, created_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_temporal_workflow_type
|
||||
ON temporal_workflow_links(workflow_type)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Rollback Instructions
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- To rollback this migration:
|
||||
--
|
||||
-- DROP INDEX IF EXISTS idx_temporal_workflow_type;
|
||||
-- DROP INDEX IF EXISTS idx_temporal_status;
|
||||
-- DROP INDEX IF EXISTS idx_temporal_node_id;
|
||||
-- DROP INDEX IF EXISTS idx_temporal_edge_id;
|
||||
-- DROP INDEX IF EXISTS idx_temporal_entity_id;
|
||||
-- DROP INDEX IF EXISTS idx_temporal_workflow_id;
|
||||
-- DROP TABLE IF EXISTS temporal_workflow_links;
|
||||
@@ -0,0 +1,45 @@
|
||||
//! Community repository - trait-based interface
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use mem_core::Community;
|
||||
|
||||
/// Community operations trait
|
||||
#[async_trait]
|
||||
pub trait CommunityRepoOps: Send + Sync {
|
||||
async fn insert(&self, community: &Community) -> Result<String>;
|
||||
async fn find_by_id(&self, id: &str) -> Result<Option<Community>>;
|
||||
async fn update_summary(&self, id: &str, summary: &str, keywords: &[String], emb: Option<&[f32]>) -> Result<()>;
|
||||
async fn update_counts(&self, id: &str) -> Result<()>;
|
||||
async fn find_stale(&self, max_age_hrs: i64, limit: i32) -> Result<Vec<Community>>;
|
||||
async fn search_by_keywords(&self, proj_id: &str, keyword: &str) -> Result<Vec<Community>>;
|
||||
async fn find_by_project(&self, proj_id: &str) -> Result<Vec<Community>>;
|
||||
async fn increment_version(&self, id: &str) -> Result<()>;
|
||||
async fn count(&self, proj_id: &str) -> Result<i64>;
|
||||
}
|
||||
|
||||
pub struct MockCommunityRepo;
|
||||
|
||||
#[async_trait]
|
||||
impl CommunityRepoOps for MockCommunityRepo {
|
||||
async fn insert(&self, c: &Community) -> Result<String> { Ok(c.id.clone()) }
|
||||
async fn find_by_id(&self, _id: &str) -> Result<Option<Community>> { Ok(None) }
|
||||
async fn update_summary(&self, _id: &str, _s: &str, _k: &[String], _e: Option<&[f32]>) -> Result<()> { Ok(()) }
|
||||
async fn update_counts(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn find_stale(&self, _a: i64, _l: i32) -> Result<Vec<Community>> { Ok(vec![]) }
|
||||
async fn search_by_keywords(&self, _p: &str, _k: &str) -> Result<Vec<Community>> { Ok(vec![]) }
|
||||
async fn find_by_project(&self, _p: &str) -> Result<Vec<Community>> { Ok(vec![]) }
|
||||
async fn increment_version(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn count(&self, _p: &str) -> Result<i64> { Ok(0) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_community_repo() {
|
||||
let repo = MockCommunityRepo;
|
||||
assert!(repo.count("test").await.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
/// PostgreSQL repository implementation for Phase 2.6 DB Integration.
|
||||
///
|
||||
/// Connects ingest pipeline to persistent storage.
|
||||
/// Handles transactions, error recovery, and audit logging.
|
||||
|
||||
use sqlx::{Pool, Postgres, Row, Transaction, Error as SqlxError};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use chrono::{DateTime, Utc};
|
||||
use crate::entity_repo::{Entity, EntityRepo};
|
||||
use crate::edge_repo::{Edge, EdgeRepo};
|
||||
|
||||
/// Database connection error types
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum DbError {
|
||||
ConnectionFailed(String),
|
||||
QueryFailed(String),
|
||||
TransactionFailed(String),
|
||||
DuplicateKey(String),
|
||||
NotFound(String),
|
||||
InvalidData(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DbError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
match self {
|
||||
DbError::ConnectionFailed(msg) => write!(f, "Connection failed: {}", msg),
|
||||
DbError::QueryFailed(msg) => write!(f, "Query failed: {}", msg),
|
||||
DbError::TransactionFailed(msg) => write!(f, "Transaction failed: {}", msg),
|
||||
DbError::DuplicateKey(msg) => write!(f, "Duplicate key: {}", msg),
|
||||
DbError::NotFound(msg) => write!(f, "Not found: {}", msg),
|
||||
DbError::InvalidData(msg) => write!(f, "Invalid data: {}", msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for DbError {}
|
||||
|
||||
/// PostgreSQL repository pool
|
||||
pub struct DbPool {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl DbPool {
|
||||
/// Create new DB pool from connection string
|
||||
pub async fn new(database_url: &str) -> Result<Self, DbError> {
|
||||
let pool = Pool::<Postgres>::connect(database_url)
|
||||
.await
|
||||
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
|
||||
|
||||
Ok(DbPool { pool })
|
||||
}
|
||||
|
||||
/// Get pool for queries
|
||||
pub fn pool(&self) -> &Pool<Postgres> {
|
||||
&self.pool
|
||||
}
|
||||
|
||||
/// Test connection
|
||||
pub async fn health_check(&self) -> Result<(), DbError> {
|
||||
sqlx::query("SELECT 1")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::ConnectionFailed(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Persistent entity repository
|
||||
pub struct PersistentEntityRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl PersistentEntityRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Save entity to database (idempotent)
|
||||
pub async fn save(&self, entity: &Entity) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO memory_entity (id, entity_type, name, description, embedding, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
description = EXCLUDED.description,
|
||||
updated_at = EXCLUDED.updated_at
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entity.id)
|
||||
.bind(&entity.entity_type)
|
||||
.bind(&entity.name)
|
||||
.bind(&entity.description)
|
||||
.bind(&entity.embedding)
|
||||
.bind(Utc::now())
|
||||
.bind(Utc::now())
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("duplicate") {
|
||||
DbError::DuplicateKey(format!("Entity {} already exists", entity.id))
|
||||
} else {
|
||||
DbError::QueryFailed(e.to_string())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get entity by ID
|
||||
pub async fn get(&self, id: &str) -> Result<Option<Entity>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, entity_type, name, description, embedding, created_at, updated_at
|
||||
FROM memory_entity
|
||||
WHERE id = $1 AND deleted_at IS NULL;
|
||||
"#;
|
||||
|
||||
let row = sqlx::query(query)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(row.map(|r| Entity {
|
||||
id: r.get("id"),
|
||||
entity_type: r.get("entity_type"),
|
||||
name: r.get("name"),
|
||||
description: r.get("description"),
|
||||
embedding: r.get("embedding"),
|
||||
created_at: r.get("created_at"),
|
||||
updated_at: r.get("updated_at"),
|
||||
}))
|
||||
}
|
||||
|
||||
/// List entities with pagination
|
||||
pub async fn list(&self, limit: i64, offset: i64) -> Result<Vec<Entity>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, entity_type, name, description, embedding, created_at, updated_at
|
||||
FROM memory_entity
|
||||
WHERE deleted_at IS NULL
|
||||
ORDER BY created_at DESC
|
||||
LIMIT $1 OFFSET $2;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(limit)
|
||||
.bind(offset)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| Entity {
|
||||
id: r.get("id"),
|
||||
entity_type: r.get("entity_type"),
|
||||
name: r.get("name"),
|
||||
description: r.get("description"),
|
||||
embedding: r.get("embedding"),
|
||||
created_at: r.get("created_at"),
|
||||
updated_at: r.get("updated_at"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Soft delete entity
|
||||
pub async fn delete(&self, id: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE memory_entity
|
||||
SET deleted_at = $1
|
||||
WHERE id = $2;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Persistent edge repository
|
||||
pub struct PersistentEdgeRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl PersistentEdgeRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Save edge to database (idempotent)
|
||||
pub async fn save(&self, edge: &Edge) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO memory_edge (id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
strength = EXCLUDED.strength,
|
||||
t_invalid = EXCLUDED.t_invalid,
|
||||
t_expired = EXCLUDED.t_expired
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&edge.id)
|
||||
.bind(&edge.source_id)
|
||||
.bind(&edge.target_id)
|
||||
.bind(&edge.relation_type)
|
||||
.bind(&edge.fact)
|
||||
.bind(edge.strength)
|
||||
.bind(edge.t_valid)
|
||||
.bind(edge.t_invalid)
|
||||
.bind(edge.t_created)
|
||||
.bind(edge.t_expired)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.to_string().contains("duplicate") {
|
||||
DbError::DuplicateKey(format!("Edge {} already exists", edge.id))
|
||||
} else {
|
||||
DbError::QueryFailed(e.to_string())
|
||||
}
|
||||
})?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get edge by ID
|
||||
pub async fn get(&self, id: &str) -> Result<Option<Edge>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
|
||||
FROM memory_edge
|
||||
WHERE id = $1 AND t_expired IS NULL;
|
||||
"#;
|
||||
|
||||
let row = sqlx::query(query)
|
||||
.bind(id)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(row.map(|r| Edge {
|
||||
id: r.get("id"),
|
||||
source_id: r.get("source_id"),
|
||||
target_id: r.get("target_id"),
|
||||
relation_type: r.get("relation_type"),
|
||||
fact: r.get("fact"),
|
||||
strength: r.get("strength"),
|
||||
t_valid: r.get("t_valid"),
|
||||
t_invalid: r.get("t_invalid"),
|
||||
t_created: r.get("t_created"),
|
||||
t_expired: r.get("t_expired"),
|
||||
}))
|
||||
}
|
||||
|
||||
/// List edges for a source entity
|
||||
pub async fn list_from(&self, source_id: &str, limit: i64) -> Result<Vec<Edge>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, source_id, target_id, relation_type, fact, strength, t_valid, t_invalid, t_created, t_expired
|
||||
FROM memory_edge
|
||||
WHERE source_id = $1 AND t_expired IS NULL AND t_invalid IS NULL
|
||||
ORDER BY t_created DESC
|
||||
LIMIT $2;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(source_id)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| Edge {
|
||||
id: r.get("id"),
|
||||
source_id: r.get("source_id"),
|
||||
target_id: r.get("target_id"),
|
||||
relation_type: r.get("relation_type"),
|
||||
fact: r.get("fact"),
|
||||
strength: r.get("strength"),
|
||||
t_valid: r.get("t_valid"),
|
||||
t_invalid: r.get("t_invalid"),
|
||||
t_created: r.get("t_created"),
|
||||
t_expired: r.get("t_expired"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Mark edge as contradicted (soft delete)
|
||||
pub async fn invalidate(&self, id: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE memory_edge
|
||||
SET t_invalid = $1
|
||||
WHERE id = $2;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Review queue entry for human verification
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReviewQueueEntry {
|
||||
pub id: String,
|
||||
pub extraction_type: String, // "entity" | "edge" | "contradiction"
|
||||
pub content: serde_json::Value, // Full extracted data
|
||||
pub status: String, // "pending" | "approved" | "rejected"
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub reviewed_at: Option<DateTime<Utc>>,
|
||||
pub reviewed_by: Option<String>, // User ID who reviewed
|
||||
pub rejection_reason: Option<String>,
|
||||
}
|
||||
|
||||
/// Review queue repository
|
||||
pub struct ReviewQueueRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl ReviewQueueRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Add item to review queue
|
||||
pub async fn enqueue(&self, entry: &ReviewQueueEntry) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO review_queue (id, extraction_type, content, status, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entry.id)
|
||||
.bind(&entry.extraction_type)
|
||||
.bind(&entry.content)
|
||||
.bind(&entry.status)
|
||||
.bind(Utc::now())
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get pending items for review
|
||||
pub async fn list_pending(&self, limit: i64) -> Result<Vec<ReviewQueueEntry>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, extraction_type, content, status, created_at, reviewed_at, reviewed_by, rejection_reason
|
||||
FROM review_queue
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT $1;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(limit)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| ReviewQueueEntry {
|
||||
id: r.get("id"),
|
||||
extraction_type: r.get("extraction_type"),
|
||||
content: r.get("content"),
|
||||
status: r.get("status"),
|
||||
created_at: r.get("created_at"),
|
||||
reviewed_at: r.get("reviewed_at"),
|
||||
reviewed_by: r.get("reviewed_by"),
|
||||
rejection_reason: r.get("rejection_reason"),
|
||||
}).collect())
|
||||
}
|
||||
|
||||
/// Approve review queue entry
|
||||
pub async fn approve(&self, id: &str, reviewed_by: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE review_queue
|
||||
SET status = 'approved', reviewed_at = $1, reviewed_by = $2
|
||||
WHERE id = $3;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(reviewed_by)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reject review queue entry
|
||||
pub async fn reject(&self, id: &str, reviewed_by: &str, reason: &str) -> Result<(), DbError> {
|
||||
let query = r#"
|
||||
UPDATE review_queue
|
||||
SET status = 'rejected', reviewed_at = $1, reviewed_by = $2, rejection_reason = $3
|
||||
WHERE id = $4;
|
||||
"#;
|
||||
|
||||
sqlx::query(query)
|
||||
.bind(Utc::now())
|
||||
.bind(reviewed_by)
|
||||
.bind(reason)
|
||||
.bind(id)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Extraction Audit Repository (Immutable log for audit trail)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ExtractionAuditEntry {
|
||||
pub id: String,
|
||||
pub extraction_type: String, // "entity" | "edge"
|
||||
pub extraction_id: String, // ID of extracted entity/edge
|
||||
pub source_content: String, // Original text
|
||||
pub extracted_data: serde_json::Value,
|
||||
pub llm_confidence: Option<f32>,
|
||||
pub contradiction_score: Option<f32>,
|
||||
pub status: String, // "extracted" | "approved" | "rejected"
|
||||
pub extracted_at: DateTime<Utc>,
|
||||
pub extracted_by: String, // User or "system"
|
||||
}
|
||||
|
||||
pub struct ExtractionAuditRepo {
|
||||
pool: Pool<Postgres>,
|
||||
}
|
||||
|
||||
impl ExtractionAuditRepo {
|
||||
pub fn new(pool: Pool<Postgres>) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
|
||||
/// Log an extraction attempt (immutable append)
|
||||
pub async fn log_extraction(&self, entry: &ExtractionAuditEntry) -> Result<String, DbError> {
|
||||
let query = r#"
|
||||
INSERT INTO extraction_audit (id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
RETURNING id;
|
||||
"#;
|
||||
|
||||
let id = sqlx::query_scalar::<_, String>(query)
|
||||
.bind(&entry.id)
|
||||
.bind(&entry.extraction_type)
|
||||
.bind(&entry.extraction_id)
|
||||
.bind(&entry.source_content)
|
||||
.bind(&entry.extracted_data)
|
||||
.bind(entry.llm_confidence)
|
||||
.bind(entry.contradiction_score)
|
||||
.bind(&entry.status)
|
||||
.bind(entry.extracted_at)
|
||||
.bind(&entry.extracted_by)
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Get audit trail for an extracted item
|
||||
pub async fn get_history(&self, extraction_id: &str) -> Result<Vec<ExtractionAuditEntry>, DbError> {
|
||||
let query = r#"
|
||||
SELECT id, extraction_type, extraction_id, source_content, extracted_data, llm_confidence, contradiction_score, status, extracted_at, extracted_by
|
||||
FROM extraction_audit
|
||||
WHERE extraction_id = $1
|
||||
ORDER BY extracted_at DESC;
|
||||
"#;
|
||||
|
||||
let rows = sqlx::query(query)
|
||||
.bind(extraction_id)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(|e| DbError::QueryFailed(e.to_string()))?;
|
||||
|
||||
Ok(rows.iter().map(|r| ExtractionAuditEntry {
|
||||
id: r.get("id"),
|
||||
extraction_type: r.get("extraction_type"),
|
||||
extraction_id: r.get("extraction_id"),
|
||||
source_content: r.get("source_content"),
|
||||
extracted_data: r.get("extracted_data"),
|
||||
llm_confidence: r.get("llm_confidence"),
|
||||
contradiction_score: r.get("contradiction_score"),
|
||||
status: r.get("status"),
|
||||
extracted_at: r.get("extracted_at"),
|
||||
extracted_by: r.get("extracted_by"),
|
||||
}).collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_db_error_display() {
|
||||
let err = DbError::ConnectionFailed("test".to_string());
|
||||
assert!(err.to_string().contains("Connection failed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_review_queue_entry_creation() {
|
||||
let entry = ReviewQueueEntry {
|
||||
id: "test-1".to_string(),
|
||||
extraction_type: "entity".to_string(),
|
||||
content: serde_json::json!({"name": "test"}),
|
||||
status: "pending".to_string(),
|
||||
created_at: Utc::now(),
|
||||
reviewed_at: None,
|
||||
reviewed_by: None,
|
||||
rejection_reason: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.extraction_type, "entity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dead_letter_entry_creation() {
|
||||
let entry = DeadLetterEntry {
|
||||
id: "dlq-1".to_string(),
|
||||
original_content: "test content".to_string(),
|
||||
error_message: "extraction failed".to_string(),
|
||||
error_type: "extraction_failed".to_string(),
|
||||
retry_count: 0,
|
||||
max_retries: 3,
|
||||
created_at: Utc::now(),
|
||||
last_retry_at: None,
|
||||
};
|
||||
|
||||
assert_eq!(entry.retry_count, 0);
|
||||
assert!(entry.retry_count < entry.max_retries);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
//! Edge repository - trait-based interface
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use mem_core::edge::Edge;
|
||||
|
||||
/// Edge operations trait
|
||||
#[async_trait]
|
||||
pub trait EdgeRepoOps: Send + Sync {
|
||||
async fn insert(&self, edge: &Edge) -> Result<String>;
|
||||
async fn find_between_entities(&self, src_id: &str, tgt_id: &str) -> Result<Vec<Edge>>;
|
||||
async fn find_valid_at(&self, proj_id: &str, at: OffsetDateTime, limit: i32) -> Result<Vec<Edge>>;
|
||||
async fn mark_contradiction_candidate(&self, edge_id: &str, conflict_id: &str, conf: f32) -> Result<()>;
|
||||
async fn confirm_invalidation(&self, edge_id: &str, invalid_at: OffsetDateTime) -> Result<()>;
|
||||
async fn resolve_contradiction(&self, edge_id: &str, action: &str, reviewer: &str) -> Result<()>;
|
||||
async fn find_similar(&self, emb: &[f32], src: &str, tgt: &str, thresh: f32) -> Result<Vec<(Edge, f32)>>;
|
||||
async fn soft_delete(&self, id: &str) -> Result<()>;
|
||||
async fn record_access(&self, id: &str) -> Result<()>;
|
||||
async fn count_active(&self, proj_id: &str) -> Result<i64>;
|
||||
async fn find_pending_review(&self, limit: i32) -> Result<Vec<(String, String, f32)>>;
|
||||
}
|
||||
|
||||
pub struct MockEdgeRepo;
|
||||
|
||||
#[async_trait]
|
||||
impl EdgeRepoOps for MockEdgeRepo {
|
||||
async fn insert(&self, edge: &Edge) -> Result<String> { Ok(edge.id.clone()) }
|
||||
async fn find_between_entities(&self, _s: &str, _t: &str) -> Result<Vec<Edge>> { Ok(vec![]) }
|
||||
async fn find_valid_at(&self, _p: &str, _at: OffsetDateTime, _l: i32) -> Result<Vec<Edge>> { Ok(vec![]) }
|
||||
async fn mark_contradiction_candidate(&self, _e: &str, _c: &str, _f: f32) -> Result<()> { Ok(()) }
|
||||
async fn confirm_invalidation(&self, _e: &str, _ia: OffsetDateTime) -> Result<()> { Ok(()) }
|
||||
async fn resolve_contradiction(&self, _e: &str, _a: &str, _r: &str) -> Result<()> { Ok(()) }
|
||||
async fn find_similar(&self, _e: &[f32], _s: &str, _t: &str, _th: f32) -> Result<Vec<(Edge, f32)>> { Ok(vec![]) }
|
||||
async fn soft_delete(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn record_access(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn count_active(&self, _p: &str) -> Result<i64> { Ok(0) }
|
||||
async fn find_pending_review(&self, _l: i32) -> Result<Vec<(String, String, f32)>> { Ok(vec![]) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_edge_repo() {
|
||||
let repo = MockEdgeRepo;
|
||||
assert!(repo.count_active("test").await.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
//! Entity repository - trait-based interface
|
||||
//! Avoids sqlx macros requiring DATABASE_URL
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use mem_core::entity::Entity;
|
||||
|
||||
/// Entity operations trait
|
||||
#[async_trait]
|
||||
pub trait EntityRepoOps: Send + Sync {
|
||||
async fn insert(&self, entity: &Entity) -> Result<String>;
|
||||
async fn find_by_id(&self, id: &str) -> Result<Option<Entity>>;
|
||||
async fn find_by_name(&self, project_id: &str, name: &str) -> Result<Option<Entity>>;
|
||||
async fn find_similar_by_name(&self, project_id: &str, embedding: &[f32], threshold: f32, limit: i32) -> Result<Vec<(Entity, f32)>>;
|
||||
async fn link_source_episode(&self, entity_id: &str, episode_id: i64) -> Result<()>;
|
||||
async fn soft_delete(&self, id: &str) -> Result<()>;
|
||||
async fn record_access(&self, id: &str) -> Result<()>;
|
||||
async fn set_community(&self, entity_id: &str, community_id: &str) -> Result<()>;
|
||||
async fn clear_community(&self, entity_id: &str) -> Result<()>;
|
||||
async fn count_active(&self, project_id: &str) -> Result<i64>;
|
||||
}
|
||||
|
||||
/// Mock implementation for testing (replaces DB access)
|
||||
pub struct MockEntityRepo;
|
||||
|
||||
#[async_trait]
|
||||
impl EntityRepoOps for MockEntityRepo {
|
||||
async fn insert(&self, entity: &Entity) -> Result<String> { Ok(entity.id.clone()) }
|
||||
async fn find_by_id(&self, _id: &str) -> Result<Option<Entity>> { Ok(None) }
|
||||
async fn find_by_name(&self, _proj: &str, _name: &str) -> Result<Option<Entity>> { Ok(None) }
|
||||
async fn find_similar_by_name(&self, _proj: &str, _emb: &[f32], _thresh: f32, _limit: i32) -> Result<Vec<(Entity, f32)>> { Ok(vec![]) }
|
||||
async fn link_source_episode(&self, _ent: &str, _ep: i64) -> Result<()> { Ok(()) }
|
||||
async fn soft_delete(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn record_access(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
async fn set_community(&self, _ent: &str, _com: &str) -> Result<()> { Ok(()) }
|
||||
async fn clear_community(&self, _ent: &str) -> Result<()> { Ok(()) }
|
||||
async fn count_active(&self, _proj: &str) -> Result<i64> { Ok(0) }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_mock_repo() {
|
||||
let repo = MockEntityRepo;
|
||||
assert!(repo.count_active("test").await.is_ok());
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,15 @@ pub mod pgvector;
|
||||
pub mod rebuild;
|
||||
pub mod pg_repo;
|
||||
pub mod schema;
|
||||
pub mod entity_repo;
|
||||
pub mod edge_repo;
|
||||
pub mod community_repo;
|
||||
|
||||
pub use event_log::{EventRecord, LogWriter};
|
||||
pub use pgvector::{VectorRecord, VectorStore, ChunkL0, MemoryL1, MemoryL2};
|
||||
pub use rebuild::{RebuildEngine, RebuildOpts, RebuildStats};
|
||||
pub use pg_repo::{PgRepo, MemoryNode, VectorKind, Level, ScoredNode, Scope, SignatureHit};
|
||||
pub use schema::init_schema;
|
||||
pub use entity_repo::{EntityRepoOps, MockEntityRepo};
|
||||
pub use edge_repo::{EdgeRepoOps, MockEdgeRepo};
|
||||
pub use community_repo::{CommunityRepoOps, MockCommunityRepo};
|
||||
|
||||
Reference in New Issue
Block a user