Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7da1cffec6 | ||
|
|
a4c445b6ba | ||
|
|
6b35b04d20 | ||
|
|
c41cef0ca5 | ||
|
|
d52b99f0b1 | ||
|
|
0c6abc3fef | ||
|
|
b40cc47729 | ||
|
|
31f5265603 | ||
|
|
5cfff3990a | ||
|
|
9807e7ec97 | ||
|
|
d8b7adf2fc | ||
|
|
9062c51545 | ||
|
|
1793fe39f0 | ||
|
|
49a82e2caf |
@@ -4,9 +4,6 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- main
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
test:
|
test:
|
||||||
@@ -17,7 +14,7 @@ jobs:
|
|||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Cargo test
|
- name: Cargo test
|
||||||
run: cargo test -p mem-ingest --lib 2>&1 | tail -30
|
run: cargo test -p mem-ingest --lib 2>&1 | tail -20
|
||||||
|
|
||||||
- name: Cargo check
|
- name: Cargo check
|
||||||
run: cargo check -p mem-ingest 2>&1 | grep -E "error|warning: unused|Finished" || true
|
run: cargo check -p mem-ingest 2>&1 | grep -E "error|warning: unused|Finished" || true
|
||||||
@@ -26,7 +23,6 @@ jobs:
|
|||||||
name: Build & Push Image
|
name: Build & Push Image
|
||||||
runs-on: rust
|
runs-on: rust
|
||||||
needs: test
|
needs: test
|
||||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
/// Configuration management for inter-pod URLs via environment variables (ConfigMap)
|
||||||
|
/// All service URLs come from K8s ConfigMap, never hardcoded
|
||||||
|
///
|
||||||
|
/// ConfigMap in K8s:
|
||||||
|
/// ```yaml
|
||||||
|
/// apiVersion: v1
|
||||||
|
/// kind: ConfigMap
|
||||||
|
/// metadata:
|
||||||
|
/// name: memory-service-config
|
||||||
|
/// namespace: poimen
|
||||||
|
/// data:
|
||||||
|
/// MEMORY_SERVICE_ADDR: "http://memory-service.poimen.svc.cluster.local:8080"
|
||||||
|
/// AUTHENTIK_ISSUER: "http://authentik.iam.svc.cluster.local/application/o/poimen-memory/"
|
||||||
|
/// WEBHOOK_URL: "http://temporal-webhook.temporal.svc.cluster.local:9000/webhook"
|
||||||
|
/// ```
|
||||||
|
|
||||||
|
use anyhow::{anyhow, Result};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ServiceConfig {
|
||||||
|
/// Memory service address (this service itself)
|
||||||
|
pub memory_service_addr: String,
|
||||||
|
|
||||||
|
/// Authentik OIDC issuer endpoint
|
||||||
|
pub authentik_issuer: String,
|
||||||
|
|
||||||
|
/// Temporal webhook callback URL
|
||||||
|
pub webhook_url: String,
|
||||||
|
|
||||||
|
/// OpenSearch cluster endpoint
|
||||||
|
pub opensearch_url: String,
|
||||||
|
|
||||||
|
/// PostgreSQL connection string
|
||||||
|
pub database_url: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ServiceConfig {
|
||||||
|
/// Load configuration from environment variables (set by K8s ConfigMap)
|
||||||
|
/// Fails if required env vars are missing
|
||||||
|
pub fn from_env() -> Result<Self> {
|
||||||
|
let memory_service_addr = std::env::var("MEMORY_SERVICE_ADDR")
|
||||||
|
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||||
|
|
||||||
|
let authentik_issuer = std::env::var("AUTHENTIK_ISSUER")
|
||||||
|
.map_err(|_| anyhow!("AUTHENTIK_ISSUER env var not set (configure in ConfigMap)"))?;
|
||||||
|
|
||||||
|
let webhook_url = std::env::var("WEBHOOK_URL")
|
||||||
|
.map_err(|_| anyhow!("WEBHOOK_URL env var not set (configure in ConfigMap)"))?;
|
||||||
|
|
||||||
|
let opensearch_url = std::env::var("OPENSEARCH_URL")
|
||||||
|
.map_err(|_| anyhow!("OPENSEARCH_URL env var not set (configure in ConfigMap)"))?;
|
||||||
|
|
||||||
|
let database_url = std::env::var("DATABASE_URL")
|
||||||
|
.map_err(|_| anyhow!("DATABASE_URL env var not set (configure Secret or ConfigMap)"))?;
|
||||||
|
|
||||||
|
Ok(ServiceConfig {
|
||||||
|
memory_service_addr,
|
||||||
|
authentik_issuer,
|
||||||
|
webhook_url,
|
||||||
|
opensearch_url,
|
||||||
|
database_url,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Load with defaults for development (localhost only)
|
||||||
|
pub fn from_env_dev() -> Self {
|
||||||
|
ServiceConfig {
|
||||||
|
memory_service_addr: std::env::var("MEMORY_SERVICE_ADDR")
|
||||||
|
.unwrap_or_else(|_| "http://localhost:8080".to_string()),
|
||||||
|
authentik_issuer: std::env::var("AUTHENTIK_ISSUER")
|
||||||
|
.unwrap_or_else(|_| "http://localhost:8080/application/o/poimen-memory/".to_string()),
|
||||||
|
webhook_url: std::env::var("WEBHOOK_URL")
|
||||||
|
.unwrap_or_else(|_| "http://localhost:9000/webhook".to_string()),
|
||||||
|
opensearch_url: std::env::var("OPENSEARCH_URL")
|
||||||
|
.unwrap_or_else(|_| "http://localhost:9200".to_string()),
|
||||||
|
database_url: std::env::var("DATABASE_URL")
|
||||||
|
.unwrap_or_else(|_| "postgres://localhost/memory".to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_config_from_env_dev() {
|
||||||
|
let config = ServiceConfig::from_env_dev();
|
||||||
|
assert_eq!(config.memory_service_addr, "http://localhost:8080");
|
||||||
|
assert!(config.authentik_issuer.contains("localhost"));
|
||||||
|
assert!(config.webhook_url.contains("localhost"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
pub mod config;
|
||||||
pub mod endpoints;
|
pub mod endpoints;
|
||||||
pub mod handlers;
|
pub mod handlers;
|
||||||
pub mod http_server;
|
pub mod http_server;
|
||||||
|
|||||||
@@ -448,8 +448,10 @@ async fn cmd_learn(
|
|||||||
all_files.sort();
|
all_files.sort();
|
||||||
println!("Found {} markdown files", all_files.len());
|
println!("Found {} markdown files", all_files.len());
|
||||||
|
|
||||||
|
// Load configuration from environment (set by K8s ConfigMap)
|
||||||
|
let config = mem_cli::config::ServiceConfig::from_env_dev();
|
||||||
let api_url = std::env::var("MEM_API_URL")
|
let api_url = std::env::var("MEM_API_URL")
|
||||||
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
.unwrap_or_else(|_| config.memory_service_addr.clone());
|
||||||
let api_token = std::env::var("MEM_API_TOKEN").ok();
|
let api_token = std::env::var("MEM_API_TOKEN").ok();
|
||||||
let http = reqwest::Client::builder()
|
let http = reqwest::Client::builder()
|
||||||
.timeout(std::time::Duration::from_secs(120))
|
.timeout(std::time::Duration::from_secs(120))
|
||||||
|
|||||||
@@ -1,234 +0,0 @@
|
|||||||
-- Phase 4: Community Detection Schema
|
|
||||||
-- Extends memory_community with label propagation execution and statistics
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 1: Create label propagation run tracking
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS label_propagation_run (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
run_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
algorithm VARCHAR(50) DEFAULT 'label_propagation',
|
|
||||||
max_iterations INT DEFAULT 10,
|
|
||||||
convergence_threshold FLOAT DEFAULT 0.01,
|
|
||||||
iterations_completed INT,
|
|
||||||
converged BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
-- Execution metadata
|
|
||||||
status VARCHAR(20) DEFAULT 'running'
|
|
||||||
CHECK (status IN ('running', 'completed', 'failed')),
|
|
||||||
error_message TEXT,
|
|
||||||
duration_ms INT,
|
|
||||||
|
|
||||||
-- Statistics
|
|
||||||
communities_detected INT,
|
|
||||||
communities_merged INT,
|
|
||||||
communities_split INT,
|
|
||||||
nodes_processed INT,
|
|
||||||
edges_processed INT,
|
|
||||||
|
|
||||||
-- Execution mode
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
CONSTRAINT chk_iterations_valid CHECK (iterations_completed >= 0 AND iterations_completed <= max_iterations)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_project
|
|
||||||
ON label_propagation_run(project_id, run_at DESC);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_status
|
|
||||||
ON label_propagation_run(project_id, status)
|
|
||||||
WHERE status IN ('running', 'failed');
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 2: Create community member map
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS community_member_map (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
|
||||||
entity_id UUID NOT NULL REFERENCES memory_entity(id) ON DELETE CASCADE,
|
|
||||||
label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Label strength (0-1, higher = stronger membership)
|
|
||||||
label_strength FLOAT DEFAULT 1.0,
|
|
||||||
|
|
||||||
-- Membership tracking
|
|
||||||
is_seed BOOLEAN DEFAULT FALSE,
|
|
||||||
joined_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
left_at TIMESTAMPTZ,
|
|
||||||
|
|
||||||
-- Consistency
|
|
||||||
CONSTRAINT uq_community_entity_project UNIQUE (project_id, community_id, entity_id),
|
|
||||||
CONSTRAINT chk_label_strength CHECK (label_strength >= 0 AND label_strength <= 1)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_project
|
|
||||||
ON community_member_map(project_id, community_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_entity_lookup
|
|
||||||
ON community_member_map(entity_id, community_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_strength
|
|
||||||
ON community_member_map(community_id, label_strength DESC)
|
|
||||||
WHERE left_at IS NULL;
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_seeds
|
|
||||||
ON community_member_map(project_id, is_seed)
|
|
||||||
WHERE is_seed = TRUE;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 3: Create community statistics table
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS community_statistics (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
community_id UUID NOT NULL UNIQUE REFERENCES memory_community(id) ON DELETE CASCADE,
|
|
||||||
label_propagation_run_id UUID NOT NULL REFERENCES label_propagation_run(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Membership stats
|
|
||||||
member_count INT DEFAULT 0,
|
|
||||||
active_member_count INT DEFAULT 0,
|
|
||||||
seed_member_count INT DEFAULT 0,
|
|
||||||
|
|
||||||
-- Graph structure
|
|
||||||
internal_edge_count INT DEFAULT 0,
|
|
||||||
external_edge_count INT DEFAULT 0,
|
|
||||||
|
|
||||||
-- Cohesion metrics
|
|
||||||
density FLOAT DEFAULT 0.0,
|
|
||||||
modularity FLOAT DEFAULT 0.0,
|
|
||||||
|
|
||||||
-- Edge types within community
|
|
||||||
relation_type_distribution JSONB DEFAULT '{}',
|
|
||||||
|
|
||||||
-- Temporal metrics
|
|
||||||
first_entity_created TIMESTAMPTZ,
|
|
||||||
last_entity_accessed TIMESTAMPTZ,
|
|
||||||
avg_entity_age_days FLOAT DEFAULT 0.0,
|
|
||||||
|
|
||||||
-- Quality scores
|
|
||||||
coherence_score FLOAT DEFAULT 0.5,
|
|
||||||
stability_score FLOAT DEFAULT 0.5,
|
|
||||||
significance_score FLOAT DEFAULT 0.5,
|
|
||||||
|
|
||||||
CONSTRAINT chk_stats_nonnegative CHECK (
|
|
||||||
member_count >= 0 AND
|
|
||||||
internal_edge_count >= 0 AND
|
|
||||||
external_edge_count >= 0
|
|
||||||
),
|
|
||||||
CONSTRAINT chk_stats_bounded CHECK (
|
|
||||||
density >= 0 AND density <= 1 AND
|
|
||||||
modularity >= -1 AND modularity <= 1 AND
|
|
||||||
coherence_score >= 0 AND coherence_score <= 1 AND
|
|
||||||
stability_score >= 0 AND stability_score <= 1 AND
|
|
||||||
significance_score >= 0 AND significance_score <= 1
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_project
|
|
||||||
ON community_statistics(project_id, community_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_run
|
|
||||||
ON community_statistics(label_propagation_run_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_quality
|
|
||||||
ON community_statistics(project_id, coherence_score DESC, significance_score DESC)
|
|
||||||
WHERE coherence_score > 0.7;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 4: Create community merge history
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS community_merge_history (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
source_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
|
||||||
target_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
|
||||||
merge_reason VARCHAR(100),
|
|
||||||
merged_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Rollback capability
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
-- Statistics before merge
|
|
||||||
source_member_count INT,
|
|
||||||
target_member_count INT,
|
|
||||||
|
|
||||||
-- Impact
|
|
||||||
members_moved INT,
|
|
||||||
edges_reattached INT
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_project
|
|
||||||
ON community_merge_history(project_id, merged_at DESC);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_communities
|
|
||||||
ON community_merge_history(source_community_id, target_community_id);
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 5: Add community detection status to memory_community
|
|
||||||
-- ============================================
|
|
||||||
ALTER TABLE memory_community
|
|
||||||
ADD COLUMN IF NOT EXISTS last_detection_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
|
||||||
ADD COLUMN IF NOT EXISTS detection_score FLOAT DEFAULT 0.5,
|
|
||||||
ADD COLUMN IF NOT EXISTS is_permanent BOOLEAN DEFAULT FALSE,
|
|
||||||
ADD COLUMN IF NOT EXISTS merge_into_id UUID REFERENCES memory_community(id) ON DELETE SET NULL;
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_detection_run
|
|
||||||
ON memory_community(last_detection_run_id, detection_score DESC)
|
|
||||||
WHERE detection_score > 0.7;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 6: Add community-level summary generation tracking
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS community_summary_generation (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
|
||||||
generated_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
generated_by VARCHAR(255),
|
|
||||||
|
|
||||||
-- LLM usage
|
|
||||||
llm_model VARCHAR(100),
|
|
||||||
input_tokens INT,
|
|
||||||
output_tokens INT,
|
|
||||||
cost_usd FLOAT,
|
|
||||||
|
|
||||||
-- Generation method
|
|
||||||
method VARCHAR(50) DEFAULT 'extractive', -- 'extractive' or 'abstractive'
|
|
||||||
|
|
||||||
-- Quality
|
|
||||||
coherence_rating INT CHECK (coherence_rating >= 1 AND coherence_rating <= 5),
|
|
||||||
user_feedback TEXT,
|
|
||||||
|
|
||||||
-- Result
|
|
||||||
summary_text TEXT NOT NULL,
|
|
||||||
summary_embedding VECTOR(768),
|
|
||||||
|
|
||||||
-- Versioning
|
|
||||||
version INT DEFAULT 1,
|
|
||||||
is_latest BOOLEAN DEFAULT TRUE
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_latest
|
|
||||||
ON community_summary_generation(community_id, generated_at DESC)
|
|
||||||
WHERE is_latest = TRUE;
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_embedding
|
|
||||||
ON community_summary_generation USING hnsw (summary_embedding vector_cosine_ops)
|
|
||||||
WITH (m = 16, ef_construction = 200)
|
|
||||||
WHERE is_latest = TRUE;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- ROLLBACK INSTRUCTIONS
|
|
||||||
-- ============================================
|
|
||||||
-- DROP TABLE IF EXISTS community_summary_generation;
|
|
||||||
-- DROP TABLE IF EXISTS community_merge_history;
|
|
||||||
-- DROP TABLE IF EXISTS community_statistics;
|
|
||||||
-- DROP TABLE IF EXISTS community_member_map;
|
|
||||||
-- DROP TABLE IF EXISTS label_propagation_run;
|
|
||||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS last_detection_run_id;
|
|
||||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS detection_score;
|
|
||||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS is_permanent;
|
|
||||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS merge_into_id;
|
|
||||||
@@ -1,293 +0,0 @@
|
|||||||
-- Phase 3: Compaction Schema
|
|
||||||
-- T3.1-T3.4: Deduplication, GC, and dry-run support
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 1: Exact dedup tracking (T3.1)
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS exact_dedup_record (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
|
|
||||||
-- Source and target edges
|
|
||||||
source_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
|
||||||
target_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Match criteria (all must match for exact dedup)
|
|
||||||
source_match BOOLEAN NOT NULL,
|
|
||||||
target_match BOOLEAN NOT NULL,
|
|
||||||
relation_match BOOLEAN NOT NULL,
|
|
||||||
fact_match BOOLEAN NOT NULL,
|
|
||||||
|
|
||||||
-- Dedup decision
|
|
||||||
dedup_action VARCHAR(20) DEFAULT 'pending'
|
|
||||||
CHECK (dedup_action IN ('pending', 'merged', 'kept_separate', 'manual_review')),
|
|
||||||
|
|
||||||
-- Metadata
|
|
||||||
detected_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
processed_at TIMESTAMPTZ,
|
|
||||||
compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Dry-run support
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
CONSTRAINT chk_unique_edge_pair UNIQUE (source_edge_id, target_edge_id, project_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_project
|
|
||||||
ON exact_dedup_record(project_id, dedup_action);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_edges
|
|
||||||
ON exact_dedup_record(source_edge_id, target_edge_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_pending
|
|
||||||
ON exact_dedup_record(project_id, detected_at)
|
|
||||||
WHERE dedup_action = 'pending';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 2: Stale GC tracking (T3.1)
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS stale_gc_record (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
|
|
||||||
-- Entity or edge marked for GC
|
|
||||||
entity_id UUID REFERENCES memory_entity(id) ON DELETE CASCADE,
|
|
||||||
edge_id UUID REFERENCES memory_edge(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Staleness criteria
|
|
||||||
age_days INT NOT NULL,
|
|
||||||
t_invalid_at TIMESTAMPTZ,
|
|
||||||
access_count BIGINT DEFAULT 0,
|
|
||||||
|
|
||||||
-- GC decision
|
|
||||||
gc_action VARCHAR(20) DEFAULT 'pending'
|
|
||||||
CHECK (gc_action IN ('pending', 'deleted', 'archived', 'kept')),
|
|
||||||
|
|
||||||
-- Metadata
|
|
||||||
detected_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
processed_at TIMESTAMPTZ,
|
|
||||||
compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Dry-run support
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
CONSTRAINT chk_entity_or_edge CHECK (
|
|
||||||
(entity_id IS NOT NULL AND edge_id IS NULL) OR
|
|
||||||
(entity_id IS NULL AND edge_id IS NOT NULL)
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_project
|
|
||||||
ON stale_gc_record(project_id, gc_action);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_age
|
|
||||||
ON stale_gc_record(project_id, age_days DESC)
|
|
||||||
WHERE gc_action = 'pending';
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_invalid
|
|
||||||
ON stale_gc_record(t_invalid_at)
|
|
||||||
WHERE t_invalid_at IS NOT NULL AND gc_action = 'pending';
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 3: Semantic dedup with LLM verification (T3.2)
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS semantic_dedup_record (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
|
|
||||||
-- Source and target edges
|
|
||||||
source_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
|
||||||
target_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Pre-filter score (0-1, eliminates 60-70% of candidates)
|
|
||||||
prefilter_score FLOAT NOT NULL,
|
|
||||||
prefilter_passed BOOLEAN NOT NULL,
|
|
||||||
|
|
||||||
-- LLM verification (if prefilter_passed = true)
|
|
||||||
llm_model VARCHAR(100),
|
|
||||||
llm_prompt TEXT,
|
|
||||||
llm_response TEXT,
|
|
||||||
llm_confidence FLOAT,
|
|
||||||
llm_cost_usd FLOAT,
|
|
||||||
|
|
||||||
-- Dedup decision
|
|
||||||
dedup_action VARCHAR(50) DEFAULT 'pending'
|
|
||||||
CHECK (dedup_action IN (
|
|
||||||
'pending', 'auto_merged', 'auto_kept_separate',
|
|
||||||
'manual_review', 'llm_error', 'below_threshold'
|
|
||||||
)),
|
|
||||||
|
|
||||||
-- Merge strategy (if auto-merged)
|
|
||||||
merge_strategy VARCHAR(50), -- 'keep_superset', 'keep_newer', 'keep_higher_confidence'
|
|
||||||
merged_edge_id UUID REFERENCES memory_edge(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Metadata
|
|
||||||
detected_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
processed_at TIMESTAMPTZ,
|
|
||||||
compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
|
|
||||||
|
|
||||||
-- Dry-run support
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
CONSTRAINT chk_confidence_valid CHECK (
|
|
||||||
llm_confidence IS NULL OR (llm_confidence >= 0 AND llm_confidence <= 1)
|
|
||||||
),
|
|
||||||
CONSTRAINT chk_prefilter_valid CHECK (prefilter_score >= 0 AND prefilter_score <= 1)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_project
|
|
||||||
ON semantic_dedup_record(project_id, dedup_action);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_pending
|
|
||||||
ON semantic_dedup_record(project_id, llm_confidence DESC NULLS LAST)
|
|
||||||
WHERE dedup_action = 'manual_review' OR dedup_action = 'pending';
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_edges
|
|
||||||
ON semantic_dedup_record(source_edge_id, target_edge_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_merged
|
|
||||||
ON semantic_dedup_record(project_id, merged_edge_id)
|
|
||||||
WHERE merged_edge_id IS NOT NULL;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 4: Compaction audit trail (T3.3)
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS compaction_audit (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
compaction_run_id UUID NOT NULL REFERENCES compaction_log(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Action details
|
|
||||||
action_type VARCHAR(50) NOT NULL, -- 'exact_dedup', 'semantic_dedup', 'stale_gc', etc.
|
|
||||||
source_id UUID,
|
|
||||||
target_id UUID,
|
|
||||||
|
|
||||||
-- Before state
|
|
||||||
before_state JSONB NOT NULL,
|
|
||||||
before_hash VARCHAR(64),
|
|
||||||
|
|
||||||
-- After state
|
|
||||||
after_state JSONB NOT NULL,
|
|
||||||
after_hash VARCHAR(64),
|
|
||||||
|
|
||||||
-- Provenance
|
|
||||||
initiated_by VARCHAR(255),
|
|
||||||
approval_status VARCHAR(50) DEFAULT 'pending'
|
|
||||||
CHECK (approval_status IN ('pending', 'approved', 'rejected', 'auto')),
|
|
||||||
approved_by VARCHAR(255),
|
|
||||||
approval_reason TEXT,
|
|
||||||
|
|
||||||
-- Rollback capability
|
|
||||||
is_reversible BOOLEAN DEFAULT TRUE,
|
|
||||||
reversal_instructions JSONB,
|
|
||||||
|
|
||||||
-- Dry-run tracking
|
|
||||||
dry_run BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
-- Timestamp
|
|
||||||
recorded_at TIMESTAMPTZ DEFAULT NOW()
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_run
|
|
||||||
ON compaction_audit(compaction_run_id);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_project
|
|
||||||
ON compaction_audit(project_id, recorded_at DESC);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_reversible
|
|
||||||
ON compaction_audit(project_id, recorded_at DESC)
|
|
||||||
WHERE is_reversible = TRUE;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 5: Compaction dry-run validation
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS compaction_dryrun_result (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
compaction_run_id UUID NOT NULL REFERENCES compaction_log(id) ON DELETE CASCADE,
|
|
||||||
|
|
||||||
-- Dry-run metadata
|
|
||||||
started_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
completed_at TIMESTAMPTZ,
|
|
||||||
|
|
||||||
-- Statistics
|
|
||||||
exact_dedup_candidates INT DEFAULT 0,
|
|
||||||
exact_dedup_safe INT DEFAULT 0,
|
|
||||||
|
|
||||||
semantic_dedup_candidates INT DEFAULT 0,
|
|
||||||
semantic_dedup_safe INT DEFAULT 0,
|
|
||||||
semantic_dedup_manual_review INT DEFAULT 0,
|
|
||||||
|
|
||||||
stale_gc_candidates INT DEFAULT 0,
|
|
||||||
stale_gc_safe INT DEFAULT 0,
|
|
||||||
|
|
||||||
-- Predicted impact
|
|
||||||
predicted_space_freed_mb FLOAT DEFAULT 0.0,
|
|
||||||
predicted_edge_count_reduction INT DEFAULT 0,
|
|
||||||
predicted_entity_count_reduction INT DEFAULT 0,
|
|
||||||
|
|
||||||
-- Validation issues found
|
|
||||||
issues_found INT DEFAULT 0,
|
|
||||||
issue_details JSONB DEFAULT '[]',
|
|
||||||
|
|
||||||
-- Decision
|
|
||||||
approval_recommended BOOLEAN DEFAULT FALSE,
|
|
||||||
approval_reason TEXT
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_dryrun_project
|
|
||||||
ON compaction_dryrun_result(project_id, completed_at DESC);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_dryrun_run
|
|
||||||
ON compaction_dryrun_result(compaction_run_id);
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- STEP 6: Scheduled compaction jobs (T3.4)
|
|
||||||
-- ============================================
|
|
||||||
CREATE TABLE IF NOT EXISTS compaction_schedule (
|
|
||||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
||||||
project_id VARCHAR(255) NOT NULL,
|
|
||||||
|
|
||||||
-- Schedule config
|
|
||||||
cron_expression VARCHAR(100) NOT NULL, -- e.g., "0 2 * * *" for daily at 2 AM UTC
|
|
||||||
timezone VARCHAR(50) DEFAULT 'UTC',
|
|
||||||
|
|
||||||
-- Execution config
|
|
||||||
tier INT DEFAULT 1, -- 1 = exact dedup, 2 = semantic dedup, 3 = both
|
|
||||||
dry_run_first BOOLEAN DEFAULT TRUE,
|
|
||||||
auto_approve_safe_actions BOOLEAN DEFAULT FALSE,
|
|
||||||
|
|
||||||
-- Resource limits
|
|
||||||
max_execution_time_minutes INT DEFAULT 60,
|
|
||||||
max_llm_cost_usd FLOAT DEFAULT 10.0,
|
|
||||||
|
|
||||||
-- Status
|
|
||||||
enabled BOOLEAN DEFAULT TRUE,
|
|
||||||
|
|
||||||
-- Metadata
|
|
||||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
||||||
last_run_at TIMESTAMPTZ,
|
|
||||||
next_run_at TIMESTAMPTZ,
|
|
||||||
|
|
||||||
-- Notifications
|
|
||||||
notify_on_completion BOOLEAN DEFAULT TRUE,
|
|
||||||
notify_emails TEXT[] DEFAULT '{}'
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_schedule_project
|
|
||||||
ON compaction_schedule(project_id, enabled)
|
|
||||||
WHERE enabled = TRUE;
|
|
||||||
|
|
||||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_schedule_next_run
|
|
||||||
ON compaction_schedule(next_run_at)
|
|
||||||
WHERE enabled = TRUE;
|
|
||||||
|
|
||||||
-- ============================================
|
|
||||||
-- ROLLBACK INSTRUCTIONS
|
|
||||||
-- ============================================
|
|
||||||
-- DROP TABLE IF EXISTS compaction_schedule;
|
|
||||||
-- DROP TABLE IF EXISTS compaction_dryrun_result;
|
|
||||||
-- DROP TABLE IF EXISTS compaction_audit;
|
|
||||||
-- DROP TABLE IF EXISTS semantic_dedup_record;
|
|
||||||
-- DROP TABLE IF EXISTS stale_gc_record;
|
|
||||||
-- DROP TABLE IF EXISTS exact_dedup_record;
|
|
||||||
Reference in New Issue
Block a user