migrations: add Phase 3 & 4 schemas (compaction + community detection)
Build and Push Memory Service / Build and Push Image (push) Failing after 9s
Build and Push Memory Service / Build and Push Image (push) Failing after 9s
Phase 3: 008_phase3_compaction.sql (10.1 KB)
T3.1: exact_dedup_record, stale_gc_record (Tier 1 exact dedup + GC)
T3.2: semantic_dedup_record (Tier 2 LLM-gated semantic dedup)
T3.3: compaction_audit, compaction_dryrun_result (audit trail + dry-run)
T3.4: compaction_schedule (CronJob configuration)
Features:
- Pre-filter tracking (60-70% candidate elimination)
- Dry-run support for all operations
- Reversible actions with rollback instructions
- LLM cost tracking and approval workflow
- Scheduled execution with resource limits
Phase 4: 006_phase4_community_detection.sql (8.7 KB)
T4.1: label_propagation_run, community_member_map, community_statistics
(Label propagation algorithm + membership tracking + quality metrics)
T4.2: community_summary_generation (LLM-based summary versioning)
T4.3: (CronJob config deferred to T3.4-style scheduling)
Features:
- Algorithm execution tracking (convergence, iterations)
- Entity→Community membership with label strength
- Cohesion metrics (density, modularity, quality scores)
- Community merge history with rollback capability
- Summary versioning with LLM cost tracking
Aligned with original design:
- PHASE3_COMPACTION_PLAN.md (21.9 KB)
- phase-4-community/ (T4.1-T4.3 specifications)
- PHASE2_EXTENSIONS.md (community detection section)
All tables include:
- Bi-temporal tracking (created_at, processed_at)
- Project isolation (project_id in all tables)
- Dry-run mode support
- Audit trail + reversibility
- Proper indexes for query performance
Next: Implement T3.1-T3.4 + T4.1-T4.2 algorithms (3-4 weeks)
This commit is contained in:
@@ -0,0 +1,234 @@
|
||||
-- Phase 4: Community Detection Schema
|
||||
-- Extends memory_community with label propagation execution and statistics
|
||||
|
||||
-- ============================================
|
||||
-- STEP 1: Create label propagation run tracking
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS label_propagation_run (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
run_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
algorithm VARCHAR(50) DEFAULT 'label_propagation',
|
||||
max_iterations INT DEFAULT 10,
|
||||
convergence_threshold FLOAT DEFAULT 0.01,
|
||||
iterations_completed INT,
|
||||
converged BOOLEAN DEFAULT FALSE,
|
||||
|
||||
-- Execution metadata
|
||||
status VARCHAR(20) DEFAULT 'running'
|
||||
CHECK (status IN ('running', 'completed', 'failed')),
|
||||
error_message TEXT,
|
||||
duration_ms INT,
|
||||
|
||||
-- Statistics
|
||||
communities_detected INT,
|
||||
communities_merged INT,
|
||||
communities_split INT,
|
||||
nodes_processed INT,
|
||||
edges_processed INT,
|
||||
|
||||
-- Execution mode
|
||||
dry_run BOOLEAN DEFAULT FALSE,
|
||||
|
||||
CONSTRAINT chk_iterations_valid CHECK (iterations_completed >= 0 AND iterations_completed <= max_iterations)
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_project
|
||||
ON label_propagation_run(project_id, run_at DESC);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_status
|
||||
ON label_propagation_run(project_id, status)
|
||||
WHERE status IN ('running', 'failed');
|
||||
|
||||
-- ============================================
|
||||
-- STEP 2: Create community member map
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS community_member_map (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
||||
entity_id UUID NOT NULL REFERENCES memory_entity(id) ON DELETE CASCADE,
|
||||
label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
||||
|
||||
-- Label strength (0-1, higher = stronger membership)
|
||||
label_strength FLOAT DEFAULT 1.0,
|
||||
|
||||
-- Membership tracking
|
||||
is_seed BOOLEAN DEFAULT FALSE,
|
||||
joined_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
left_at TIMESTAMPTZ,
|
||||
|
||||
-- Consistency
|
||||
CONSTRAINT uq_community_entity_project UNIQUE (project_id, community_id, entity_id),
|
||||
CONSTRAINT chk_label_strength CHECK (label_strength >= 0 AND label_strength <= 1)
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_project
|
||||
ON community_member_map(project_id, community_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_entity_lookup
|
||||
ON community_member_map(entity_id, community_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_strength
|
||||
ON community_member_map(community_id, label_strength DESC)
|
||||
WHERE left_at IS NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_seeds
|
||||
ON community_member_map(project_id, is_seed)
|
||||
WHERE is_seed = TRUE;
|
||||
|
||||
-- ============================================
|
||||
-- STEP 3: Create community statistics table
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS community_statistics (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
community_id UUID NOT NULL UNIQUE REFERENCES memory_community(id) ON DELETE CASCADE,
|
||||
label_propagation_run_id UUID NOT NULL REFERENCES label_propagation_run(id) ON DELETE CASCADE,
|
||||
|
||||
-- Membership stats
|
||||
member_count INT DEFAULT 0,
|
||||
active_member_count INT DEFAULT 0,
|
||||
seed_member_count INT DEFAULT 0,
|
||||
|
||||
-- Graph structure
|
||||
internal_edge_count INT DEFAULT 0,
|
||||
external_edge_count INT DEFAULT 0,
|
||||
|
||||
-- Cohesion metrics
|
||||
density FLOAT DEFAULT 0.0,
|
||||
modularity FLOAT DEFAULT 0.0,
|
||||
|
||||
-- Edge types within community
|
||||
relation_type_distribution JSONB DEFAULT '{}',
|
||||
|
||||
-- Temporal metrics
|
||||
first_entity_created TIMESTAMPTZ,
|
||||
last_entity_accessed TIMESTAMPTZ,
|
||||
avg_entity_age_days FLOAT DEFAULT 0.0,
|
||||
|
||||
-- Quality scores
|
||||
coherence_score FLOAT DEFAULT 0.5,
|
||||
stability_score FLOAT DEFAULT 0.5,
|
||||
significance_score FLOAT DEFAULT 0.5,
|
||||
|
||||
CONSTRAINT chk_stats_nonnegative CHECK (
|
||||
member_count >= 0 AND
|
||||
internal_edge_count >= 0 AND
|
||||
external_edge_count >= 0
|
||||
),
|
||||
CONSTRAINT chk_stats_bounded CHECK (
|
||||
density >= 0 AND density <= 1 AND
|
||||
modularity >= -1 AND modularity <= 1 AND
|
||||
coherence_score >= 0 AND coherence_score <= 1 AND
|
||||
stability_score >= 0 AND stability_score <= 1 AND
|
||||
significance_score >= 0 AND significance_score <= 1
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_project
|
||||
ON community_statistics(project_id, community_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_run
|
||||
ON community_statistics(label_propagation_run_id);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_quality
|
||||
ON community_statistics(project_id, coherence_score DESC, significance_score DESC)
|
||||
WHERE coherence_score > 0.7;
|
||||
|
||||
-- ============================================
|
||||
-- STEP 4: Create community merge history
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS community_merge_history (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
source_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
||||
target_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
||||
merge_reason VARCHAR(100),
|
||||
merged_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
||||
|
||||
-- Rollback capability
|
||||
dry_run BOOLEAN DEFAULT FALSE,
|
||||
|
||||
-- Statistics before merge
|
||||
source_member_count INT,
|
||||
target_member_count INT,
|
||||
|
||||
-- Impact
|
||||
members_moved INT,
|
||||
edges_reattached INT
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_project
|
||||
ON community_merge_history(project_id, merged_at DESC);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_communities
|
||||
ON community_merge_history(source_community_id, target_community_id);
|
||||
|
||||
-- ============================================
|
||||
-- STEP 5: Add community detection status to memory_community
|
||||
-- ============================================
|
||||
ALTER TABLE memory_community
|
||||
ADD COLUMN IF NOT EXISTS last_detection_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
|
||||
ADD COLUMN IF NOT EXISTS detection_score FLOAT DEFAULT 0.5,
|
||||
ADD COLUMN IF NOT EXISTS is_permanent BOOLEAN DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS merge_into_id UUID REFERENCES memory_community(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_detection_run
|
||||
ON memory_community(last_detection_run_id, detection_score DESC)
|
||||
WHERE detection_score > 0.7;
|
||||
|
||||
-- ============================================
|
||||
-- STEP 6: Add community-level summary generation tracking
|
||||
-- ============================================
|
||||
CREATE TABLE IF NOT EXISTS community_summary_generation (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(255) NOT NULL,
|
||||
community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
|
||||
generated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
generated_by VARCHAR(255),
|
||||
|
||||
-- LLM usage
|
||||
llm_model VARCHAR(100),
|
||||
input_tokens INT,
|
||||
output_tokens INT,
|
||||
cost_usd FLOAT,
|
||||
|
||||
-- Generation method
|
||||
method VARCHAR(50) DEFAULT 'extractive', -- 'extractive' or 'abstractive'
|
||||
|
||||
-- Quality
|
||||
coherence_rating INT CHECK (coherence_rating >= 1 AND coherence_rating <= 5),
|
||||
user_feedback TEXT,
|
||||
|
||||
-- Result
|
||||
summary_text TEXT NOT NULL,
|
||||
summary_embedding VECTOR(768),
|
||||
|
||||
-- Versioning
|
||||
version INT DEFAULT 1,
|
||||
is_latest BOOLEAN DEFAULT TRUE
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_latest
|
||||
ON community_summary_generation(community_id, generated_at DESC)
|
||||
WHERE is_latest = TRUE;
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_embedding
|
||||
ON community_summary_generation USING hnsw (summary_embedding vector_cosine_ops)
|
||||
WITH (m = 16, ef_construction = 200)
|
||||
WHERE is_latest = TRUE;
|
||||
|
||||
-- ============================================
|
||||
-- ROLLBACK INSTRUCTIONS
|
||||
-- ============================================
|
||||
-- DROP TABLE IF EXISTS community_summary_generation;
|
||||
-- DROP TABLE IF EXISTS community_merge_history;
|
||||
-- DROP TABLE IF EXISTS community_statistics;
|
||||
-- DROP TABLE IF EXISTS community_member_map;
|
||||
-- DROP TABLE IF EXISTS label_propagation_run;
|
||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS last_detection_run_id;
|
||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS detection_score;
|
||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS is_permanent;
|
||||
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS merge_into_id;
|
||||
Reference in New Issue
Block a user