poimen-memory/crates/mem-store/migrations/006_phase4_community_detection.sql

-- Phase 4: Community Detection Schema
-- Extends memory_community with label propagation execution and statistics

-- ============================================
-- STEP 1: Create label propagation run tracking
-- ============================================
CREATE TABLE IF NOT EXISTS label_propagation_run (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    run_at TIMESTAMPTZ DEFAULT NOW(),
    algorithm VARCHAR(50) DEFAULT 'label_propagation',
    max_iterations INT DEFAULT 10,
    convergence_threshold FLOAT DEFAULT 0.01,
    iterations_completed INT,
    converged BOOLEAN DEFAULT FALSE,
    
    -- Execution metadata
    status VARCHAR(20) DEFAULT 'running'
        CHECK (status IN ('running', 'completed', 'failed')),
    error_message TEXT,
    duration_ms INT,
    
    -- Statistics
    communities_detected INT,
    communities_merged INT,
    communities_split INT,
    nodes_processed INT,
    edges_processed INT,
    
    -- Execution mode
    dry_run BOOLEAN DEFAULT FALSE,
    
    CONSTRAINT chk_iterations_valid CHECK (iterations_completed >= 0 AND iterations_completed <= max_iterations)
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_project 
    ON label_propagation_run(project_id, run_at DESC);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_label_prop_run_status 
    ON label_propagation_run(project_id, status)
    WHERE status IN ('running', 'failed');

-- ============================================
-- STEP 2: Create community member map
-- ============================================
CREATE TABLE IF NOT EXISTS community_member_map (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
    entity_id UUID NOT NULL REFERENCES memory_entity(id) ON DELETE CASCADE,
    label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
    
    -- Label strength (0-1, higher = stronger membership)
    label_strength FLOAT DEFAULT 1.0,
    
    -- Membership tracking
    is_seed BOOLEAN DEFAULT FALSE,
    joined_at TIMESTAMPTZ DEFAULT NOW(),
    left_at TIMESTAMPTZ,
    
    -- Consistency
    CONSTRAINT uq_community_entity_project UNIQUE (project_id, community_id, entity_id),
    CONSTRAINT chk_label_strength CHECK (label_strength >= 0 AND label_strength <= 1)
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_project 
    ON community_member_map(project_id, community_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_entity_lookup 
    ON community_member_map(entity_id, community_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_member_strength 
    ON community_member_map(community_id, label_strength DESC)
    WHERE left_at IS NULL;

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_seeds 
    ON community_member_map(project_id, is_seed)
    WHERE is_seed = TRUE;

-- ============================================
-- STEP 3: Create community statistics table
-- ============================================
CREATE TABLE IF NOT EXISTS community_statistics (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    community_id UUID NOT NULL UNIQUE REFERENCES memory_community(id) ON DELETE CASCADE,
    label_propagation_run_id UUID NOT NULL REFERENCES label_propagation_run(id) ON DELETE CASCADE,
    
    -- Membership stats
    member_count INT DEFAULT 0,
    active_member_count INT DEFAULT 0,
    seed_member_count INT DEFAULT 0,
    
    -- Graph structure
    internal_edge_count INT DEFAULT 0,
    external_edge_count INT DEFAULT 0,
    
    -- Cohesion metrics
    density FLOAT DEFAULT 0.0,
    modularity FLOAT DEFAULT 0.0,
    
    -- Edge types within community
    relation_type_distribution JSONB DEFAULT '{}',
    
    -- Temporal metrics
    first_entity_created TIMESTAMPTZ,
    last_entity_accessed TIMESTAMPTZ,
    avg_entity_age_days FLOAT DEFAULT 0.0,
    
    -- Quality scores
    coherence_score FLOAT DEFAULT 0.5,
    stability_score FLOAT DEFAULT 0.5,
    significance_score FLOAT DEFAULT 0.5,
    
    CONSTRAINT chk_stats_nonnegative CHECK (
        member_count >= 0 AND 
        internal_edge_count >= 0 AND 
        external_edge_count >= 0
    ),
    CONSTRAINT chk_stats_bounded CHECK (
        density >= 0 AND density <= 1 AND
        modularity >= -1 AND modularity <= 1 AND
        coherence_score >= 0 AND coherence_score <= 1 AND
        stability_score >= 0 AND stability_score <= 1 AND
        significance_score >= 0 AND significance_score <= 1
    )
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_project 
    ON community_statistics(project_id, community_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_run 
    ON community_statistics(label_propagation_run_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_stats_quality 
    ON community_statistics(project_id, coherence_score DESC, significance_score DESC)
    WHERE coherence_score > 0.7;

-- ============================================
-- STEP 4: Create community merge history
-- ============================================
CREATE TABLE IF NOT EXISTS community_merge_history (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    source_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
    target_community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
    merge_reason VARCHAR(100),
    merged_at TIMESTAMPTZ DEFAULT NOW(),
    label_propagation_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
    
    -- Rollback capability
    dry_run BOOLEAN DEFAULT FALSE,
    
    -- Statistics before merge
    source_member_count INT,
    target_member_count INT,
    
    -- Impact
    members_moved INT,
    edges_reattached INT
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_project 
    ON community_merge_history(project_id, merged_at DESC);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_merge_history_communities 
    ON community_merge_history(source_community_id, target_community_id);

-- ============================================
-- STEP 5: Add community detection status to memory_community
-- ============================================
ALTER TABLE memory_community 
    ADD COLUMN IF NOT EXISTS last_detection_run_id UUID REFERENCES label_propagation_run(id) ON DELETE SET NULL,
    ADD COLUMN IF NOT EXISTS detection_score FLOAT DEFAULT 0.5,
    ADD COLUMN IF NOT EXISTS is_permanent BOOLEAN DEFAULT FALSE,
    ADD COLUMN IF NOT EXISTS merge_into_id UUID REFERENCES memory_community(id) ON DELETE SET NULL;

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_detection_run 
    ON memory_community(last_detection_run_id, detection_score DESC)
    WHERE detection_score > 0.7;

-- ============================================
-- STEP 6: Add community-level summary generation tracking
-- ============================================
CREATE TABLE IF NOT EXISTS community_summary_generation (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    community_id UUID NOT NULL REFERENCES memory_community(id) ON DELETE CASCADE,
    generated_at TIMESTAMPTZ DEFAULT NOW(),
    generated_by VARCHAR(255),
    
    -- LLM usage
    llm_model VARCHAR(100),
    input_tokens INT,
    output_tokens INT,
    cost_usd FLOAT,
    
    -- Generation method
    method VARCHAR(50) DEFAULT 'extractive',  -- 'extractive' or 'abstractive'
    
    -- Quality
    coherence_rating INT CHECK (coherence_rating >= 1 AND coherence_rating <= 5),
    user_feedback TEXT,
    
    -- Result
    summary_text TEXT NOT NULL,
    summary_embedding VECTOR(768),
    
    -- Versioning
    version INT DEFAULT 1,
    is_latest BOOLEAN DEFAULT TRUE
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_latest 
    ON community_summary_generation(community_id, generated_at DESC)
    WHERE is_latest = TRUE;

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_community_summary_embedding 
    ON community_summary_generation USING hnsw (summary_embedding vector_cosine_ops)
    WITH (m = 16, ef_construction = 200)
    WHERE is_latest = TRUE;

-- ============================================
-- ROLLBACK INSTRUCTIONS
-- ============================================
-- DROP TABLE IF EXISTS community_summary_generation;
-- DROP TABLE IF EXISTS community_merge_history;
-- DROP TABLE IF EXISTS community_statistics;
-- DROP TABLE IF EXISTS community_member_map;
-- DROP TABLE IF EXISTS label_propagation_run;
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS last_detection_run_id;
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS detection_score;
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS is_permanent;
-- ALTER TABLE memory_community DROP COLUMN IF EXISTS merge_into_id;