poimen-memory/crates/mem-store/migrations/008_phase3_compaction.sql

-- Phase 3: Compaction Schema
-- T3.1-T3.4: Deduplication, GC, and dry-run support

-- ============================================
-- STEP 1: Exact dedup tracking (T3.1)
-- ============================================
CREATE TABLE IF NOT EXISTS exact_dedup_record (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    
    -- Source and target edges
    source_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
    target_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
    
    -- Match criteria (all must match for exact dedup)
    source_match BOOLEAN NOT NULL,
    target_match BOOLEAN NOT NULL,
    relation_match BOOLEAN NOT NULL,
    fact_match BOOLEAN NOT NULL,
    
    -- Dedup decision
    dedup_action VARCHAR(20) DEFAULT 'pending'
        CHECK (dedup_action IN ('pending', 'merged', 'kept_separate', 'manual_review')),
    
    -- Metadata
    detected_at TIMESTAMPTZ DEFAULT NOW(),
    processed_at TIMESTAMPTZ,
    compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
    
    -- Dry-run support
    dry_run BOOLEAN DEFAULT FALSE,
    
    CONSTRAINT chk_unique_edge_pair UNIQUE (source_edge_id, target_edge_id, project_id)
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_project 
    ON exact_dedup_record(project_id, dedup_action);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_edges 
    ON exact_dedup_record(source_edge_id, target_edge_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_exact_dedup_pending 
    ON exact_dedup_record(project_id, detected_at)
    WHERE dedup_action = 'pending';

-- ============================================
-- STEP 2: Stale GC tracking (T3.1)
-- ============================================
CREATE TABLE IF NOT EXISTS stale_gc_record (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    
    -- Entity or edge marked for GC
    entity_id UUID REFERENCES memory_entity(id) ON DELETE CASCADE,
    edge_id UUID REFERENCES memory_edge(id) ON DELETE CASCADE,
    
    -- Staleness criteria
    age_days INT NOT NULL,
    t_invalid_at TIMESTAMPTZ,
    access_count BIGINT DEFAULT 0,
    
    -- GC decision
    gc_action VARCHAR(20) DEFAULT 'pending'
        CHECK (gc_action IN ('pending', 'deleted', 'archived', 'kept')),
    
    -- Metadata
    detected_at TIMESTAMPTZ DEFAULT NOW(),
    processed_at TIMESTAMPTZ,
    compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
    
    -- Dry-run support
    dry_run BOOLEAN DEFAULT FALSE,
    
    CONSTRAINT chk_entity_or_edge CHECK (
        (entity_id IS NOT NULL AND edge_id IS NULL) OR
        (entity_id IS NULL AND edge_id IS NOT NULL)
    )
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_project 
    ON stale_gc_record(project_id, gc_action);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_age 
    ON stale_gc_record(project_id, age_days DESC)
    WHERE gc_action = 'pending';

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_stale_gc_invalid 
    ON stale_gc_record(t_invalid_at)
    WHERE t_invalid_at IS NOT NULL AND gc_action = 'pending';

-- ============================================
-- STEP 3: Semantic dedup with LLM verification (T3.2)
-- ============================================
CREATE TABLE IF NOT EXISTS semantic_dedup_record (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    
    -- Source and target edges
    source_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
    target_edge_id UUID NOT NULL REFERENCES memory_edge(id) ON DELETE CASCADE,
    
    -- Pre-filter score (0-1, eliminates 60-70% of candidates)
    prefilter_score FLOAT NOT NULL,
    prefilter_passed BOOLEAN NOT NULL,
    
    -- LLM verification (if prefilter_passed = true)
    llm_model VARCHAR(100),
    llm_prompt TEXT,
    llm_response TEXT,
    llm_confidence FLOAT,
    llm_cost_usd FLOAT,
    
    -- Dedup decision
    dedup_action VARCHAR(50) DEFAULT 'pending'
        CHECK (dedup_action IN (
            'pending', 'auto_merged', 'auto_kept_separate', 
            'manual_review', 'llm_error', 'below_threshold'
        )),
    
    -- Merge strategy (if auto-merged)
    merge_strategy VARCHAR(50),  -- 'keep_superset', 'keep_newer', 'keep_higher_confidence'
    merged_edge_id UUID REFERENCES memory_edge(id) ON DELETE SET NULL,
    
    -- Metadata
    detected_at TIMESTAMPTZ DEFAULT NOW(),
    processed_at TIMESTAMPTZ,
    compaction_run_id UUID REFERENCES compaction_log(id) ON DELETE SET NULL,
    
    -- Dry-run support
    dry_run BOOLEAN DEFAULT FALSE,
    
    CONSTRAINT chk_confidence_valid CHECK (
        llm_confidence IS NULL OR (llm_confidence >= 0 AND llm_confidence <= 1)
    ),
    CONSTRAINT chk_prefilter_valid CHECK (prefilter_score >= 0 AND prefilter_score <= 1)
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_project 
    ON semantic_dedup_record(project_id, dedup_action);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_pending 
    ON semantic_dedup_record(project_id, llm_confidence DESC NULLS LAST)
    WHERE dedup_action = 'manual_review' OR dedup_action = 'pending';

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_edges 
    ON semantic_dedup_record(source_edge_id, target_edge_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_semantic_dedup_merged 
    ON semantic_dedup_record(project_id, merged_edge_id)
    WHERE merged_edge_id IS NOT NULL;

-- ============================================
-- STEP 4: Compaction audit trail (T3.3)
-- ============================================
CREATE TABLE IF NOT EXISTS compaction_audit (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    compaction_run_id UUID NOT NULL REFERENCES compaction_log(id) ON DELETE CASCADE,
    
    -- Action details
    action_type VARCHAR(50) NOT NULL,  -- 'exact_dedup', 'semantic_dedup', 'stale_gc', etc.
    source_id UUID,
    target_id UUID,
    
    -- Before state
    before_state JSONB NOT NULL,
    before_hash VARCHAR(64),
    
    -- After state
    after_state JSONB NOT NULL,
    after_hash VARCHAR(64),
    
    -- Provenance
    initiated_by VARCHAR(255),
    approval_status VARCHAR(50) DEFAULT 'pending'
        CHECK (approval_status IN ('pending', 'approved', 'rejected', 'auto')),
    approved_by VARCHAR(255),
    approval_reason TEXT,
    
    -- Rollback capability
    is_reversible BOOLEAN DEFAULT TRUE,
    reversal_instructions JSONB,
    
    -- Dry-run tracking
    dry_run BOOLEAN DEFAULT FALSE,
    
    -- Timestamp
    recorded_at TIMESTAMPTZ DEFAULT NOW()
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_run 
    ON compaction_audit(compaction_run_id);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_project 
    ON compaction_audit(project_id, recorded_at DESC);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_compaction_audit_reversible 
    ON compaction_audit(project_id, recorded_at DESC)
    WHERE is_reversible = TRUE;

-- ============================================
-- STEP 5: Compaction dry-run validation
-- ============================================
CREATE TABLE IF NOT EXISTS compaction_dryrun_result (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    compaction_run_id UUID NOT NULL REFERENCES compaction_log(id) ON DELETE CASCADE,
    
    -- Dry-run metadata
    started_at TIMESTAMPTZ DEFAULT NOW(),
    completed_at TIMESTAMPTZ,
    
    -- Statistics
    exact_dedup_candidates INT DEFAULT 0,
    exact_dedup_safe INT DEFAULT 0,
    
    semantic_dedup_candidates INT DEFAULT 0,
    semantic_dedup_safe INT DEFAULT 0,
    semantic_dedup_manual_review INT DEFAULT 0,
    
    stale_gc_candidates INT DEFAULT 0,
    stale_gc_safe INT DEFAULT 0,
    
    -- Predicted impact
    predicted_space_freed_mb FLOAT DEFAULT 0.0,
    predicted_edge_count_reduction INT DEFAULT 0,
    predicted_entity_count_reduction INT DEFAULT 0,
    
    -- Validation issues found
    issues_found INT DEFAULT 0,
    issue_details JSONB DEFAULT '[]',
    
    -- Decision
    approval_recommended BOOLEAN DEFAULT FALSE,
    approval_reason TEXT
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_dryrun_project 
    ON compaction_dryrun_result(project_id, completed_at DESC);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_dryrun_run 
    ON compaction_dryrun_result(compaction_run_id);

-- ============================================
-- STEP 6: Scheduled compaction jobs (T3.4)
-- ============================================
CREATE TABLE IF NOT EXISTS compaction_schedule (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    project_id VARCHAR(255) NOT NULL,
    
    -- Schedule config
    cron_expression VARCHAR(100) NOT NULL,  -- e.g., "0 2 * * *" for daily at 2 AM UTC
    timezone VARCHAR(50) DEFAULT 'UTC',
    
    -- Execution config
    tier INT DEFAULT 1,  -- 1 = exact dedup, 2 = semantic dedup, 3 = both
    dry_run_first BOOLEAN DEFAULT TRUE,
    auto_approve_safe_actions BOOLEAN DEFAULT FALSE,
    
    -- Resource limits
    max_execution_time_minutes INT DEFAULT 60,
    max_llm_cost_usd FLOAT DEFAULT 10.0,
    
    -- Status
    enabled BOOLEAN DEFAULT TRUE,
    
    -- Metadata
    created_at TIMESTAMPTZ DEFAULT NOW(),
    last_run_at TIMESTAMPTZ,
    next_run_at TIMESTAMPTZ,
    
    -- Notifications
    notify_on_completion BOOLEAN DEFAULT TRUE,
    notify_emails TEXT[] DEFAULT '{}'
);

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_schedule_project 
    ON compaction_schedule(project_id, enabled)
    WHERE enabled = TRUE;

CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_schedule_next_run 
    ON compaction_schedule(next_run_at)
    WHERE enabled = TRUE;

-- ============================================
-- ROLLBACK INSTRUCTIONS
-- ============================================
-- DROP TABLE IF EXISTS compaction_schedule;
-- DROP TABLE IF EXISTS compaction_dryrun_result;
-- DROP TABLE IF EXISTS compaction_audit;
-- DROP TABLE IF EXISTS semantic_dedup_record;
-- DROP TABLE IF EXISTS stale_gc_record;
-- DROP TABLE IF EXISTS exact_dedup_record;