63 lines
2.7 KiB
SQL
63 lines
2.7 KiB
SQL
|
-- M8.2 — Dual-write indexing pipeline
|
||
|
|
-- Unified chunk table for pgvector (embedding) and OpenSearch (text) coordination
|
||
|
|
-- Both stores write same chunk_id; OpenSearch failure marked for eventual consistency retry
|
||
|
|
|
||
|
|
CREATE TABLE IF NOT EXISTS chunks (
|
||
|
|
-- Identity
|
||
|
|
id UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(),
|
||
|
|
chunk_hash TEXT NOT NULL UNIQUE, -- SHA256(content) for deduplication
|
||
|
|
|
||
|
|
-- Content
|
||
|
|
content TEXT NOT NULL,
|
||
|
|
source TEXT NOT NULL, -- "ingest", "vault", "transcript", etc.
|
||
|
|
project TEXT NOT NULL,
|
||
|
|
|
||
|
|
-- Metadata
|
||
|
|
level TEXT NOT NULL CHECK (level IN ('L0', 'L1', 'L2', 'R')),
|
||
|
|
breadcrumb TEXT[] DEFAULT '{}', -- hierarchical path for context display
|
||
|
|
|
||
|
|
-- Vector storage (pgvector)
|
||
|
|
embedding vector(768), -- 768-dim nomic embeddings (nullable for pending writes)
|
||
|
|
|
||
|
|
-- Dual-write tracking
|
||
|
|
indexed_in_pgvector BOOLEAN NOT NULL DEFAULT false,
|
||
|
|
indexed_in_opensearch BOOLEAN NOT NULL DEFAULT false,
|
||
|
|
opensearch_pending BOOLEAN NOT NULL DEFAULT false, -- retry marker
|
||
|
|
|
||
|
|
-- Timestamps
|
||
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
|
|
pgvector_indexed_at TIMESTAMPTZ,
|
||
|
|
opensearch_indexed_at TIMESTAMPTZ,
|
||
|
|
opensearch_retry_count INT NOT NULL DEFAULT 0,
|
||
|
|
opensearch_last_retry_at TIMESTAMPTZ,
|
||
|
|
|
||
|
|
CONSTRAINT valid_pgvector CHECK (
|
||
|
|
-- Either both stored, or pgvector done + opensearch pending
|
||
|
|
(indexed_in_pgvector AND indexed_in_opensearch AND NOT opensearch_pending) OR
|
||
|
|
(indexed_in_pgvector AND NOT indexed_in_opensearch AND opensearch_pending)
|
||
|
|
)
|
||
|
|
);
|
||
|
|
|
||
|
|
-- Indexes for common queries
|
||
|
|
CREATE INDEX idx_chunks_project ON chunks(project);
|
||
|
|
CREATE INDEX idx_chunks_level ON chunks(project, level);
|
||
|
|
CREATE INDEX idx_chunks_source ON chunks(source);
|
||
|
|
CREATE INDEX idx_chunks_opensearch_pending ON chunks(project) WHERE opensearch_pending = true;
|
||
|
|
CREATE INDEX idx_chunks_created_at ON chunks(created_at DESC);
|
||
|
|
|
||
|
|
-- Vector search index (only for successfully stored embeddings)
|
||
|
|
CREATE INDEX idx_chunks_embedding ON chunks USING hnsw (embedding vector_cosine_ops)
|
||
|
|
WHERE indexed_in_pgvector = true AND embedding IS NOT NULL;
|
||
|
|
|
||
|
|
-- Deduplication index
|
||
|
|
CREATE INDEX idx_chunks_hash ON chunks(chunk_hash);
|
||
|
|
|
||
|
|
-- Breadcrumb GiST index for hierarchical path queries
|
||
|
|
CREATE INDEX idx_chunks_breadcrumb ON chunks USING gin (breadcrumb);
|
||
|
|
|
||
|
|
COMMENT ON TABLE chunks IS 'M8.2 unified chunk store: dual-write to pgvector (embedding) + OpenSearch (text)';
|
||
|
|
COMMENT ON COLUMN chunks.id IS 'Same chunk_id written to both pgvector and OpenSearch';
|
||
|
|
COMMENT ON COLUMN chunks.chunk_hash IS 'SHA256(content) for dedup before dual write';
|
||
|
|
COMMENT ON COLUMN chunks.opensearch_pending IS 'Retry marker: OpenSearch write failed, needs eventual consistency retry';
|
||
|
|
COMMENT ON COLUMN chunks.embedding IS 'pgvector embedding (768-dim nomic), NULL until pgvector write succeeds';
|