From e2f7ee1144efb36e9ce842f030c88e9e0a0bcdd3 Mon Sep 17 00:00:00 2001 From: poimen Date: Fri, 28 Aug 2026 13:54:46 -0700 Subject: [PATCH] chore: Remove outdated design docs (old query optimization, hybrid search design, API review) --- docs/API_REVIEW.md | 501 -------------------- docs/HYBRID_SEARCH_DESIGN.md | 762 ------------------------------ docs/IMPLEMENTATION_NOTES.md | 329 ------------- docs/QUERY_OPTIMIZATION_ENGINE.md | 698 --------------------------- 4 files changed, 2290 deletions(-) delete mode 100644 docs/API_REVIEW.md delete mode 100644 docs/HYBRID_SEARCH_DESIGN.md delete mode 100644 docs/IMPLEMENTATION_NOTES.md delete mode 100644 docs/QUERY_OPTIMIZATION_ENGINE.md diff --git a/docs/API_REVIEW.md b/docs/API_REVIEW.md deleted file mode 100644 index 8908099..0000000 --- a/docs/API_REVIEW.md +++ /dev/null @@ -1,501 +0,0 @@ -# API Review: Query APIs vs Retrieval APIs - -## Current API Endpoints (9 routes) - -### Overview Table - -| # | Endpoint | Method | Category | Auth | Rate Limit | Purpose | -|---|----------|--------|----------|------|-----------|---------| -| 1 | `/health` | GET | System | ❌ No | ❌ No | Health check | -| 2 | `/memory/ingest` | POST | **Write** | ✅ (write) | 100/hr | Queue ingest job | -| 3 | `/memory/ingest/{id}` | GET | **Status** | ✅ (read) | ✅ Query | Check ingest progress | -| 4 | `/memory/query` | GET | **QUERY API** ⭐ | ✅ (read) | 1000/hr | Semantic search | -| 5 | `/memory/projects` | GET | Metadata | ✅ (read) | 100/hr | List projects | -| 6 | `/memory/skills` | GET | Metadata | ✅ (read) | ✅ Query | List skills | -| 7 | `/memory/vault/generate` | POST | **Write** | ✅ (write) | 100/hr | Generate vault | -| 8 | `/memory/vault` | GET | **RETRIEVAL API** ⭐ | ✅ (read) | ✅ Query | Browse vault root | -| 9 | `/memory/vault/{project}` | GET | **RETRIEVAL API** ⭐ | ✅ (read) | ✅ Query | Browse project | -| 10 | `/memory/vault/{project}/{file}` | GET | **RETRIEVAL API** ⭐ | ✅ (read) | ✅ Query | Read file | - ---- - -## Distinction: Query APIs vs Retrieval APIs - -### Query APIs (Search/Semantic) - -#### 🔍 `/memory/query` — Semantic Search (PRIMARY) - -``` -GET /memory/query?query=&project=&limit= -Authorization: Bearer - -Query Parameters: - - query (required): Search string (e.g., "fix kubernetes port 8080") - - project (required): Filter by project - - limit (optional): Top-K results (default: 5) - -Returns: -{ - "query": "fix kubernetes port 8080", - "project": "poimen", - "results": [ - { - "id": "chunk-123", - "text": "kubectl port-forward...", - "source": "runbooks/port-forward.md", - "level": "L1", - "breadcrumb": ["runbooks", "kubernetes"], - "score": 0.92, - "sem_component": 0.88, - "lex_component": 0.96 // [NEW] when hybrid enabled - }, - ... - ] -} -``` - -**Current Implementation:** -- ✅ JWT auth enforced -- ✅ Rate limited: 1000/hr -- ✅ Capability check: `memory:read` -- ⏳ **MISSING**: Hybrid search integration -- ⏳ **MISSING**: Query routing decision tree -- ⏳ **MISSING**: OpenSearch lexical search - -**Flow:** -``` -GET /memory/query - │ - ├─ Validate JWT / ApiKey - ├─ Check capability: memory:read - ├─ Check rate limit: 1000/hr per user - │ - ├─ (Current) QueryWorker.query() - │ └─ Only semantic via pgvector - │ - └─ (NEEDED) HybridQueryWorker.query() - ├─ Parallel pgvector search (top-50) - ├─ Parallel OpenSearch BM25 (top-50) + JWT forward - ├─ Normalize scores to [0.0, 1.0] - ├─ Fusion: 0.6*sem + 0.4*lex - └─ Return top-10 results with breakdown -``` - ---- - -### Retrieval APIs (Browse/Read) - -#### 📂 `/memory/vault` — List Vault Root - -``` -GET /memory/vault -Authorization: Bearer - -Returns: -{ - "projects": [ - { - "name": "poimen", - "path": "vault/poimen", - "file_count": 42, - "updated_at": "2024-01-15T10:30:00Z" - }, - ... - ] -} -``` - -**Purpose:** Browse vault directory structure (top-level) - ---- - -#### 📂 `/memory/vault/{project}` — List Project Files - -``` -GET /memory/vault/poimen -Authorization: Bearer - -Returns: -{ - "project": "poimen", - "files": [ - { - "name": "runbooks", - "type": "directory", - "path": "vault/poimen/runbooks", - "file_count": 15 - }, - { - "name": "deployment.md", - "type": "file", - "path": "vault/poimen/deployment.md", - "size_bytes": 4096, - "updated_at": "2024-01-15T10:30:00Z" - }, - ... - ] -} -``` - -**Purpose:** Browse files in a project (file tree view) - ---- - -#### 📄 `/memory/vault/{project}/{file}` — Read File Content - -``` -GET /memory/vault/poimen/deployment.md -Authorization: Bearer - -Returns: -{ - "project": "poimen", - "file": "deployment.md", - "path": "vault/poimen/deployment.md", - "content": "# Deployment Guide\n\n## Overview\n...", - "size_bytes": 4096, - "updated_at": "2024-01-15T10:30:00Z", - "sections": [ - { - "title": "Overview", - "level": 1, - "content": "..." - }, - ... - ] -} -``` - -**Purpose:** Read/view full file content (no AI processing) - ---- - -## Current Issues & Gaps - -### ❌ Issue 1: `/memory/query` is Semantic-Only - -**Problem:** -- Current `/memory/query` calls `QueryWorker.query()` which ONLY does pgvector -- No access to OpenSearch (lexical search) -- No fusion/reranking logic -- No score breakdown for debugging - -**Solution:** -```rust -// Instead of: -state.query_worker.query(&project, &question, Some(limit)).await - -// Should be: -state.hybrid_query_worker.hybrid_query( - &project, - &question, - limit, - SearchMethod::Hybrid, // or Semantic, Lexical - HybridWeights { semantic: 0.6, lexical: 0.4 }, - claims, - &token // Forward JWT to OpenSearch -).await -``` - ---- - -### ❌ Issue 2: No Query Routing - -**Problem:** -- All queries use hybrid (once implemented) -- No decision tree for: - - Short queries (< 3 tokens) → lexical better - - Special syntax (#tag) → lexical better - - LLM unavailable → fallback to lexical - -**Solution:** -```rust -pub fn route_query(query: &str, method_override: Option) -> SearchMethod { - // If user explicitly requested a method, use it - if let Some(method) = method_override { - return method; - } - - // Otherwise, auto-route based on query characteristics - let token_count = query.split_whitespace().count(); - - if token_count < 3 { - SearchMethod::Lexical // Short queries - } else if query.contains('#') || query.contains('@') { - SearchMethod::Lexical // Special syntax - } else { - SearchMethod::Hybrid // Normal queries - } -} -``` - ---- - -### ❌ Issue 3: Retrieval APIs Don't Integrate with Search - -**Problem:** -- `/memory/vault/*` only reads files from PVC -- No integration with indexed chunks -- User can't "click through" from search results to source -- No breadcrumb context - -**Opportunity:** -``` -Query Result (from /memory/query): - - source: "runbooks/kubernetes/networking.md" - - breadcrumb: ["runbooks", "kubernetes"] - - section: "Port Forwarding" - -User Clicks "View Full Document" - → /memory/vault/poimen/runbooks/kubernetes/networking.md - → Server highlights the relevant section - → Shows context (adjacent sections) -``` - ---- - -## Recommended API Enhancements - -### Phase 1: Enhance `/memory/query` (Now) - -``` -GET /memory/query?query=&project=&limit=&method= -Authorization: Bearer - -Returns: -{ - "query": "fix kubernetes port", - "project": "poimen", - "search_method": "hybrid", - "results": [ - { - "id": "chunk-123", - "text": "...", - "source": "...", - "score": 0.92, - "breakdown": { - "semantic": 0.88, - "lexical": 0.96, - "semantic_weight": 0.6, - "lexical_weight": 0.4, - "reason": "High semantic + lexical match" - } - } - ], - "metrics": { - "retrieval_time_ms": 245, - "semantic_engine": "pgvector", - "lexical_engine": "opensearch", - "total_docs_searched": 100 - } -} -``` - -**Implementation:** -1. Update `query_handler` to accept `method` parameter -2. Implement hybrid retrieval in `HybridQueryWorker` -3. Forward JWT token to OpenSearch -4. Return score breakdown - ---- - -### Phase 2: Add Context-Aware Retrieval (Week 2) - -**New Endpoint:** `/memory/query/with-context` - -``` -GET /memory/query/with-context?query=&project=&context_chunks=2 -Authorization: Bearer - -Returns: -{ - "query": "...", - "results": [ - { - "id": "chunk-123", - "text": "...", - "score": 0.92, - "context": { - "previous_chunk": { "id": "chunk-122", "text": "..." }, - "next_chunk": { "id": "chunk-124", "text": "..." }, - "section_title": "Port Forwarding", - "breadcrumb_full": ["runbooks", "kubernetes", "troubleshooting"] - } - } - ] -} -``` - -**Purpose:** -- Return adjacent chunks for full context -- Help LLM (agent) understand query context -- Support follow-up questions - ---- - -### Phase 3: Add Faceted Search (Week 3) - -**Enhancement to `/memory/query`:** - -``` -GET /memory/query?query=&project=&filters= -Authorization: Bearer - -Query: - - filters: {"level": "L1", "source_pattern": "runbooks/*", "updated_since": "2024-01-01"} - -Returns: -{ - "query": "...", - "filters_applied": { "level": "L1", ... }, - "results": [...], - "facets": { - "levels": { "L1": 15, "L2": 8 }, - "sources": { "runbooks": 12, "docs": 11 }, - "dates": { "2024-01": 14, "2024-02": 9 } - } -} -``` - ---- - -### Phase 4: Add Relevance Feedback (Week 4) - -**New Endpoint:** `/memory/query/feedback` - -``` -POST /memory/query/feedback -Authorization: Bearer - -Body: -{ - "query_id": "q-123", - "query_text": "fix kubernetes port", - "result_id": "chunk-123", - "relevant": true, // or false - "rating": 4, // 1-5 stars - "feedback_text": "This was exactly what I needed" -} - -Returns: -{ - "query_id": "q-123", - "status": "recorded", - "message": "Thank you for feedback" -} -``` - -**Purpose:** -- Train weight tuning (0.6/0.4 optimal?) -- Detect broken results -- Improve future rankings - ---- - -## Architecture Changes Needed - -### Current State (Semantic-Only) - -``` -/memory/query - │ - └─ QueryWorker.query() - └─ pgvector (only) -``` - -### Target State (Hybrid) - -``` -/memory/query - │ - ├─ Route query (decision tree) - │ - ├─ If HYBRID: - │ └─ HybridQueryWorker.hybrid_query() - │ ├─ Parallel pgvector (top-50) - │ ├─ Parallel OpenSearch + JWT (top-50) - │ ├─ Normalize scores - │ ├─ Fusion: 0.6*sem + 0.4*lex - │ └─ Return top-10 + breakdown - │ - ├─ If SEMANTIC: - │ └─ QueryWorker.query() [fallback] - │ - └─ If LEXICAL: - └─ OpenSearchWorker.query() + JWT [new] -``` - ---- - -## Implementation Checklist - -### ✅ Already Implemented -- [x] `/health` — Health check -- [x] `/memory/ingest` — Queue ingest -- [x] `/memory/ingest/{id}` — Check status -- [x] `/memory/query` — Semantic search (pgvector only) -- [x] `/memory/projects` — List projects -- [x] `/memory/skills` — List skills -- [x] `/memory/vault/generate` — Generate vault -- [x] `/memory/vault` — Browse vault root -- [x] `/memory/vault/{project}` — Browse project -- [x] `/memory/vault/{project}/{file}` — Read file -- [x] JWT auth (validates Authentik tokens) -- [x] Rate limiting (per user, per endpoint) -- [x] Capability checks (memory:read, memory:write) -- [x] Idempotency (ingest deduplication) - -### 📋 TODO: Hybrid Query Support -- [ ] Implement `HybridQueryWorker` -- [ ] Add OpenSearch client integration -- [ ] Implement score normalization -- [ ] Implement fusion strategy (weighted linear) -- [ ] Add query routing decision tree -- [ ] Add `?method=` parameter to `/memory/query` -- [ ] Return score breakdown in response -- [ ] Test with multiple weights (A/B testing) -- [ ] Add metrics endpoint for latency tracking - -### 📋 TODO: Enhanced Retrieval -- [ ] Add `/memory/query/with-context` endpoint -- [ ] Implement adjacent chunk retrieval -- [ ] Add faceted search to `/memory/query` -- [ ] Add `/memory/query/feedback` for relevance feedback -- [ ] Link search results to vault files - ---- - -## Summary - -### Query APIs (Search/AI) -- **`/memory/query`** ⭐ Primary semantic search - - Searches indexed chunks (pgvector) - - Should become hybrid (pgvector + OpenSearch) - - Returns: scored, ranked results + breakdown - - Used by: LLM agents, frontend search UI - - **Target**: 0-500ms latency - -### Retrieval APIs (Browse/Read) -- **`/memory/vault`** ⭐ Browse vault tree - - Lists projects and files from PVC - - Returns: file structure (no content) - - Used by: Web UI vault browser - - **Target**: 100-200ms latency - -- **`/memory/vault/{project}`** ⭐ Browse project - - Lists files in a project - - Returns: file tree with metadata - - Used by: Web UI file picker - -- **`/memory/vault/{project}/{file}`** ⭐ Read file - - Reads full file content from PVC - - Returns: markdown + sections - - Used by: Web UI file viewer - - **Target**: 50-100ms latency - -### Gap -- Query APIs and Retrieval APIs are **disconnected** -- Search results should link to vault files -- No way to "view full context" after search -- **Solution**: Add `/memory/query/with-context` endpoint - diff --git a/docs/HYBRID_SEARCH_DESIGN.md b/docs/HYBRID_SEARCH_DESIGN.md deleted file mode 100644 index a85e815..0000000 --- a/docs/HYBRID_SEARCH_DESIGN.md +++ /dev/null @@ -1,762 +0,0 @@ -# Hybrid Search Design: Retrieval Pipeline + Index Optimization - -## Goal -Maximize retrieval accuracy and relevance by combining: -- **Semantic search** (pgvector): Understanding query intent -- **Lexical search** (OpenSearch BM25): Exact term matching -- **Ranking fusion**: Intelligent combination for best results - ---- - -## 1. Retrieval Pipeline Architecture - -### Stage 1: Query Normalization (Entry Point) - -``` -User Query: "fix kubernetes port 8080 conflict" - ↓ -├─ Tokenize & clean -├─ Expand abbreviations (k8s → kubernetes) -├─ Extract entities (port:8080, service:kubernetes) -└─ Generate embedding (for semantic) -``` - -**Implementation:** -```rust -pub struct QueryContext { - raw_query: String, - normalized: String, // Lowercase, trimmed - tokens: Vec, // ["fix", "kubernetes", ...] - entities: HashMap, // {port: "8080"} - embedding: Vec, // 384-dim or 1536-dim - timestamp: Instant, -} -``` - ---- - -### Stage 2: Parallel Retrieval (Both Engines) - -``` -Query Context - │ - ├─ SEMANTIC PATH (pgvector) - │ ├─ Query PostgreSQL with embedding - │ ├─ SELECT chunks WHERE embedding <-> query_vec < distance_threshold - │ ├─ ORDER BY cosine_similarity DESC LIMIT 50 - │ └─ Return: [(chunk_id, score_0_to_1, chunk_text)] - │ - └─ LEXICAL PATH (OpenSearch + JWT) - ├─ Tokenize query - ├─ POST vault-*/_search with BM25 - ├─ Query: multi_match on [content, breadcrumb, source] - ├─ ORDER BY BM25 score DESC LIMIT 50 - └─ Return: [(doc_id, bm25_score_raw, chunk_text)] -``` - -**Key: Execute both in parallel (tokio::join! or similar)** - -```rust -pub async fn hybrid_retrieve( - query_ctx: &QueryContext, - pg: &PgClient, - opensearch: &OpenSearchClient, - jwt_token: &str, -) -> Result { - let semantic_fut = pg.semantic_search(&query_ctx.embedding, 50); - let lexical_fut = opensearch.lexical_search(&query_ctx.normalized, 50, jwt_token); - - let (semantic_results, lexical_results) = tokio::try_join!(semantic_fut, lexical_fut)?; - - // Stage 3: Normalize & Rank - let ranked = rank_and_fuse(&semantic_results, &lexical_results)?; - - Ok(ranked) -} -``` - ---- - -### Stage 3: Score Normalization & Ranking Fusion - -**Problem:** Scores are incompatible -- pgvector: cosine similarity (0.0 to 1.0) -- BM25: raw TF-IDF scores (unbounded, typically 0-10+) - -**Solution: Min-Max Normalization** - -```rust -pub fn normalize_scores(results: &[(String, f32)]) -> Vec<(String, f32)> { - let min_score = results.iter().map(|(_, s)| s).fold(f32::INFINITY, f32::min); - let max_score = results.iter().map(|(_, s)| s).fold(f32::NEG_INFINITY, f32::max); - - let range = max_score - min_score; - if range < 0.001 { - // All scores identical → uniform - return results.iter().map(|(id, _)| (id.clone(), 0.5)).collect(); - } - - results - .iter() - .map(|(id, score)| { - let normalized = (score - min_score) / range; - (id.clone(), normalized) - }) - .collect() -} -``` - ---- - -### Stage 4: Fusion Strategy - -#### Option A: Weighted Linear Combination (Recommended for Now) - -```rust -pub fn weighted_fusion( - semantic: Vec<(String, f32)>, - lexical: Vec<(String, f32)>, - semantic_weight: f32, // 0.6 - lexical_weight: f32, // 0.4 -) -> Vec { - // Normalize both - let sem_norm = normalize_scores(&semantic); - let lex_norm = normalize_scores(&lexical); - - // Create maps for O(1) lookup - let sem_map: HashMap = sem_norm.into_iter().collect(); - let lex_map: HashMap = lex_norm.into_iter().collect(); - - // Merge all document IDs - let mut all_ids: HashSet = sem_map.keys().cloned().collect(); - all_ids.extend(lex_map.keys().cloned()); - - // Compute fusion scores - let mut results: Vec = all_ids - .into_iter() - .map(|id| { - let sem_score = sem_map.get(&id).copied().unwrap_or(0.0); - let lex_score = lex_map.get(&id).copied().unwrap_or(0.0); - - let fused_score = - semantic_weight * sem_score + - lexical_weight * lex_score; - - SearchResult { - id, - score: fused_score, - sem_component: sem_score, - lex_component: lex_score, - // ... other fields - } - }) - .collect(); - - // Sort by fused score - results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap()); - results.truncate(10); // Top-k - - results -} -``` - -**Pros:** -- ✅ Simple, interpretable -- ✅ Easy to tune weights -- ✅ Transparent scoring - -**Cons:** -- ❌ Assumes linear relationship -- ❌ Sensitive to weight tuning - ---- - -#### Option B: Reciprocal Rank Fusion (RRF) - Alternative - -```rust -pub fn reciprocal_rank_fusion( - semantic: Vec<(String, f32)>, - lexical: Vec<(String, f32)>, -) -> Vec { - // Convert to ranks (position in sorted list) - let k = 60; // Constant (typically 60) - - let mut fused_scores: HashMap = HashMap::new(); - - // Add semantic ranks - for (rank, (id, _)) in semantic.into_iter().enumerate() { - let rrf_score = 1.0 / (k as f32 + (rank as f32 + 1.0)); - fused_scores.insert(id, rrf_score); - } - - // Add lexical ranks (combine if already present) - for (rank, (id, _)) in lexical.into_iter().enumerate() { - let rrf_score = 1.0 / (k as f32 + (rank as f32 + 1.0)); - *fused_scores.entry(id).or_insert(0.0) += rrf_score; - } - - // Sort by combined RRF score - let mut results: Vec<_> = fused_scores.into_iter().collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); - results.truncate(10); - - results - .into_iter() - .map(|(id, score)| SearchResult { id, score, ..Default::default() }) - .collect() -} -``` - -**Pros:** -- ✅ No parameter tuning needed -- ✅ Robust to score distribution differences -- ✅ Academic consensus (best for diverse rankers) - -**Cons:** -- ❌ Less transparent (harder to debug) -- ❌ Loses score magnitudes - ---- - -## 2. Index Optimization - -### 2.1 PostgreSQL (pgvector) Index Schema - -#### Chunk Storage Table - -```sql -CREATE TABLE chunks ( - id UUID PRIMARY KEY, - - -- Content - text TEXT NOT NULL, -- Full chunk text - section_id UUID, -- Which section of doc - position_in_doc INT, -- Order for context - - -- Metadata for retrieval - document_id UUID NOT NULL, -- Source document - source VARCHAR(255) NOT NULL, -- File path: "runbooks/port-forward.md" - project_id UUID NOT NULL, -- Project filter - - -- Hierarchy for breadcrumb - level VARCHAR(10), -- "L0", "L1", "L2" - breadcrumb JSONB, -- ["runbooks", "kubernetes", "networking"] - - -- Embedding - embedding vector(384), -- 384-dim (all-MiniLM-L6-v2) - -- OR vector(1536) for OpenAI - - -- Metadata - chunk_hash VARCHAR(64), -- SHA256 for dedup - created_at TIMESTAMP, - updated_at TIMESTAMP, - - -- Index hints - is_indexed BOOLEAN DEFAULT FALSE, - INDEX_score FLOAT, -- For quality ranking - - CONSTRAINT fk_document FOREIGN KEY (document_id) REFERENCES documents(id) -); - --- PRIMARY INDEX: Vector similarity search -CREATE INDEX chunks_embedding_idx - ON chunks - USING ivfflat (embedding vector_cosine_ops) - WITH (lists = 100); -- Adjust based on data size - --- SECONDARY INDEXES: Filtering/metadata -CREATE INDEX chunks_document_id_idx ON chunks(document_id); -CREATE INDEX chunks_project_id_idx ON chunks(project_id); -CREATE INDEX chunks_source_idx ON chunks(source); -CREATE INDEX chunks_level_idx ON chunks(level); - --- For deduplication during indexing -CREATE INDEX chunks_hash_idx ON chunks(chunk_hash); -``` - -**Tuning Notes:** -- **lists parameter**: - - Small dataset (<10k): 50-100 - - Medium (10k-100k): 100-200 - - Large (>100k): 200-500 -- **Index Type**: ivfflat (fast, approximate) vs hnsw (more accurate, slower) - -#### Retrieval Query - -```sql --- Semantic search with filtering -SELECT - id, - text, - source, - breadcrumb, - level, - 1 - (embedding <-> $1::vector) as similarity_score, - position_in_doc -FROM chunks -WHERE - project_id = $2 -- Filter by project first - AND level IN ('L0', 'L1', 'L2') -- Exclude deep sections - AND created_at > NOW() - INTERVAL '1 year' -- Recency -ORDER BY - embedding <-> $1::vector -- Cosine distance -LIMIT 50; - --- Add context: retrieve adjacent chunks -WITH target_chunk AS ( - SELECT section_id, position_in_doc FROM chunks WHERE id = $1 -) -SELECT * FROM chunks -WHERE - section_id = (SELECT section_id FROM target_chunk) - AND position_in_doc BETWEEN - (SELECT position_in_doc FROM target_chunk) - 2 - AND (SELECT position_in_doc FROM target_chunk) + 2 -ORDER BY position_in_doc; -``` - ---- - -### 2.2 OpenSearch (Lexical) Index Schema - -#### Index Mapping (vault-* indices) - -```json -{ - "settings": { - "number_of_shards": 2, - "number_of_replicas": 1, - "index.codec": "best_compression", - "analysis": { - "analyzer": { - "standard_analyzer": { - "type": "standard", - "stopwords": "_english_" - }, - "ngram_analyzer": { - "type": "custom", - "tokenizer": "ngram_tokenizer", - "filter": ["lowercase"] - }, - "ngram_tokenizer": { - "type": "ngram", - "min_gram": 3, - "max_gram": 4, - "token_chars": ["letter", "digit"] - } - } - } - }, - "mappings": { - "properties": { - "content": { - "type": "text", - "analyzer": "standard_analyzer", - "fields": { - "raw": { "type": "keyword" }, - "ngram": { "type": "text", "analyzer": "ngram_analyzer" } - }, - "boost": 2.0 -- Content gets higher weight - }, - "source": { - "type": "keyword", - "boost": 1.5 - }, - "breadcrumb": { - "type": "keyword", - "boost": 1.2 - }, - "level": { - "type": "keyword" - }, - "section_title": { - "type": "text", - "analyzer": "standard_analyzer", - "boost": 1.8 - }, - "document_id": { - "type": "keyword" - }, - "project_id": { - "type": "keyword" - }, - "indexed_at": { - "type": "date" - } - } - } -} -``` - -#### Retrieval Query - -```json -{ - "size": 50, - "query": { - "bool": { - "must": [ - { - "multi_match": { - "query": "fix kubernetes port 8080", - "fields": [ - "content^2", -- Content gets 2x weight - "section_title^1.5", - "breadcrumb", - "source" - ], - "type": "best_fields", -- Match best field, not sum - "operator": "or", - "fuzziness": "AUTO", - "max_expansions": 50 - } - } - ], - "filter": [ - { "term": { "project_id": "poimen" } }, - { "terms": { "level": ["L0", "L1", "L2"] } }, - { "range": { "indexed_at": { "gte": "now-1y" } } } - ] - } - }, - "_source": ["content", "source", "breadcrumb", "level", "document_id"] -} -``` - -**Analyzer Choices:** -- `standard`: Good for most cases -- `ngram`: Better for typos/misspellings -- `edge_ngram`: Better for autocomplete - ---- - -## 3. Retrieval Accuracy Optimization - -### 3.1 Score Calculation Breakdown - -```rust -pub struct ScoreBreakdown { - pub doc_id: String, - pub semantic_score: f32, // 0.88 (cosine similarity) - pub lexical_score: f32, // 0.96 (BM25 normalized) - pub semantic_weight: f32, // 0.6 - pub lexical_weight: f32, // 0.4 - pub final_score: f32, // 0.92 - pub rank: usize, // Position in results - pub retrieval_path: String, // "hybrid" | "semantic_only" | "lexical_only" -} - -pub struct SearchResult { - pub id: String, - pub chunk: String, - pub source: String, - pub breadcrumb: Vec, - pub level: String, - pub breakdown: ScoreBreakdown, -} -``` - -### 3.2 Quality Metrics - -#### Metric 1: Mean Reciprocal Rank (MRR) - -```rust -// How early is the correct answer ranked? -fn mean_reciprocal_rank(results: &[SearchResult], ground_truth_id: &str) -> f32 { - results - .iter() - .position(|r| r.id == ground_truth_id) - .map(|pos| 1.0 / (pos + 1) as f32) - .unwrap_or(0.0) -} - -// MRR@10 = average of top 10 positions across queries -``` - -#### Metric 2: Normalized Discounted Cumulative Gain (NDCG) - -```rust -// How good are rankings, accounting for position? -fn ndcg(results: &[SearchResult], relevance_scores: &[u32]) -> f32 { - let dcg: f32 = results - .iter() - .enumerate() - .zip(relevance_scores) - .map(|((pos, _), rel)| (*rel as f32) / (pos as f32 + 2.0).log2()) - .sum(); - - let idcg: f32 = { - let mut sorted = relevance_scores.to_vec(); - sorted.sort_by(|a, b| b.cmp(a)); - sorted - .iter() - .enumerate() - .map(|(pos, rel)| (*rel as f32) / (pos as f32 + 2.0).log2()) - .sum() - }; - - if idcg == 0.0 { 0.0 } else { dcg / idcg } -} - -// Score: 0.0 (worst) to 1.0 (perfect ranking) -``` - -#### Metric 3: Precision@K and Recall@K - -```rust -fn precision_at_k(results: &[SearchResult], ground_truth: &HashSet, k: usize) -> f32 { - let retrieved_truth: HashSet<_> = results - .iter() - .take(k) - .filter(|r| ground_truth.contains(&r.id)) - .map(|r| r.id.clone()) - .collect(); - - retrieved_truth.len() as f32 / k as f32 -} - -fn recall_at_k(results: &[SearchResult], ground_truth: &HashSet, k: usize) -> f32 { - let retrieved_truth: HashSet<_> = results - .iter() - .take(k) - .filter(|r| ground_truth.contains(&r.id)) - .map(|r| r.id.clone()) - .collect(); - - retrieved_truth.len() as f32 / ground_truth.len() as f32 -} -``` - ---- - -### 3.3 Weight Tuning Strategy - -**Start with defaults:** semantic=0.6, lexical=0.4 - -**Then A/B test:** - -```rust -pub async fn evaluate_weights( - test_queries: &[(String, Vec)], // (query, ground_truth_ids) - pg: &PgClient, - opensearch: &OpenSearchClient, -) -> Result { - let weight_combinations = vec![ - (0.5, 0.5), // Equal - (0.6, 0.4), // Semantic bias (default) - (0.7, 0.3), // Heavy semantic - (0.4, 0.6), // Lexical bias - ]; - - for (sem_w, lex_w) in weight_combinations { - let mut ndcg_scores = Vec::new(); - - for (query, ground_truth) in test_queries { - let results = hybrid_retrieve(query, sem_w, lex_w, pg, opensearch).await?; - let relevance = ground_truth.iter() - .map(|id| if results.iter().any(|r| &r.id == id) { 1 } else { 0 }) - .collect::>(); - - let score = ndcg(&results, &relevance); - ndcg_scores.push(score); - } - - let avg_ndcg = ndcg_scores.iter().sum::() / ndcg_scores.len() as f32; - println!("Weights ({}, {}): NDCG = {:.3}", sem_w, lex_w, avg_ndcg); - } - - Ok(BestWeights { semantic: 0.6, lexical: 0.4 }) -} -``` - ---- - -## 4. Query Routing & Fallback - -### Decision Tree - -``` -User Query - │ - ├─ Is query very short (<3 tokens)? - │ ├─ YES → Use LEXICAL only (BM25 better for keywords) - │ │ "fix port" → Exact term match - │ │ - │ └─ NO → Continue... - │ - ├─ Does query contain special syntax (#hashtag, @mention)? - │ ├─ YES → Use LEXICAL + filter - │ │ - │ └─ NO → Continue... - │ - ├─ Can we embed the query? (check LLM availability) - │ ├─ YES → Use HYBRID (both engines) - │ │ - │ └─ NO → Fallback to LEXICAL only - │ - └─ Execute chosen strategy -``` - -**Implementation:** - -```rust -pub async fn route_query(query: &str, openai: &LLMClient) -> QueryStrategy { - let token_count = query.split_whitespace().count(); - - // Very short queries: lexical is better - if token_count < 3 { - return QueryStrategy::LexicalOnly; - } - - // Special syntax: use lexical to preserve exact matches - if query.contains('#') || query.contains('@') { - return QueryStrategy::LexicalWithFilters; - } - - // Try to embed - match openai.embed(query).await { - Ok(_embedding) => QueryStrategy::Hybrid, - Err(_) => { - // LLM unavailable: fallback to lexical - QueryStrategy::LexicalOnly - } - } -} -``` - ---- - -## 5. Indexing Pipeline (Write Side) - -When documents change (via git merge): - -``` -Git Merge Event - │ - ├─ Parse new/changed document - ├─ Split into chunks (by heading) - │ - ├─ For each chunk: - │ ├─ Compute embedding (send to LLM) - │ │ └─ Cache: avoid re-embedding identical chunks - │ │ - │ ├─ Write to PostgreSQL (chunks table) - │ │ └─ INSERT with embedding vector - │ │ - │ └─ Index to OpenSearch - │ ├─ POST vault-*/_doc/{id} - │ └─ With JWT token (Memory Service → OpenSearch) - │ - ├─ Update chunk_hash (for dedup) - └─ Mark is_indexed = TRUE -``` - -**Deduplication:** If chunk_hash exists and is_indexed=TRUE, skip. - ---- - -## 6. Testing Strategy - -### Test Fixture: Query + Expected Results - -```yaml -test_queries: - - query: "fix kubernetes port 8080 conflict" - expected_docs: ["runbooks/port-forward.md", "docs/troubleshooting.md"] - min_mrr: 0.5 # Top 2 expected - min_ndcg: 0.7 - - - query: "how to debug deployment issues" - expected_docs: ["runbooks/deployment-debug.md"] - min_mrr: 0.8 # Top 1 expected - min_ndcg: 0.85 -``` - -### Test Execution - -```rust -#[tokio::test] -async fn test_hybrid_search_accuracy() { - let test_queries = load_test_fixtures("tests/fixtures/search_queries.yaml"); - let pg = setup_pg_for_test().await; - let opensearch = setup_opensearch_for_test().await; - - for test in test_queries { - let results = hybrid_retrieve( - &test.query, - &pg, - &opensearch, - 0.6, // semantic weight - 0.4, // lexical weight - ).await.unwrap(); - - let ndcg = calculate_ndcg(&results, &test.expected_docs); - assert!(ndcg >= test.min_ndcg, "NDCG {:.3} < {:.3}", ndcg, test.min_ndcg); - } -} -``` - ---- - -## 7. Deployment Phases - -### Phase 1: Lexical-Only (Week 1) -- Deploy OpenSearch + JWT -- Use LEXICAL strategy only -- Benchmark: Precision, Recall -- Goal: Ensure BM25 works reliably - -### Phase 2: Hybrid with Fallback (Week 2-3) -- Deploy hybrid retrieval code -- Weight tuning: 50/50, 60/40, 70/30 -- A/B test: 10% traffic hybrid, 90% semantic -- Metrics: Compare NDCG, MRR - -### Phase 3: Gradual Rollout (Week 4+) -- 10% → 25% → 50% → 100% -- Monitor latency (parallel = slightly slower) -- Monitor accuracy (should be better) - -### Phase 4: Optimization (Week 5+) -- Tune index parameters (lists, refresh_interval) -- Optimize query routing -- Feature flag: weights, strategy - ---- - -## Summary: Configuration - -```yaml -# k8s/app/memory-deployment.yaml -env: - # Search strategy - - name: SEARCH_STRATEGY - value: "hybrid" # hybrid | semantic | lexical - - # Hybrid weights - - name: HYBRID_SEMANTIC_WEIGHT - value: "0.6" - - name: HYBRID_LEXICAL_WEIGHT - value: "0.4" - - # Ranking algorithm - - name: RANKING_ALGORITHM - value: "weighted_linear" # weighted_linear | rrf - - # Retrieval limits - - name: SEMANTIC_RETRIEVE_K - value: "50" # Retrieve top-50 from pgvector - - name: LEXICAL_RETRIEVE_K - value: "50" # Retrieve top-50 from OpenSearch - - name: FINAL_RESULT_K - value: "10" # Return top-10 to user - - # Indexing - - name: PGVECTOR_INDEX_LISTS - value: "100" - - name: OPENSEARCH_REFRESH_INTERVAL - value: "30s" - - name: CHUNK_DEDUP_ENABLED - value: "true" -``` - diff --git a/docs/IMPLEMENTATION_NOTES.md b/docs/IMPLEMENTATION_NOTES.md deleted file mode 100644 index 56f849d..0000000 --- a/docs/IMPLEMENTATION_NOTES.md +++ /dev/null @@ -1,329 +0,0 @@ -# Implementation Notes: Query Optimization Engine - -## Status - -✅ **Design Phase COMPLETE** -- Query Optimizer (query_optimizer.rs) — READY -- Hybrid Query Worker (hybrid_query_worker.rs) — STUB (needs API integration) -- Design Documentation (QUERY_OPTIMIZATION_ENGINE.md) — COMPLETE - -⚠️ **API Integration Notes** (for Phase 2) - ---- - -## VectorStore API Corrections - -### Current Methods (Confirmed) - -The actual `VectorStore` has level-based search methods: - -```rust -// NOT available: -vector_store.search(&embedding, project, limit, None) -vector_store.search_with_ids(&embedding, project, limit, chunk_ids) - -// ACTUALLY available: -vector_store.search_l1(project, &embedding, limit) -vector_store.search_l2(project, &embedding, limit) -vector_store.search_l3(project, &embedding, limit) -``` - -### Updated Semantic Retrieval - -```rust -async fn retrieve_semantic( - &self, - project: &str, - query_ctx: &QueryContext, - limit: i64, -) -> Result> { - let embedding = query_ctx - .embedding - .as_ref() - .ok_or_else(|| anyhow::anyhow!("no embedding"))?; - - // Use L1 (most specific level) - let results = self.vector_store.search_l1(project, embedding, limit).await?; - - // Convert to (id, score) tuples - let scored: Vec<(String, f32)> = results - .into_iter() - .map(|r| (r.id, r.score)) - .collect(); - - Ok(scored) -} -``` - ---- - -## OpenSearchClient API Corrections - -### Issue: Method Visibility - -The `lexical_search` method in `opensearch_client.rs` is private: - -```rust -// NOT public (private): -async fn lexical_search(...) - -// NEEDED: -pub async fn lexical_search(...) -``` - -### Fix - -Make the method public: - -```rust -// In crates/mem-cli/src/opensearch_client.rs -pub async fn lexical_search( - &self, - query: &str, - limit: usize, - jwt_token: &str, -) -> Result)>> -``` - ---- - -## Simplified Phase 2 Implementation - -For Phase 2, instead of modifying http_server.rs extensively, create a wrapper: - -```rust -/// In crates/mem-cli/src/http_server.rs - -async fn query_handler(...) -> HttpResponse { - let (claims, token) = match validate_auth(&req, &state).await { - Ok(c) => c, - Err(e) => return e, - }; - - // ... existing checks ... - - // Try hybrid if available - #[cfg(feature = "hybrid_search")] - { - match state.hybrid_query_worker.query( - &project, - &question, - limit, - &token, - ).await { - Ok(response) => return HttpResponse::Ok().json(response), - Err(e) => { - tracing::warn!("Hybrid query failed: {}, falling back", e); - } - } - } - - // Fallback: existing semantic search - match state.query_worker.query(&project, &question, Some(limit)).await { - Ok(results) => HttpResponse::Ok().json(json!({ - "query": question, - "project": project, - "results": results, - "method": "semantic_only" // Indicate fallback - })), - Err(e) => { - tracing::error!("Query failed: {}", e); - HttpResponse::InternalServerError().json(json!({"error": "query_failed"})) - } - } -} -``` - ---- - -## Build Status - -### Current Issues (Non-Blocking Design) - -1. **hybrid_query_worker.rs** uses placeholder VectorStore API - - Fix: Use `search_l1()` instead of `search()` - - Status: TRIVIAL (rename methods) - -2. **OpenSearchClient::lexical_search** is private - - Fix: Remove `async fn`, change to `pub async fn` - - Status: TRIVIAL (add `pub`) - -3. **Embedding type mismatch** in line 89 - - Fix: Use correct return type from embeddings crate - - Status: TRIVIAL (type annotation) - -### Estimated Fix Time - -**15-20 minutes** to update API calls and make methods public. - -### No Design Changes Needed - -All architectural decisions are sound: -- ✅ QueryOptimizer (6-stage pipeline) — No API dependency -- ✅ RRF Fusion algorithm — No API dependency -- ✅ Search strategy routing — No API dependency -- ⚠️ HybridQueryWorker — Needs VectorStore + OpenSearchClient API fixes - ---- - -## Phase 2 Checklist (1 Week) - -### Day 1-2: Fix Compilation - -- [ ] Update `hybrid_query_worker.rs` to use actual VectorStore API -- [ ] Make `OpenSearchClient::lexical_search` public -- [ ] Fix type mismatches in embedding handling -- [ ] `cargo check` passes without errors - -### Day 3-4: Integration - -- [ ] Update `/memory/query` handler to attempt hybrid search -- [ ] Add fallback strategy (hybrid → semantic → error) -- [ ] Forward JWT token through query pipeline -- [ ] Update response format to include metrics + score breakdown - -### Day 5: Testing - -- [ ] Write 10+ integration tests -- [ ] Test fallback scenarios (OpenSearch unavailable) -- [ ] Measure latency (hybrid vs semantic) -- [ ] Verify score breakdown accuracy - -### Day 6-7: Buffer + Deployment - -- [ ] Performance profiling -- [ ] Documentation updates -- [ ] Deploy to staging -- [ ] Manual E2E testing - ---- - -## Reference: Actual API Signatures - -### VectorStore (from mem-store) - -```rust -pub async fn search_l1( - &self, - project: &str, - embedding: &[f32], - limit: i64, -) -> Result>; - -pub struct SearchResult { - pub id: String, - pub score: f32, - pub item: ChunkWithMetadata, -} - -pub struct ChunkWithMetadata { - pub id: String, - pub content: String, - pub source: String, - pub level: Option, - pub breadcrumb: Option>, -} -``` - -### EmbeddingsClient (from mem-llm) - -```rust -pub async fn embed(&self, text: &str) -> Result>; - -// Returns: 384-dim (all-MiniLM) or 1536-dim (OpenAI) -``` - -### OpenSearchClient (current, needs pub) - -```rust -pub async fn lexical_search( - &self, - query: &str, - limit: usize, - jwt_token: &str, -) -> Result)>>; -// (id, score, chunk, source, breadcrumb) -``` - ---- - -## Code Diff Preview (Phase 2) - -### Fix 1: Make OpenSearch method public - -```diff -- async fn lexical_search( -+ pub async fn lexical_search( -``` - -### Fix 2: Update HybridQueryWorker to use real API - -```diff -- let results = self.vector_store.search(&embedding, project, limit, None).await?; -+ let results = self.vector_store.search_l1(project, embedding, limit).await?; - -- let scored: Vec<(String, f32)> = results -- .into_iter() -- .map(|(id, score, _)| (id, score)) -- .collect(); - -+ let scored: Vec<(String, f32)> = results -+ .into_iter() -+ .map(|r| (r.id, r.score)) -+ .collect(); -``` - -### Fix 3: Update http_server.rs to use HybridQueryWorker - -```diff -- match state.query_worker.query(&project, &question, Some(limit)).await { -+ // Try hybrid first -+ match state.hybrid_query_worker.query( -+ &project, -+ &question, -+ limit, -+ &jwt_token, -+ ).await { - Ok(response) => return HttpResponse::Ok().json(response), -+ Err(_) => { -+ // Fallback to semantic - } -+ } -+ -+ match state.query_worker.query(...).await { -``` - ---- - -## Design Validation - -✅ **All design decisions validated:** - -| Component | Status | Notes | -|-----------|--------|-------| -| Query Optimizer (6-stage) | ✅ READY | No API dependency | -| RRF Algorithm | ✅ READY | No API dependency | -| Query Routing | ✅ READY | Pure logic | -| Cascading Strategy | ✅ READY | Uses existing APIs | -| Hybrid Strategy | ⚠️ STUB | Needs VectorStore API fix | -| Fallback Pattern | ✅ READY | Uses existing query_worker | - -**None of these require architectural changes.** - ---- - -## Next: Week 2 Task - -**"Implement Phase 2: Hybrid Integration"** - -**Time estimate:** 3-4 days (15-20 min for compilation fixes + 2-3 days for integration + testing) - -**Deliverables:** -1. ✅ `/memory/query` now attempts hybrid search -2. ✅ Score breakdown + metrics in response -3. ✅ Fallback to semantic if OpenSearch unavailable -4. ✅ 10+ integration tests -5. ✅ Latency measurements (hybrid vs semantic) - ---- - diff --git a/docs/QUERY_OPTIMIZATION_ENGINE.md b/docs/QUERY_OPTIMIZATION_ENGINE.md deleted file mode 100644 index 4dcff27..0000000 --- a/docs/QUERY_OPTIMIZATION_ENGINE.md +++ /dev/null @@ -1,698 +0,0 @@ -# Query Optimization Engine: Design & Implementation Guide - -## Executive Summary - -This document specifies the **Query Optimization Engine** and **Query Context Constructor** for Poimen Memory's hybrid search. We are implementing **Approach A (Parallel RRF)** for: - -- ✅ **Highest accuracy** (mission-critical for agent reasoning) -- ✅ **Fault tolerance** (hybrid + semantic-only fallback) -- ✅ **No false negatives** (semantic catches synonyms lexical misses) -- ✅ **Transparent scoring** (debug + optimize) -- ✅ **Decoupled systems** (embedding model changes don't break architecture) - ---- - -## Architecture Overview - -``` -User Query - │ - ├─ QueryOptimizer (Query Context Constructor) - │ ├─ Normalize - │ ├─ Tokenize - │ ├─ Extract entities (years, names, keywords) - │ ├─ Analyze characteristics (special syntax, dates, negation) - │ ├─ Classify question type (procedural, factual, troubleshooting, etc) - │ └─ Route to search strategy (Hybrid, Semantic, Lexical, or Cascading) - │ - └─ HybridQueryWorker (Parallel Retrieval Orchestration) - ├─ Generate embedding (LLM) - │ - ├─ Execute Strategy - │ ├─ HYBRID: Parallel pgvector + OpenSearch - │ ├─ SEMANTIC: pgvector only (fallback) - │ ├─ LEXICAL: OpenSearch only (fallback) - │ └─ CASCADING: OpenSearch (narrow) → pgvector (rerank) - │ - ├─ Fuse Results (RRF Algorithm) - │ ├─ Normalize scores to [0-1] range - │ ├─ Apply RRF formula: 1 / (k + rank) - │ └─ Merge and re-rank - │ - └─ Return Response - └─ Top-10 results with score breakdown + metrics -``` - ---- - -## Part 1: Query Context Constructor (QueryOptimizer) - -### Why This Matters - -Before executing a query against the hybrid system, we need to: -1. **Understand** the query intent -2. **Extract** relevant context (years, entities, exact phrases) -3. **Classify** the question type -4. **Route** to the best search strategy - -This prevents "garbage in, garbage out" retrieval. - -### Stage 1: Query Normalization - -```rust -Input: " HOW do I FIX kubernetes PORT 8080 ??? " -Output: "how do i fix kubernetes port 8080 ???" -``` - -**Purpose:** -- Lowercase for consistent matching -- Trim whitespace -- Remove double spaces - ---- - -### Stage 2: Tokenization - -``` -Input: "how do i fix kubernetes port 8080" -Output: ["how", "do", "i", "fix", "kubernetes", "port", "8080"] -``` - -**Tokens used for:** -- Entity extraction -- Date filter detection -- Negation detection -- Token count heuristic - ---- - -### Stage 3: Entity Extraction - -Extract structured information from query: - -```json -{ - "year": "2024", - "exact_phrase": "kubernetes port forwarding", - "dates": ["january", "2024"], - "tags": ["#networking", "@devops"] -} -``` - -**Entities detected:** -- **Years**: YYYY format (2000-2100) -- **Quoted phrases**: "exact text" (preserve for lexical search) -- **Date keywords**: month names, relative dates ("this month", "last week") -- **Tags**: #hashtags, @mentions (preserve for special syntax routing) - ---- - -### Stage 4: Query Characteristic Analysis - -```rust -pub struct QueryContext { - pub has_special_syntax: bool, // #tag, @mention, "phrase" - pub has_date_filters: bool, // years, month names - pub has_negation: bool, // -word, "NOT", "no" - pub token_count: usize, // ["how", "do", "i", ...].len() - pub question_type: QuestionType, // Factual, Procedural, etc - pub search_strategy: SearchStrategy, // Hybrid, Semantic, etc -} -``` - -**Importance:** -- Short queries (< 3 tokens) → lexical better -- Special syntax (#tag) → preserve exact matches -- Date filters → use cascading (lexical to narrow, then semantic) -- Negation → more complex semantic reasoning needed - ---- - -### Stage 5: Question Type Classification - -Classify into one of 6 types: - -| Type | Indicators | Best Strategy | Example | -|------|-----------|------------------|---------| -| **Factual** | "What", "Define", "When" | Hybrid | "What is Kubernetes?" | -| **Procedural** | "How", "Steps", "Guide" | Hybrid ⭐⭐ | "How do I deploy to K8s?" | -| **Comparative** | "Compare", "Difference", "vs" | Hybrid | "Compare Docker vs Kubernetes" | -| **Troubleshooting** | "Fix", "Error", "Debug", "Broken" | Hybrid ⭐⭐ | "Fix port 8080 conflict" | -| **Navigational** | "Where", "Find", "Show" | Lexical First | "Where is the deployment guide?" | -| **Open** | General, conversational | Hybrid | "Tell me about networking" | - -**Classification algorithm:** -```rust -fn classify_question(raw_query: &str, tokens: &[String]) -> QuestionType { - match first_token { - "how" => Procedural, - "what" => { - if contains("difference") { Comparative } else { Factual } - } - "fix" | "error" => Troubleshooting, - "where" | "find" => Navigational, - _ => Open, - } -} -``` - ---- - -### Stage 6: Search Strategy Routing - -Decision tree to choose optimal search strategy: - -``` -Query Characteristics - │ - ├─ Token count < 3? - │ └─ YES → LEXICAL_ONLY (keywords better than embeddings) - │ - ├─ Has special syntax (#tag, "phrase")? - │ ├─ YES + date filters → LEXICAL_FIRST (narrow by keywords, rerank by semantic) - │ └─ YES alone → LEXICAL_ONLY (preserve exact syntax) - │ - ├─ Has date filters? - │ └─ YES → LEXICAL_FIRST (OpenSearch to filter by date, pgvector reranks) - │ - └─ Question Type? - ├─ Procedural → HYBRID ⭐⭐ (need both exact steps + understanding) - ├─ Troubleshooting → HYBRID ⭐⭐ (need errors + semantic understanding) - ├─ Navigational → LEXICAL_FIRST (find specific docs, then semantic rank) - └─ Others → HYBRID (good default) -``` - -**Output:** -```rust -SearchStrategy::Hybrid // or Semantic, Lexical, LexicalFirst -confidence: 0.95 // 0.0-1.0, how confident in routing decision -``` - -**Confidence scores:** -- Date filters + cascading: 0.9 (high confidence) -- Procedural questions: 0.95 (very high confidence) -- Special syntax: 0.8-0.85 (moderate, but clear signal) -- Generic queries: 0.8 (default, reasonable) - ---- - -## Part 2: Hybrid Query Worker (Parallel Orchestration) - -### Overview - -The `HybridQueryWorker` executes the query against selected engines and fuses results. - -### Flow: HYBRID Strategy - -``` -Input: "How do I fix kubernetes port conflict?" - Strategy: HYBRID, Confidence: 0.95 - - ↓ Generate Embedding (LLM) - ↓ - ┌──────────────────────────────────────┐ - │ PARALLEL Execution │ - ├──────────────────────────────────────┤ - │ │ - │ Thread 1: pgvector │ - │ ├─ Query: embedding <-> vector │ - │ ├─ Filter: project_id, level, date │ - │ └─ Retrieve: Top-50 chunks │ - │ Results: [(doc1, 0.95), ...] │ - │ │ - │ Thread 2: OpenSearch + JWT │ - │ ├─ Query: multi_match BM25 │ - │ ├─ Filter: project_id, level, date │ - │ └─ Retrieve: Top-50 documents │ - │ Results: [(doc1, 8.5), ...] │ - │ │ - └──────────────────────────────────────┘ - ↓ ~200-300ms total - ↓ - Normalize Scores - ├─ pgvector: already [0.0, 1.0] - └─ OpenSearch: min-max to [0.0, 1.0] - ↓ - RRF Fusion - ├─ doc1: 1/(60+1) + 1/(60+1) = 0.033 - ├─ doc2: 1/(60+2) + 0 = 0.016 - └─ doc3: 0 + 1/(60+1) = 0.016 - ↓ Sort by score - ↓ - Return Top-10: [doc1, doc2, doc3, ...] -``` - -### Flow: CASCADING Strategy (Lexical First) - -``` -Input: "Fix kubernetes networking in 2024" - Strategy: LEXICAL_FIRST, Confidence: 0.9 - - ↓ Generate Embedding (LLM) - ↓ - Stage 1: OpenSearch (Narrow) - ├─ Query: multi_match + filter(year=2024) - ├─ Retrieve: Top-200 documents - └─ Extract chunk_ids: [id1, id2, id3, ...] - ↓ - Stage 2: pgvector (Rerank) - ├─ Query: embedding <-> vector WHERE chunk_id IN (top-200) - ├─ Retrieve: Top-10 from narrowed set - └─ Results: [(id1, 0.95), (id3, 0.92), ...] - ↓ - Return Top-10 Results -``` - -**Benefit of cascading:** -- OpenSearch quickly filters by date/keywords (100ms) -- pgvector only ranks the 200 most relevant (not all 10,000+) -- Much faster than hybrid (150ms vs 300ms) -- No false negatives (semantic still sees all matches) - ---- - -## Part 3: Reciprocal Rank Fusion (RRF) - -### Why RRF? - -OpenSearch BM25 and pgvector cosine distance use **completely different mathematical distributions**. You cannot simply average them. - -**Example:** -``` -pgvector scores: [0.95, 0.88, 0.82, ...] (0.0 to 1.0) -OpenSearch scores: [8.5, 7.2, 6.1, ...] (0.0 to 50+, unbounded) - -Naive average would bias towards OpenSearch (much larger numbers) -``` - -**RRF Solution:** -Convert both to **ranks** (positions), then fuse ranks: - -``` -pgvector results: - Rank 1: doc1 (score 0.95) - Rank 2: doc2 (score 0.88) - Rank 3: doc3 (score 0.82) - -OpenSearch results: - Rank 1: doc1 (score 8.5) - Rank 2: doc4 (score 7.2) - Rank 3: doc2 (score 6.8) - -RRF Fusion: - For each document, calculate: 1 / (k + rank) - where k = 60 (constant) - - doc1: 1/(60+1) + 1/(60+1) = 0.0164 + 0.0164 = 0.0328 - doc2: 1/(60+2) + 1/(60+3) = 0.0159 + 0.0158 = 0.0317 - doc4: 1/(60+2) = 0.0159 - doc3: 1/(60+3) = 0.0158 - - Sorted: [doc1 (0.0328), doc2 (0.0317), doc4 (0.0159), doc3 (0.0158)] -``` - -### RRF Algorithm (Rust) - -```rust -pub fn fuse_rrf( - semantic_results: Vec<(String, f32)>, // [(id, score), ...] - lexical_results: Vec<(String, f32)>, - k: f32, // Usually 60 - final_k: usize, // Return top-k (usually 10) -) -> Vec<(String, f32)> { - let mut fused_scores: HashMap = HashMap::new(); - - // Add semantic ranks - for (rank, (id, _)) in semantic_results.into_iter().enumerate() { - let rrf_score = 1.0 / (k + (rank as f32) + 1.0); - fused_scores.insert(id, rrf_score); - } - - // Add lexical ranks (combine if already present) - for (rank, (id, _)) in lexical_results.into_iter().enumerate() { - let rrf_score = 1.0 / (k + (rank as f32) + 1.0); - *fused_scores.entry(id).or_insert(0.0) += rrf_score; - } - - // Sort by combined score - let mut results: Vec<_> = fused_scores.into_iter().collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap()); - results.truncate(final_k); - - results -} -``` - -### Why RRF Over Weighted Linear? - -| Factor | RRF | Weighted Linear (0.6/0.4) | -|--------|-----|-----| -| **Stability** | ✅ Rank-based, not score-value-dependent | ⚠️ Sensitive to score magnitude differences | -| **Parameter tuning** | ❌ None needed | ✅ Can tune 0.6/0.4 weights | -| **Academic backing** | ✅ Proven in information retrieval | ⚠️ Arbitrary without data | -| **Robustness** | ✅ Works if embedding model changes | ⚠️ May need re-tuning | -| **Debugging** | ✅ Clear: "in top 1, top 3" | ⚠️ Harder: "score 0.88 vs 0.96" | - -**Recommendation:** Use RRF by default. If A/B testing shows 0.6/0.4 weighted linear works better, switch to that. - ---- - -## Part 4: Response Format - -### API Response with Score Breakdown - -```json -{ - "query": "how do i fix kubernetes port 8080", - "project": "poimen", - "search_strategy": "Hybrid", - "strategy_confidence": 0.95, - - "results": [ - { - "id": "chunk-123", - "rank": 1, - "final_score": 0.0328, - "semantic_score": 0.95, - "lexical_score": 8.5, - "fusion_method": "rrf", - "retrieval_engine": "Hybrid", - - "text": "kubectl port-forward service port:8080...", - "source": "runbooks/kubernetes/networking.md", - "level": "L1", - "breadcrumb": ["runbooks", "kubernetes", "troubleshooting"], - - "score_breakdown": { - "semantic_rank": 1, - "lexical_rank": 1, - "rrf_components": { - "semantic_contribution": 0.0164, - "lexical_contribution": 0.0164 - } - } - } - ], - - "metrics": { - "total_time_ms": 245, - "semantic_time_ms": 120, - "lexical_time_ms": 118, - "fusion_time_ms": 7, - "semantic_results_count": 50, - "lexical_results_count": 50, - "final_results_count": 10 - } -} -``` - ---- - -## Part 5: Integration with Existing API - -### Current Endpoint - -``` -GET /memory/query?query=...&project=...&limit=... -``` - -### Updated Implementation - -Replace: -```rust -state.query_worker.query(&project, &question, Some(limit)) -``` - -With: -```rust -state.hybrid_query_worker.query( - &project, - &question, - limit, - &jwt_token, // Forward to OpenSearch -) -``` - -### Fallback Strategy - -```rust -async fn query_handler(...) -> HttpResponse { - // Try hybrid - match state.hybrid_query_worker.query(...).await { - Ok(response) => return HttpResponse::Ok().json(response), - Err(e) => { - tracing::warn!("Hybrid query failed: {}, falling back to semantic", e); - - // Fallback: semantic only - match state.query_worker.query(...).await { - Ok(results) => return HttpResponse::Ok().json(results), - Err(e2) => return HttpResponse::InternalServerError().json(...) - } - } - } -} -``` - -This ensures availability: -- **Hybrid works** → Use hybrid (best accuracy) -- **OpenSearch down** → Fallback to semantic (still good) -- **Both down** → Error (clear signal) - ---- - -## Part 6: Implementation Phases - -### Phase 1: Query Optimization Engine (Week 1) - -**Deliverables:** -- ✅ QueryOptimizer (6-stage pipeline) -- ✅ QueryContext data structure -- ✅ Question classification -- ✅ Search strategy routing -- ✅ RRFusion algorithm -- ✅ Unit tests (15+ tests) - -**Files:** -- `crates/mem-cli/src/query_optimizer.rs` (16KB) -- `crates/mem-cli/src/hybrid_query_worker.rs` (13KB) - -**Status:** ✅ COMPLETE (code committed) - ---- - -### Phase 2: Hybrid Query Integration (Week 2) - -**Tasks:** -1. Update `/memory/query` handler to use `HybridQueryWorker` -2. Add fallback strategy (hybrid → semantic → error) -3. Forward JWT token to OpenSearch -4. Return score breakdown in response -5. Add metrics to response -6. Integration tests (10+ tests) - -**Files to modify:** -- `crates/mem-cli/src/http_server.rs` -- `tests/it_hybrid_query.rs` (new) - -**Timeline:** 2-3 days - ---- - -### Phase 3: Performance Optimization (Week 3) - -**Tasks:** -1. Measure baseline latency (semantic vs lexical vs hybrid) -2. Optimize pgvector query (HNSW index tuning) -3. Optimize OpenSearch query (field boosts, analyzers) -4. Add query caching (1hr TTL) -5. Benchmark with 1000-query test set - -**Metrics to track:** -- Semantic search: 80-120ms -- Lexical search: 60-100ms -- Hybrid search: 150-250ms (parallel) -- Cascading search: 100-180ms - ---- - -### Phase 4: Testing & Validation (Week 4) - -**Test Fixtures:** Create query dataset with ground truth - -```yaml -queries: - - query: "How do I fix kubernetes port 8080?" - expected_docs: ["runbooks/networking.md", "docs/troubleshooting.md"] - min_ndcg: 0.85 - question_type: Procedural - expected_strategy: Hybrid - - - query: "#networking @devops" - expected_docs: ["docs/network-policies.md"] - min_ndcg: 0.9 - question_type: Navigational - expected_strategy: LexicalOnly - - - query: "fix port 2024" - expected_docs: ["runbooks/deployment-2024.md"] - min_ndcg: 0.8 - question_type: Troubleshooting - expected_strategy: LexicalFirst -``` - -**Accuracy metrics:** -- NDCG@10 (ranking quality) -- MRR (how early is first correct result) -- Precision@5, @10 -- Recall@10 - -**A/B Testing:** -- Hybrid vs semantic-only -- RRF (k=60) vs weighted linear (0.6/0.4) -- Different question types - ---- - -## Part 7: Configuration & Tuning - -### Environment Variables - -```bash -# Query optimization -QUERY_OPTIMIZATION_ENABLED=true -QUERY_QUESTION_CLASSIFICATION=true -QUERY_ENTITY_EXTRACTION=true - -# RRF configuration -RRF_K_CONSTANT=60 -RRF_RETRIEVE_K=50 # Top-50 from each engine -RRF_FINAL_K=10 # Return top-10 - -# Cascading strategy -CASCADING_LEXICAL_MULTIPLIER=4 # Fetch 4x results in stage 1 - -# Query caching -QUERY_CACHE_TTL_SECS=3600 -QUERY_CACHE_MAX_SIZE=10000 - -# Fallback strategy -ENABLE_SEMANTIC_FALLBACK=true -``` - -### RRF Tuning - -**k parameter (constant):** -- Lower k → Earlier ranks weighted more -- k=60 is standard (academic consensus) -- k=20-30 → Aggressively weights top results -- k=100-200 → More uniform weighting - -**retrieve_k (top-K from each engine):** -- Default: 50 -- Can increase to 100 for more diversity -- Trade-off: 50 is good balance (speed vs coverage) - -**final_k (return top-K):** -- Default: 10 -- Agents typically use 5-10 -- Can lower to 5 for faster inference - ---- - -## Part 8: Testing Checklist - -### Unit Tests - -```rust -#[tokio::test] -async fn test_query_optimizer_procedural() { } - -#[tokio::test] -async fn test_query_optimizer_short_query() { } - -#[tokio::test] -async fn test_query_optimizer_special_syntax() { } - -#[tokio::test] -async fn test_rrf_fusion_basic() { } - -#[tokio::test] -async fn test_rrf_fusion_single_engine() { } - -#[tokio::test] -async fn test_hybrid_query_integration() { } - -#[tokio::test] -async fn test_cascading_query_integration() { } - -#[tokio::test] -async fn test_fallback_semantic_only() { } -``` - -### Integration Tests - -```rust -#[tokio::test] -async fn test_end_to_end_hybrid_query() { - // Setup: ingest test documents - // Execute: /memory/query with hybrid strategy - // Verify: top result is expected document - // Assert: NDCG >= 0.85 -} - -#[tokio::test] -async fn test_cascading_vs_hybrid() { - // Compare latency and accuracy - // Cascading should be faster - // Accuracy should be similar -} -``` - -### A/B Testing Queries - -```yaml -test_dataset: - - category: "Procedural" - queries: - - "How do I deploy to Kubernetes?" - - "Steps to fix port conflicts" - - "Deploy application guide" - - - category: "Troubleshooting" - queries: - - "Fix OOMKilled error" - - "Debug networking issue" - - "Resolve timeout errors" - - - category: "Navigational" - queries: - - "Where is the deployment runbook?" - - "Find kubernetes best practices" - - "#networking documents" - - - category: "Factual" - queries: - - "What is a StatefulSet?" - - "Define PVC" - - "What does idempotency mean?" -``` - ---- - -## Summary: Architecture Decision - -| Aspect | Approach A (Parallel RRF) | -|--------|---------------------------| -| **Retrieval Method** | Parallel pgvector + OpenSearch | -| **Fusion Algorithm** | RRF (Reciprocal Rank Fusion) | -| **Accuracy** | ⭐⭐⭐⭐⭐ Highest | -| **Latency** | ⭐⭐⭐ Moderate (150-250ms for hybrid) | -| **Complexity** | Moderate (RRF logic, parallel orchestration) | -| **Fault Tolerance** | ✅ Fallback to semantic if OpenSearch down | -| **Debugging** | ✅ Clear score breakdown | -| **Recommended for** | Mission-critical LLM agent reasoning | - -**Status:** ✅ DESIGN COMPLETE, CODE IMPLEMENTED - -Next: Phase 2 (Integrate into `/memory/query` endpoint) -