crates/mem-cli/Cargo.toml

@@ -46,3 +46,4 @@ futures-util = "0.3"
 async-stream = "0.3"
 rand = "0.8"
 lru = "0.12"
+once_cell = { workspace = true }

crates/mem-cli/src/http_server.rs

@@ -382,6 +382,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
             .app_data(state.clone())
             .wrap(Logger::default())
             .route("/health", web::get().to(health_check))
+            .route("/metrics", web::get().to(crate::metrics::metrics_handler))
             .route("/memory/ingest", web::post().to(ingest_handler))
             .route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
             .route("/memory/query", web::get().to(query_handler))

crates/mem-cli/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod endpoints;
 pub mod handlers;
 pub mod http_server;
+pub mod metrics;
 pub mod query;
 pub mod auth;
 pub mod ingest_worker;

crates/mem-cli/src/metrics.rs

@@ -0,0 +1,618 @@
+//! Prometheus metrics module (O10)
+//!
+//! Centralized metrics registry for poimen-memory observability.
+//! All handlers instrument via these shared metrics.
+//! Exposed at GET /metrics in Prometheus text format.
+
+use once_cell::sync::Lazy;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::time::Instant;
+
+// ─── Metric Types ───────────────────────────────────────────
+
+/// Simple counter (monotonically increasing)
+pub struct Counter {
+    value: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl Counter {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { value: AtomicU64::new(0), name, help }
+    }
+    pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
+    pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); }
+    pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
+}
+
+/// Gauge (can go up and down)
+pub struct Gauge {
+    value: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl Gauge {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { value: AtomicU64::new(0), name, help }
+    }
+    pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); }
+    pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
+    pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); }
+    pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
+}
+
+/// Gauge for f64 values (stored as bits)
+pub struct GaugeF64 {
+    bits: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl GaugeF64 {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { bits: AtomicU64::new(0), name, help }
+    }
+    pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); }
+    pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) }
+}
+
+/// Histogram with fixed buckets for latency tracking
+pub struct Histogram {
+    buckets: &'static [f64],
+    counts: Vec<AtomicU64>,
+    sum: AtomicU64, // stored as f64 bits
+    count: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl Histogram {
+    pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self {
+        let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect();
+        Self {
+            buckets, counts, name, help,
+            sum: AtomicU64::new(0f64.to_bits()),
+            count: AtomicU64::new(0),
+        }
+    }
+
+    pub fn observe(&self, value: f64) {
+        self.count.fetch_add(1, Ordering::Relaxed);
+        // Add to sum (CAS loop for f64)
+        loop {
+            let old_bits = self.sum.load(Ordering::Relaxed);
+            let old = f64::from_bits(old_bits);
+            let new = old + value;
+            if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() {
+                break;
+            }
+        }
+        // Increment bucket counters
+        for (i, &bound) in self.buckets.iter().enumerate() {
+            if value <= bound {
+                self.counts[i].fetch_add(1, Ordering::Relaxed);
+            }
+        }
+        // +Inf bucket
+        self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed);
+    }
+}
+
+/// Labeled counter (key = label combination string)
+pub struct LabeledCounter {
+    values: Mutex<HashMap<String, u64>>,
+    name: &'static str,
+    help: &'static str,
+    label_names: &'static [&'static str],
+}
+
+impl LabeledCounter {
+    pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self {
+        Self { values: Mutex::new(HashMap::new()), name, help, label_names }
+    }
+    pub fn inc(&self, labels: &[&str]) {
+        let key = labels.join(",");
+        let mut map = self.values.lock().unwrap();
+        *map.entry(key).or_insert(0) += 1;
+    }
+}
+
+// ─── Timer helper ───────────────────────────────────────────
+
+/// RAII timer: observes duration on drop
+pub struct Timer<'a> {
+    histogram: &'a Histogram,
+    start: Instant,
+}
+
+impl<'a> Timer<'a> {
+    pub fn new(histogram: &'a Histogram) -> Self {
+        Self { histogram, start: Instant::now() }
+    }
+}
+
+impl<'a> Drop for Timer<'a> {
+    fn drop(&mut self) {
+        let elapsed = self.start.elapsed().as_secs_f64();
+        self.histogram.observe(elapsed);
+    }
+}
+
+// ─── Default buckets ────────────────────────────────────────
+
+/// Latency buckets for HTTP handlers (seconds)
+pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
+/// Latency buckets for LLM calls (seconds)
+pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0];
+/// Latency buckets for DB queries (seconds)
+pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0];
+
+// ═══════════════════════════════════════════════════════════
+// O1: Ingest handler metrics (I1-I12)
+// ═══════════════════════════════════════════════════════════
+
+pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_ingest_requests_total", "Total ingest requests received");
+pub static INGEST_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_ingest_errors_total", "Total ingest request errors");
+pub static INGEST_RECORDS_TOTAL: Counter = Counter::new(
+    "memory_ingest_records_total", "Total records ingested");
+pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new(
+    "memory_ingest_entities_extracted_total", "Total entities extracted during ingest");
+pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new(
+    "memory_ingest_edges_extracted_total", "Total edges extracted during ingest");
+pub static INGEST_IN_FLIGHT: Gauge = Gauge::new(
+    "memory_ingest_in_flight", "Currently processing ingest jobs");
+pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new(
+    "memory_ingest_queue_size", "Number of jobs waiting in ingest queue");
+pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new(
+    "memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)");
+pub static INGEST_BYTES_TOTAL: Counter = Counter::new(
+    "memory_ingest_bytes_total", "Total bytes ingested");
+pub static INGEST_AUTH_FAILURES: Counter = Counter::new(
+    "memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint");
+pub static INGEST_RATE_LIMITED: Counter = Counter::new(
+    "memory_ingest_rate_limited_total", "Total rate-limited ingest requests");
+
+pub static INGEST_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O2: Query handler metrics (Q1-Q12)
+// ═══════════════════════════════════════════════════════════
+
+pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_query_requests_total", "Total query requests received");
+pub static QUERY_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_query_errors_total", "Total query request errors");
+pub static QUERY_RESULTS_TOTAL: Counter = Counter::new(
+    "memory_query_results_total", "Total results returned across all queries");
+pub static QUERY_EMPTY_RESULTS: Counter = Counter::new(
+    "memory_query_empty_results_total", "Queries returning zero results");
+pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new(
+    "memory_query_embedding_failures_total", "Total embedding failures during query");
+pub static QUERY_IN_FLIGHT: Gauge = Gauge::new(
+    "memory_query_in_flight", "Currently processing queries");
+pub static QUERY_AUTH_FAILURES: Counter = Counter::new(
+    "memory_query_auth_failures_total", "Total auth failures on query endpoint");
+pub static QUERY_RATE_LIMITED: Counter = Counter::new(
+    "memory_query_rate_limited_total", "Total rate-limited query requests");
+pub static QUERY_CACHE_HITS: Counter = Counter::new(
+    "memory_query_cache_hits_total", "Total query cache hits");
+pub static QUERY_CACHE_MISSES: Counter = Counter::new(
+    "memory_query_cache_misses_total", "Total query cache misses");
+
+pub static QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS));
+pub static QUERY_EMBEDDING_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O3: Context endpoint metrics (C1-C8)
+// ═══════════════════════════════════════════════════════════
+
+pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_context_requests_total", "Total context retrieval requests");
+pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_context_errors_total", "Total context retrieval errors");
+pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new(
+    "memory_context_semantic_hits_total", "Results from semantic (cosine) tier");
+pub static CONTEXT_BM25_HITS: Counter = Counter::new(
+    "memory_context_bm25_hits_total", "Results from BM25 (lexical) tier");
+pub static CONTEXT_GRAPH_HITS: Counter = Counter::new(
+    "memory_context_graph_hits_total", "Results from graph traversal tier");
+pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new(
+    "memory_context_empty_results_total", "Context requests returning zero results");
+
+pub static CONTEXT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS));
+pub static CONTEXT_TIER_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O4: Relevance judge metrics (R1-R9)
+// ═══════════════════════════════════════════════════════════
+
+pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new(
+    "memory_relevance_evals_total", "Total relevance evaluations performed");
+pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_relevance_errors_total", "Total relevance evaluation errors");
+pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new(
+    "memory_relevance_relevant_total", "Results judged relevant");
+pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new(
+    "memory_relevance_irrelevant_total", "Results judged irrelevant");
+
+pub static RELEVANCE_SCORE: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_relevance_score", "Distribution of relevance scores",
+        &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]));
+pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new(
+    "memory_relevance_precision", "Current precision (relevant/retrieved)");
+pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new(
+    "memory_relevance_recall", "Current recall (relevant/total_relevant)");
+pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new(
+    "memory_relevance_f1_score", "Current F1 score");
+pub static RELEVANCE_EVAL_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O5: Write volume and storage metrics (W1-W12)
+// ═══════════════════════════════════════════════════════════
+
+pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new(
+    "memory_write_entities_total", "Total entities written to DB");
+pub static WRITE_EDGES_TOTAL: Counter = Counter::new(
+    "memory_write_edges_total", "Total edges written to DB");
+pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new(
+    "memory_write_chunks_total", "Total chunks written to DB");
+pub static WRITE_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_write_errors_total", "Total write errors");
+pub static WRITE_BYTES_TOTAL: Counter = Counter::new(
+    "memory_write_bytes_total", "Total bytes written to storage");
+
+pub static DB_ENTITY_COUNT: Gauge = Gauge::new(
+    "memory_db_entity_count", "Current entity count in memory_entity table");
+pub static DB_EDGE_COUNT: Gauge = Gauge::new(
+    "memory_db_edge_count", "Current edge count in memory_edge table");
+pub static DB_CHUNK_COUNT: Gauge = Gauge::new(
+    "memory_db_chunk_count", "Current chunk count in memory_chunks table");
+
+pub static WRITE_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS));
+pub static WRITE_BATCH_SIZE: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_write_batch_size", "Write batch sizes",
+        &[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]));
+
+// Storage gauges (updated periodically)
+pub static DB_SIZE_BYTES: Gauge = Gauge::new(
+    "memory_db_size_bytes", "Total database size in bytes");
+pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new(
+    "memory_db_index_size_bytes", "Total index size in bytes");
+
+// ═══════════════════════════════════════════════════════════
+// O6: Pod resource observability (P1-P13)
+// (Most collected by node-exporter/cAdvisor, but we track app-level)
+// ═══════════════════════════════════════════════════════════
+
+pub static APP_UPTIME_SECONDS: Gauge = Gauge::new(
+    "memory_app_uptime_seconds", "Application uptime in seconds");
+pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new(
+    "memory_app_active_connections", "Active HTTP connections");
+pub static APP_GOROUTINES: Gauge = Gauge::new(
+    "memory_app_tokio_tasks", "Active tokio tasks (approximate)");
+pub static APP_HEAP_BYTES: Gauge = Gauge::new(
+    "memory_app_heap_bytes", "Approximate heap memory usage");
+
+// ═══════════════════════════════════════════════════════════
+// O7: Availability metrics and dependency health (A1-A10)
+// ═══════════════════════════════════════════════════════════
+
+pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new(
+    "memory_health_checks_total", "Total health check requests");
+pub static HEALTH_CHECK_FAILURES: Counter = Counter::new(
+    "memory_health_check_failures_total", "Total health check failures");
+
+pub static DEP_DB_UP: Gauge = Gauge::new(
+    "memory_dependency_db_up", "Database dependency health (1=up, 0=down)");
+pub static DEP_EMBEDDING_UP: Gauge = Gauge::new(
+    "memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)");
+pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new(
+    "memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)");
+pub static DEP_LLM_UP: Gauge = Gauge::new(
+    "memory_dependency_llm_up", "LLM service health (1=up, 0=down)");
+
+pub static DEP_DB_LATENCY: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS));
+pub static DEP_EMBEDDING_LATENCY: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS));
+
+pub static REQUEST_ERRORS_BY_STATUS: Lazy<LabeledCounter> = Lazy::new(||
+    LabeledCounter::new(
+        "memory_request_errors_by_status", "Request errors by HTTP status code",
+        &["status", "endpoint"]));
+
+// ═══════════════════════════════════════════════════════════
+// O8: Ingest rate pattern tracking (IR1-IR10)
+// ═══════════════════════════════════════════════════════════
+
+pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new(
+    "memory_ingest_rate_1m", "Ingest rate per second (1-minute window)");
+pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new(
+    "memory_ingest_rate_5m", "Ingest rate per second (5-minute window)");
+pub static INGEST_LLM_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS));
+pub static INGEST_FACT_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS));
+pub static INGEST_DEDUP_TOTAL: Counter = Counter::new(
+    "memory_ingest_dedup_total", "Total entities deduplicated");
+pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new(
+    "memory_ingest_contradiction_total", "Total contradictions detected");
+pub static INGEST_PROJECTS: Gauge = Gauge::new(
+    "memory_ingest_active_projects", "Number of active projects with ingested data");
+
+// ═══════════════════════════════════════════════════════════
+// O9: Postgres internal observability (PG1-PG33)
+// (Most collected by pg_exporter, we expose app-visible DB stats)
+// ═══════════════════════════════════════════════════════════
+
+pub static DB_POOL_SIZE: Gauge = Gauge::new(
+    "memory_db_pool_size", "Current connection pool size");
+pub static DB_POOL_IDLE: Gauge = Gauge::new(
+    "memory_db_pool_idle", "Idle connections in pool");
+pub static DB_POOL_ACTIVE: Gauge = Gauge::new(
+    "memory_db_pool_active", "Active connections in pool");
+pub static DB_QUERY_TOTAL: Counter = Counter::new(
+    "memory_db_queries_total", "Total DB queries executed");
+pub static DB_QUERY_ERRORS: Counter = Counter::new(
+    "memory_db_query_errors_total", "Total DB query errors");
+pub static DB_QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS));
+pub static DB_TRANSACTION_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS));
+
+// Table-specific row counts (updated periodically)
+pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new(
+    "memory_db_table_entity_rows", "Rows in memory_entity table");
+pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new(
+    "memory_db_table_edge_rows", "Rows in memory_edge table");
+pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new(
+    "memory_db_table_chunk_rows", "Rows in memory_chunks table");
+
+// ═══════════════════════════════════════════════════════════
+// Metrics export (Prometheus text format)
+// ═══════════════════════════════════════════════════════════
+
+/// Render all metrics in Prometheus text exposition format
+pub fn render_metrics() -> String {
+    let mut out = String::with_capacity(8192);
+
+    // Helper macros
+    macro_rules! counter {
+        ($c:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n",
+                $c.name, $c.help, $c.name, $c.name, $c.get()));
+        };
+    }
+    macro_rules! gauge {
+        ($g:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n",
+                $g.name, $g.help, $g.name, $g.name, $g.get()));
+        };
+    }
+    macro_rules! gauge_f64 {
+        ($g:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n",
+                $g.name, $g.help, $g.name, $g.name, $g.get()));
+        };
+    }
+    macro_rules! histogram {
+        ($h:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name));
+            for (i, &bound) in $h.buckets.iter().enumerate() {
+                out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n",
+                    $h.name, bound, $h.counts[i].load(Ordering::Relaxed)));
+            }
+            out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n",
+                $h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed)));
+            out.push_str(&format!("{}_sum {:.6}\n", $h.name,
+                f64::from_bits($h.sum.load(Ordering::Relaxed))));
+            out.push_str(&format!("{}_count {}\n", $h.name,
+                $h.count.load(Ordering::Relaxed)));
+        };
+    }
+
+    // O1: Ingest
+    counter!(INGEST_REQUESTS_TOTAL);
+    counter!(INGEST_ERRORS_TOTAL);
+    counter!(INGEST_RECORDS_TOTAL);
+    counter!(INGEST_ENTITIES_EXTRACTED);
+    counter!(INGEST_EDGES_EXTRACTED);
+    gauge!(INGEST_IN_FLIGHT);
+    gauge!(INGEST_QUEUE_SIZE);
+    counter!(INGEST_DUPLICATES_TOTAL);
+    counter!(INGEST_BYTES_TOTAL);
+    counter!(INGEST_AUTH_FAILURES);
+    counter!(INGEST_RATE_LIMITED);
+    histogram!(INGEST_DURATION);
+
+    // O2: Query
+    counter!(QUERY_REQUESTS_TOTAL);
+    counter!(QUERY_ERRORS_TOTAL);
+    counter!(QUERY_RESULTS_TOTAL);
+    counter!(QUERY_EMPTY_RESULTS);
+    counter!(QUERY_EMBEDDING_FAILURES);
+    gauge!(QUERY_IN_FLIGHT);
+    counter!(QUERY_AUTH_FAILURES);
+    counter!(QUERY_RATE_LIMITED);
+    counter!(QUERY_CACHE_HITS);
+    counter!(QUERY_CACHE_MISSES);
+    histogram!(QUERY_DURATION);
+    histogram!(QUERY_EMBEDDING_DURATION);
+
+    // O3: Context
+    counter!(CONTEXT_REQUESTS_TOTAL);
+    counter!(CONTEXT_ERRORS_TOTAL);
+    counter!(CONTEXT_SEMANTIC_HITS);
+    counter!(CONTEXT_BM25_HITS);
+    counter!(CONTEXT_GRAPH_HITS);
+    counter!(CONTEXT_EMPTY_RESULTS);
+    histogram!(CONTEXT_DURATION);
+    histogram!(CONTEXT_TIER_DURATION);
+
+    // O4: Relevance
+    counter!(RELEVANCE_EVALS_TOTAL);
+    counter!(RELEVANCE_ERRORS_TOTAL);
+    counter!(RELEVANCE_RELEVANT_TOTAL);
+    counter!(RELEVANCE_IRRELEVANT_TOTAL);
+    histogram!(RELEVANCE_SCORE);
+    gauge_f64!(RELEVANCE_PRECISION);
+    gauge_f64!(RELEVANCE_RECALL);
+    gauge_f64!(RELEVANCE_F1);
+    histogram!(RELEVANCE_EVAL_DURATION);
+
+    // O5: Write volume
+    counter!(WRITE_ENTITIES_TOTAL);
+    counter!(WRITE_EDGES_TOTAL);
+    counter!(WRITE_CHUNKS_TOTAL);
+    counter!(WRITE_ERRORS_TOTAL);
+    counter!(WRITE_BYTES_TOTAL);
+    gauge!(DB_ENTITY_COUNT);
+    gauge!(DB_EDGE_COUNT);
+    gauge!(DB_CHUNK_COUNT);
+    histogram!(WRITE_DURATION);
+    histogram!(WRITE_BATCH_SIZE);
+    gauge!(DB_SIZE_BYTES);
+    gauge!(DB_INDEX_SIZE_BYTES);
+
+    // O6: Pod resources
+    gauge!(APP_UPTIME_SECONDS);
+    gauge!(APP_ACTIVE_CONNECTIONS);
+    gauge!(APP_GOROUTINES);
+    gauge!(APP_HEAP_BYTES);
+
+    // O7: Availability
+    counter!(HEALTH_CHECKS_TOTAL);
+    counter!(HEALTH_CHECK_FAILURES);
+    gauge!(DEP_DB_UP);
+    gauge!(DEP_EMBEDDING_UP);
+    gauge!(DEP_OPENSEARCH_UP);
+    gauge!(DEP_LLM_UP);
+    histogram!(DEP_DB_LATENCY);
+    histogram!(DEP_EMBEDDING_LATENCY);
+
+    // O8: Ingest rate
+    gauge_f64!(INGEST_RATE_1M);
+    gauge_f64!(INGEST_RATE_5M);
+    histogram!(INGEST_LLM_EXTRACT_DURATION);
+    histogram!(INGEST_FACT_EXTRACT_DURATION);
+    counter!(INGEST_DEDUP_TOTAL);
+    counter!(INGEST_CONTRADICTION_TOTAL);
+    gauge!(INGEST_PROJECTS);
+
+    // O9: Postgres
+    gauge!(DB_POOL_SIZE);
+    gauge!(DB_POOL_IDLE);
+    gauge!(DB_POOL_ACTIVE);
+    counter!(DB_QUERY_TOTAL);
+    counter!(DB_QUERY_ERRORS);
+    histogram!(DB_QUERY_DURATION);
+    histogram!(DB_TRANSACTION_DURATION);
+    gauge!(DB_TABLE_ENTITY_ROWS);
+    gauge!(DB_TABLE_EDGE_ROWS);
+    gauge!(DB_TABLE_CHUNK_ROWS);
+
+    // Labeled counter: errors by status
+    {
+        let map = REQUEST_ERRORS_BY_STATUS.values.lock().unwrap();
+        if !map.is_empty() {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n",
+                REQUEST_ERRORS_BY_STATUS.name, REQUEST_ERRORS_BY_STATUS.help,
+                REQUEST_ERRORS_BY_STATUS.name));
+            for (key, val) in map.iter() {
+                let parts: Vec<&str> = key.split(',').collect();
+                if parts.len() == 2 {
+                    out.push_str(&format!("{}{{status=\"{}\",endpoint=\"{}\"}} {}\n",
+                        REQUEST_ERRORS_BY_STATUS.name, parts[0], parts[1], val));
+                }
+            }
+        }
+    }
+
+    out
+}
+
+/// GET /metrics handler
+pub async fn metrics_handler() -> actix_web::HttpResponse {
+    actix_web::HttpResponse::Ok()
+        .content_type("text/plain; version=0.0.4; charset=utf-8")
+        .body(render_metrics())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_counter() {
+        let c = Counter::new("test_counter", "test");
+        assert_eq!(c.get(), 0);
+        c.inc();
+        assert_eq!(c.get(), 1);
+        c.inc_by(5);
+        assert_eq!(c.get(), 6);
+    }
+
+    #[test]
+    fn test_gauge() {
+        let g = Gauge::new("test_gauge", "test");
+        assert_eq!(g.get(), 0);
+        g.set(42);
+        assert_eq!(g.get(), 42);
+        g.inc();
+        assert_eq!(g.get(), 43);
+        g.dec();
+        assert_eq!(g.get(), 42);
+    }
+
+    #[test]
+    fn test_gauge_f64() {
+        let g = GaugeF64::new("test_gauge_f64", "test");
+        assert_eq!(g.get(), 0.0);
+        g.set(3.14);
+        assert!((g.get() - 3.14).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_histogram() {
+        let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]);
+        h.observe(0.05);
+        h.observe(0.3);
+        h.observe(0.8);
+        h.observe(2.0);
+        assert_eq!(h.count.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    fn test_render_metrics_not_empty() {
+        INGEST_REQUESTS_TOTAL.inc();
+        QUERY_REQUESTS_TOTAL.inc();
+        let output = render_metrics();
+        assert!(output.contains("memory_ingest_requests_total"));
+        assert!(output.contains("memory_query_requests_total"));
+        assert!(output.contains("# HELP"));
+        assert!(output.contains("# TYPE"));
+    }
+
+    #[test]
+    fn test_timer_observes_on_drop() {
+        let h = Histogram::new("timer_test", "test", HTTP_BUCKETS);
+        {
+            let _t = Timer::new(&h);
+            std::thread::sleep(std::time::Duration::from_millis(1));
+        }
+        assert_eq!(h.count.load(Ordering::Relaxed), 1);
+    }
+}