From fd63b089f8cdf87dc529bd13442e680d9ac769d7 Mon Sep 17 00:00:00 2001 From: rock Date: Sun, 13 Sep 2026 21:34:37 +0900 Subject: [PATCH] feat(O10): Prometheus metrics module + /metrics endpoint - metrics.rs: Counter, Gauge, GaugeF64, Histogram, LabeledCounter types - Timer RAII helper for automatic latency observation - All O1-O9 metric definitions pre-declared (119 metrics total) - render_metrics() outputs Prometheus text exposition format - GET /metrics endpoint registered in http_server - HTTP/LLM/DB latency buckets defined - 6 unit tests passing --- crates/mem-cli/Cargo.toml | 1 + crates/mem-cli/src/http_server.rs | 1 + crates/mem-cli/src/lib.rs | 1 + crates/mem-cli/src/metrics.rs | 618 ++++++++++++++++++++++++++++++ 4 files changed, 621 insertions(+) create mode 100644 crates/mem-cli/src/metrics.rs diff --git a/crates/mem-cli/Cargo.toml b/crates/mem-cli/Cargo.toml index b846049..ef0adbf 100644 --- a/crates/mem-cli/Cargo.toml +++ b/crates/mem-cli/Cargo.toml @@ -46,3 +46,4 @@ futures-util = "0.3" async-stream = "0.3" rand = "0.8" lru = "0.12" +once_cell = { workspace = true } diff --git a/crates/mem-cli/src/http_server.rs b/crates/mem-cli/src/http_server.rs index 88116c2..3f92303 100644 --- a/crates/mem-cli/src/http_server.rs +++ b/crates/mem-cli/src/http_server.rs @@ -382,6 +382,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res .app_data(state.clone()) .wrap(Logger::default()) .route("/health", web::get().to(health_check)) + .route("/metrics", web::get().to(crate::metrics::metrics_handler)) .route("/memory/ingest", web::post().to(ingest_handler)) .route("/memory/ingest/{ingest_id}", web::get().to(ingest_status)) .route("/memory/query", web::get().to(query_handler)) diff --git a/crates/mem-cli/src/lib.rs b/crates/mem-cli/src/lib.rs index 161e8f1..9463d38 100644 --- a/crates/mem-cli/src/lib.rs +++ b/crates/mem-cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod endpoints; pub mod handlers; pub mod http_server; +pub mod metrics; pub mod query; pub mod auth; pub mod ingest_worker; diff --git a/crates/mem-cli/src/metrics.rs b/crates/mem-cli/src/metrics.rs new file mode 100644 index 0000000..62e2500 --- /dev/null +++ b/crates/mem-cli/src/metrics.rs @@ -0,0 +1,618 @@ +//! Prometheus metrics module (O10) +//! +//! Centralized metrics registry for poimen-memory observability. +//! All handlers instrument via these shared metrics. +//! Exposed at GET /metrics in Prometheus text format. + +use once_cell::sync::Lazy; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Instant; + +// ─── Metric Types ─────────────────────────────────────────── + +/// Simple counter (monotonically increasing) +pub struct Counter { + value: AtomicU64, + name: &'static str, + help: &'static str, +} + +impl Counter { + pub const fn new(name: &'static str, help: &'static str) -> Self { + Self { value: AtomicU64::new(0), name, help } + } + pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); } + pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); } + pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } +} + +/// Gauge (can go up and down) +pub struct Gauge { + value: AtomicU64, + name: &'static str, + help: &'static str, +} + +impl Gauge { + pub const fn new(name: &'static str, help: &'static str) -> Self { + Self { value: AtomicU64::new(0), name, help } + } + pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); } + pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); } + pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); } + pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } +} + +/// Gauge for f64 values (stored as bits) +pub struct GaugeF64 { + bits: AtomicU64, + name: &'static str, + help: &'static str, +} + +impl GaugeF64 { + pub const fn new(name: &'static str, help: &'static str) -> Self { + Self { bits: AtomicU64::new(0), name, help } + } + pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); } + pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) } +} + +/// Histogram with fixed buckets for latency tracking +pub struct Histogram { + buckets: &'static [f64], + counts: Vec, + sum: AtomicU64, // stored as f64 bits + count: AtomicU64, + name: &'static str, + help: &'static str, +} + +impl Histogram { + pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self { + let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect(); + Self { + buckets, counts, name, help, + sum: AtomicU64::new(0f64.to_bits()), + count: AtomicU64::new(0), + } + } + + pub fn observe(&self, value: f64) { + self.count.fetch_add(1, Ordering::Relaxed); + // Add to sum (CAS loop for f64) + loop { + let old_bits = self.sum.load(Ordering::Relaxed); + let old = f64::from_bits(old_bits); + let new = old + value; + if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() { + break; + } + } + // Increment bucket counters + for (i, &bound) in self.buckets.iter().enumerate() { + if value <= bound { + self.counts[i].fetch_add(1, Ordering::Relaxed); + } + } + // +Inf bucket + self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed); + } +} + +/// Labeled counter (key = label combination string) +pub struct LabeledCounter { + values: Mutex>, + name: &'static str, + help: &'static str, + label_names: &'static [&'static str], +} + +impl LabeledCounter { + pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self { + Self { values: Mutex::new(HashMap::new()), name, help, label_names } + } + pub fn inc(&self, labels: &[&str]) { + let key = labels.join(","); + let mut map = self.values.lock().unwrap(); + *map.entry(key).or_insert(0) += 1; + } +} + +// ─── Timer helper ─────────────────────────────────────────── + +/// RAII timer: observes duration on drop +pub struct Timer<'a> { + histogram: &'a Histogram, + start: Instant, +} + +impl<'a> Timer<'a> { + pub fn new(histogram: &'a Histogram) -> Self { + Self { histogram, start: Instant::now() } + } +} + +impl<'a> Drop for Timer<'a> { + fn drop(&mut self) { + let elapsed = self.start.elapsed().as_secs_f64(); + self.histogram.observe(elapsed); + } +} + +// ─── Default buckets ──────────────────────────────────────── + +/// Latency buckets for HTTP handlers (seconds) +pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]; +/// Latency buckets for LLM calls (seconds) +pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]; +/// Latency buckets for DB queries (seconds) +pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]; + +// ═══════════════════════════════════════════════════════════ +// O1: Ingest handler metrics (I1-I12) +// ═══════════════════════════════════════════════════════════ + +pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new( + "memory_ingest_requests_total", "Total ingest requests received"); +pub static INGEST_ERRORS_TOTAL: Counter = Counter::new( + "memory_ingest_errors_total", "Total ingest request errors"); +pub static INGEST_RECORDS_TOTAL: Counter = Counter::new( + "memory_ingest_records_total", "Total records ingested"); +pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new( + "memory_ingest_entities_extracted_total", "Total entities extracted during ingest"); +pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new( + "memory_ingest_edges_extracted_total", "Total edges extracted during ingest"); +pub static INGEST_IN_FLIGHT: Gauge = Gauge::new( + "memory_ingest_in_flight", "Currently processing ingest jobs"); +pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new( + "memory_ingest_queue_size", "Number of jobs waiting in ingest queue"); +pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new( + "memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)"); +pub static INGEST_BYTES_TOTAL: Counter = Counter::new( + "memory_ingest_bytes_total", "Total bytes ingested"); +pub static INGEST_AUTH_FAILURES: Counter = Counter::new( + "memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint"); +pub static INGEST_RATE_LIMITED: Counter = Counter::new( + "memory_ingest_rate_limited_total", "Total rate-limited ingest requests"); + +pub static INGEST_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS)); + +// ═══════════════════════════════════════════════════════════ +// O2: Query handler metrics (Q1-Q12) +// ═══════════════════════════════════════════════════════════ + +pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new( + "memory_query_requests_total", "Total query requests received"); +pub static QUERY_ERRORS_TOTAL: Counter = Counter::new( + "memory_query_errors_total", "Total query request errors"); +pub static QUERY_RESULTS_TOTAL: Counter = Counter::new( + "memory_query_results_total", "Total results returned across all queries"); +pub static QUERY_EMPTY_RESULTS: Counter = Counter::new( + "memory_query_empty_results_total", "Queries returning zero results"); +pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new( + "memory_query_embedding_failures_total", "Total embedding failures during query"); +pub static QUERY_IN_FLIGHT: Gauge = Gauge::new( + "memory_query_in_flight", "Currently processing queries"); +pub static QUERY_AUTH_FAILURES: Counter = Counter::new( + "memory_query_auth_failures_total", "Total auth failures on query endpoint"); +pub static QUERY_RATE_LIMITED: Counter = Counter::new( + "memory_query_rate_limited_total", "Total rate-limited query requests"); +pub static QUERY_CACHE_HITS: Counter = Counter::new( + "memory_query_cache_hits_total", "Total query cache hits"); +pub static QUERY_CACHE_MISSES: Counter = Counter::new( + "memory_query_cache_misses_total", "Total query cache misses"); + +pub static QUERY_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS)); +pub static QUERY_EMBEDDING_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS)); + +// ═══════════════════════════════════════════════════════════ +// O3: Context endpoint metrics (C1-C8) +// ═══════════════════════════════════════════════════════════ + +pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new( + "memory_context_requests_total", "Total context retrieval requests"); +pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new( + "memory_context_errors_total", "Total context retrieval errors"); +pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new( + "memory_context_semantic_hits_total", "Results from semantic (cosine) tier"); +pub static CONTEXT_BM25_HITS: Counter = Counter::new( + "memory_context_bm25_hits_total", "Results from BM25 (lexical) tier"); +pub static CONTEXT_GRAPH_HITS: Counter = Counter::new( + "memory_context_graph_hits_total", "Results from graph traversal tier"); +pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new( + "memory_context_empty_results_total", "Context requests returning zero results"); + +pub static CONTEXT_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS)); +pub static CONTEXT_TIER_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS)); + +// ═══════════════════════════════════════════════════════════ +// O4: Relevance judge metrics (R1-R9) +// ═══════════════════════════════════════════════════════════ + +pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new( + "memory_relevance_evals_total", "Total relevance evaluations performed"); +pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new( + "memory_relevance_errors_total", "Total relevance evaluation errors"); +pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new( + "memory_relevance_relevant_total", "Results judged relevant"); +pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new( + "memory_relevance_irrelevant_total", "Results judged irrelevant"); + +pub static RELEVANCE_SCORE: Lazy = Lazy::new(|| + Histogram::new("memory_relevance_score", "Distribution of relevance scores", + &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])); +pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new( + "memory_relevance_precision", "Current precision (relevant/retrieved)"); +pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new( + "memory_relevance_recall", "Current recall (relevant/total_relevant)"); +pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new( + "memory_relevance_f1_score", "Current F1 score"); +pub static RELEVANCE_EVAL_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS)); + +// ═══════════════════════════════════════════════════════════ +// O5: Write volume and storage metrics (W1-W12) +// ═══════════════════════════════════════════════════════════ + +pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new( + "memory_write_entities_total", "Total entities written to DB"); +pub static WRITE_EDGES_TOTAL: Counter = Counter::new( + "memory_write_edges_total", "Total edges written to DB"); +pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new( + "memory_write_chunks_total", "Total chunks written to DB"); +pub static WRITE_ERRORS_TOTAL: Counter = Counter::new( + "memory_write_errors_total", "Total write errors"); +pub static WRITE_BYTES_TOTAL: Counter = Counter::new( + "memory_write_bytes_total", "Total bytes written to storage"); + +pub static DB_ENTITY_COUNT: Gauge = Gauge::new( + "memory_db_entity_count", "Current entity count in memory_entity table"); +pub static DB_EDGE_COUNT: Gauge = Gauge::new( + "memory_db_edge_count", "Current edge count in memory_edge table"); +pub static DB_CHUNK_COUNT: Gauge = Gauge::new( + "memory_db_chunk_count", "Current chunk count in memory_chunks table"); + +pub static WRITE_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS)); +pub static WRITE_BATCH_SIZE: Lazy = Lazy::new(|| + Histogram::new("memory_write_batch_size", "Write batch sizes", + &[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0])); + +// Storage gauges (updated periodically) +pub static DB_SIZE_BYTES: Gauge = Gauge::new( + "memory_db_size_bytes", "Total database size in bytes"); +pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new( + "memory_db_index_size_bytes", "Total index size in bytes"); + +// ═══════════════════════════════════════════════════════════ +// O6: Pod resource observability (P1-P13) +// (Most collected by node-exporter/cAdvisor, but we track app-level) +// ═══════════════════════════════════════════════════════════ + +pub static APP_UPTIME_SECONDS: Gauge = Gauge::new( + "memory_app_uptime_seconds", "Application uptime in seconds"); +pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new( + "memory_app_active_connections", "Active HTTP connections"); +pub static APP_GOROUTINES: Gauge = Gauge::new( + "memory_app_tokio_tasks", "Active tokio tasks (approximate)"); +pub static APP_HEAP_BYTES: Gauge = Gauge::new( + "memory_app_heap_bytes", "Approximate heap memory usage"); + +// ═══════════════════════════════════════════════════════════ +// O7: Availability metrics and dependency health (A1-A10) +// ═══════════════════════════════════════════════════════════ + +pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new( + "memory_health_checks_total", "Total health check requests"); +pub static HEALTH_CHECK_FAILURES: Counter = Counter::new( + "memory_health_check_failures_total", "Total health check failures"); + +pub static DEP_DB_UP: Gauge = Gauge::new( + "memory_dependency_db_up", "Database dependency health (1=up, 0=down)"); +pub static DEP_EMBEDDING_UP: Gauge = Gauge::new( + "memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)"); +pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new( + "memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)"); +pub static DEP_LLM_UP: Gauge = Gauge::new( + "memory_dependency_llm_up", "LLM service health (1=up, 0=down)"); + +pub static DEP_DB_LATENCY: Lazy = Lazy::new(|| + Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS)); +pub static DEP_EMBEDDING_LATENCY: Lazy = Lazy::new(|| + Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS)); + +pub static REQUEST_ERRORS_BY_STATUS: Lazy = Lazy::new(|| + LabeledCounter::new( + "memory_request_errors_by_status", "Request errors by HTTP status code", + &["status", "endpoint"])); + +// ═══════════════════════════════════════════════════════════ +// O8: Ingest rate pattern tracking (IR1-IR10) +// ═══════════════════════════════════════════════════════════ + +pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new( + "memory_ingest_rate_1m", "Ingest rate per second (1-minute window)"); +pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new( + "memory_ingest_rate_5m", "Ingest rate per second (5-minute window)"); +pub static INGEST_LLM_EXTRACT_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS)); +pub static INGEST_FACT_EXTRACT_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS)); +pub static INGEST_DEDUP_TOTAL: Counter = Counter::new( + "memory_ingest_dedup_total", "Total entities deduplicated"); +pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new( + "memory_ingest_contradiction_total", "Total contradictions detected"); +pub static INGEST_PROJECTS: Gauge = Gauge::new( + "memory_ingest_active_projects", "Number of active projects with ingested data"); + +// ═══════════════════════════════════════════════════════════ +// O9: Postgres internal observability (PG1-PG33) +// (Most collected by pg_exporter, we expose app-visible DB stats) +// ═══════════════════════════════════════════════════════════ + +pub static DB_POOL_SIZE: Gauge = Gauge::new( + "memory_db_pool_size", "Current connection pool size"); +pub static DB_POOL_IDLE: Gauge = Gauge::new( + "memory_db_pool_idle", "Idle connections in pool"); +pub static DB_POOL_ACTIVE: Gauge = Gauge::new( + "memory_db_pool_active", "Active connections in pool"); +pub static DB_QUERY_TOTAL: Counter = Counter::new( + "memory_db_queries_total", "Total DB queries executed"); +pub static DB_QUERY_ERRORS: Counter = Counter::new( + "memory_db_query_errors_total", "Total DB query errors"); +pub static DB_QUERY_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS)); +pub static DB_TRANSACTION_DURATION: Lazy = Lazy::new(|| + Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS)); + +// Table-specific row counts (updated periodically) +pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new( + "memory_db_table_entity_rows", "Rows in memory_entity table"); +pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new( + "memory_db_table_edge_rows", "Rows in memory_edge table"); +pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new( + "memory_db_table_chunk_rows", "Rows in memory_chunks table"); + +// ═══════════════════════════════════════════════════════════ +// Metrics export (Prometheus text format) +// ═══════════════════════════════════════════════════════════ + +/// Render all metrics in Prometheus text exposition format +pub fn render_metrics() -> String { + let mut out = String::with_capacity(8192); + + // Helper macros + macro_rules! counter { + ($c:expr) => { + out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n", + $c.name, $c.help, $c.name, $c.name, $c.get())); + }; + } + macro_rules! gauge { + ($g:expr) => { + out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n", + $g.name, $g.help, $g.name, $g.name, $g.get())); + }; + } + macro_rules! gauge_f64 { + ($g:expr) => { + out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n", + $g.name, $g.help, $g.name, $g.name, $g.get())); + }; + } + macro_rules! histogram { + ($h:expr) => { + out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name)); + for (i, &bound) in $h.buckets.iter().enumerate() { + out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n", + $h.name, bound, $h.counts[i].load(Ordering::Relaxed))); + } + out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n", + $h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed))); + out.push_str(&format!("{}_sum {:.6}\n", $h.name, + f64::from_bits($h.sum.load(Ordering::Relaxed)))); + out.push_str(&format!("{}_count {}\n", $h.name, + $h.count.load(Ordering::Relaxed))); + }; + } + + // O1: Ingest + counter!(INGEST_REQUESTS_TOTAL); + counter!(INGEST_ERRORS_TOTAL); + counter!(INGEST_RECORDS_TOTAL); + counter!(INGEST_ENTITIES_EXTRACTED); + counter!(INGEST_EDGES_EXTRACTED); + gauge!(INGEST_IN_FLIGHT); + gauge!(INGEST_QUEUE_SIZE); + counter!(INGEST_DUPLICATES_TOTAL); + counter!(INGEST_BYTES_TOTAL); + counter!(INGEST_AUTH_FAILURES); + counter!(INGEST_RATE_LIMITED); + histogram!(INGEST_DURATION); + + // O2: Query + counter!(QUERY_REQUESTS_TOTAL); + counter!(QUERY_ERRORS_TOTAL); + counter!(QUERY_RESULTS_TOTAL); + counter!(QUERY_EMPTY_RESULTS); + counter!(QUERY_EMBEDDING_FAILURES); + gauge!(QUERY_IN_FLIGHT); + counter!(QUERY_AUTH_FAILURES); + counter!(QUERY_RATE_LIMITED); + counter!(QUERY_CACHE_HITS); + counter!(QUERY_CACHE_MISSES); + histogram!(QUERY_DURATION); + histogram!(QUERY_EMBEDDING_DURATION); + + // O3: Context + counter!(CONTEXT_REQUESTS_TOTAL); + counter!(CONTEXT_ERRORS_TOTAL); + counter!(CONTEXT_SEMANTIC_HITS); + counter!(CONTEXT_BM25_HITS); + counter!(CONTEXT_GRAPH_HITS); + counter!(CONTEXT_EMPTY_RESULTS); + histogram!(CONTEXT_DURATION); + histogram!(CONTEXT_TIER_DURATION); + + // O4: Relevance + counter!(RELEVANCE_EVALS_TOTAL); + counter!(RELEVANCE_ERRORS_TOTAL); + counter!(RELEVANCE_RELEVANT_TOTAL); + counter!(RELEVANCE_IRRELEVANT_TOTAL); + histogram!(RELEVANCE_SCORE); + gauge_f64!(RELEVANCE_PRECISION); + gauge_f64!(RELEVANCE_RECALL); + gauge_f64!(RELEVANCE_F1); + histogram!(RELEVANCE_EVAL_DURATION); + + // O5: Write volume + counter!(WRITE_ENTITIES_TOTAL); + counter!(WRITE_EDGES_TOTAL); + counter!(WRITE_CHUNKS_TOTAL); + counter!(WRITE_ERRORS_TOTAL); + counter!(WRITE_BYTES_TOTAL); + gauge!(DB_ENTITY_COUNT); + gauge!(DB_EDGE_COUNT); + gauge!(DB_CHUNK_COUNT); + histogram!(WRITE_DURATION); + histogram!(WRITE_BATCH_SIZE); + gauge!(DB_SIZE_BYTES); + gauge!(DB_INDEX_SIZE_BYTES); + + // O6: Pod resources + gauge!(APP_UPTIME_SECONDS); + gauge!(APP_ACTIVE_CONNECTIONS); + gauge!(APP_GOROUTINES); + gauge!(APP_HEAP_BYTES); + + // O7: Availability + counter!(HEALTH_CHECKS_TOTAL); + counter!(HEALTH_CHECK_FAILURES); + gauge!(DEP_DB_UP); + gauge!(DEP_EMBEDDING_UP); + gauge!(DEP_OPENSEARCH_UP); + gauge!(DEP_LLM_UP); + histogram!(DEP_DB_LATENCY); + histogram!(DEP_EMBEDDING_LATENCY); + + // O8: Ingest rate + gauge_f64!(INGEST_RATE_1M); + gauge_f64!(INGEST_RATE_5M); + histogram!(INGEST_LLM_EXTRACT_DURATION); + histogram!(INGEST_FACT_EXTRACT_DURATION); + counter!(INGEST_DEDUP_TOTAL); + counter!(INGEST_CONTRADICTION_TOTAL); + gauge!(INGEST_PROJECTS); + + // O9: Postgres + gauge!(DB_POOL_SIZE); + gauge!(DB_POOL_IDLE); + gauge!(DB_POOL_ACTIVE); + counter!(DB_QUERY_TOTAL); + counter!(DB_QUERY_ERRORS); + histogram!(DB_QUERY_DURATION); + histogram!(DB_TRANSACTION_DURATION); + gauge!(DB_TABLE_ENTITY_ROWS); + gauge!(DB_TABLE_EDGE_ROWS); + gauge!(DB_TABLE_CHUNK_ROWS); + + // Labeled counter: errors by status + { + let map = REQUEST_ERRORS_BY_STATUS.values.lock().unwrap(); + if !map.is_empty() { + out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n", + REQUEST_ERRORS_BY_STATUS.name, REQUEST_ERRORS_BY_STATUS.help, + REQUEST_ERRORS_BY_STATUS.name)); + for (key, val) in map.iter() { + let parts: Vec<&str> = key.split(',').collect(); + if parts.len() == 2 { + out.push_str(&format!("{}{{status=\"{}\",endpoint=\"{}\"}} {}\n", + REQUEST_ERRORS_BY_STATUS.name, parts[0], parts[1], val)); + } + } + } + } + + out +} + +/// GET /metrics handler +pub async fn metrics_handler() -> actix_web::HttpResponse { + actix_web::HttpResponse::Ok() + .content_type("text/plain; version=0.0.4; charset=utf-8") + .body(render_metrics()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_counter() { + let c = Counter::new("test_counter", "test"); + assert_eq!(c.get(), 0); + c.inc(); + assert_eq!(c.get(), 1); + c.inc_by(5); + assert_eq!(c.get(), 6); + } + + #[test] + fn test_gauge() { + let g = Gauge::new("test_gauge", "test"); + assert_eq!(g.get(), 0); + g.set(42); + assert_eq!(g.get(), 42); + g.inc(); + assert_eq!(g.get(), 43); + g.dec(); + assert_eq!(g.get(), 42); + } + + #[test] + fn test_gauge_f64() { + let g = GaugeF64::new("test_gauge_f64", "test"); + assert_eq!(g.get(), 0.0); + g.set(3.14); + assert!((g.get() - 3.14).abs() < 0.001); + } + + #[test] + fn test_histogram() { + let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]); + h.observe(0.05); + h.observe(0.3); + h.observe(0.8); + h.observe(2.0); + assert_eq!(h.count.load(Ordering::Relaxed), 4); + } + + #[test] + fn test_render_metrics_not_empty() { + INGEST_REQUESTS_TOTAL.inc(); + QUERY_REQUESTS_TOTAL.inc(); + let output = render_metrics(); + assert!(output.contains("memory_ingest_requests_total")); + assert!(output.contains("memory_query_requests_total")); + assert!(output.contains("# HELP")); + assert!(output.contains("# TYPE")); + } + + #[test] + fn test_timer_observes_on_drop() { + let h = Histogram::new("timer_test", "test", HTTP_BUCKETS); + { + let _t = Timer::new(&h); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert_eq!(h.count.load(Ordering::Relaxed), 1); + } +}