//! Prometheus metrics module (O10) //! //! Centralized metrics registry for poimen-memory observability. //! All handlers instrument via these shared metrics. //! Exposed at GET /metrics in Prometheus text format. use once_cell::sync::Lazy; use std::sync::atomic::{AtomicU64, Ordering}; use std::collections::HashMap; use std::sync::Mutex; use std::time::Instant; // ─── Metric Types ─────────────────────────────────────────── /// Simple counter (monotonically increasing) pub struct Counter { value: AtomicU64, name: &'static str, help: &'static str, } impl Counter { pub const fn new(name: &'static str, help: &'static str) -> Self { Self { value: AtomicU64::new(0), name, help } } pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); } pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); } pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } } /// Gauge (can go up and down) pub struct Gauge { value: AtomicU64, name: &'static str, help: &'static str, } impl Gauge { pub const fn new(name: &'static str, help: &'static str) -> Self { Self { value: AtomicU64::new(0), name, help } } pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); } pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); } pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); } pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) } } /// Gauge for f64 values (stored as bits) pub struct GaugeF64 { bits: AtomicU64, name: &'static str, help: &'static str, } impl GaugeF64 { pub const fn new(name: &'static str, help: &'static str) -> Self { Self { bits: AtomicU64::new(0), name, help } } pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); } pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) } } /// Histogram with fixed buckets for latency tracking pub struct Histogram { pub buckets: &'static [f64], pub counts: Vec, pub sum: AtomicU64, // stored as f64 bits pub count: AtomicU64, pub name: &'static str, pub help: &'static str, } impl Histogram { pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self { let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect(); Self { buckets, counts, name, help, sum: AtomicU64::new(0f64.to_bits()), count: AtomicU64::new(0), } } pub fn observe(&self, value: f64) { self.count.fetch_add(1, Ordering::Relaxed); // Add to sum (CAS loop for f64) loop { let old_bits = self.sum.load(Ordering::Relaxed); let old = f64::from_bits(old_bits); let new = old + value; if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() { break; } } // Increment bucket counters for (i, &bound) in self.buckets.iter().enumerate() { if value <= bound { self.counts[i].fetch_add(1, Ordering::Relaxed); } } // +Inf bucket self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed); } } /// Labeled counter (key = label combination string) pub struct LabeledCounter { values: Mutex>, name: &'static str, help: &'static str, label_names: &'static [&'static str], } impl LabeledCounter { pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self { Self { values: Mutex::new(HashMap::new()), name, help, label_names } } pub fn inc(&self, labels: &[&str]) { let key = labels.join(","); let mut map = self.values.lock().unwrap(); *map.entry(key).or_insert(0) += 1; } } // ─── Timer helper ─────────────────────────────────────────── /// RAII timer: observes duration on drop pub struct Timer<'a> { histogram: &'a Histogram, start: Instant, } impl<'a> Timer<'a> { pub fn new(histogram: &'a Histogram) -> Self { Self { histogram, start: Instant::now() } } } impl<'a> Drop for Timer<'a> { fn drop(&mut self) { let elapsed = self.start.elapsed().as_secs_f64(); self.histogram.observe(elapsed); } } // ─── Default buckets ──────────────────────────────────────── /// Latency buckets for HTTP handlers (seconds) pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]; /// Latency buckets for LLM calls (seconds) pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]; /// Latency buckets for DB queries (seconds) pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]; // ═══════════════════════════════════════════════════════════ // O1: Ingest handler metrics (I1-I12) // ═══════════════════════════════════════════════════════════ pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new( "memory_ingest_requests_total", "Total ingest requests received"); pub static INGEST_ERRORS_TOTAL: Counter = Counter::new( "memory_ingest_errors_total", "Total ingest request errors"); pub static INGEST_RECORDS_TOTAL: Counter = Counter::new( "memory_ingest_records_total", "Total records ingested"); pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new( "memory_ingest_entities_extracted_total", "Total entities extracted during ingest"); pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new( "memory_ingest_edges_extracted_total", "Total edges extracted during ingest"); pub static INGEST_IN_FLIGHT: Gauge = Gauge::new( "memory_ingest_in_flight", "Currently processing ingest jobs"); pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new( "memory_ingest_queue_size", "Number of jobs waiting in ingest queue"); pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new( "memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)"); pub static INGEST_BYTES_TOTAL: Counter = Counter::new( "memory_ingest_bytes_total", "Total bytes ingested"); pub static INGEST_AUTH_FAILURES: Counter = Counter::new( "memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint"); pub static INGEST_RATE_LIMITED: Counter = Counter::new( "memory_ingest_rate_limited_total", "Total rate-limited ingest requests"); pub static INGEST_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS)); // ═══════════════════════════════════════════════════════════ // O2: Query handler metrics (Q1-Q12) // ═══════════════════════════════════════════════════════════ pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new( "memory_query_requests_total", "Total query requests received"); pub static QUERY_ERRORS_TOTAL: Counter = Counter::new( "memory_query_errors_total", "Total query request errors"); pub static QUERY_RESULTS_TOTAL: Counter = Counter::new( "memory_query_results_total", "Total results returned across all queries"); pub static QUERY_EMPTY_RESULTS: Counter = Counter::new( "memory_query_empty_results_total", "Queries returning zero results"); pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new( "memory_query_embedding_failures_total", "Total embedding failures during query"); pub static QUERY_IN_FLIGHT: Gauge = Gauge::new( "memory_query_in_flight", "Currently processing queries"); pub static QUERY_AUTH_FAILURES: Counter = Counter::new( "memory_query_auth_failures_total", "Total auth failures on query endpoint"); pub static QUERY_RATE_LIMITED: Counter = Counter::new( "memory_query_rate_limited_total", "Total rate-limited query requests"); pub static QUERY_CACHE_HITS: Counter = Counter::new( "memory_query_cache_hits_total", "Total query cache hits"); pub static QUERY_CACHE_MISSES: Counter = Counter::new( "memory_query_cache_misses_total", "Total query cache misses"); pub static QUERY_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS)); pub static QUERY_EMBEDDING_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS)); // ═══════════════════════════════════════════════════════════ // O3: Context endpoint metrics (C1-C8) // ═══════════════════════════════════════════════════════════ pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new( "memory_context_requests_total", "Total context retrieval requests"); pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new( "memory_context_errors_total", "Total context retrieval errors"); pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new( "memory_context_semantic_hits_total", "Results from semantic (cosine) tier"); pub static CONTEXT_BM25_HITS: Counter = Counter::new( "memory_context_bm25_hits_total", "Results from BM25 (lexical) tier"); pub static CONTEXT_GRAPH_HITS: Counter = Counter::new( "memory_context_graph_hits_total", "Results from graph traversal tier"); pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new( "memory_context_empty_results_total", "Context requests returning zero results"); pub static CONTEXT_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS)); pub static CONTEXT_TIER_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS)); // ═══════════════════════════════════════════════════════════ // O4: Relevance judge metrics (R1-R9) // ═══════════════════════════════════════════════════════════ pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new( "memory_relevance_evals_total", "Total relevance evaluations performed"); pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new( "memory_relevance_errors_total", "Total relevance evaluation errors"); pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new( "memory_relevance_relevant_total", "Results judged relevant"); pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new( "memory_relevance_irrelevant_total", "Results judged irrelevant"); pub static RELEVANCE_SCORE: Lazy = Lazy::new(|| Histogram::new("memory_relevance_score", "Distribution of relevance scores", &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])); pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new( "memory_relevance_precision", "Current precision (relevant/retrieved)"); pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new( "memory_relevance_recall", "Current recall (relevant/total_relevant)"); pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new( "memory_relevance_f1_score", "Current F1 score"); pub static RELEVANCE_EVAL_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS)); // ═══════════════════════════════════════════════════════════ // O5: Write volume and storage metrics (W1-W12) // ═══════════════════════════════════════════════════════════ pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new( "memory_write_entities_total", "Total entities written to DB"); pub static WRITE_EDGES_TOTAL: Counter = Counter::new( "memory_write_edges_total", "Total edges written to DB"); pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new( "memory_write_chunks_total", "Total chunks written to DB"); pub static WRITE_ERRORS_TOTAL: Counter = Counter::new( "memory_write_errors_total", "Total write errors"); pub static WRITE_BYTES_TOTAL: Counter = Counter::new( "memory_write_bytes_total", "Total bytes written to storage"); pub static DB_ENTITY_COUNT: Gauge = Gauge::new( "memory_db_entity_count", "Current entity count in memory_entity table"); pub static DB_EDGE_COUNT: Gauge = Gauge::new( "memory_db_edge_count", "Current edge count in memory_edge table"); pub static DB_CHUNK_COUNT: Gauge = Gauge::new( "memory_db_chunk_count", "Current chunk count in memory_chunks table"); pub static WRITE_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS)); pub static WRITE_BATCH_SIZE: Lazy = Lazy::new(|| Histogram::new("memory_write_batch_size", "Write batch sizes", &[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0])); // Storage gauges (updated periodically) pub static DB_SIZE_BYTES: Gauge = Gauge::new( "memory_db_size_bytes", "Total database size in bytes"); pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new( "memory_db_index_size_bytes", "Total index size in bytes"); // ═══════════════════════════════════════════════════════════ // O6: Pod resource observability (P1-P13) // (Most collected by node-exporter/cAdvisor, but we track app-level) // ═══════════════════════════════════════════════════════════ pub static APP_UPTIME_SECONDS: Gauge = Gauge::new( "memory_app_uptime_seconds", "Application uptime in seconds"); pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new( "memory_app_active_connections", "Active HTTP connections"); pub static APP_GOROUTINES: Gauge = Gauge::new( "memory_app_tokio_tasks", "Active tokio tasks (approximate)"); pub static APP_HEAP_BYTES: Gauge = Gauge::new( "memory_app_heap_bytes", "Approximate heap memory usage"); // ═══════════════════════════════════════════════════════════ // O7: Availability metrics and dependency health (A1-A10) // ═══════════════════════════════════════════════════════════ pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new( "memory_health_checks_total", "Total health check requests"); pub static HEALTH_CHECK_FAILURES: Counter = Counter::new( "memory_health_check_failures_total", "Total health check failures"); pub static DEP_DB_UP: Gauge = Gauge::new( "memory_dependency_db_up", "Database dependency health (1=up, 0=down)"); pub static DEP_EMBEDDING_UP: Gauge = Gauge::new( "memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)"); pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new( "memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)"); pub static DEP_LLM_UP: Gauge = Gauge::new( "memory_dependency_llm_up", "LLM service health (1=up, 0=down)"); pub static DEP_DB_LATENCY: Lazy = Lazy::new(|| Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS)); pub static DEP_EMBEDDING_LATENCY: Lazy = Lazy::new(|| Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS)); pub static REQUEST_ERRORS_BY_STATUS: Lazy = Lazy::new(|| LabeledCounter::new( "memory_request_errors_by_status", "Request errors by HTTP status code", &["status", "endpoint"])); // ═══════════════════════════════════════════════════════════ // Named error counters (per error type, per endpoint) // Format: memory_error_{ERROR_NAME}_{ENDPOINT}_total // ═══════════════════════════════════════════════════════════ // Ingest errors pub static ERROR_AUTH_FAILURE_INGEST: Counter = Counter::new( "memory_error_auth_failure_ingest_total", "Auth failures on ingest endpoint"); pub static ERROR_FORBIDDEN_INGEST: Counter = Counter::new( "memory_error_forbidden_ingest_total", "Forbidden (missing capability) on ingest"); pub static ERROR_RATE_LIMITED_INGEST: Counter = Counter::new( "memory_error_rate_limited_ingest_total", "Rate limited on ingest"); pub static ERROR_BAD_REQUEST_INGEST: Counter = Counter::new( "memory_error_bad_request_ingest_total", "Bad request on ingest"); pub static ERROR_DB_ERROR_INGEST: Counter = Counter::new( "memory_error_db_error_ingest_total", "Database error during ingest"); // Query errors pub static ERROR_AUTH_FAILURE_QUERY: Counter = Counter::new( "memory_error_auth_failure_query_total", "Auth failures on query endpoint"); pub static ERROR_FORBIDDEN_QUERY: Counter = Counter::new( "memory_error_forbidden_query_total", "Forbidden (missing capability) on query"); pub static ERROR_BAD_REQUEST_QUERY: Counter = Counter::new( "memory_error_bad_request_query_total", "Bad request on query"); pub static ERROR_EMBEDDING_FAILURE_QUERY: Counter = Counter::new( "memory_error_embedding_failure_query_total", "Embedding service failure during query"); pub static ERROR_SEARCH_FAILURE_QUERY: Counter = Counter::new( "memory_error_search_failure_query_total", "Search execution failure during query"); // Context errors pub static ERROR_AUTH_FAILURE_CONTEXT: Counter = Counter::new( "memory_error_auth_failure_context_total", "Auth failures on context endpoint"); pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new( "memory_error_forbidden_context_total", "Forbidden (missing capability) on context"); pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new( "memory_error_lookup_failure_context_total", "Context lookup failure"); // Unexpected errors (unhandled 500s, panics, unknown failures) pub static ERROR_UNEXPECTED_TOTAL: Counter = Counter::new( "memory_error_unexpected_total", "Total unexpected/unhandled errors (500s)"); pub static ERROR_UNEXPECTED_INGEST: Counter = Counter::new( "memory_error_unexpected_ingest_total", "Unexpected errors during ingest"); pub static ERROR_UNEXPECTED_QUERY: Counter = Counter::new( "memory_error_unexpected_query_total", "Unexpected errors during query"); pub static ERROR_UNEXPECTED_CONTEXT: Counter = Counter::new( "memory_error_unexpected_context_total", "Unexpected errors during context"); // Agent endpoint error counters pub static ERROR_AUTH_FAILURE_AGENT: Counter = Counter::new( "memory_error_auth_failure_agent_total", "Auth failures on agent endpoints"); pub static ERROR_BAD_REQUEST_AGENT: Counter = Counter::new( "memory_error_bad_request_agent_total", "Bad request errors on agent endpoints (expected)"); pub static ERROR_NOT_FOUND_AGENT: Counter = Counter::new( "memory_error_not_found_agent_total", "Not found errors on agent endpoints (expected)"); pub static ERROR_UNEXPECTED_AGENT: Counter = Counter::new( "memory_error_unexpected_agent_total", "Unexpected errors on agent endpoints (DB failures, 500s)"); // Last error info (most recent error for debugging) pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new( "memory_last_error_timestamp_seconds", "Unix timestamp of most recent error"); // ═══════════════════════════════════════════════════════════ // O8: Ingest rate pattern tracking (IR1-IR10) // ═══════════════════════════════════════════════════════════ pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new( "memory_ingest_rate_1m", "Ingest rate per second (1-minute window)"); pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new( "memory_ingest_rate_5m", "Ingest rate per second (5-minute window)"); pub static INGEST_LLM_EXTRACT_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS)); pub static INGEST_FACT_EXTRACT_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS)); pub static INGEST_DEDUP_TOTAL: Counter = Counter::new( "memory_ingest_dedup_total", "Total entities deduplicated"); pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new( "memory_ingest_contradiction_total", "Total contradictions detected"); pub static INGEST_PROJECTS: Gauge = Gauge::new( "memory_ingest_active_projects", "Number of active projects with ingested data"); // ═══════════════════════════════════════════════════════════ // O9: Postgres internal observability (PG1-PG33) // (Most collected by pg_exporter, we expose app-visible DB stats) // ═══════════════════════════════════════════════════════════ pub static DB_POOL_SIZE: Gauge = Gauge::new( "memory_db_pool_size", "Current connection pool size"); pub static DB_POOL_IDLE: Gauge = Gauge::new( "memory_db_pool_idle", "Idle connections in pool"); pub static DB_POOL_ACTIVE: Gauge = Gauge::new( "memory_db_pool_active", "Active connections in pool"); pub static DB_QUERY_TOTAL: Counter = Counter::new( "memory_db_queries_total", "Total DB queries executed"); pub static DB_QUERY_ERRORS: Counter = Counter::new( "memory_db_query_errors_total", "Total DB query errors"); pub static DB_QUERY_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS)); pub static DB_TRANSACTION_DURATION: Lazy = Lazy::new(|| Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS)); // Table-specific row counts (updated periodically) pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new( "memory_db_table_entity_rows", "Rows in memory_entity table"); pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new( "memory_db_table_edge_rows", "Rows in memory_edge table"); pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new( "memory_db_table_chunk_rows", "Rows in memory_chunks table"); // ═══════════════════════════════════════════════════════════ // Metrics export (Prometheus text format) // ═══════════════════════════════════════════════════════════ /// Render all metrics in Prometheus text exposition format pub fn render_metrics() -> String { let mut out = String::with_capacity(8192); // Helper macros macro_rules! counter { ($c:expr) => { out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n", $c.name, $c.help, $c.name, $c.name, $c.get())); }; } macro_rules! gauge { ($g:expr) => { out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n", $g.name, $g.help, $g.name, $g.name, $g.get())); }; } macro_rules! gauge_f64 { ($g:expr) => { out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n", $g.name, $g.help, $g.name, $g.name, $g.get())); }; } macro_rules! histogram { ($h:expr) => { out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name)); for (i, &bound) in $h.buckets.iter().enumerate() { out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n", $h.name, bound, $h.counts[i].load(Ordering::Relaxed))); } out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n", $h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed))); out.push_str(&format!("{}_sum {:.6}\n", $h.name, f64::from_bits($h.sum.load(Ordering::Relaxed)))); out.push_str(&format!("{}_count {}\n", $h.name, $h.count.load(Ordering::Relaxed))); }; } // O1: Ingest counter!(INGEST_REQUESTS_TOTAL); counter!(INGEST_ERRORS_TOTAL); counter!(INGEST_RECORDS_TOTAL); counter!(INGEST_ENTITIES_EXTRACTED); counter!(INGEST_EDGES_EXTRACTED); gauge!(INGEST_IN_FLIGHT); gauge!(INGEST_QUEUE_SIZE); counter!(INGEST_DUPLICATES_TOTAL); counter!(INGEST_BYTES_TOTAL); counter!(INGEST_AUTH_FAILURES); counter!(INGEST_RATE_LIMITED); histogram!(INGEST_DURATION); // O2: Query counter!(QUERY_REQUESTS_TOTAL); counter!(QUERY_ERRORS_TOTAL); counter!(QUERY_RESULTS_TOTAL); counter!(QUERY_EMPTY_RESULTS); counter!(QUERY_EMBEDDING_FAILURES); gauge!(QUERY_IN_FLIGHT); counter!(QUERY_AUTH_FAILURES); counter!(QUERY_RATE_LIMITED); counter!(QUERY_CACHE_HITS); counter!(QUERY_CACHE_MISSES); histogram!(QUERY_DURATION); histogram!(QUERY_EMBEDDING_DURATION); // O3: Context counter!(CONTEXT_REQUESTS_TOTAL); counter!(CONTEXT_ERRORS_TOTAL); counter!(CONTEXT_SEMANTIC_HITS); counter!(CONTEXT_BM25_HITS); counter!(CONTEXT_GRAPH_HITS); counter!(CONTEXT_EMPTY_RESULTS); histogram!(CONTEXT_DURATION); histogram!(CONTEXT_TIER_DURATION); // O4: Relevance counter!(RELEVANCE_EVALS_TOTAL); counter!(RELEVANCE_ERRORS_TOTAL); counter!(RELEVANCE_RELEVANT_TOTAL); counter!(RELEVANCE_IRRELEVANT_TOTAL); histogram!(RELEVANCE_SCORE); gauge_f64!(RELEVANCE_PRECISION); gauge_f64!(RELEVANCE_RECALL); gauge_f64!(RELEVANCE_F1); histogram!(RELEVANCE_EVAL_DURATION); // O5: Write volume counter!(WRITE_ENTITIES_TOTAL); counter!(WRITE_EDGES_TOTAL); counter!(WRITE_CHUNKS_TOTAL); counter!(WRITE_ERRORS_TOTAL); counter!(WRITE_BYTES_TOTAL); gauge!(DB_ENTITY_COUNT); gauge!(DB_EDGE_COUNT); gauge!(DB_CHUNK_COUNT); histogram!(WRITE_DURATION); histogram!(WRITE_BATCH_SIZE); gauge!(DB_SIZE_BYTES); gauge!(DB_INDEX_SIZE_BYTES); // O6: Pod resources gauge!(APP_UPTIME_SECONDS); gauge!(APP_ACTIVE_CONNECTIONS); gauge!(APP_GOROUTINES); gauge!(APP_HEAP_BYTES); // O7: Availability counter!(HEALTH_CHECKS_TOTAL); counter!(HEALTH_CHECK_FAILURES); gauge!(DEP_DB_UP); gauge!(DEP_EMBEDDING_UP); gauge!(DEP_OPENSEARCH_UP); gauge!(DEP_LLM_UP); histogram!(DEP_DB_LATENCY); histogram!(DEP_EMBEDDING_LATENCY); // O8: Ingest rate gauge_f64!(INGEST_RATE_1M); gauge_f64!(INGEST_RATE_5M); histogram!(INGEST_LLM_EXTRACT_DURATION); histogram!(INGEST_FACT_EXTRACT_DURATION); counter!(INGEST_DEDUP_TOTAL); counter!(INGEST_CONTRADICTION_TOTAL); gauge!(INGEST_PROJECTS); // O9: Postgres gauge!(DB_POOL_SIZE); gauge!(DB_POOL_IDLE); gauge!(DB_POOL_ACTIVE); counter!(DB_QUERY_TOTAL); counter!(DB_QUERY_ERRORS); histogram!(DB_QUERY_DURATION); histogram!(DB_TRANSACTION_DURATION); gauge!(DB_TABLE_ENTITY_ROWS); gauge!(DB_TABLE_EDGE_ROWS); gauge!(DB_TABLE_CHUNK_ROWS); // Named error counters counter!(ERROR_AUTH_FAILURE_INGEST); counter!(ERROR_FORBIDDEN_INGEST); counter!(ERROR_RATE_LIMITED_INGEST); counter!(ERROR_BAD_REQUEST_INGEST); counter!(ERROR_DB_ERROR_INGEST); counter!(ERROR_AUTH_FAILURE_QUERY); counter!(ERROR_FORBIDDEN_QUERY); counter!(ERROR_BAD_REQUEST_QUERY); counter!(ERROR_EMBEDDING_FAILURE_QUERY); counter!(ERROR_SEARCH_FAILURE_QUERY); counter!(ERROR_AUTH_FAILURE_CONTEXT); counter!(ERROR_FORBIDDEN_CONTEXT); counter!(ERROR_LOOKUP_FAILURE_CONTEXT); counter!(ERROR_UNEXPECTED_TOTAL); counter!(ERROR_UNEXPECTED_INGEST); counter!(ERROR_UNEXPECTED_QUERY); counter!(ERROR_UNEXPECTED_CONTEXT); counter!(ERROR_AUTH_FAILURE_AGENT); counter!(ERROR_BAD_REQUEST_AGENT); counter!(ERROR_NOT_FOUND_AGENT); counter!(ERROR_UNEXPECTED_AGENT); gauge!(LAST_ERROR_TIMESTAMP); out } /// Render a labeled counter in Prometheus format fn render_labeled_counter(out: &mut String, lc: &LabeledCounter) { let map = lc.values.lock().unwrap(); if map.is_empty() { return; } out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n", lc.name, lc.help, lc.name)); for (key, val) in map.iter() { let parts: Vec<&str> = key.split(',').collect(); let labels: Vec = lc.label_names.iter().zip(parts.iter()) .map(|(name, val)| format!("{}=\"{}\"", name, val)) .collect(); out.push_str(&format!("{}{{{}}} {}\n", lc.name, labels.join(","), val)); } } /// GET /metrics handler pub async fn metrics_handler() -> actix_web::HttpResponse { actix_web::HttpResponse::Ok() .content_type("text/plain; version=0.0.4; charset=utf-8") .body(render_metrics()) } #[cfg(test)] mod tests { use super::*; #[test] fn test_counter() { let c = Counter::new("test_counter", "test"); assert_eq!(c.get(), 0); c.inc(); assert_eq!(c.get(), 1); c.inc_by(5); assert_eq!(c.get(), 6); } #[test] fn test_gauge() { let g = Gauge::new("test_gauge", "test"); assert_eq!(g.get(), 0); g.set(42); assert_eq!(g.get(), 42); g.inc(); assert_eq!(g.get(), 43); g.dec(); assert_eq!(g.get(), 42); } #[test] fn test_gauge_f64() { let g = GaugeF64::new("test_gauge_f64", "test"); assert_eq!(g.get(), 0.0); g.set(3.14); assert!((g.get() - 3.14).abs() < 0.001); } #[test] fn test_histogram() { let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]); h.observe(0.05); h.observe(0.3); h.observe(0.8); h.observe(2.0); assert_eq!(h.count.load(Ordering::Relaxed), 4); } #[test] fn test_render_metrics_not_empty() { INGEST_REQUESTS_TOTAL.inc(); QUERY_REQUESTS_TOTAL.inc(); let output = render_metrics(); assert!(output.contains("memory_ingest_requests_total")); assert!(output.contains("memory_query_requests_total")); assert!(output.contains("# HELP")); assert!(output.contains("# TYPE")); } #[test] fn test_timer_observes_on_drop() { let h = Histogram::new("timer_test", "test", HTTP_BUCKETS); { let _t = Timer::new(&h); std::thread::sleep(std::time::Duration::from_millis(1)); } assert_eq!(h.count.load(Ordering::Relaxed), 1); } }