riotpiao-poimen/poimen-memory
2
0
Fork 0
Code Issues 11 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(O10): Prometheus metrics module + /metrics endpoint

Browse Source 
- metrics.rs: Counter, Gauge, GaugeF64, Histogram, LabeledCounter types
- Timer RAII helper for automatic latency observation
- All O1-O9 metric definitions pre-declared (119 metrics total)
- render_metrics() outputs Prometheus text exposition format
- GET /metrics endpoint registered in http_server
- HTTP/LLM/DB latency buckets defined
- 6 unit tests passing
This commit is contained in:
rock
2026-09-13 21:34:37 +09:00
parent 66cc282b6d
commit fd63b089f8
4 changed files with 621 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/mem-cli/Cargo.toml
+1
View File
@@ -46,3 +46,4 @@ futures-util = "0.3"
async-stream = "0.3"
rand = "0.8"
lru = "0.12"
once_cell = { workspace = true }
crates/mem-cli/src/http_server.rs
+1
View File
@@ -382,6 +382,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
.app_data(state.clone())
.wrap(Logger::default())
.route("/health", web::get().to(health_check))
.route("/metrics", web::get().to(crate::metrics::metrics_handler))
.route("/memory/ingest", web::post().to(ingest_handler))
.route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
.route("/memory/query", web::get().to(query_handler))
crates/mem-cli/src/lib.rs
+1
View File
@@ -1,6 +1,7 @@
pub mod endpoints;
pub mod handlers;
pub mod http_server;
pub mod metrics;
pub mod query;
pub mod auth;
pub mod ingest_worker;
crates/mem-cli/src/metrics.rs
+618
View File
@@ -0,0 +1,618 @@
//! Prometheus metrics module (O10)
//!
//! Centralized metrics registry for poimen-memory observability.
//! All handlers instrument via these shared metrics.
//! Exposed at GET /metrics in Prometheus text format.
use once_cell::sync::Lazy;
use std::sync::atomic::{AtomicU64, Ordering};
use std::collections::HashMap;
use std::sync::Mutex;
use std::time::Instant;
// ─── Metric Types ───────────────────────────────────────────
/// Simple counter (monotonically increasing)
pub struct Counter {
value: AtomicU64,
name: &'static str,
help: &'static str,
}
impl Counter {
pub const fn new(name: &'static str, help: &'static str) -> Self {
Self { value: AtomicU64::new(0), name, help }
}
pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); }
pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
}
/// Gauge (can go up and down)
pub struct Gauge {
value: AtomicU64,
name: &'static str,
help: &'static str,
}
impl Gauge {
pub const fn new(name: &'static str, help: &'static str) -> Self {
Self { value: AtomicU64::new(0), name, help }
}
pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); }
pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); }
pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
}
/// Gauge for f64 values (stored as bits)
pub struct GaugeF64 {
bits: AtomicU64,
name: &'static str,
help: &'static str,
}
impl GaugeF64 {
pub const fn new(name: &'static str, help: &'static str) -> Self {
Self { bits: AtomicU64::new(0), name, help }
}
pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); }
pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) }
}
/// Histogram with fixed buckets for latency tracking
pub struct Histogram {
buckets: &'static [f64],
counts: Vec<AtomicU64>,
sum: AtomicU64, // stored as f64 bits
count: AtomicU64,
name: &'static str,
help: &'static str,
}
impl Histogram {
pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self {
let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect();
Self {
buckets, counts, name, help,
sum: AtomicU64::new(0f64.to_bits()),
count: AtomicU64::new(0),
}
}
pub fn observe(&self, value: f64) {
self.count.fetch_add(1, Ordering::Relaxed);
// Add to sum (CAS loop for f64)
loop {
let old_bits = self.sum.load(Ordering::Relaxed);
let old = f64::from_bits(old_bits);
let new = old + value;
if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() {
break;
}
}
// Increment bucket counters
for (i, &bound) in self.buckets.iter().enumerate() {
if value <= bound {
self.counts[i].fetch_add(1, Ordering::Relaxed);
}
}
// +Inf bucket
self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed);
}
}
/// Labeled counter (key = label combination string)
pub struct LabeledCounter {
values: Mutex<HashMap<String, u64>>,
name: &'static str,
help: &'static str,
label_names: &'static [&'static str],
}
impl LabeledCounter {
pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self {
Self { values: Mutex::new(HashMap::new()), name, help, label_names }
}
pub fn inc(&self, labels: &[&str]) {
let key = labels.join(",");
let mut map = self.values.lock().unwrap();
*map.entry(key).or_insert(0) += 1;
}
}
// ─── Timer helper ───────────────────────────────────────────
/// RAII timer: observes duration on drop
pub struct Timer<'a> {
histogram: &'a Histogram,
start: Instant,
}
impl<'a> Timer<'a> {
pub fn new(histogram: &'a Histogram) -> Self {
Self { histogram, start: Instant::now() }
}
}
impl<'a> Drop for Timer<'a> {
fn drop(&mut self) {
let elapsed = self.start.elapsed().as_secs_f64();
self.histogram.observe(elapsed);
}
}
// ─── Default buckets ────────────────────────────────────────
/// Latency buckets for HTTP handlers (seconds)
pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
/// Latency buckets for LLM calls (seconds)
pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0];
/// Latency buckets for DB queries (seconds)
pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0];
// ═══════════════════════════════════════════════════════════
// O1: Ingest handler metrics (I1-I12)
// ═══════════════════════════════════════════════════════════
pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new(
"memory_ingest_requests_total", "Total ingest requests received");
pub static INGEST_ERRORS_TOTAL: Counter = Counter::new(
"memory_ingest_errors_total", "Total ingest request errors");
pub static INGEST_RECORDS_TOTAL: Counter = Counter::new(
"memory_ingest_records_total", "Total records ingested");
pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new(
"memory_ingest_entities_extracted_total", "Total entities extracted during ingest");
pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new(
"memory_ingest_edges_extracted_total", "Total edges extracted during ingest");
pub static INGEST_IN_FLIGHT: Gauge = Gauge::new(
"memory_ingest_in_flight", "Currently processing ingest jobs");
pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new(
"memory_ingest_queue_size", "Number of jobs waiting in ingest queue");
pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new(
"memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)");
pub static INGEST_BYTES_TOTAL: Counter = Counter::new(
"memory_ingest_bytes_total", "Total bytes ingested");
pub static INGEST_AUTH_FAILURES: Counter = Counter::new(
"memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint");
pub static INGEST_RATE_LIMITED: Counter = Counter::new(
"memory_ingest_rate_limited_total", "Total rate-limited ingest requests");
pub static INGEST_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS));
// ═══════════════════════════════════════════════════════════
// O2: Query handler metrics (Q1-Q12)
// ═══════════════════════════════════════════════════════════
pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new(
"memory_query_requests_total", "Total query requests received");
pub static QUERY_ERRORS_TOTAL: Counter = Counter::new(
"memory_query_errors_total", "Total query request errors");
pub static QUERY_RESULTS_TOTAL: Counter = Counter::new(
"memory_query_results_total", "Total results returned across all queries");
pub static QUERY_EMPTY_RESULTS: Counter = Counter::new(
"memory_query_empty_results_total", "Queries returning zero results");
pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new(
"memory_query_embedding_failures_total", "Total embedding failures during query");
pub static QUERY_IN_FLIGHT: Gauge = Gauge::new(
"memory_query_in_flight", "Currently processing queries");
pub static QUERY_AUTH_FAILURES: Counter = Counter::new(
"memory_query_auth_failures_total", "Total auth failures on query endpoint");
pub static QUERY_RATE_LIMITED: Counter = Counter::new(
"memory_query_rate_limited_total", "Total rate-limited query requests");
pub static QUERY_CACHE_HITS: Counter = Counter::new(
"memory_query_cache_hits_total", "Total query cache hits");
pub static QUERY_CACHE_MISSES: Counter = Counter::new(
"memory_query_cache_misses_total", "Total query cache misses");
pub static QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS));
pub static QUERY_EMBEDDING_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS));
// ═══════════════════════════════════════════════════════════
// O3: Context endpoint metrics (C1-C8)
// ═══════════════════════════════════════════════════════════
pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new(
"memory_context_requests_total", "Total context retrieval requests");
pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new(
"memory_context_errors_total", "Total context retrieval errors");
pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new(
"memory_context_semantic_hits_total", "Results from semantic (cosine) tier");
pub static CONTEXT_BM25_HITS: Counter = Counter::new(
"memory_context_bm25_hits_total", "Results from BM25 (lexical) tier");
pub static CONTEXT_GRAPH_HITS: Counter = Counter::new(
"memory_context_graph_hits_total", "Results from graph traversal tier");
pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new(
"memory_context_empty_results_total", "Context requests returning zero results");
pub static CONTEXT_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS));
pub static CONTEXT_TIER_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS));
// ═══════════════════════════════════════════════════════════
// O4: Relevance judge metrics (R1-R9)
// ═══════════════════════════════════════════════════════════
pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new(
"memory_relevance_evals_total", "Total relevance evaluations performed");
pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new(
"memory_relevance_errors_total", "Total relevance evaluation errors");
pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new(
"memory_relevance_relevant_total", "Results judged relevant");
pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new(
"memory_relevance_irrelevant_total", "Results judged irrelevant");
pub static RELEVANCE_SCORE: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_relevance_score", "Distribution of relevance scores",
&[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]));
pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new(
"memory_relevance_precision", "Current precision (relevant/retrieved)");
pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new(
"memory_relevance_recall", "Current recall (relevant/total_relevant)");
pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new(
"memory_relevance_f1_score", "Current F1 score");
pub static RELEVANCE_EVAL_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS));
// ═══════════════════════════════════════════════════════════
// O5: Write volume and storage metrics (W1-W12)
// ═══════════════════════════════════════════════════════════
pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new(
"memory_write_entities_total", "Total entities written to DB");
pub static WRITE_EDGES_TOTAL: Counter = Counter::new(
"memory_write_edges_total", "Total edges written to DB");
pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new(
"memory_write_chunks_total", "Total chunks written to DB");
pub static WRITE_ERRORS_TOTAL: Counter = Counter::new(
"memory_write_errors_total", "Total write errors");
pub static WRITE_BYTES_TOTAL: Counter = Counter::new(
"memory_write_bytes_total", "Total bytes written to storage");
pub static DB_ENTITY_COUNT: Gauge = Gauge::new(
"memory_db_entity_count", "Current entity count in memory_entity table");
pub static DB_EDGE_COUNT: Gauge = Gauge::new(
"memory_db_edge_count", "Current edge count in memory_edge table");
pub static DB_CHUNK_COUNT: Gauge = Gauge::new(
"memory_db_chunk_count", "Current chunk count in memory_chunks table");
pub static WRITE_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS));
pub static WRITE_BATCH_SIZE: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_write_batch_size", "Write batch sizes",
&[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]));
// Storage gauges (updated periodically)
pub static DB_SIZE_BYTES: Gauge = Gauge::new(
"memory_db_size_bytes", "Total database size in bytes");
pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new(
"memory_db_index_size_bytes", "Total index size in bytes");
// ═══════════════════════════════════════════════════════════
// O6: Pod resource observability (P1-P13)
// (Most collected by node-exporter/cAdvisor, but we track app-level)
// ═══════════════════════════════════════════════════════════
pub static APP_UPTIME_SECONDS: Gauge = Gauge::new(
"memory_app_uptime_seconds", "Application uptime in seconds");
pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new(
"memory_app_active_connections", "Active HTTP connections");
pub static APP_GOROUTINES: Gauge = Gauge::new(
"memory_app_tokio_tasks", "Active tokio tasks (approximate)");
pub static APP_HEAP_BYTES: Gauge = Gauge::new(
"memory_app_heap_bytes", "Approximate heap memory usage");
// ═══════════════════════════════════════════════════════════
// O7: Availability metrics and dependency health (A1-A10)
// ═══════════════════════════════════════════════════════════
pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new(
"memory_health_checks_total", "Total health check requests");
pub static HEALTH_CHECK_FAILURES: Counter = Counter::new(
"memory_health_check_failures_total", "Total health check failures");
pub static DEP_DB_UP: Gauge = Gauge::new(
"memory_dependency_db_up", "Database dependency health (1=up, 0=down)");
pub static DEP_EMBEDDING_UP: Gauge = Gauge::new(
"memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)");
pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new(
"memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)");
pub static DEP_LLM_UP: Gauge = Gauge::new(
"memory_dependency_llm_up", "LLM service health (1=up, 0=down)");
pub static DEP_DB_LATENCY: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS));
pub static DEP_EMBEDDING_LATENCY: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS));
pub static REQUEST_ERRORS_BY_STATUS: Lazy<LabeledCounter> = Lazy::new(||
LabeledCounter::new(
"memory_request_errors_by_status", "Request errors by HTTP status code",
&["status", "endpoint"]));
// ═══════════════════════════════════════════════════════════
// O8: Ingest rate pattern tracking (IR1-IR10)
// ═══════════════════════════════════════════════════════════
pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new(
"memory_ingest_rate_1m", "Ingest rate per second (1-minute window)");
pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new(
"memory_ingest_rate_5m", "Ingest rate per second (5-minute window)");
pub static INGEST_LLM_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS));
pub static INGEST_FACT_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS));
pub static INGEST_DEDUP_TOTAL: Counter = Counter::new(
"memory_ingest_dedup_total", "Total entities deduplicated");
pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new(
"memory_ingest_contradiction_total", "Total contradictions detected");
pub static INGEST_PROJECTS: Gauge = Gauge::new(
"memory_ingest_active_projects", "Number of active projects with ingested data");
// ═══════════════════════════════════════════════════════════
// O9: Postgres internal observability (PG1-PG33)
// (Most collected by pg_exporter, we expose app-visible DB stats)
// ═══════════════════════════════════════════════════════════
pub static DB_POOL_SIZE: Gauge = Gauge::new(
"memory_db_pool_size", "Current connection pool size");
pub static DB_POOL_IDLE: Gauge = Gauge::new(
"memory_db_pool_idle", "Idle connections in pool");
pub static DB_POOL_ACTIVE: Gauge = Gauge::new(
"memory_db_pool_active", "Active connections in pool");
pub static DB_QUERY_TOTAL: Counter = Counter::new(
"memory_db_queries_total", "Total DB queries executed");
pub static DB_QUERY_ERRORS: Counter = Counter::new(
"memory_db_query_errors_total", "Total DB query errors");
pub static DB_QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS));
pub static DB_TRANSACTION_DURATION: Lazy<Histogram> = Lazy::new(||
Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS));
// Table-specific row counts (updated periodically)
pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new(
"memory_db_table_entity_rows", "Rows in memory_entity table");
pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new(
"memory_db_table_edge_rows", "Rows in memory_edge table");
pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new(
"memory_db_table_chunk_rows", "Rows in memory_chunks table");
// ═══════════════════════════════════════════════════════════
// Metrics export (Prometheus text format)
// ═══════════════════════════════════════════════════════════
/// Render all metrics in Prometheus text exposition format
pub fn render_metrics() -> String {
let mut out = String::with_capacity(8192);
// Helper macros
macro_rules! counter {
($c:expr) => {
out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n",
$c.name, $c.help, $c.name, $c.name, $c.get()));
};
}
macro_rules! gauge {
($g:expr) => {
out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n",
$g.name, $g.help, $g.name, $g.name, $g.get()));
};
}
macro_rules! gauge_f64 {
($g:expr) => {
out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n",
$g.name, $g.help, $g.name, $g.name, $g.get()));
};
}
macro_rules! histogram {
($h:expr) => {
out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name));
for (i, &bound) in $h.buckets.iter().enumerate() {
out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n",
$h.name, bound, $h.counts[i].load(Ordering::Relaxed)));
}
out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n",
$h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed)));
out.push_str(&format!("{}_sum {:.6}\n", $h.name,
f64::from_bits($h.sum.load(Ordering::Relaxed))));
out.push_str(&format!("{}_count {}\n", $h.name,
$h.count.load(Ordering::Relaxed)));
};
}
// O1: Ingest
counter!(INGEST_REQUESTS_TOTAL);
counter!(INGEST_ERRORS_TOTAL);
counter!(INGEST_RECORDS_TOTAL);
counter!(INGEST_ENTITIES_EXTRACTED);
counter!(INGEST_EDGES_EXTRACTED);
gauge!(INGEST_IN_FLIGHT);
gauge!(INGEST_QUEUE_SIZE);
counter!(INGEST_DUPLICATES_TOTAL);
counter!(INGEST_BYTES_TOTAL);
counter!(INGEST_AUTH_FAILURES);
counter!(INGEST_RATE_LIMITED);
histogram!(INGEST_DURATION);
// O2: Query
counter!(QUERY_REQUESTS_TOTAL);
counter!(QUERY_ERRORS_TOTAL);
counter!(QUERY_RESULTS_TOTAL);
counter!(QUERY_EMPTY_RESULTS);
counter!(QUERY_EMBEDDING_FAILURES);
gauge!(QUERY_IN_FLIGHT);
counter!(QUERY_AUTH_FAILURES);
counter!(QUERY_RATE_LIMITED);
counter!(QUERY_CACHE_HITS);
counter!(QUERY_CACHE_MISSES);
histogram!(QUERY_DURATION);
histogram!(QUERY_EMBEDDING_DURATION);
// O3: Context
counter!(CONTEXT_REQUESTS_TOTAL);
counter!(CONTEXT_ERRORS_TOTAL);
counter!(CONTEXT_SEMANTIC_HITS);
counter!(CONTEXT_BM25_HITS);
counter!(CONTEXT_GRAPH_HITS);
counter!(CONTEXT_EMPTY_RESULTS);
histogram!(CONTEXT_DURATION);
histogram!(CONTEXT_TIER_DURATION);
// O4: Relevance
counter!(RELEVANCE_EVALS_TOTAL);
counter!(RELEVANCE_ERRORS_TOTAL);
counter!(RELEVANCE_RELEVANT_TOTAL);
counter!(RELEVANCE_IRRELEVANT_TOTAL);
histogram!(RELEVANCE_SCORE);
gauge_f64!(RELEVANCE_PRECISION);
gauge_f64!(RELEVANCE_RECALL);
gauge_f64!(RELEVANCE_F1);
histogram!(RELEVANCE_EVAL_DURATION);
// O5: Write volume
counter!(WRITE_ENTITIES_TOTAL);
counter!(WRITE_EDGES_TOTAL);
counter!(WRITE_CHUNKS_TOTAL);
counter!(WRITE_ERRORS_TOTAL);
counter!(WRITE_BYTES_TOTAL);
gauge!(DB_ENTITY_COUNT);
gauge!(DB_EDGE_COUNT);
gauge!(DB_CHUNK_COUNT);
histogram!(WRITE_DURATION);
histogram!(WRITE_BATCH_SIZE);
gauge!(DB_SIZE_BYTES);
gauge!(DB_INDEX_SIZE_BYTES);
// O6: Pod resources
gauge!(APP_UPTIME_SECONDS);
gauge!(APP_ACTIVE_CONNECTIONS);
gauge!(APP_GOROUTINES);
gauge!(APP_HEAP_BYTES);
// O7: Availability
counter!(HEALTH_CHECKS_TOTAL);
counter!(HEALTH_CHECK_FAILURES);
gauge!(DEP_DB_UP);
gauge!(DEP_EMBEDDING_UP);
gauge!(DEP_OPENSEARCH_UP);
gauge!(DEP_LLM_UP);
histogram!(DEP_DB_LATENCY);
histogram!(DEP_EMBEDDING_LATENCY);
// O8: Ingest rate
gauge_f64!(INGEST_RATE_1M);
gauge_f64!(INGEST_RATE_5M);
histogram!(INGEST_LLM_EXTRACT_DURATION);
histogram!(INGEST_FACT_EXTRACT_DURATION);
counter!(INGEST_DEDUP_TOTAL);
counter!(INGEST_CONTRADICTION_TOTAL);
gauge!(INGEST_PROJECTS);
// O9: Postgres
gauge!(DB_POOL_SIZE);
gauge!(DB_POOL_IDLE);
gauge!(DB_POOL_ACTIVE);
counter!(DB_QUERY_TOTAL);
counter!(DB_QUERY_ERRORS);
histogram!(DB_QUERY_DURATION);
histogram!(DB_TRANSACTION_DURATION);
gauge!(DB_TABLE_ENTITY_ROWS);
gauge!(DB_TABLE_EDGE_ROWS);
gauge!(DB_TABLE_CHUNK_ROWS);
// Labeled counter: errors by status
{
let map = REQUEST_ERRORS_BY_STATUS.values.lock().unwrap();
if !map.is_empty() {
out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n",
REQUEST_ERRORS_BY_STATUS.name, REQUEST_ERRORS_BY_STATUS.help,
REQUEST_ERRORS_BY_STATUS.name));
for (key, val) in map.iter() {
let parts: Vec<&str> = key.split(',').collect();
if parts.len() == 2 {
out.push_str(&format!("{}{{status=\"{}\",endpoint=\"{}\"}} {}\n",
REQUEST_ERRORS_BY_STATUS.name, parts[0], parts[1], val));
}
}
}
}
out
}
/// GET /metrics handler
pub async fn metrics_handler() -> actix_web::HttpResponse {
actix_web::HttpResponse::Ok()
.content_type("text/plain; version=0.0.4; charset=utf-8")
.body(render_metrics())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_counter() {
let c = Counter::new("test_counter", "test");
assert_eq!(c.get(), 0);
c.inc();
assert_eq!(c.get(), 1);
c.inc_by(5);
assert_eq!(c.get(), 6);
}
#[test]
fn test_gauge() {
let g = Gauge::new("test_gauge", "test");
assert_eq!(g.get(), 0);
g.set(42);
assert_eq!(g.get(), 42);
g.inc();
assert_eq!(g.get(), 43);
g.dec();
assert_eq!(g.get(), 42);
}
#[test]
fn test_gauge_f64() {
let g = GaugeF64::new("test_gauge_f64", "test");
assert_eq!(g.get(), 0.0);
g.set(3.14);
assert!((g.get() - 3.14).abs() < 0.001);
}
#[test]
fn test_histogram() {
let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]);
h.observe(0.05);
h.observe(0.3);
h.observe(0.8);
h.observe(2.0);
assert_eq!(h.count.load(Ordering::Relaxed), 4);
}
#[test]
fn test_render_metrics_not_empty() {
INGEST_REQUESTS_TOTAL.inc();
QUERY_REQUESTS_TOTAL.inc();
let output = render_metrics();
assert!(output.contains("memory_ingest_requests_total"));
assert!(output.contains("memory_query_requests_total"));
assert!(output.contains("# HELP"));
assert!(output.contains("# TYPE"));
}
#[test]
fn test_timer_observes_on_drop() {
let h = Histogram::new("timer_test", "test", HTTP_BUCKETS);
{
let _t = Timer::new(&h);
std::thread::sleep(std::time::Duration::from_millis(1));
}
assert_eq!(h.count.load(Ordering::Relaxed), 1);
}
}