diff --git a/crates/mem-cli/src/handlers/unified_query.rs b/crates/mem-cli/src/handlers/unified_query.rs index b42e9f0..0a7ed00 100644 --- a/crates/mem-cli/src/handlers/unified_query.rs +++ b/crates/mem-cli/src/handlers/unified_query.rs @@ -130,19 +130,15 @@ pub async fn unified_query_handler( ) { QUERY_AUTH_FAILURES.inc(); QUERY_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&["unknown", "/memory/query", "auth_failure"]); + ERROR_AUTH_FAILURE_QUERY.inc(); QUERY_IN_FLIGHT.dec(); return response; } - // Extract user_id from JWT sub claim via state (set by validate_and_rate_limit) - let user_id = crate::handlers::middleware::extract_user_id(&req, &state); - REQUESTS_BY_USER.inc(&[&user_id, "/memory/query"]); - // 2. Validate input if let Err(response) = validate_unified_request(&body) { QUERY_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&[&user_id, "/memory/query", "bad_request"]); + ERROR_BAD_REQUEST_QUERY.inc(); QUERY_IN_FLIGHT.dec(); return response; } @@ -160,9 +156,9 @@ pub async fn unified_query_handler( Err(e) => { QUERY_EMBEDDING_FAILURES.inc(); QUERY_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&[&user_id, "/memory/query", "embedding_failure"]); + ERROR_EMBEDDING_FAILURE_QUERY.inc(); QUERY_IN_FLIGHT.dec(); - error!("Embedding failed for user={}: {}", user_id, e); + error!("Embedding failed: {}", e); return crate::handlers::response_builder::internal_error( "Failed to embed query" ); diff --git a/crates/mem-cli/src/http_server.rs b/crates/mem-cli/src/http_server.rs index 173abbc..03b9b7e 100644 --- a/crates/mem-cli/src/http_server.rs +++ b/crates/mem-cli/src/http_server.rs @@ -491,19 +491,17 @@ pub async fn ingest_handler( Err(e) => { INGEST_AUTH_FAILURES.inc(); INGEST_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&["unknown", "/memory/ingest", "auth_failure"]); + ERROR_AUTH_FAILURE_INGEST.inc(); INGEST_IN_FLIGHT.dec(); return e; } }; let user_id = &claims.sub; - REQUESTS_BY_USER.inc(&[user_id, "/memory/ingest"]); - if !has_capability(&claims, "memory:write") { INGEST_AUTH_FAILURES.inc(); INGEST_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&[user_id, "/memory/ingest", "forbidden"]); + ERROR_FORBIDDEN_INGEST.inc(); INGEST_IN_FLIGHT.dec(); return HttpResponse::Forbidden().json(json!({ "error": "forbidden", @@ -512,7 +510,7 @@ pub async fn ingest_handler( } if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") { INGEST_RATE_LIMITED.inc(); - ERRORS_BY_USER.inc(&[user_id, "/memory/ingest", "rate_limited"]); + ERROR_RATE_LIMITED_INGEST.inc(); INGEST_IN_FLIGHT.dec(); return e; } @@ -1051,17 +1049,15 @@ pub async fn context_handler( Ok(c) => c, Err(e) => { CONTEXT_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&["unknown", "/memory/context", "auth_failure"]); + ERROR_AUTH_FAILURE_CONTEXT.inc(); return e; } }; let user_id = &claims.sub; - REQUESTS_BY_USER.inc(&[user_id, "/memory/context"]); - if !has_capability(&claims, "memory:read") { CONTEXT_ERRORS_TOTAL.inc(); - ERRORS_BY_USER.inc(&[user_id, "/memory/context", "forbidden"]); + ERROR_FORBIDDEN_CONTEXT.inc(); return HttpResponse::Forbidden().json(json!({ "error": "forbidden", "reason": "missing capability: memory:read" @@ -1093,6 +1089,7 @@ pub async fn context_handler( } Err(e) => { CONTEXT_ERRORS_TOTAL.inc(); + ERROR_LOOKUP_FAILURE_CONTEXT.inc(); tracing::error!("context lookup error: {}", e); HttpResponse::BadRequest().json(json!({ "error": "lookup_failed", diff --git a/crates/mem-cli/src/metrics.rs b/crates/mem-cli/src/metrics.rs index bf60ff0..84937b2 100644 --- a/crates/mem-cli/src/metrics.rs +++ b/crates/mem-cli/src/metrics.rs @@ -334,19 +334,46 @@ pub static REQUEST_ERRORS_BY_STATUS: Lazy = Lazy::new(|| "memory_request_errors_by_status", "Request errors by HTTP status code", &["status", "endpoint"])); -/// Errors with user identity and error name -/// Labels: [user_id, endpoint, error_name] -pub static ERRORS_BY_USER: Lazy = Lazy::new(|| - LabeledCounter::new( - "memory_errors_by_user", "Errors by user identity and error type", - &["user_id", "endpoint", "error_name"])); +// ═══════════════════════════════════════════════════════════ +// Named error counters (per error type, per endpoint) +// Format: memory_error_{ERROR_NAME}_{ENDPOINT}_total +// ═══════════════════════════════════════════════════════════ -/// Requests by user identity -/// Labels: [user_id, endpoint] -pub static REQUESTS_BY_USER: Lazy = Lazy::new(|| - LabeledCounter::new( - "memory_requests_by_user", "Requests by user identity", - &["user_id", "endpoint"])); +// Ingest errors +pub static ERROR_AUTH_FAILURE_INGEST: Counter = Counter::new( + "memory_error_auth_failure_ingest_total", "Auth failures on ingest endpoint"); +pub static ERROR_FORBIDDEN_INGEST: Counter = Counter::new( + "memory_error_forbidden_ingest_total", "Forbidden (missing capability) on ingest"); +pub static ERROR_RATE_LIMITED_INGEST: Counter = Counter::new( + "memory_error_rate_limited_ingest_total", "Rate limited on ingest"); +pub static ERROR_BAD_REQUEST_INGEST: Counter = Counter::new( + "memory_error_bad_request_ingest_total", "Bad request on ingest"); +pub static ERROR_DB_ERROR_INGEST: Counter = Counter::new( + "memory_error_db_error_ingest_total", "Database error during ingest"); + +// Query errors +pub static ERROR_AUTH_FAILURE_QUERY: Counter = Counter::new( + "memory_error_auth_failure_query_total", "Auth failures on query endpoint"); +pub static ERROR_FORBIDDEN_QUERY: Counter = Counter::new( + "memory_error_forbidden_query_total", "Forbidden (missing capability) on query"); +pub static ERROR_BAD_REQUEST_QUERY: Counter = Counter::new( + "memory_error_bad_request_query_total", "Bad request on query"); +pub static ERROR_EMBEDDING_FAILURE_QUERY: Counter = Counter::new( + "memory_error_embedding_failure_query_total", "Embedding service failure during query"); +pub static ERROR_SEARCH_FAILURE_QUERY: Counter = Counter::new( + "memory_error_search_failure_query_total", "Search execution failure during query"); + +// Context errors +pub static ERROR_AUTH_FAILURE_CONTEXT: Counter = Counter::new( + "memory_error_auth_failure_context_total", "Auth failures on context endpoint"); +pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new( + "memory_error_forbidden_context_total", "Forbidden (missing capability) on context"); +pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new( + "memory_error_lookup_failure_context_total", "Context lookup failure"); + +// Last error info (most recent error for debugging) +pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new( + "memory_last_error_timestamp_seconds", "Unix timestamp of most recent error"); // ═══════════════════════════════════════════════════════════ // O8: Ingest rate pattern tracking (IR1-IR10) @@ -538,12 +565,21 @@ pub fn render_metrics() -> String { gauge!(DB_TABLE_EDGE_ROWS); gauge!(DB_TABLE_CHUNK_ROWS); - // Labeled counter: errors by status - render_labeled_counter(&mut out, &REQUEST_ERRORS_BY_STATUS); - // Labeled counter: errors by user - render_labeled_counter(&mut out, &ERRORS_BY_USER); - // Labeled counter: requests by user - render_labeled_counter(&mut out, &REQUESTS_BY_USER); + // Named error counters + counter!(ERROR_AUTH_FAILURE_INGEST); + counter!(ERROR_FORBIDDEN_INGEST); + counter!(ERROR_RATE_LIMITED_INGEST); + counter!(ERROR_BAD_REQUEST_INGEST); + counter!(ERROR_DB_ERROR_INGEST); + counter!(ERROR_AUTH_FAILURE_QUERY); + counter!(ERROR_FORBIDDEN_QUERY); + counter!(ERROR_BAD_REQUEST_QUERY); + counter!(ERROR_EMBEDDING_FAILURE_QUERY); + counter!(ERROR_SEARCH_FAILURE_QUERY); + counter!(ERROR_AUTH_FAILURE_CONTEXT); + counter!(ERROR_FORBIDDEN_CONTEXT); + counter!(ERROR_LOOKUP_FAILURE_CONTEXT); + gauge!(LAST_ERROR_TIMESTAMP); out }