feat: complete observability stack (O1-O13) #52
@@ -130,19 +130,15 @@ pub async fn unified_query_handler(
|
|||||||
) {
|
) {
|
||||||
QUERY_AUTH_FAILURES.inc();
|
QUERY_AUTH_FAILURES.inc();
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
QUERY_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&["unknown", "/memory/query", "auth_failure"]);
|
ERROR_AUTH_FAILURE_QUERY.inc();
|
||||||
QUERY_IN_FLIGHT.dec();
|
QUERY_IN_FLIGHT.dec();
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract user_id from JWT sub claim via state (set by validate_and_rate_limit)
|
|
||||||
let user_id = crate::handlers::middleware::extract_user_id(&req, &state);
|
|
||||||
REQUESTS_BY_USER.inc(&[&user_id, "/memory/query"]);
|
|
||||||
|
|
||||||
// 2. Validate input
|
// 2. Validate input
|
||||||
if let Err(response) = validate_unified_request(&body) {
|
if let Err(response) = validate_unified_request(&body) {
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
QUERY_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&[&user_id, "/memory/query", "bad_request"]);
|
ERROR_BAD_REQUEST_QUERY.inc();
|
||||||
QUERY_IN_FLIGHT.dec();
|
QUERY_IN_FLIGHT.dec();
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
@@ -160,9 +156,9 @@ pub async fn unified_query_handler(
|
|||||||
Err(e) => {
|
Err(e) => {
|
||||||
QUERY_EMBEDDING_FAILURES.inc();
|
QUERY_EMBEDDING_FAILURES.inc();
|
||||||
QUERY_ERRORS_TOTAL.inc();
|
QUERY_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&[&user_id, "/memory/query", "embedding_failure"]);
|
ERROR_EMBEDDING_FAILURE_QUERY.inc();
|
||||||
QUERY_IN_FLIGHT.dec();
|
QUERY_IN_FLIGHT.dec();
|
||||||
error!("Embedding failed for user={}: {}", user_id, e);
|
error!("Embedding failed: {}", e);
|
||||||
return crate::handlers::response_builder::internal_error(
|
return crate::handlers::response_builder::internal_error(
|
||||||
"Failed to embed query"
|
"Failed to embed query"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -491,19 +491,17 @@ pub async fn ingest_handler(
|
|||||||
Err(e) => {
|
Err(e) => {
|
||||||
INGEST_AUTH_FAILURES.inc();
|
INGEST_AUTH_FAILURES.inc();
|
||||||
INGEST_ERRORS_TOTAL.inc();
|
INGEST_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&["unknown", "/memory/ingest", "auth_failure"]);
|
ERROR_AUTH_FAILURE_INGEST.inc();
|
||||||
INGEST_IN_FLIGHT.dec();
|
INGEST_IN_FLIGHT.dec();
|
||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let user_id = &claims.sub;
|
let user_id = &claims.sub;
|
||||||
REQUESTS_BY_USER.inc(&[user_id, "/memory/ingest"]);
|
|
||||||
|
|
||||||
if !has_capability(&claims, "memory:write") {
|
if !has_capability(&claims, "memory:write") {
|
||||||
INGEST_AUTH_FAILURES.inc();
|
INGEST_AUTH_FAILURES.inc();
|
||||||
INGEST_ERRORS_TOTAL.inc();
|
INGEST_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&[user_id, "/memory/ingest", "forbidden"]);
|
ERROR_FORBIDDEN_INGEST.inc();
|
||||||
INGEST_IN_FLIGHT.dec();
|
INGEST_IN_FLIGHT.dec();
|
||||||
return HttpResponse::Forbidden().json(json!({
|
return HttpResponse::Forbidden().json(json!({
|
||||||
"error": "forbidden",
|
"error": "forbidden",
|
||||||
@@ -512,7 +510,7 @@ pub async fn ingest_handler(
|
|||||||
}
|
}
|
||||||
if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
|
if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
|
||||||
INGEST_RATE_LIMITED.inc();
|
INGEST_RATE_LIMITED.inc();
|
||||||
ERRORS_BY_USER.inc(&[user_id, "/memory/ingest", "rate_limited"]);
|
ERROR_RATE_LIMITED_INGEST.inc();
|
||||||
INGEST_IN_FLIGHT.dec();
|
INGEST_IN_FLIGHT.dec();
|
||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
@@ -1051,17 +1049,15 @@ pub async fn context_handler(
|
|||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
CONTEXT_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&["unknown", "/memory/context", "auth_failure"]);
|
ERROR_AUTH_FAILURE_CONTEXT.inc();
|
||||||
return e;
|
return e;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let user_id = &claims.sub;
|
let user_id = &claims.sub;
|
||||||
REQUESTS_BY_USER.inc(&[user_id, "/memory/context"]);
|
|
||||||
|
|
||||||
if !has_capability(&claims, "memory:read") {
|
if !has_capability(&claims, "memory:read") {
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
CONTEXT_ERRORS_TOTAL.inc();
|
||||||
ERRORS_BY_USER.inc(&[user_id, "/memory/context", "forbidden"]);
|
ERROR_FORBIDDEN_CONTEXT.inc();
|
||||||
return HttpResponse::Forbidden().json(json!({
|
return HttpResponse::Forbidden().json(json!({
|
||||||
"error": "forbidden",
|
"error": "forbidden",
|
||||||
"reason": "missing capability: memory:read"
|
"reason": "missing capability: memory:read"
|
||||||
@@ -1093,6 +1089,7 @@ pub async fn context_handler(
|
|||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
CONTEXT_ERRORS_TOTAL.inc();
|
CONTEXT_ERRORS_TOTAL.inc();
|
||||||
|
ERROR_LOOKUP_FAILURE_CONTEXT.inc();
|
||||||
tracing::error!("context lookup error: {}", e);
|
tracing::error!("context lookup error: {}", e);
|
||||||
HttpResponse::BadRequest().json(json!({
|
HttpResponse::BadRequest().json(json!({
|
||||||
"error": "lookup_failed",
|
"error": "lookup_failed",
|
||||||
|
|||||||
@@ -334,19 +334,46 @@ pub static REQUEST_ERRORS_BY_STATUS: Lazy<LabeledCounter> = Lazy::new(||
|
|||||||
"memory_request_errors_by_status", "Request errors by HTTP status code",
|
"memory_request_errors_by_status", "Request errors by HTTP status code",
|
||||||
&["status", "endpoint"]));
|
&["status", "endpoint"]));
|
||||||
|
|
||||||
/// Errors with user identity and error name
|
// ═══════════════════════════════════════════════════════════
|
||||||
/// Labels: [user_id, endpoint, error_name]
|
// Named error counters (per error type, per endpoint)
|
||||||
pub static ERRORS_BY_USER: Lazy<LabeledCounter> = Lazy::new(||
|
// Format: memory_error_{ERROR_NAME}_{ENDPOINT}_total
|
||||||
LabeledCounter::new(
|
// ═══════════════════════════════════════════════════════════
|
||||||
"memory_errors_by_user", "Errors by user identity and error type",
|
|
||||||
&["user_id", "endpoint", "error_name"]));
|
|
||||||
|
|
||||||
/// Requests by user identity
|
// Ingest errors
|
||||||
/// Labels: [user_id, endpoint]
|
pub static ERROR_AUTH_FAILURE_INGEST: Counter = Counter::new(
|
||||||
pub static REQUESTS_BY_USER: Lazy<LabeledCounter> = Lazy::new(||
|
"memory_error_auth_failure_ingest_total", "Auth failures on ingest endpoint");
|
||||||
LabeledCounter::new(
|
pub static ERROR_FORBIDDEN_INGEST: Counter = Counter::new(
|
||||||
"memory_requests_by_user", "Requests by user identity",
|
"memory_error_forbidden_ingest_total", "Forbidden (missing capability) on ingest");
|
||||||
&["user_id", "endpoint"]));
|
pub static ERROR_RATE_LIMITED_INGEST: Counter = Counter::new(
|
||||||
|
"memory_error_rate_limited_ingest_total", "Rate limited on ingest");
|
||||||
|
pub static ERROR_BAD_REQUEST_INGEST: Counter = Counter::new(
|
||||||
|
"memory_error_bad_request_ingest_total", "Bad request on ingest");
|
||||||
|
pub static ERROR_DB_ERROR_INGEST: Counter = Counter::new(
|
||||||
|
"memory_error_db_error_ingest_total", "Database error during ingest");
|
||||||
|
|
||||||
|
// Query errors
|
||||||
|
pub static ERROR_AUTH_FAILURE_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_auth_failure_query_total", "Auth failures on query endpoint");
|
||||||
|
pub static ERROR_FORBIDDEN_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_forbidden_query_total", "Forbidden (missing capability) on query");
|
||||||
|
pub static ERROR_BAD_REQUEST_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_bad_request_query_total", "Bad request on query");
|
||||||
|
pub static ERROR_EMBEDDING_FAILURE_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_embedding_failure_query_total", "Embedding service failure during query");
|
||||||
|
pub static ERROR_SEARCH_FAILURE_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_search_failure_query_total", "Search execution failure during query");
|
||||||
|
|
||||||
|
// Context errors
|
||||||
|
pub static ERROR_AUTH_FAILURE_CONTEXT: Counter = Counter::new(
|
||||||
|
"memory_error_auth_failure_context_total", "Auth failures on context endpoint");
|
||||||
|
pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new(
|
||||||
|
"memory_error_forbidden_context_total", "Forbidden (missing capability) on context");
|
||||||
|
pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new(
|
||||||
|
"memory_error_lookup_failure_context_total", "Context lookup failure");
|
||||||
|
|
||||||
|
// Last error info (most recent error for debugging)
|
||||||
|
pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new(
|
||||||
|
"memory_last_error_timestamp_seconds", "Unix timestamp of most recent error");
|
||||||
|
|
||||||
// ═══════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════
|
||||||
// O8: Ingest rate pattern tracking (IR1-IR10)
|
// O8: Ingest rate pattern tracking (IR1-IR10)
|
||||||
@@ -538,12 +565,21 @@ pub fn render_metrics() -> String {
|
|||||||
gauge!(DB_TABLE_EDGE_ROWS);
|
gauge!(DB_TABLE_EDGE_ROWS);
|
||||||
gauge!(DB_TABLE_CHUNK_ROWS);
|
gauge!(DB_TABLE_CHUNK_ROWS);
|
||||||
|
|
||||||
// Labeled counter: errors by status
|
// Named error counters
|
||||||
render_labeled_counter(&mut out, &REQUEST_ERRORS_BY_STATUS);
|
counter!(ERROR_AUTH_FAILURE_INGEST);
|
||||||
// Labeled counter: errors by user
|
counter!(ERROR_FORBIDDEN_INGEST);
|
||||||
render_labeled_counter(&mut out, &ERRORS_BY_USER);
|
counter!(ERROR_RATE_LIMITED_INGEST);
|
||||||
// Labeled counter: requests by user
|
counter!(ERROR_BAD_REQUEST_INGEST);
|
||||||
render_labeled_counter(&mut out, &REQUESTS_BY_USER);
|
counter!(ERROR_DB_ERROR_INGEST);
|
||||||
|
counter!(ERROR_AUTH_FAILURE_QUERY);
|
||||||
|
counter!(ERROR_FORBIDDEN_QUERY);
|
||||||
|
counter!(ERROR_BAD_REQUEST_QUERY);
|
||||||
|
counter!(ERROR_EMBEDDING_FAILURE_QUERY);
|
||||||
|
counter!(ERROR_SEARCH_FAILURE_QUERY);
|
||||||
|
counter!(ERROR_AUTH_FAILURE_CONTEXT);
|
||||||
|
counter!(ERROR_FORBIDDEN_CONTEXT);
|
||||||
|
counter!(ERROR_LOOKUP_FAILURE_CONTEXT);
|
||||||
|
gauge!(LAST_ERROR_TIMESTAMP);
|
||||||
|
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user