- ERROR_UNEXPECTED_TOTAL: global unexpected error counter - ERROR_UNEXPECTED_INGEST/QUERY/CONTEXT: per-endpoint unexpected errors - All 500 error paths now increment unexpected counter - Error logs include user_id for customer association: tracing::error!(user_id = claims.sub, "Unexpected error: ...") - Covers: DB errors, search failures, temporal query failures - 515 tests passing
This commit is contained in:
@@ -203,7 +203,9 @@ async fn search_entities(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Entity search failed: {}", e);
|
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
||||||
|
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
||||||
|
error!("Unexpected error: entity search failed: {}", e);
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -305,7 +307,9 @@ async fn search_edges(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Edge search failed: {}", e);
|
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
||||||
|
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
||||||
|
error!("Unexpected error: edge search failed: {}", e);
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@@ -367,7 +371,9 @@ async fn search_hybrid(
|
|||||||
).await {
|
).await {
|
||||||
Ok(r) => r,
|
Ok(r) => r,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Hybrid search failed: {}", e);
|
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
||||||
|
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
||||||
|
error!("Unexpected error: hybrid search failed: {}", e);
|
||||||
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -581,7 +581,9 @@ async fn execute_ingest(
|
|||||||
HttpResponse::Accepted().json(response)
|
HttpResponse::Accepted().json(response)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("DB error: {}", e);
|
crate::metrics::ERROR_UNEXPECTED_INGEST.inc();
|
||||||
|
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
||||||
|
tracing::error!(user_id = body.project.as_str(), "Unexpected DB error during ingest: {}", e);
|
||||||
HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
|
HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -905,7 +907,9 @@ pub async fn query_handler(
|
|||||||
match query_temporal_graph(&state, ¶ms).await {
|
match query_temporal_graph(&state, ¶ms).await {
|
||||||
Ok(response) => HttpResponse::Ok().json(response),
|
Ok(response) => HttpResponse::Ok().json(response),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::error!("Temporal graph query failed: {}", e);
|
crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
|
||||||
|
crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
|
||||||
|
tracing::error!(user_id = claims.sub.as_str(), "Unexpected error: temporal graph query failed: {}", e);
|
||||||
HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
|
HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -371,6 +371,16 @@ pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new(
|
|||||||
pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new(
|
pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new(
|
||||||
"memory_error_lookup_failure_context_total", "Context lookup failure");
|
"memory_error_lookup_failure_context_total", "Context lookup failure");
|
||||||
|
|
||||||
|
// Unexpected errors (unhandled 500s, panics, unknown failures)
|
||||||
|
pub static ERROR_UNEXPECTED_TOTAL: Counter = Counter::new(
|
||||||
|
"memory_error_unexpected_total", "Total unexpected/unhandled errors (500s)");
|
||||||
|
pub static ERROR_UNEXPECTED_INGEST: Counter = Counter::new(
|
||||||
|
"memory_error_unexpected_ingest_total", "Unexpected errors during ingest");
|
||||||
|
pub static ERROR_UNEXPECTED_QUERY: Counter = Counter::new(
|
||||||
|
"memory_error_unexpected_query_total", "Unexpected errors during query");
|
||||||
|
pub static ERROR_UNEXPECTED_CONTEXT: Counter = Counter::new(
|
||||||
|
"memory_error_unexpected_context_total", "Unexpected errors during context");
|
||||||
|
|
||||||
// Last error info (most recent error for debugging)
|
// Last error info (most recent error for debugging)
|
||||||
pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new(
|
pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new(
|
||||||
"memory_last_error_timestamp_seconds", "Unix timestamp of most recent error");
|
"memory_last_error_timestamp_seconds", "Unix timestamp of most recent error");
|
||||||
@@ -579,6 +589,10 @@ pub fn render_metrics() -> String {
|
|||||||
counter!(ERROR_AUTH_FAILURE_CONTEXT);
|
counter!(ERROR_AUTH_FAILURE_CONTEXT);
|
||||||
counter!(ERROR_FORBIDDEN_CONTEXT);
|
counter!(ERROR_FORBIDDEN_CONTEXT);
|
||||||
counter!(ERROR_LOOKUP_FAILURE_CONTEXT);
|
counter!(ERROR_LOOKUP_FAILURE_CONTEXT);
|
||||||
|
counter!(ERROR_UNEXPECTED_TOTAL);
|
||||||
|
counter!(ERROR_UNEXPECTED_INGEST);
|
||||||
|
counter!(ERROR_UNEXPECTED_QUERY);
|
||||||
|
counter!(ERROR_UNEXPECTED_CONTEXT);
|
||||||
gauge!(LAST_ERROR_TIMESTAMP);
|
gauge!(LAST_ERROR_TIMESTAMP);
|
||||||
|
|
||||||
out
|
out
|
||||||
|
|||||||
Reference in New Issue
Block a user