feat: complete observability stack (O1-O13) (#52)

## Complete Observability Stack (O1-O13)

Implements all 13 observability issues in a single PR. 119 metrics total.

### Commits (one per issue)

| Issue | Title | Metrics |
|-------|-------|---------|
| **O10** | Prometheus metrics module + /metrics endpoint | Foundation |
| **O1** | Instrument ingest handler | I1-I12 (12) |
| **O2** | Instrument query handler | Q1-Q12 (12) |
| **O3** | Instrument context endpoint | C1-C8 (8) |
| **O4** | Relevance judge | R1-R9 (9) |
| **O5** | Write volume + storage metrics | W1-W12 (12) |
| **O6** | Pod resource observability | P1-P13 |
| **O7** | Availability + dependency health | A1-A10 (10) |
| **O8** | Ingest rate pattern tracking | IR1-IR10 (10) |
| **O9** | Postgres internal observability | PG1-PG33 |
| **O11** | Grafana dashboard | 12 panels |
| **O12** | Prometheus alerting rules | 11 alerts |
| **O13** | Relevance evaluation CronJob | K8s manifest |

### Key Changes

- **metrics.rs**: Zero-dependency Prometheus metrics (Counter, Gauge, Histogram, Timer)
- **GET /metrics**: Prometheus text exposition format endpoint
- **Ingest/Query/Context handlers**: Instrumented with latency, errors, auth failures
- **Health check**: DB dependency check with latency tracking
- **Background task**: Periodic DB stats collection (entity/edge counts, pool stats)
- **Relevance judge**: Threshold-based eval with precision/recall/F1 tracking
- **Grafana dashboard**: 12 panels covering all metric groups
- **Alert rules**: 11 PrometheusRule alerts (availability, latency, errors, quality)
- **CronJob**: Periodic relevance evaluation with sample queries

### Testing

- 506 tests passing (0 failures)
- All metrics modules have unit tests
- Relevance judge: 4 tests

### Deploy

```bash
# Grafana dashboard
kubectl apply -f k8s/infra/grafana-dashboard.json

# Prometheus alerts
kubectl apply -f k8s/infra/prometheus-alerts.yaml

# Relevance eval CronJob
kubectl apply -f k8s/infra/relevance-eval-cronjob.yaml
```

Closes #27 #28 #29 #30 #31 #32 #33 #34 #35 #36 #37 #38 #39

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #52
Co-authored-by: poimen <[email protected]>

crates/mem-cli/src/handlers/unified_query.rs

@@ -118,17 +118,28 @@ pub async fn unified_query_handler(
     body: web::Json<UnifiedQueryRequest>,
     state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    QUERY_REQUESTS_TOTAL.inc();
+    QUERY_IN_FLIGHT.inc();
+    let _timer = Timer::new(&QUERY_DURATION);
     let start_time = std::time::Instant::now();
 
     // 1. Validate JWT + rate limit
     if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
         &req, &state, "query", 500
     ) {
+        QUERY_AUTH_FAILURES.inc();
+        QUERY_ERRORS_TOTAL.inc();
+        ERROR_AUTH_FAILURE_QUERY.inc();
+        QUERY_IN_FLIGHT.dec();
         return response;
     }
 
     // 2. Validate input
     if let Err(response) = validate_unified_request(&body) {
+        QUERY_ERRORS_TOTAL.inc();
+        ERROR_BAD_REQUEST_QUERY.inc();
+        QUERY_IN_FLIGHT.dec();
         return response;
     }
 
@@ -136,9 +147,17 @@ pub async fn unified_query_handler(
            body.search_type, body.query, body.entity_type, body.relation_type);
 
     // 3. Embed query once (reused for all search types)
+    let embed_start = std::time::Instant::now();
     let query_embedding = match state.embeddings.embed_one(&body.query).await {
-        Ok(emb) => emb.to_vec(),
+        Ok(emb) => {
+            QUERY_EMBEDDING_DURATION.observe(embed_start.elapsed().as_secs_f64());
+            emb.to_vec()
+        }
         Err(e) => {
+            QUERY_EMBEDDING_FAILURES.inc();
+            QUERY_ERRORS_TOTAL.inc();
+            ERROR_EMBEDDING_FAILURE_QUERY.inc();
+            QUERY_IN_FLIGHT.dec();
             error!("Embedding failed: {}", e);
             return crate::handlers::response_builder::internal_error(
                 "Failed to embed query"
@@ -152,12 +171,15 @@ pub async fn unified_query_handler(
         "edges" => search_edges(&body, &state, &query_embedding, start_time).await,
         "hybrid" => search_hybrid(&body, &state, &query_embedding, start_time).await,
         _ => {
+            QUERY_ERRORS_TOTAL.inc();
+            QUERY_IN_FLIGHT.dec();
             return crate::handlers::response_builder::bad_request(
                 "search_type must be 'entities', 'edges', or 'hybrid'"
             );
         }
     };
 
+    QUERY_IN_FLIGHT.dec();
     response
 }
 
@@ -181,7 +203,9 @@ async fn search_entities(
     ).await {
         Ok(r) => r,
         Err(e) => {
-            error!("Entity search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: entity search failed: {}", e);
             return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
         }
     };
@@ -247,6 +271,10 @@ async fn search_entities(
 
     info!("Unified query (entities): {} results in {}ms", count, elapsed);
 
+    // O2: Track result counts
+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
     let response = UnifiedQueryResponse {
         query: req.query.clone(),
         search_type: "entities".to_string(),
@@ -279,7 +307,9 @@ async fn search_edges(
     ).await {
         Ok(r) => r,
         Err(e) => {
-            error!("Edge search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: edge search failed: {}", e);
             return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
         }
     };
@@ -305,6 +335,9 @@ async fn search_edges(
 
     info!("Unified query (edges): {} results in {}ms", count, elapsed);
 
+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
     let response = UnifiedQueryResponse {
         query: req.query.clone(),
         search_type: "edges".to_string(),
@@ -338,7 +371,9 @@ async fn search_hybrid(
     ).await {
         Ok(r) => r,
         Err(e) => {
-            error!("Hybrid search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: hybrid search failed: {}", e);
             return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
         }
     };
@@ -350,6 +385,9 @@ async fn search_hybrid(
 
     info!("Unified query (hybrid): {} results in {}ms", count, elapsed);
 
+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
     let response = UnifiedQueryResponse {
         query: req.query.clone(),
         search_type: "hybrid".to_string(),