feat: complete observability stack (O1-O13) (#52)

## Complete Observability Stack (O1-O13)

Implements all 13 observability issues in a single PR. 119 metrics total.

### Commits (one per issue)

| Issue | Title | Metrics |
|-------|-------|---------|
| **O10** | Prometheus metrics module + /metrics endpoint | Foundation |
| **O1** | Instrument ingest handler | I1-I12 (12) |
| **O2** | Instrument query handler | Q1-Q12 (12) |
| **O3** | Instrument context endpoint | C1-C8 (8) |
| **O4** | Relevance judge | R1-R9 (9) |
| **O5** | Write volume + storage metrics | W1-W12 (12) |
| **O6** | Pod resource observability | P1-P13 |
| **O7** | Availability + dependency health | A1-A10 (10) |
| **O8** | Ingest rate pattern tracking | IR1-IR10 (10) |
| **O9** | Postgres internal observability | PG1-PG33 |
| **O11** | Grafana dashboard | 12 panels |
| **O12** | Prometheus alerting rules | 11 alerts |
| **O13** | Relevance evaluation CronJob | K8s manifest |

### Key Changes

- **metrics.rs**: Zero-dependency Prometheus metrics (Counter, Gauge, Histogram, Timer)
- **GET /metrics**: Prometheus text exposition format endpoint
- **Ingest/Query/Context handlers**: Instrumented with latency, errors, auth failures
- **Health check**: DB dependency check with latency tracking
- **Background task**: Periodic DB stats collection (entity/edge counts, pool stats)
- **Relevance judge**: Threshold-based eval with precision/recall/F1 tracking
- **Grafana dashboard**: 12 panels covering all metric groups
- **Alert rules**: 11 PrometheusRule alerts (availability, latency, errors, quality)
- **CronJob**: Periodic relevance evaluation with sample queries

### Testing

- 506 tests passing (0 failures)
- All metrics modules have unit tests
- Relevance judge: 4 tests

### Deploy

```bash
# Grafana dashboard
kubectl apply -f k8s/infra/grafana-dashboard.json

# Prometheus alerts
kubectl apply -f k8s/infra/prometheus-alerts.yaml

# Relevance eval CronJob
kubectl apply -f k8s/infra/relevance-eval-cronjob.yaml
```

Closes #27 #28 #29 #30 #31 #32 #33 #34 #35 #36 #37 #38 #39

---------

Co-authored-by: rock <[email protected]>
Reviewed-on: #52
Co-authored-by: poimen <[email protected]>

crates/mem-cli/src/http_server.rs

@@ -374,6 +374,30 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
     });
 
     tracing::info!("Starting HTTP server on port {}", port);
+
+    // O5/O7/O9: Background stats collector (every 60s)
+    {
+        let stats_pool = state.get_ref().pool.clone();
+        tokio::spawn(async move {
+            let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
+            loop {
+                interval.tick().await;
+                // O5: Table row counts
+                if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_entity")
+                    .fetch_one(&stats_pool).await {
+                    crate::metrics::DB_TABLE_ENTITY_ROWS.set(row.0 as u64);
+                }
+                if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_edge")
+                    .fetch_one(&stats_pool).await {
+                    crate::metrics::DB_TABLE_EDGE_ROWS.set(row.0 as u64);
+                }
+                // O9: Pool stats
+                crate::metrics::DB_POOL_SIZE.set(stats_pool.size() as u64);
+                crate::metrics::DB_POOL_IDLE.set(stats_pool.num_idle() as u64);
+            }
+        });
+    }
+
     tracing::info!("Creating HttpServer instance...");
 
     let server = HttpServer::new(move || {
@@ -382,6 +406,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
             .app_data(state.clone())
             .wrap(Logger::default())
             .route("/health", web::get().to(health_check))
+            .route("/metrics", web::get().to(crate::metrics::metrics_handler))
             .route("/memory/ingest", web::post().to(ingest_handler))
             .route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
             .route("/memory/query", web::get().to(query_handler))
@@ -428,7 +453,24 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
 
 /// Health check (no auth)
 pub async fn health_check(state: web::Data<AppState>) -> HttpResponse {
+    use crate::metrics::*;
+    HEALTH_CHECKS_TOTAL.inc();
     let uptime = state.start_time.elapsed().as_secs();
+    APP_UPTIME_SECONDS.set(uptime);
+
+    // O7: Check DB dependency
+    let db_start = std::time::Instant::now();
+    match sqlx::query("SELECT 1").execute(&state.pool).await {
+        Ok(_) => {
+            DEP_DB_UP.set(1);
+            DEP_DB_LATENCY.observe(db_start.elapsed().as_secs_f64());
+        }
+        Err(_) => {
+            DEP_DB_UP.set(0);
+            HEALTH_CHECK_FAILURES.inc();
+        }
+    }
+
     HttpResponse::Ok().json(json!({"status": "ok", "uptime_seconds": uptime}))
 }
 
@@ -438,29 +480,57 @@ pub async fn ingest_handler(
     body: web::Json<IngestRequest>,
     state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    INGEST_REQUESTS_TOTAL.inc();
+    INGEST_IN_FLIGHT.inc();
+    let _timer = Timer::new(&INGEST_DURATION);
+
     // Auth + capability check
     let (claims, _token) = match validate_auth(&req, &state).await {
         Ok(c) => c,
-        Err(e) => return e,
+        Err(e) => {
+            INGEST_AUTH_FAILURES.inc();
+            INGEST_ERRORS_TOTAL.inc();
+            ERROR_AUTH_FAILURE_INGEST.inc();
+            INGEST_IN_FLIGHT.dec();
+            return e;
+        }
     };
+
+    let user_id = &claims.sub;
     if !has_capability(&claims, "memory:write") {
+        INGEST_AUTH_FAILURES.inc();
+        INGEST_ERRORS_TOTAL.inc();
+        ERROR_FORBIDDEN_INGEST.inc();
+        INGEST_IN_FLIGHT.dec();
         return HttpResponse::Forbidden().json(json!({
             "error": "forbidden",
             "reason": "missing capability: memory:write"
         }));
     }
     if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
+        INGEST_RATE_LIMITED.inc();
+        ERROR_RATE_LIMITED_INGEST.inc();
+        INGEST_IN_FLIGHT.dec();
         return e;
     }
 
     // Check idempotency
     if let Some(cached) = state.idempotency_store.get(&body.ingest_id) {
         tracing::info!("Returning cached response for ingest_id: {}", body.ingest_id);
+        INGEST_DUPLICATES_TOTAL.inc();
+        INGEST_IN_FLIGHT.dec();
         return HttpResponse::Accepted().json(cached);
     }
 
+    let byte_count: usize = body.records.iter().map(|r| r.text.len()).sum();
+    INGEST_BYTES_TOTAL.inc_by(byte_count as u64);
+    INGEST_RECORDS_TOTAL.inc_by(body.records.len() as u64);
+
     // Execute ingest
-    execute_ingest(&state, &body).await
+    let resp = execute_ingest(&state, &body).await;
+    INGEST_IN_FLIGHT.dec();
+    resp
 }
 
 /// Execute ingest job creation and spawn worker
@@ -511,7 +581,9 @@ async fn execute_ingest(
             HttpResponse::Accepted().json(response)
         }
         Err(e) => {
-            tracing::error!("DB error: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_INGEST.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            tracing::error!(user_id = body.project.as_str(), "Unexpected DB error during ingest: {}", e);
             HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
         }
     }
@@ -768,8 +840,13 @@ async fn store_compacted_memory(
     .await;
 
     match result {
-        Ok(_) => true,
+        Ok(_) => {
+            crate::metrics::WRITE_CHUNKS_TOTAL.inc();
+            crate::metrics::WRITE_BYTES_TOTAL.inc_by(memory.len() as u64);
+            true
+        }
         Err(e) => {
+            crate::metrics::WRITE_ERRORS_TOTAL.inc();
             tracing::error!("Failed to store compacted memory: {}", e);
             false
         }
@@ -830,7 +907,9 @@ pub async fn query_handler(
     match query_temporal_graph(&state, &params).await {
         Ok(response) => HttpResponse::Ok().json(response),
         Err(e) => {
-            tracing::error!("Temporal graph query failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            tracing::error!(user_id = claims.sub.as_str(), "Unexpected error: temporal graph query failed: {}", e);
             HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
         }
     }
@@ -966,13 +1045,23 @@ pub async fn context_handler(
     body: web::Json<crate::context_endpoint::ContextRequest>,
     state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    CONTEXT_REQUESTS_TOTAL.inc();
+    let _timer = Timer::new(&CONTEXT_DURATION);
+
     let (claims, _token) = match validate_auth(&req, &state).await {
         Ok(c) => c,
-        Err(e) => return e,
+        Err(e) => {
+            CONTEXT_ERRORS_TOTAL.inc();
+            ERROR_AUTH_FAILURE_CONTEXT.inc();
+            return e;
+        }
     };
 
-    // Check read capability
+    let user_id = &claims.sub;
     if !has_capability(&claims, "memory:read") {
+        CONTEXT_ERRORS_TOTAL.inc();
+        ERROR_FORBIDDEN_CONTEXT.inc();
         return HttpResponse::Forbidden().json(json!({
             "error": "forbidden",
             "reason": "missing capability: memory:read"
@@ -997,9 +1086,14 @@ pub async fn context_handler(
                 skills = response.skills.len(),
                 "context lookup successful"
             );
+            // O3: Track tier hits
+            let total = response.lessons.len() + response.skills.len();
+            if total == 0 { CONTEXT_EMPTY_RESULTS.inc(); }
             HttpResponse::Ok().json(response)
         }
         Err(e) => {
+            CONTEXT_ERRORS_TOTAL.inc();
+            ERROR_LOOKUP_FAILURE_CONTEXT.inc();
             tracing::error!("context lookup error: {}", e);
             HttpResponse::BadRequest().json(json!({
                 "error": "lookup_failed",