test: verify embedding response parsing against real service format

- 6 parsing tests for EmbeddingResponse struct - test_parse_real_embedding_response: exact format from embeddings-predictor - test_parse_768_dim_response: full 768-dim vector - test_parse_multi_input_response: array input returns multiple embeddings - test_parse_embedding_error_response: error format - test_parse_html_fails_gracefully: HTML error page correctly rejected - Confirms: parsing is correct, 'expected ident' error is non-JSON response
feat: complete observability stack (O1-O13) (#52 )
2026-09-14 08:46:03 +09:00 · 2026-09-13 13:53:50 +00:00 · 2026-09-13 05:42:01 +00:00
27 changed files with 2547 additions and 81 deletions
@@ -1,12 +1,55 @@
+# Git
 .git
 .gitignore
+.gitattributes
+
+# CI/CD
+.github
+.gitea
+.gitlab-ci.yml
+
+# Kubernetes
+k8s/
+helm/
+
+# Documentation
 *.md
-__pycache__
-*.pyc
-.env.local
-.venv
-venv/
-.pytest_cache
-.coverage
-htmlcov
+docs/
+
+# IDE
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# OS
 .DS_Store
+Thumbs.db
+
+# Build artifacts
+target/
+dist/
+build/
+
+# Dependencies (will be downloaded fresh)
+.cargo/
+Cargo.lock.bak
+
+# Testing
+.coverage
+coverage/
+
+# Secrets
+.env
+.env.local
+.env.*.local
+
+# Archives
+*.tar
+*.tar.gz
+*.zip
+
+# Node (if any)
+node_modules/
+*.log
@@ -0,0 +1,19 @@
+MEM_AUTH_MODE=none
+MEM_RATE_LIMIT_INGEST=1000
+MEM_RATE_LIMIT_QUERY=10000
+MEM_IDEMPOTENCY_TTL_SECS=86400
+MEM_EMBEDDING_BATCH_SIZE=4
+
+DATABASE_URL=postgresql://app:katFpWYB4EH9KU9NABOglnE9ekea5rBxyOY9WZeUTi1ujhFS1pVzNxrXbB7A4qGc@127.0.0.1:5433/memory
+
+# Embedding via direct port-forward (skip gateway auth)
+LLM_ENDPOINT=http://localhost:9090/v1/chat/completions
+LLM_API_BASE=http://localhost:9090
+LLM_MODEL=nomic-ai/nomic-embed-text-v2-moe
+LLM_TIMEOUT_SECS=60
+ENABLE_LLM_EXTRACTION=true
+EMBEDDINGS_MODEL=nomic-ai/nomic-embed-text-v2-moe
+
+MEM_PORT=8081
+MEM_API_KEY=test-key
+MEM_HOME=/tmp
@@ -0,0 +1,50 @@
+# Local development environment (.env file)
+# Copy to .env and fill in your local/dev URLs
+# .env is gitignored - never commit
+
+# Auth mode: jwt | apikey | none
+MEM_AUTH_MODE=none
+
+# Rate limiting
+MEM_RATE_LIMIT_INGEST=1000
+MEM_RATE_LIMIT_QUERY=10000
+MEM_IDEMPOTENCY_TTL_SECS=86400
+
+# Embeddings
+MEM_EMBEDDING_BATCH_SIZE=32
+
+# Database (local or remote)
+DATABASE_URL=postgresql://user:password@localhost:5432/memory
+
+# Downstream services - point to your local/dev endpoints
+
+# LLM Service (entity extraction, fact extraction)
+LLM_ENDPOINT=http://localhost:11434/v1/chat/completions
+LLM_API_BASE=http://localhost:11434/v1
+LLM_MODEL=qwen:7b
+LLM_TIMEOUT_SECS=60
+ENABLE_LLM_EXTRACTION=true
+
+# OpenSearch (vector store, BM25)
+OPENSEARCH_HOST=localhost:9200
+OPENSEARCH_SCHEME=http
+OPENSEARCH_VERIFY_CERTS=false
+
+# Authentik (OIDC - optional for local dev)
+AUTHENTIK_ISSUER=https://authentik.riotpiao.com/application/o/poimen/
+AUTHENTIK_CLIENT_ID=
+AUTHENTIK_CLIENT_SECRET=
+TOKEN_URL=https://authentik.riotpiao.com/application/o/token/
+AUTHENTIK_VERIFY_SSL=false
+
+# Temporal (workflow orchestration - future)
+TEMPORAL_ENDPOINT=localhost:7233
+TEMPORAL_NAMESPACE=poimen
+
+# API Gateway (route optimization - future)
+GATEWAY_URL=http://localhost:8080
+
+# Server config
+MEM_PORT=8080
+MEM_API_KEY=test-key
+MEM_HOME=/tmp
@@ -18,6 +18,15 @@ jobs:
    name: CI
    runs-on: rust
    steps:
+      - name: Clean disk space (runner GC)
+        run: |
+          df -h /
+          echo "Cleaning docker, cargo cache..."
+          docker system prune -af --volumes || true
+          rm -rf ~/.cargo/registry/cache ~/.cargo/registry/index ~/.cargo/git || true
+          rm -rf /tmp/* || true
+          df -h /
+
      - name: Install Node.js and Docker
        run: |
          apt-get update
@@ -26,17 +35,11 @@ jobs:
      - name: Checkout code
        uses: actions/checkout@v4

-      - name: Cargo build all
-        run: cargo build --all --verbose
-
-      - name: Cargo test all
-        run: cargo test --all --lib --verbose 2>&1 | tail -150 || true
-
-      - name: Cargo clippy
-        run: cargo clippy --all --all-targets -- -D warnings 2>&1 | tail -50 || true
-
-      - name: Clean build artifacts before Docker
-        run: cargo clean
+      - name: Cargo build, test, clippy (single compile pass)
+        run: |
+          cargo build --all --verbose
+          cargo test --all --lib --verbose 2>&1 | tail -150 || true
+          cargo clippy --all --all-targets -- -D warnings 2>&1 | tail -50 || true

      - name: Get short SHA
        id: sha
@@ -44,12 +47,22 @@ jobs:

      - name: Registry login
        run: |
+          if [ -z "${REGISTRY_USER}" ] || [ -z "${REGISTRY_TOKEN}" ]; then
+            echo "ERROR: Missing REGISTRY_USER or REGISTRY_TOKEN secrets"
+            exit 1
+          fi
          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
            --username "${REGISTRY_USER}" --password-stdin
        env:
          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}

+      - name: Clean cargo before Docker build
+        run: |
+          cargo clean || true
+          rm -rf ~/.cargo/registry/cache ~/.cargo/registry/index ~/.cargo/git || true
+          df -h /
+
      - name: Build and push Docker image (SHA tag only)
        run: |
          docker build --no-cache --progress=plain \
@@ -58,5 +71,8 @@ jobs:
          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
          echo "Pushed: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"

-      - name: Prune unused images
-        run: docker image prune -a --force 2>&1 | tail -3 || true
+      - name: Prune unused images and cleanup
+        run: |
+          docker image prune -a --force 2>&1 | tail -3 || true
+          cargo clean || true
+          df -h /
@@ -15,29 +15,48 @@ jobs:
    name: Tag & Push Latest
    runs-on: rust
    steps:
-      - name: Install Docker
-        run: apt-get update && apt-get install -y docker.io
+      - name: Install Docker and curl
+        run: apt-get update && apt-get install -y docker.io curl

-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Get short SHA
+      - name: Get short SHA via Gitea API
        id: sha
-        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+        run: |
+          # Fetch latest commit SHA for main branch from Gitea API
+          COMMIT_SHA=$(curl -s -H "Authorization: token ${REGISTRY_TOKEN}" \
+            "https://forgejo.riotpiao.com/api/v1/repos/riotpiao-poimen/poimen-memory/commits?sha=main&limit=1" | \
+            grep -o '"sha":"[^"]*' | head -1 | cut -d'"' -f4)
+          
+          if [ -z "$COMMIT_SHA" ]; then
+            echo "ERROR: Failed to fetch commit SHA from Gitea API"
+            exit 1
+          fi
+          
+          SHORT_SHA=$(echo "$COMMIT_SHA" | cut -c1-7)
+          echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
+          echo "Full SHA: $COMMIT_SHA, Short: $SHORT_SHA"
+        env:
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}

      - name: Registry login
        run: |
+          if [ -z "${REGISTRY_USER}" ] || [ -z "${REGISTRY_TOKEN}" ]; then
+            echo "ERROR: Missing REGISTRY_USER or REGISTRY_TOKEN secrets"
+            exit 1
+          fi
          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
            --username "${REGISTRY_USER}" --password-stdin
        env:
          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}

-      - name: Pull SHA image and tag as latest
+      - name: Verify SHA image exists, tag as latest
        run: |
-          docker pull "${IMAGE}:${{ steps.sha.outputs.short_sha }}" && \
-          docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest" && \
-          docker push "${IMAGE}:latest" && \
+          if ! docker pull "${IMAGE}:${{ steps.sha.outputs.short_sha }}"; then
+            echo "ERROR: Image ${IMAGE}:${{ steps.sha.outputs.short_sha }} not found. Check build.yaml passed."
+            exit 1
+          fi
+          docker tag "${IMAGE}:${{ steps.sha.outputs.short_sha }}" "${IMAGE}:latest"
+          docker push "${IMAGE}:latest"
          echo "Tagged and pushed: ${IMAGE}:latest (from ${{ steps.sha.outputs.short_sha }})"

      - name: Prune images
@@ -31,7 +31,7 @@ jobs:
          echo "Changed migrations: $CHANGED"
          echo "CHANGED_MIGRATIONS=$CHANGED" >> $GITHUB_ENV

-      - name: Run migrations
+      - name: Run changed migrations and verify schema
        if: env.CHANGED_MIGRATIONS != ''
        run: |
          export PGPASSWORD="${DB_PASSWORD}"
@@ -55,18 +55,27 @@ jobs:
          DB_USER: ${{ secrets.DB_USER }}
          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}

-      - name: Run all migrations (manual trigger)
+      - name: Run all migrations and verify schema (manual trigger)
        if: github.event_name == 'workflow_dispatch'
        run: |
          export PGPASSWORD="${DB_PASSWORD}"

          echo "=== Running all migrations in order ==="
+          FAILED=0
          for f in $(ls crates/mem-store/migrations/*.sql | sort); do
            echo "--- Applying: $f ---"
-            psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1 || true
-            echo "--- Done: $f ---"
+            if ! psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$f" 2>&1; then
+              echo "ERROR: Migration $f failed!"
+              FAILED=1
+            else
+              echo "--- OK: $f ---"
+            fi
          done

+          if [ $FAILED -eq 1 ]; then
+            exit 1
+          fi
+
          echo "=== Final schema ==="
          psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\dt memory*"
          psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -c "\d memory_entity"
@@ -2053,6 +2053,7 @@ dependencies = [
 "mem-ingest",
 "mem-llm",
 "mem-store",
+ "once_cell",
 "pgvector",
 "rand 0.8.7",
 "redis",
@@ -5,14 +5,23 @@ FROM rust:1-bookworm as builder

 WORKDIR /build

+# Build settings
+ENV SQLX_OFFLINE=true
+
 # Copy source
 COPY . .

-# Build the mem binary (offline sqlx - uses .sqlx/ cache)
-ENV SQLX_OFFLINE=true
-RUN cargo build --release -p mem-cli && \
+# Build release binary with space-efficient cleanup
+RUN cargo build --release -p mem-cli --locked && \
    strip target/release/mem && \
-    rm -rf target/release/deps target/release/build target/release/incremental target/release/.fingerprint
+    # Aggressive cleanup to free disk space
+    rm -rf target/release/deps && \
+    rm -rf target/release/build && \
+    rm -rf target/release/incremental && \
+    rm -rf target/release/.fingerprint && \
+    rm -rf .cargo/registry/cache && \
+    rm -rf .cargo/registry/index && \
+    rm -rf .cargo/git

 # Stage 2: Runtime
 FROM debian:bookworm-slim
@@ -0,0 +1,84 @@
+# Local Development Setup
+
+Running poimen-memory locally for development.
+
+## Quick Start
+
+1. **Copy env template**:
+   ```bash
+   cp .env.example .env
+   ```
+
+2. **Edit `.env`** with your local endpoints:
+   ```bash
+   # Edit .env with your local/dev service URLs
+   # Example: LLM service on localhost:11434, OpenSearch on localhost:9200
+   ```
+
+3. **Run the service**:
+   ```bash
+   cargo run --release -- serve --port 8080
+   ```
+
+The application loads configuration from `.env` (via `dotenvy` or similar).
+
+## `.env` File
+
+**Location**: Project root (`.env`)
+**Status**: Gitignored - never committed
+**Template**: `.env.example` (included in repo, shows all available variables)
+
+### Key Variables
+
+```bash
+# Database
+DATABASE_URL=postgresql://user:pass@localhost:5432/memory
+
+# LLM (point to your local LLM service)
+LLM_ENDPOINT=http://localhost:11434/v1/chat/completions
+LLM_MODEL=qwen:7b
+
+# OpenSearch (local vector store)
+OPENSEARCH_HOST=localhost:9200
+
+# Auth (disabled for local dev)
+MEM_AUTH_MODE=none
+
+# API Key (test key for local dev)
+MEM_API_KEY=test-key
+```
+
+## Local Service Stack (Example)
+
+```bash
+# Terminal 1: OpenSearch
+docker run -d -p 9200:9200 -e OPENSEARCH_JAVA_OPTS="-Xms512m -Xmx512m" \
+  opensearchproject/opensearch:latest
+
+# Terminal 2: Ollama (LLM)
+ollama serve
+
+# Terminal 3: poimen-memory
+cargo run --release -- serve --port 8080
+```
+
+## Production vs Local
+
+| Aspect | Production (K8s) | Local Dev |
+|--------|-----------------|-----------|
+| **Config** | `k8s/app/config.yaml` (SOPS-encrypted) | `.env` (gitignored) |
+| **Injection** | ConfigMap via `envFrom:` | dotenv via `dotenvy` crate |
+| **Services** | Cluster-internal DNS | localhost/127.0.0.1 |
+| **Auth** | JWT (Authentik) | None (disabled) |
+| **Commit?** | Yes (encrypted) | No (gitignored) |
+
+## Switching to Production Config
+
+To run against production services (not recommended locally):
+1. Edit `.env` with production URLs
+2. Set credentials appropriately
+3. Ensure network access to production services
+
+---
+
+See `.env.example` for all available environment variables.
@@ -46,3 +46,4 @@ futures-util = "0.3"
 async-stream = "0.3"
 rand = "0.8"
 lru = "0.12"
+once_cell = { workspace = true }
@@ -61,6 +61,49 @@ pub fn validate_and_rate_limit(
    Ok(())
 }

+/// Extract user identity from JWT claims (sub field)
+///
+/// Tries to decode JWT from Authorization header to get `sub` claim.
+/// Falls back to "anonymous" if auth is disabled or header missing.
+/// Used by metrics to track errors/requests per user.
+pub fn extract_user_id(req: &HttpRequest, state: &AppState) -> String {
+    // If auth disabled, check synthetic claims
+    if state.jwt_validator.is_none() {
+        return "anonymous".to_string();
+    }
+
+    // Try to extract sub from JWT
+    let token = req.headers()
+        .get("Authorization")
+        .and_then(|h| h.to_str().ok())
+        .and_then(|h| h.strip_prefix("Bearer "))
+        .unwrap_or("");
+
+    if token.is_empty() {
+        return "anonymous".to_string();
+    }
+
+    // Decode JWT payload without validation (already validated by validate_and_rate_limit)
+    // JWT format: header.payload.signature
+    let parts: Vec<&str> = token.split('.').collect();
+    if parts.len() != 3 {
+        return "anonymous".to_string();
+    }
+
+    // Decode base64 payload
+    use base64::Engine;
+    let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD;
+    if let Ok(payload_bytes) = engine.decode(parts[1]) {
+        if let Ok(payload) = serde_json::from_slice::<serde_json::Value>(&payload_bytes) {
+            if let Some(sub) = payload.get("sub").and_then(|s| s.as_str()) {
+                return sub.to_string();
+            }
+        }
+    }
+
+    "anonymous".to_string()
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -118,17 +118,28 @@ pub async fn unified_query_handler(
    body: web::Json<UnifiedQueryRequest>,
    state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    QUERY_REQUESTS_TOTAL.inc();
+    QUERY_IN_FLIGHT.inc();
+    let _timer = Timer::new(&QUERY_DURATION);
    let start_time = std::time::Instant::now();

    // 1. Validate JWT + rate limit
    if let Err(response) = crate::handlers::middleware::validate_and_rate_limit(
        &req, &state, "query", 500
    ) {
+        QUERY_AUTH_FAILURES.inc();
+        QUERY_ERRORS_TOTAL.inc();
+        ERROR_AUTH_FAILURE_QUERY.inc();
+        QUERY_IN_FLIGHT.dec();
        return response;
    }

    // 2. Validate input
    if let Err(response) = validate_unified_request(&body) {
+        QUERY_ERRORS_TOTAL.inc();
+        ERROR_BAD_REQUEST_QUERY.inc();
+        QUERY_IN_FLIGHT.dec();
        return response;
    }

@@ -136,9 +147,17 @@ pub async fn unified_query_handler(
           body.search_type, body.query, body.entity_type, body.relation_type);

    // 3. Embed query once (reused for all search types)
+    let embed_start = std::time::Instant::now();
    let query_embedding = match state.embeddings.embed_one(&body.query).await {
-        Ok(emb) => emb.to_vec(),
+        Ok(emb) => {
+            QUERY_EMBEDDING_DURATION.observe(embed_start.elapsed().as_secs_f64());
+            emb.to_vec()
+        }
        Err(e) => {
+            QUERY_EMBEDDING_FAILURES.inc();
+            QUERY_ERRORS_TOTAL.inc();
+            ERROR_EMBEDDING_FAILURE_QUERY.inc();
+            QUERY_IN_FLIGHT.dec();
            error!("Embedding failed: {}", e);
            return crate::handlers::response_builder::internal_error(
                "Failed to embed query"
@@ -152,12 +171,15 @@ pub async fn unified_query_handler(
        "edges" => search_edges(&body, &state, &query_embedding, start_time).await,
        "hybrid" => search_hybrid(&body, &state, &query_embedding, start_time).await,
        _ => {
+            QUERY_ERRORS_TOTAL.inc();
+            QUERY_IN_FLIGHT.dec();
            return crate::handlers::response_builder::bad_request(
                "search_type must be 'entities', 'edges', or 'hybrid'"
            );
        }
    };

+    QUERY_IN_FLIGHT.dec();
    response
 }

@@ -181,7 +203,9 @@ async fn search_entities(
    ).await {
        Ok(r) => r,
        Err(e) => {
-            error!("Entity search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: entity search failed: {}", e);
            return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
        }
    };
@@ -247,6 +271,10 @@ async fn search_entities(

    info!("Unified query (entities): {} results in {}ms", count, elapsed);

+    // O2: Track result counts
+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
    let response = UnifiedQueryResponse {
        query: req.query.clone(),
        search_type: "entities".to_string(),
@@ -279,7 +307,9 @@ async fn search_edges(
    ).await {
        Ok(r) => r,
        Err(e) => {
-            error!("Edge search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: edge search failed: {}", e);
            return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
        }
    };
@@ -305,6 +335,9 @@ async fn search_edges(

    info!("Unified query (edges): {} results in {}ms", count, elapsed);

+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
    let response = UnifiedQueryResponse {
        query: req.query.clone(),
        search_type: "edges".to_string(),
@@ -338,7 +371,9 @@ async fn search_hybrid(
    ).await {
        Ok(r) => r,
        Err(e) => {
-            error!("Hybrid search failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            error!("Unexpected error: hybrid search failed: {}", e);
            return crate::handlers::response_builder::internal_error(&format!("Search failed: {}", e));
        }
    };
@@ -350,6 +385,9 @@ async fn search_hybrid(

    info!("Unified query (hybrid): {} results in {}ms", count, elapsed);

+    crate::metrics::QUERY_RESULTS_TOTAL.inc_by(count as u64);
+    if count == 0 { crate::metrics::QUERY_EMPTY_RESULTS.inc(); }
+
    let response = UnifiedQueryResponse {
        query: req.query.clone(),
        search_type: "hybrid".to_string(),
@@ -374,6 +374,30 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
    });

    tracing::info!("Starting HTTP server on port {}", port);
+
+    // O5/O7/O9: Background stats collector (every 60s)
+    {
+        let stats_pool = state.get_ref().pool.clone();
+        tokio::spawn(async move {
+            let mut interval = tokio::time::interval(std::time::Duration::from_secs(60));
+            loop {
+                interval.tick().await;
+                // O5: Table row counts
+                if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_entity")
+                    .fetch_one(&stats_pool).await {
+                    crate::metrics::DB_TABLE_ENTITY_ROWS.set(row.0 as u64);
+                }
+                if let Ok(row) = sqlx::query_as::<_, (i64,)>("SELECT COUNT(*) FROM memory_edge")
+                    .fetch_one(&stats_pool).await {
+                    crate::metrics::DB_TABLE_EDGE_ROWS.set(row.0 as u64);
+                }
+                // O9: Pool stats
+                crate::metrics::DB_POOL_SIZE.set(stats_pool.size() as u64);
+                crate::metrics::DB_POOL_IDLE.set(stats_pool.num_idle() as u64);
+            }
+        });
+    }
+
    tracing::info!("Creating HttpServer instance...");

    let server = HttpServer::new(move || {
@@ -382,6 +406,7 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
            .app_data(state.clone())
            .wrap(Logger::default())
            .route("/health", web::get().to(health_check))
+            .route("/metrics", web::get().to(crate::metrics::metrics_handler))
            .route("/memory/ingest", web::post().to(ingest_handler))
            .route("/memory/ingest/{ingest_id}", web::get().to(ingest_status))
            .route("/memory/query", web::get().to(query_handler))
@@ -428,7 +453,24 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res

 /// Health check (no auth)
 pub async fn health_check(state: web::Data<AppState>) -> HttpResponse {
+    use crate::metrics::*;
+    HEALTH_CHECKS_TOTAL.inc();
    let uptime = state.start_time.elapsed().as_secs();
+    APP_UPTIME_SECONDS.set(uptime);
+
+    // O7: Check DB dependency
+    let db_start = std::time::Instant::now();
+    match sqlx::query("SELECT 1").execute(&state.pool).await {
+        Ok(_) => {
+            DEP_DB_UP.set(1);
+            DEP_DB_LATENCY.observe(db_start.elapsed().as_secs_f64());
+        }
+        Err(_) => {
+            DEP_DB_UP.set(0);
+            HEALTH_CHECK_FAILURES.inc();
+        }
+    }
+
    HttpResponse::Ok().json(json!({"status": "ok", "uptime_seconds": uptime}))
 }

@@ -438,29 +480,57 @@ pub async fn ingest_handler(
    body: web::Json<IngestRequest>,
    state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    INGEST_REQUESTS_TOTAL.inc();
+    INGEST_IN_FLIGHT.inc();
+    let _timer = Timer::new(&INGEST_DURATION);
+
    // Auth + capability check
    let (claims, _token) = match validate_auth(&req, &state).await {
        Ok(c) => c,
-        Err(e) => return e,
+        Err(e) => {
+            INGEST_AUTH_FAILURES.inc();
+            INGEST_ERRORS_TOTAL.inc();
+            ERROR_AUTH_FAILURE_INGEST.inc();
+            INGEST_IN_FLIGHT.dec();
+            return e;
+        }
    };
+
+    let user_id = &claims.sub;
    if !has_capability(&claims, "memory:write") {
+        INGEST_AUTH_FAILURES.inc();
+        INGEST_ERRORS_TOTAL.inc();
+        ERROR_FORBIDDEN_INGEST.inc();
+        INGEST_IN_FLIGHT.dec();
        return HttpResponse::Forbidden().json(json!({
            "error": "forbidden",
            "reason": "missing capability: memory:write"
        }));
    }
    if let Err(e) = check_rate_limit(&claims, &state, "/memory/ingest") {
+        INGEST_RATE_LIMITED.inc();
+        ERROR_RATE_LIMITED_INGEST.inc();
+        INGEST_IN_FLIGHT.dec();
        return e;
    }

    // Check idempotency
    if let Some(cached) = state.idempotency_store.get(&body.ingest_id) {
        tracing::info!("Returning cached response for ingest_id: {}", body.ingest_id);
+        INGEST_DUPLICATES_TOTAL.inc();
+        INGEST_IN_FLIGHT.dec();
        return HttpResponse::Accepted().json(cached);
    }

+    let byte_count: usize = body.records.iter().map(|r| r.text.len()).sum();
+    INGEST_BYTES_TOTAL.inc_by(byte_count as u64);
+    INGEST_RECORDS_TOTAL.inc_by(body.records.len() as u64);
+
    // Execute ingest
-    execute_ingest(&state, &body).await
+    let resp = execute_ingest(&state, &body).await;
+    INGEST_IN_FLIGHT.dec();
+    resp
 }

 /// Execute ingest job creation and spawn worker
@@ -511,7 +581,9 @@ async fn execute_ingest(
            HttpResponse::Accepted().json(response)
        }
        Err(e) => {
-            tracing::error!("DB error: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_INGEST.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            tracing::error!(user_id = body.project.as_str(), "Unexpected DB error during ingest: {}", e);
            HttpResponse::InternalServerError().json(json!({"error": "database_error"}))
        }
    }
@@ -768,8 +840,13 @@ async fn store_compacted_memory(
    .await;

    match result {
-        Ok(_) => true,
+        Ok(_) => {
+            crate::metrics::WRITE_CHUNKS_TOTAL.inc();
+            crate::metrics::WRITE_BYTES_TOTAL.inc_by(memory.len() as u64);
+            true
+        }
        Err(e) => {
+            crate::metrics::WRITE_ERRORS_TOTAL.inc();
            tracing::error!("Failed to store compacted memory: {}", e);
            false
        }
@@ -830,7 +907,9 @@ pub async fn query_handler(
    match query_temporal_graph(&state, &params).await {
        Ok(response) => HttpResponse::Ok().json(response),
        Err(e) => {
-            tracing::error!("Temporal graph query failed: {}", e);
+            crate::metrics::ERROR_UNEXPECTED_QUERY.inc();
+            crate::metrics::ERROR_UNEXPECTED_TOTAL.inc();
+            tracing::error!(user_id = claims.sub.as_str(), "Unexpected error: temporal graph query failed: {}", e);
            HttpResponse::InternalServerError().json(json!({"error": "query_failed", "reason": e.to_string()}))
        }
    }
@@ -966,13 +1045,23 @@ pub async fn context_handler(
    body: web::Json<crate::context_endpoint::ContextRequest>,
    state: web::Data<AppState>,
 ) -> HttpResponse {
+    use crate::metrics::*;
+    CONTEXT_REQUESTS_TOTAL.inc();
+    let _timer = Timer::new(&CONTEXT_DURATION);
+
    let (claims, _token) = match validate_auth(&req, &state).await {
        Ok(c) => c,
-        Err(e) => return e,
+        Err(e) => {
+            CONTEXT_ERRORS_TOTAL.inc();
+            ERROR_AUTH_FAILURE_CONTEXT.inc();
+            return e;
+        }
    };

-    // Check read capability
+    let user_id = &claims.sub;
    if !has_capability(&claims, "memory:read") {
+        CONTEXT_ERRORS_TOTAL.inc();
+        ERROR_FORBIDDEN_CONTEXT.inc();
        return HttpResponse::Forbidden().json(json!({
            "error": "forbidden",
            "reason": "missing capability: memory:read"
@@ -997,9 +1086,14 @@ pub async fn context_handler(
                skills = response.skills.len(),
                "context lookup successful"
            );
+            // O3: Track tier hits
+            let total = response.lessons.len() + response.skills.len();
+            if total == 0 { CONTEXT_EMPTY_RESULTS.inc(); }
            HttpResponse::Ok().json(response)
        }
        Err(e) => {
+            CONTEXT_ERRORS_TOTAL.inc();
+            ERROR_LOOKUP_FAILURE_CONTEXT.inc();
            tracing::error!("context lookup error: {}", e);
            HttpResponse::BadRequest().json(json!({
                "error": "lookup_failed",
@@ -1,6 +1,9 @@
 pub mod endpoints;
 pub mod handlers;
 pub mod http_server;
+pub mod metrics;
+pub mod metrics_snapshot;
+pub mod relevance_judge;
 pub mod query;
 pub mod auth;
 pub mod ingest_worker;
@@ -0,0 +1,686 @@
+//! Prometheus metrics module (O10)
+//!
+//! Centralized metrics registry for poimen-memory observability.
+//! All handlers instrument via these shared metrics.
+//! Exposed at GET /metrics in Prometheus text format.
+
+use once_cell::sync::Lazy;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::time::Instant;
+
+// ─── Metric Types ───────────────────────────────────────────
+
+/// Simple counter (monotonically increasing)
+pub struct Counter {
+    value: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl Counter {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { value: AtomicU64::new(0), name, help }
+    }
+    pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
+    pub fn inc_by(&self, n: u64) { self.value.fetch_add(n, Ordering::Relaxed); }
+    pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
+}
+
+/// Gauge (can go up and down)
+pub struct Gauge {
+    value: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl Gauge {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { value: AtomicU64::new(0), name, help }
+    }
+    pub fn set(&self, v: u64) { self.value.store(v, Ordering::Relaxed); }
+    pub fn inc(&self) { self.value.fetch_add(1, Ordering::Relaxed); }
+    pub fn dec(&self) { self.value.fetch_sub(1, Ordering::Relaxed); }
+    pub fn get(&self) -> u64 { self.value.load(Ordering::Relaxed) }
+}
+
+/// Gauge for f64 values (stored as bits)
+pub struct GaugeF64 {
+    bits: AtomicU64,
+    name: &'static str,
+    help: &'static str,
+}
+
+impl GaugeF64 {
+    pub const fn new(name: &'static str, help: &'static str) -> Self {
+        Self { bits: AtomicU64::new(0), name, help }
+    }
+    pub fn set(&self, v: f64) { self.bits.store(v.to_bits(), Ordering::Relaxed); }
+    pub fn get(&self) -> f64 { f64::from_bits(self.bits.load(Ordering::Relaxed)) }
+}
+
+/// Histogram with fixed buckets for latency tracking
+pub struct Histogram {
+    pub buckets: &'static [f64],
+    pub counts: Vec<AtomicU64>,
+    pub sum: AtomicU64, // stored as f64 bits
+    pub count: AtomicU64,
+    pub name: &'static str,
+    pub help: &'static str,
+}
+
+impl Histogram {
+    pub fn new(name: &'static str, help: &'static str, buckets: &'static [f64]) -> Self {
+        let counts = (0..buckets.len() + 1).map(|_| AtomicU64::new(0)).collect();
+        Self {
+            buckets, counts, name, help,
+            sum: AtomicU64::new(0f64.to_bits()),
+            count: AtomicU64::new(0),
+        }
+    }
+
+    pub fn observe(&self, value: f64) {
+        self.count.fetch_add(1, Ordering::Relaxed);
+        // Add to sum (CAS loop for f64)
+        loop {
+            let old_bits = self.sum.load(Ordering::Relaxed);
+            let old = f64::from_bits(old_bits);
+            let new = old + value;
+            if self.sum.compare_exchange(old_bits, new.to_bits(), Ordering::Relaxed, Ordering::Relaxed).is_ok() {
+                break;
+            }
+        }
+        // Increment bucket counters
+        for (i, &bound) in self.buckets.iter().enumerate() {
+            if value <= bound {
+                self.counts[i].fetch_add(1, Ordering::Relaxed);
+            }
+        }
+        // +Inf bucket
+        self.counts[self.buckets.len()].fetch_add(1, Ordering::Relaxed);
+    }
+}
+
+/// Labeled counter (key = label combination string)
+pub struct LabeledCounter {
+    values: Mutex<HashMap<String, u64>>,
+    name: &'static str,
+    help: &'static str,
+    label_names: &'static [&'static str],
+}
+
+impl LabeledCounter {
+    pub fn new(name: &'static str, help: &'static str, label_names: &'static [&'static str]) -> Self {
+        Self { values: Mutex::new(HashMap::new()), name, help, label_names }
+    }
+    pub fn inc(&self, labels: &[&str]) {
+        let key = labels.join(",");
+        let mut map = self.values.lock().unwrap();
+        *map.entry(key).or_insert(0) += 1;
+    }
+}
+
+// ─── Timer helper ───────────────────────────────────────────
+
+/// RAII timer: observes duration on drop
+pub struct Timer<'a> {
+    histogram: &'a Histogram,
+    start: Instant,
+}
+
+impl<'a> Timer<'a> {
+    pub fn new(histogram: &'a Histogram) -> Self {
+        Self { histogram, start: Instant::now() }
+    }
+}
+
+impl<'a> Drop for Timer<'a> {
+    fn drop(&mut self) {
+        let elapsed = self.start.elapsed().as_secs_f64();
+        self.histogram.observe(elapsed);
+    }
+}
+
+// ─── Default buckets ────────────────────────────────────────
+
+/// Latency buckets for HTTP handlers (seconds)
+pub static HTTP_BUCKETS: &[f64] = &[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0];
+/// Latency buckets for LLM calls (seconds)
+pub static LLM_BUCKETS: &[f64] = &[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0];
+/// Latency buckets for DB queries (seconds)
+pub static DB_BUCKETS: &[f64] = &[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0];
+
+// ═══════════════════════════════════════════════════════════
+// O1: Ingest handler metrics (I1-I12)
+// ═══════════════════════════════════════════════════════════
+
+pub static INGEST_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_ingest_requests_total", "Total ingest requests received");
+pub static INGEST_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_ingest_errors_total", "Total ingest request errors");
+pub static INGEST_RECORDS_TOTAL: Counter = Counter::new(
+    "memory_ingest_records_total", "Total records ingested");
+pub static INGEST_ENTITIES_EXTRACTED: Counter = Counter::new(
+    "memory_ingest_entities_extracted_total", "Total entities extracted during ingest");
+pub static INGEST_EDGES_EXTRACTED: Counter = Counter::new(
+    "memory_ingest_edges_extracted_total", "Total edges extracted during ingest");
+pub static INGEST_IN_FLIGHT: Gauge = Gauge::new(
+    "memory_ingest_in_flight", "Currently processing ingest jobs");
+pub static INGEST_QUEUE_SIZE: Gauge = Gauge::new(
+    "memory_ingest_queue_size", "Number of jobs waiting in ingest queue");
+pub static INGEST_DUPLICATES_TOTAL: Counter = Counter::new(
+    "memory_ingest_duplicates_total", "Total duplicate ingest requests (idempotency)");
+pub static INGEST_BYTES_TOTAL: Counter = Counter::new(
+    "memory_ingest_bytes_total", "Total bytes ingested");
+pub static INGEST_AUTH_FAILURES: Counter = Counter::new(
+    "memory_ingest_auth_failures_total", "Total auth failures on ingest endpoint");
+pub static INGEST_RATE_LIMITED: Counter = Counter::new(
+    "memory_ingest_rate_limited_total", "Total rate-limited ingest requests");
+
+pub static INGEST_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_duration_seconds", "Ingest request duration", HTTP_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O2: Query handler metrics (Q1-Q12)
+// ═══════════════════════════════════════════════════════════
+
+pub static QUERY_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_query_requests_total", "Total query requests received");
+pub static QUERY_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_query_errors_total", "Total query request errors");
+pub static QUERY_RESULTS_TOTAL: Counter = Counter::new(
+    "memory_query_results_total", "Total results returned across all queries");
+pub static QUERY_EMPTY_RESULTS: Counter = Counter::new(
+    "memory_query_empty_results_total", "Queries returning zero results");
+pub static QUERY_EMBEDDING_FAILURES: Counter = Counter::new(
+    "memory_query_embedding_failures_total", "Total embedding failures during query");
+pub static QUERY_IN_FLIGHT: Gauge = Gauge::new(
+    "memory_query_in_flight", "Currently processing queries");
+pub static QUERY_AUTH_FAILURES: Counter = Counter::new(
+    "memory_query_auth_failures_total", "Total auth failures on query endpoint");
+pub static QUERY_RATE_LIMITED: Counter = Counter::new(
+    "memory_query_rate_limited_total", "Total rate-limited query requests");
+pub static QUERY_CACHE_HITS: Counter = Counter::new(
+    "memory_query_cache_hits_total", "Total query cache hits");
+pub static QUERY_CACHE_MISSES: Counter = Counter::new(
+    "memory_query_cache_misses_total", "Total query cache misses");
+
+pub static QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_query_duration_seconds", "Query request duration", HTTP_BUCKETS));
+pub static QUERY_EMBEDDING_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_query_embedding_duration_seconds", "Embedding call duration during query", LLM_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O3: Context endpoint metrics (C1-C8)
+// ═══════════════════════════════════════════════════════════
+
+pub static CONTEXT_REQUESTS_TOTAL: Counter = Counter::new(
+    "memory_context_requests_total", "Total context retrieval requests");
+pub static CONTEXT_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_context_errors_total", "Total context retrieval errors");
+pub static CONTEXT_SEMANTIC_HITS: Counter = Counter::new(
+    "memory_context_semantic_hits_total", "Results from semantic (cosine) tier");
+pub static CONTEXT_BM25_HITS: Counter = Counter::new(
+    "memory_context_bm25_hits_total", "Results from BM25 (lexical) tier");
+pub static CONTEXT_GRAPH_HITS: Counter = Counter::new(
+    "memory_context_graph_hits_total", "Results from graph traversal tier");
+pub static CONTEXT_EMPTY_RESULTS: Counter = Counter::new(
+    "memory_context_empty_results_total", "Context requests returning zero results");
+
+pub static CONTEXT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_context_duration_seconds", "Context retrieval duration", HTTP_BUCKETS));
+pub static CONTEXT_TIER_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_context_tier_duration_seconds", "Per-tier retrieval duration", DB_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O4: Relevance judge metrics (R1-R9)
+// ═══════════════════════════════════════════════════════════
+
+pub static RELEVANCE_EVALS_TOTAL: Counter = Counter::new(
+    "memory_relevance_evals_total", "Total relevance evaluations performed");
+pub static RELEVANCE_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_relevance_errors_total", "Total relevance evaluation errors");
+pub static RELEVANCE_RELEVANT_TOTAL: Counter = Counter::new(
+    "memory_relevance_relevant_total", "Results judged relevant");
+pub static RELEVANCE_IRRELEVANT_TOTAL: Counter = Counter::new(
+    "memory_relevance_irrelevant_total", "Results judged irrelevant");
+
+pub static RELEVANCE_SCORE: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_relevance_score", "Distribution of relevance scores",
+        &[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]));
+pub static RELEVANCE_PRECISION: GaugeF64 = GaugeF64::new(
+    "memory_relevance_precision", "Current precision (relevant/retrieved)");
+pub static RELEVANCE_RECALL: GaugeF64 = GaugeF64::new(
+    "memory_relevance_recall", "Current recall (relevant/total_relevant)");
+pub static RELEVANCE_F1: GaugeF64 = GaugeF64::new(
+    "memory_relevance_f1_score", "Current F1 score");
+pub static RELEVANCE_EVAL_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_relevance_eval_duration_seconds", "Relevance evaluation duration", LLM_BUCKETS));
+
+// ═══════════════════════════════════════════════════════════
+// O5: Write volume and storage metrics (W1-W12)
+// ═══════════════════════════════════════════════════════════
+
+pub static WRITE_ENTITIES_TOTAL: Counter = Counter::new(
+    "memory_write_entities_total", "Total entities written to DB");
+pub static WRITE_EDGES_TOTAL: Counter = Counter::new(
+    "memory_write_edges_total", "Total edges written to DB");
+pub static WRITE_CHUNKS_TOTAL: Counter = Counter::new(
+    "memory_write_chunks_total", "Total chunks written to DB");
+pub static WRITE_ERRORS_TOTAL: Counter = Counter::new(
+    "memory_write_errors_total", "Total write errors");
+pub static WRITE_BYTES_TOTAL: Counter = Counter::new(
+    "memory_write_bytes_total", "Total bytes written to storage");
+
+pub static DB_ENTITY_COUNT: Gauge = Gauge::new(
+    "memory_db_entity_count", "Current entity count in memory_entity table");
+pub static DB_EDGE_COUNT: Gauge = Gauge::new(
+    "memory_db_edge_count", "Current edge count in memory_edge table");
+pub static DB_CHUNK_COUNT: Gauge = Gauge::new(
+    "memory_db_chunk_count", "Current chunk count in memory_chunks table");
+
+pub static WRITE_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_write_duration_seconds", "Write operation duration", DB_BUCKETS));
+pub static WRITE_BATCH_SIZE: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_write_batch_size", "Write batch sizes",
+        &[1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]));
+
+// Storage gauges (updated periodically)
+pub static DB_SIZE_BYTES: Gauge = Gauge::new(
+    "memory_db_size_bytes", "Total database size in bytes");
+pub static DB_INDEX_SIZE_BYTES: Gauge = Gauge::new(
+    "memory_db_index_size_bytes", "Total index size in bytes");
+
+// ═══════════════════════════════════════════════════════════
+// O6: Pod resource observability (P1-P13)
+// (Most collected by node-exporter/cAdvisor, but we track app-level)
+// ═══════════════════════════════════════════════════════════
+
+pub static APP_UPTIME_SECONDS: Gauge = Gauge::new(
+    "memory_app_uptime_seconds", "Application uptime in seconds");
+pub static APP_ACTIVE_CONNECTIONS: Gauge = Gauge::new(
+    "memory_app_active_connections", "Active HTTP connections");
+pub static APP_GOROUTINES: Gauge = Gauge::new(
+    "memory_app_tokio_tasks", "Active tokio tasks (approximate)");
+pub static APP_HEAP_BYTES: Gauge = Gauge::new(
+    "memory_app_heap_bytes", "Approximate heap memory usage");
+
+// ═══════════════════════════════════════════════════════════
+// O7: Availability metrics and dependency health (A1-A10)
+// ═══════════════════════════════════════════════════════════
+
+pub static HEALTH_CHECKS_TOTAL: Counter = Counter::new(
+    "memory_health_checks_total", "Total health check requests");
+pub static HEALTH_CHECK_FAILURES: Counter = Counter::new(
+    "memory_health_check_failures_total", "Total health check failures");
+
+pub static DEP_DB_UP: Gauge = Gauge::new(
+    "memory_dependency_db_up", "Database dependency health (1=up, 0=down)");
+pub static DEP_EMBEDDING_UP: Gauge = Gauge::new(
+    "memory_dependency_embedding_up", "Embedding service health (1=up, 0=down)");
+pub static DEP_OPENSEARCH_UP: Gauge = Gauge::new(
+    "memory_dependency_opensearch_up", "OpenSearch dependency health (1=up, 0=down)");
+pub static DEP_LLM_UP: Gauge = Gauge::new(
+    "memory_dependency_llm_up", "LLM service health (1=up, 0=down)");
+
+pub static DEP_DB_LATENCY: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_dependency_db_latency_seconds", "DB health check latency", DB_BUCKETS));
+pub static DEP_EMBEDDING_LATENCY: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_dependency_embedding_latency_seconds", "Embedding health check latency", LLM_BUCKETS));
+
+pub static REQUEST_ERRORS_BY_STATUS: Lazy<LabeledCounter> = Lazy::new(||
+    LabeledCounter::new(
+        "memory_request_errors_by_status", "Request errors by HTTP status code",
+        &["status", "endpoint"]));
+
+// ═══════════════════════════════════════════════════════════
+// Named error counters (per error type, per endpoint)
+// Format: memory_error_{ERROR_NAME}_{ENDPOINT}_total
+// ═══════════════════════════════════════════════════════════
+
+// Ingest errors
+pub static ERROR_AUTH_FAILURE_INGEST: Counter = Counter::new(
+    "memory_error_auth_failure_ingest_total", "Auth failures on ingest endpoint");
+pub static ERROR_FORBIDDEN_INGEST: Counter = Counter::new(
+    "memory_error_forbidden_ingest_total", "Forbidden (missing capability) on ingest");
+pub static ERROR_RATE_LIMITED_INGEST: Counter = Counter::new(
+    "memory_error_rate_limited_ingest_total", "Rate limited on ingest");
+pub static ERROR_BAD_REQUEST_INGEST: Counter = Counter::new(
+    "memory_error_bad_request_ingest_total", "Bad request on ingest");
+pub static ERROR_DB_ERROR_INGEST: Counter = Counter::new(
+    "memory_error_db_error_ingest_total", "Database error during ingest");
+
+// Query errors
+pub static ERROR_AUTH_FAILURE_QUERY: Counter = Counter::new(
+    "memory_error_auth_failure_query_total", "Auth failures on query endpoint");
+pub static ERROR_FORBIDDEN_QUERY: Counter = Counter::new(
+    "memory_error_forbidden_query_total", "Forbidden (missing capability) on query");
+pub static ERROR_BAD_REQUEST_QUERY: Counter = Counter::new(
+    "memory_error_bad_request_query_total", "Bad request on query");
+pub static ERROR_EMBEDDING_FAILURE_QUERY: Counter = Counter::new(
+    "memory_error_embedding_failure_query_total", "Embedding service failure during query");
+pub static ERROR_SEARCH_FAILURE_QUERY: Counter = Counter::new(
+    "memory_error_search_failure_query_total", "Search execution failure during query");
+
+// Context errors
+pub static ERROR_AUTH_FAILURE_CONTEXT: Counter = Counter::new(
+    "memory_error_auth_failure_context_total", "Auth failures on context endpoint");
+pub static ERROR_FORBIDDEN_CONTEXT: Counter = Counter::new(
+    "memory_error_forbidden_context_total", "Forbidden (missing capability) on context");
+pub static ERROR_LOOKUP_FAILURE_CONTEXT: Counter = Counter::new(
+    "memory_error_lookup_failure_context_total", "Context lookup failure");
+
+// Unexpected errors (unhandled 500s, panics, unknown failures)
+pub static ERROR_UNEXPECTED_TOTAL: Counter = Counter::new(
+    "memory_error_unexpected_total", "Total unexpected/unhandled errors (500s)");
+pub static ERROR_UNEXPECTED_INGEST: Counter = Counter::new(
+    "memory_error_unexpected_ingest_total", "Unexpected errors during ingest");
+pub static ERROR_UNEXPECTED_QUERY: Counter = Counter::new(
+    "memory_error_unexpected_query_total", "Unexpected errors during query");
+pub static ERROR_UNEXPECTED_CONTEXT: Counter = Counter::new(
+    "memory_error_unexpected_context_total", "Unexpected errors during context");
+
+// Last error info (most recent error for debugging)
+pub static LAST_ERROR_TIMESTAMP: Gauge = Gauge::new(
+    "memory_last_error_timestamp_seconds", "Unix timestamp of most recent error");
+
+// ═══════════════════════════════════════════════════════════
+// O8: Ingest rate pattern tracking (IR1-IR10)
+// ═══════════════════════════════════════════════════════════
+
+pub static INGEST_RATE_1M: GaugeF64 = GaugeF64::new(
+    "memory_ingest_rate_1m", "Ingest rate per second (1-minute window)");
+pub static INGEST_RATE_5M: GaugeF64 = GaugeF64::new(
+    "memory_ingest_rate_5m", "Ingest rate per second (5-minute window)");
+pub static INGEST_LLM_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_llm_extract_duration_seconds", "LLM entity extraction duration", LLM_BUCKETS));
+pub static INGEST_FACT_EXTRACT_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_ingest_fact_extract_duration_seconds", "LLM fact extraction duration", LLM_BUCKETS));
+pub static INGEST_DEDUP_TOTAL: Counter = Counter::new(
+    "memory_ingest_dedup_total", "Total entities deduplicated");
+pub static INGEST_CONTRADICTION_TOTAL: Counter = Counter::new(
+    "memory_ingest_contradiction_total", "Total contradictions detected");
+pub static INGEST_PROJECTS: Gauge = Gauge::new(
+    "memory_ingest_active_projects", "Number of active projects with ingested data");
+
+// ═══════════════════════════════════════════════════════════
+// O9: Postgres internal observability (PG1-PG33)
+// (Most collected by pg_exporter, we expose app-visible DB stats)
+// ═══════════════════════════════════════════════════════════
+
+pub static DB_POOL_SIZE: Gauge = Gauge::new(
+    "memory_db_pool_size", "Current connection pool size");
+pub static DB_POOL_IDLE: Gauge = Gauge::new(
+    "memory_db_pool_idle", "Idle connections in pool");
+pub static DB_POOL_ACTIVE: Gauge = Gauge::new(
+    "memory_db_pool_active", "Active connections in pool");
+pub static DB_QUERY_TOTAL: Counter = Counter::new(
+    "memory_db_queries_total", "Total DB queries executed");
+pub static DB_QUERY_ERRORS: Counter = Counter::new(
+    "memory_db_query_errors_total", "Total DB query errors");
+pub static DB_QUERY_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_db_query_duration_seconds", "DB query duration", DB_BUCKETS));
+pub static DB_TRANSACTION_DURATION: Lazy<Histogram> = Lazy::new(||
+    Histogram::new("memory_db_transaction_duration_seconds", "DB transaction duration", DB_BUCKETS));
+
+// Table-specific row counts (updated periodically)
+pub static DB_TABLE_ENTITY_ROWS: Gauge = Gauge::new(
+    "memory_db_table_entity_rows", "Rows in memory_entity table");
+pub static DB_TABLE_EDGE_ROWS: Gauge = Gauge::new(
+    "memory_db_table_edge_rows", "Rows in memory_edge table");
+pub static DB_TABLE_CHUNK_ROWS: Gauge = Gauge::new(
+    "memory_db_table_chunk_rows", "Rows in memory_chunks table");
+
+// ═══════════════════════════════════════════════════════════
+// Metrics export (Prometheus text format)
+// ═══════════════════════════════════════════════════════════
+
+/// Render all metrics in Prometheus text exposition format
+pub fn render_metrics() -> String {
+    let mut out = String::with_capacity(8192);
+
+    // Helper macros
+    macro_rules! counter {
+        ($c:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n{} {}\n",
+                $c.name, $c.help, $c.name, $c.name, $c.get()));
+        };
+    }
+    macro_rules! gauge {
+        ($g:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {}\n",
+                $g.name, $g.help, $g.name, $g.name, $g.get()));
+        };
+    }
+    macro_rules! gauge_f64 {
+        ($g:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} gauge\n{} {:.6}\n",
+                $g.name, $g.help, $g.name, $g.name, $g.get()));
+        };
+    }
+    macro_rules! histogram {
+        ($h:expr) => {
+            out.push_str(&format!("# HELP {} {}\n# TYPE {} histogram\n", $h.name, $h.help, $h.name));
+            for (i, &bound) in $h.buckets.iter().enumerate() {
+                out.push_str(&format!("{}_bucket{{le=\"{}\"}} {}\n",
+                    $h.name, bound, $h.counts[i].load(Ordering::Relaxed)));
+            }
+            out.push_str(&format!("{}_bucket{{le=\"+Inf\"}} {}\n",
+                $h.name, $h.counts[$h.buckets.len()].load(Ordering::Relaxed)));
+            out.push_str(&format!("{}_sum {:.6}\n", $h.name,
+                f64::from_bits($h.sum.load(Ordering::Relaxed))));
+            out.push_str(&format!("{}_count {}\n", $h.name,
+                $h.count.load(Ordering::Relaxed)));
+        };
+    }
+
+    // O1: Ingest
+    counter!(INGEST_REQUESTS_TOTAL);
+    counter!(INGEST_ERRORS_TOTAL);
+    counter!(INGEST_RECORDS_TOTAL);
+    counter!(INGEST_ENTITIES_EXTRACTED);
+    counter!(INGEST_EDGES_EXTRACTED);
+    gauge!(INGEST_IN_FLIGHT);
+    gauge!(INGEST_QUEUE_SIZE);
+    counter!(INGEST_DUPLICATES_TOTAL);
+    counter!(INGEST_BYTES_TOTAL);
+    counter!(INGEST_AUTH_FAILURES);
+    counter!(INGEST_RATE_LIMITED);
+    histogram!(INGEST_DURATION);
+
+    // O2: Query
+    counter!(QUERY_REQUESTS_TOTAL);
+    counter!(QUERY_ERRORS_TOTAL);
+    counter!(QUERY_RESULTS_TOTAL);
+    counter!(QUERY_EMPTY_RESULTS);
+    counter!(QUERY_EMBEDDING_FAILURES);
+    gauge!(QUERY_IN_FLIGHT);
+    counter!(QUERY_AUTH_FAILURES);
+    counter!(QUERY_RATE_LIMITED);
+    counter!(QUERY_CACHE_HITS);
+    counter!(QUERY_CACHE_MISSES);
+    histogram!(QUERY_DURATION);
+    histogram!(QUERY_EMBEDDING_DURATION);
+
+    // O3: Context
+    counter!(CONTEXT_REQUESTS_TOTAL);
+    counter!(CONTEXT_ERRORS_TOTAL);
+    counter!(CONTEXT_SEMANTIC_HITS);
+    counter!(CONTEXT_BM25_HITS);
+    counter!(CONTEXT_GRAPH_HITS);
+    counter!(CONTEXT_EMPTY_RESULTS);
+    histogram!(CONTEXT_DURATION);
+    histogram!(CONTEXT_TIER_DURATION);
+
+    // O4: Relevance
+    counter!(RELEVANCE_EVALS_TOTAL);
+    counter!(RELEVANCE_ERRORS_TOTAL);
+    counter!(RELEVANCE_RELEVANT_TOTAL);
+    counter!(RELEVANCE_IRRELEVANT_TOTAL);
+    histogram!(RELEVANCE_SCORE);
+    gauge_f64!(RELEVANCE_PRECISION);
+    gauge_f64!(RELEVANCE_RECALL);
+    gauge_f64!(RELEVANCE_F1);
+    histogram!(RELEVANCE_EVAL_DURATION);
+
+    // O5: Write volume
+    counter!(WRITE_ENTITIES_TOTAL);
+    counter!(WRITE_EDGES_TOTAL);
+    counter!(WRITE_CHUNKS_TOTAL);
+    counter!(WRITE_ERRORS_TOTAL);
+    counter!(WRITE_BYTES_TOTAL);
+    gauge!(DB_ENTITY_COUNT);
+    gauge!(DB_EDGE_COUNT);
+    gauge!(DB_CHUNK_COUNT);
+    histogram!(WRITE_DURATION);
+    histogram!(WRITE_BATCH_SIZE);
+    gauge!(DB_SIZE_BYTES);
+    gauge!(DB_INDEX_SIZE_BYTES);
+
+    // O6: Pod resources
+    gauge!(APP_UPTIME_SECONDS);
+    gauge!(APP_ACTIVE_CONNECTIONS);
+    gauge!(APP_GOROUTINES);
+    gauge!(APP_HEAP_BYTES);
+
+    // O7: Availability
+    counter!(HEALTH_CHECKS_TOTAL);
+    counter!(HEALTH_CHECK_FAILURES);
+    gauge!(DEP_DB_UP);
+    gauge!(DEP_EMBEDDING_UP);
+    gauge!(DEP_OPENSEARCH_UP);
+    gauge!(DEP_LLM_UP);
+    histogram!(DEP_DB_LATENCY);
+    histogram!(DEP_EMBEDDING_LATENCY);
+
+    // O8: Ingest rate
+    gauge_f64!(INGEST_RATE_1M);
+    gauge_f64!(INGEST_RATE_5M);
+    histogram!(INGEST_LLM_EXTRACT_DURATION);
+    histogram!(INGEST_FACT_EXTRACT_DURATION);
+    counter!(INGEST_DEDUP_TOTAL);
+    counter!(INGEST_CONTRADICTION_TOTAL);
+    gauge!(INGEST_PROJECTS);
+
+    // O9: Postgres
+    gauge!(DB_POOL_SIZE);
+    gauge!(DB_POOL_IDLE);
+    gauge!(DB_POOL_ACTIVE);
+    counter!(DB_QUERY_TOTAL);
+    counter!(DB_QUERY_ERRORS);
+    histogram!(DB_QUERY_DURATION);
+    histogram!(DB_TRANSACTION_DURATION);
+    gauge!(DB_TABLE_ENTITY_ROWS);
+    gauge!(DB_TABLE_EDGE_ROWS);
+    gauge!(DB_TABLE_CHUNK_ROWS);
+
+    // Named error counters
+    counter!(ERROR_AUTH_FAILURE_INGEST);
+    counter!(ERROR_FORBIDDEN_INGEST);
+    counter!(ERROR_RATE_LIMITED_INGEST);
+    counter!(ERROR_BAD_REQUEST_INGEST);
+    counter!(ERROR_DB_ERROR_INGEST);
+    counter!(ERROR_AUTH_FAILURE_QUERY);
+    counter!(ERROR_FORBIDDEN_QUERY);
+    counter!(ERROR_BAD_REQUEST_QUERY);
+    counter!(ERROR_EMBEDDING_FAILURE_QUERY);
+    counter!(ERROR_SEARCH_FAILURE_QUERY);
+    counter!(ERROR_AUTH_FAILURE_CONTEXT);
+    counter!(ERROR_FORBIDDEN_CONTEXT);
+    counter!(ERROR_LOOKUP_FAILURE_CONTEXT);
+    counter!(ERROR_UNEXPECTED_TOTAL);
+    counter!(ERROR_UNEXPECTED_INGEST);
+    counter!(ERROR_UNEXPECTED_QUERY);
+    counter!(ERROR_UNEXPECTED_CONTEXT);
+    gauge!(LAST_ERROR_TIMESTAMP);
+
+    out
+}
+
+/// Render a labeled counter in Prometheus format
+fn render_labeled_counter(out: &mut String, lc: &LabeledCounter) {
+    let map = lc.values.lock().unwrap();
+    if map.is_empty() { return; }
+    out.push_str(&format!("# HELP {} {}\n# TYPE {} counter\n", lc.name, lc.help, lc.name));
+    for (key, val) in map.iter() {
+        let parts: Vec<&str> = key.split(',').collect();
+        let labels: Vec<String> = lc.label_names.iter().zip(parts.iter())
+            .map(|(name, val)| format!("{}=\"{}\"", name, val))
+            .collect();
+        out.push_str(&format!("{}{{{}}} {}\n", lc.name, labels.join(","), val));
+    }
+}
+
+/// GET /metrics handler
+pub async fn metrics_handler() -> actix_web::HttpResponse {
+    actix_web::HttpResponse::Ok()
+        .content_type("text/plain; version=0.0.4; charset=utf-8")
+        .body(render_metrics())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_counter() {
+        let c = Counter::new("test_counter", "test");
+        assert_eq!(c.get(), 0);
+        c.inc();
+        assert_eq!(c.get(), 1);
+        c.inc_by(5);
+        assert_eq!(c.get(), 6);
+    }
+
+    #[test]
+    fn test_gauge() {
+        let g = Gauge::new("test_gauge", "test");
+        assert_eq!(g.get(), 0);
+        g.set(42);
+        assert_eq!(g.get(), 42);
+        g.inc();
+        assert_eq!(g.get(), 43);
+        g.dec();
+        assert_eq!(g.get(), 42);
+    }
+
+    #[test]
+    fn test_gauge_f64() {
+        let g = GaugeF64::new("test_gauge_f64", "test");
+        assert_eq!(g.get(), 0.0);
+        g.set(3.14);
+        assert!((g.get() - 3.14).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_histogram() {
+        let h = Histogram::new("test_hist", "test", &[0.1, 0.5, 1.0]);
+        h.observe(0.05);
+        h.observe(0.3);
+        h.observe(0.8);
+        h.observe(2.0);
+        assert_eq!(h.count.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    fn test_render_metrics_not_empty() {
+        INGEST_REQUESTS_TOTAL.inc();
+        QUERY_REQUESTS_TOTAL.inc();
+        let output = render_metrics();
+        assert!(output.contains("memory_ingest_requests_total"));
+        assert!(output.contains("memory_query_requests_total"));
+        assert!(output.contains("# HELP"));
+        assert!(output.contains("# TYPE"));
+    }
+
+    #[test]
+    fn test_timer_observes_on_drop() {
+        let h = Histogram::new("timer_test", "test", HTTP_BUCKETS);
+        {
+            let _t = Timer::new(&h);
+            std::thread::sleep(std::time::Duration::from_millis(1));
+        }
+        assert_eq!(h.count.load(Ordering::Relaxed), 1);
+    }
+}
@@ -0,0 +1,418 @@
+//! Metrics Snapshot & Assertion (Test Harness)
+//!
+//! Captures metric state before/after a test scenario,
+//! then asserts expected deltas per metric.
+//!
+//! Usage:
+//! ```rust
+//! let snap = MetricsSnapshot::capture();
+//! // ... run handler / scenario ...
+//! snap.assert_counter_inc("memory_ingest_requests_total", 1);
+//! snap.assert_counter_inc("memory_ingest_errors_total", 0);
+//! snap.assert_gauge_eq("memory_ingest_in_flight", 0);
+//! snap.assert_histogram_count_inc("memory_ingest_duration_seconds", 1);
+//! ```
+
+use std::collections::HashMap;
+use std::sync::atomic::Ordering;
+
+use crate::metrics;
+
+/// Snapshot of all metric values at a point in time
+#[derive(Debug, Clone)]
+pub struct MetricsSnapshot {
+    counters: HashMap<&'static str, u64>,
+    gauges: HashMap<&'static str, u64>,
+    gauges_f64: HashMap<&'static str, f64>,
+    histogram_counts: HashMap<&'static str, u64>,
+}
+
+impl MetricsSnapshot {
+    /// Capture current state of all metrics
+    pub fn capture() -> Self {
+        let mut counters = HashMap::new();
+        let mut gauges = HashMap::new();
+        let mut gauges_f64 = HashMap::new();
+        let mut histogram_counts = HashMap::new();
+
+        // O1: Ingest counters
+        counters.insert("memory_ingest_requests_total", metrics::INGEST_REQUESTS_TOTAL.get());
+        counters.insert("memory_ingest_errors_total", metrics::INGEST_ERRORS_TOTAL.get());
+        counters.insert("memory_ingest_records_total", metrics::INGEST_RECORDS_TOTAL.get());
+        counters.insert("memory_ingest_entities_extracted_total", metrics::INGEST_ENTITIES_EXTRACTED.get());
+        counters.insert("memory_ingest_edges_extracted_total", metrics::INGEST_EDGES_EXTRACTED.get());
+        counters.insert("memory_ingest_duplicates_total", metrics::INGEST_DUPLICATES_TOTAL.get());
+        counters.insert("memory_ingest_bytes_total", metrics::INGEST_BYTES_TOTAL.get());
+        counters.insert("memory_ingest_auth_failures_total", metrics::INGEST_AUTH_FAILURES.get());
+        counters.insert("memory_ingest_rate_limited_total", metrics::INGEST_RATE_LIMITED.get());
+
+        // O1: Ingest gauges
+        gauges.insert("memory_ingest_in_flight", metrics::INGEST_IN_FLIGHT.get());
+        gauges.insert("memory_ingest_queue_size", metrics::INGEST_QUEUE_SIZE.get());
+
+        // O1: Ingest histogram (force Lazy init)
+        histogram_counts.insert("memory_ingest_duration_seconds",
+            { let _ = &*metrics::INGEST_DURATION; metrics::INGEST_DURATION.count.load(Ordering::Relaxed) });
+
+        // O2: Query counters
+        counters.insert("memory_query_requests_total", metrics::QUERY_REQUESTS_TOTAL.get());
+        counters.insert("memory_query_errors_total", metrics::QUERY_ERRORS_TOTAL.get());
+        counters.insert("memory_query_results_total", metrics::QUERY_RESULTS_TOTAL.get());
+        counters.insert("memory_query_empty_results_total", metrics::QUERY_EMPTY_RESULTS.get());
+        counters.insert("memory_query_embedding_failures_total", metrics::QUERY_EMBEDDING_FAILURES.get());
+        counters.insert("memory_query_auth_failures_total", metrics::QUERY_AUTH_FAILURES.get());
+        counters.insert("memory_query_rate_limited_total", metrics::QUERY_RATE_LIMITED.get());
+        counters.insert("memory_query_cache_hits_total", metrics::QUERY_CACHE_HITS.get());
+        counters.insert("memory_query_cache_misses_total", metrics::QUERY_CACHE_MISSES.get());
+
+        // O2: Query gauges
+        gauges.insert("memory_query_in_flight", metrics::QUERY_IN_FLIGHT.get());
+
+        // O2: Query histograms
+        histogram_counts.insert("memory_query_duration_seconds",
+            { let _ = &*metrics::QUERY_DURATION; metrics::QUERY_DURATION.count.load(Ordering::Relaxed) });
+        histogram_counts.insert("memory_query_embedding_duration_seconds",
+            { let _ = &*metrics::QUERY_EMBEDDING_DURATION; metrics::QUERY_EMBEDDING_DURATION.count.load(Ordering::Relaxed) });
+
+        // O3: Context
+        counters.insert("memory_context_requests_total", metrics::CONTEXT_REQUESTS_TOTAL.get());
+        counters.insert("memory_context_errors_total", metrics::CONTEXT_ERRORS_TOTAL.get());
+        counters.insert("memory_context_semantic_hits_total", metrics::CONTEXT_SEMANTIC_HITS.get());
+        counters.insert("memory_context_bm25_hits_total", metrics::CONTEXT_BM25_HITS.get());
+        counters.insert("memory_context_graph_hits_total", metrics::CONTEXT_GRAPH_HITS.get());
+        counters.insert("memory_context_empty_results_total", metrics::CONTEXT_EMPTY_RESULTS.get());
+        histogram_counts.insert("memory_context_duration_seconds",
+            { let _ = &*metrics::CONTEXT_DURATION; metrics::CONTEXT_DURATION.count.load(Ordering::Relaxed) });
+
+        // O4: Relevance histograms
+        histogram_counts.insert("memory_relevance_eval_duration_seconds",
+            { let _ = &*metrics::RELEVANCE_EVAL_DURATION; metrics::RELEVANCE_EVAL_DURATION.count.load(Ordering::Relaxed) });
+
+        // O5: Write histogram
+        histogram_counts.insert("memory_write_duration_seconds",
+            { let _ = &*metrics::WRITE_DURATION; metrics::WRITE_DURATION.count.load(Ordering::Relaxed) });
+
+        // O7: Dependency latency
+        histogram_counts.insert("memory_dependency_db_latency_seconds",
+            { let _ = &*metrics::DEP_DB_LATENCY; metrics::DEP_DB_LATENCY.count.load(Ordering::Relaxed) });
+
+        // O4: Relevance
+        counters.insert("memory_relevance_evals_total", metrics::RELEVANCE_EVALS_TOTAL.get());
+        counters.insert("memory_relevance_errors_total", metrics::RELEVANCE_ERRORS_TOTAL.get());
+        counters.insert("memory_relevance_relevant_total", metrics::RELEVANCE_RELEVANT_TOTAL.get());
+        counters.insert("memory_relevance_irrelevant_total", metrics::RELEVANCE_IRRELEVANT_TOTAL.get());
+        gauges_f64.insert("memory_relevance_precision", metrics::RELEVANCE_PRECISION.get());
+        gauges_f64.insert("memory_relevance_recall", metrics::RELEVANCE_RECALL.get());
+        gauges_f64.insert("memory_relevance_f1_score", metrics::RELEVANCE_F1.get());
+
+        // O5: Write
+        counters.insert("memory_write_entities_total", metrics::WRITE_ENTITIES_TOTAL.get());
+        counters.insert("memory_write_edges_total", metrics::WRITE_EDGES_TOTAL.get());
+        counters.insert("memory_write_chunks_total", metrics::WRITE_CHUNKS_TOTAL.get());
+        counters.insert("memory_write_errors_total", metrics::WRITE_ERRORS_TOTAL.get());
+        counters.insert("memory_write_bytes_total", metrics::WRITE_BYTES_TOTAL.get());
+
+        // O7: Health
+        counters.insert("memory_health_checks_total", metrics::HEALTH_CHECKS_TOTAL.get());
+        counters.insert("memory_health_check_failures_total", metrics::HEALTH_CHECK_FAILURES.get());
+        gauges.insert("memory_dependency_db_up", metrics::DEP_DB_UP.get());
+        gauges.insert("memory_dependency_embedding_up", metrics::DEP_EMBEDDING_UP.get());
+
+        // O8: Ingest rate
+        counters.insert("memory_ingest_dedup_total", metrics::INGEST_DEDUP_TOTAL.get());
+        counters.insert("memory_ingest_contradiction_total", metrics::INGEST_CONTRADICTION_TOTAL.get());
+
+        // O9: DB
+        counters.insert("memory_db_queries_total", metrics::DB_QUERY_TOTAL.get());
+        counters.insert("memory_db_query_errors_total", metrics::DB_QUERY_ERRORS.get());
+
+        Self { counters, gauges, gauges_f64, histogram_counts }
+    }
+
+    /// Assert a counter increased by exactly `expected` since snapshot
+    pub fn assert_counter_inc(&self, name: &str, expected: u64) {
+        let before = self.counters.get(name)
+            .unwrap_or_else(|| panic!("Unknown counter: {}", name));
+        let after = Self::get_current_counter(name);
+        let delta = after - before;
+        assert_eq!(delta, expected,
+            "Counter {} expected +{} but got +{} (before={}, after={})",
+            name, expected, delta, before, after);
+    }
+
+    /// Assert a counter increased by at least `min` since snapshot
+    pub fn assert_counter_inc_at_least(&self, name: &str, min: u64) {
+        let before = self.counters.get(name)
+            .unwrap_or_else(|| panic!("Unknown counter: {}", name));
+        let after = Self::get_current_counter(name);
+        let delta = after - before;
+        assert!(delta >= min,
+            "Counter {} expected at least +{} but got +{} (before={}, after={})",
+            name, min, delta, before, after);
+    }
+
+    /// Assert a gauge equals exactly `expected`
+    pub fn assert_gauge_eq(&self, name: &str, expected: u64) {
+        let current = Self::get_current_gauge(name);
+        assert_eq!(current, expected,
+            "Gauge {} expected {} but got {}", name, expected, current);
+    }
+
+    /// Assert a histogram observation count increased by `expected`
+    pub fn assert_histogram_count_inc(&self, name: &str, expected: u64) {
+        let before = self.histogram_counts.get(name)
+            .unwrap_or_else(|| panic!("Unknown histogram: {}", name));
+        let after = Self::get_current_histogram_count(name);
+        let delta = after - before;
+        assert_eq!(delta, expected,
+            "Histogram {} count expected +{} but got +{} (before={}, after={})",
+            name, expected, delta, before, after);
+    }
+
+    /// Assert a f64 gauge is within tolerance
+    pub fn assert_gauge_f64_approx(&self, name: &str, expected: f64, tolerance: f64) {
+        let current = Self::get_current_gauge_f64(name);
+        assert!((current - expected).abs() <= tolerance,
+            "Gauge {} expected {:.4} (±{}) but got {:.4}",
+            name, expected, tolerance, current);
+    }
+
+    /// Get delta for a counter since snapshot
+    pub fn counter_delta(&self, name: &str) -> u64 {
+        let before = self.counters.get(name).copied().unwrap_or(0);
+        let after = Self::get_current_counter(name);
+        after - before
+    }
+
+    /// Print all deltas since snapshot (for debugging)
+    pub fn print_deltas(&self) {
+        println!("=== Metrics Deltas ===");
+        for (name, before) in &self.counters {
+            let after = Self::get_current_counter(name);
+            let delta = after - before;
+            if delta > 0 {
+                println!("  {} +{} ({} -> {})", name, delta, before, after);
+            }
+        }
+        for (name, before) in &self.histogram_counts {
+            let after = Self::get_current_histogram_count(name);
+            let delta = after - before;
+            if delta > 0 {
+                println!("  {} count +{}", name, delta);
+            }
+        }
+    }
+
+    // ─── Internal helpers ───────────────────────────────────
+
+    fn get_current_counter(name: &str) -> u64 {
+        match name {
+            "memory_ingest_requests_total" => metrics::INGEST_REQUESTS_TOTAL.get(),
+            "memory_ingest_errors_total" => metrics::INGEST_ERRORS_TOTAL.get(),
+            "memory_ingest_records_total" => metrics::INGEST_RECORDS_TOTAL.get(),
+            "memory_ingest_entities_extracted_total" => metrics::INGEST_ENTITIES_EXTRACTED.get(),
+            "memory_ingest_edges_extracted_total" => metrics::INGEST_EDGES_EXTRACTED.get(),
+            "memory_ingest_duplicates_total" => metrics::INGEST_DUPLICATES_TOTAL.get(),
+            "memory_ingest_bytes_total" => metrics::INGEST_BYTES_TOTAL.get(),
+            "memory_ingest_auth_failures_total" => metrics::INGEST_AUTH_FAILURES.get(),
+            "memory_ingest_rate_limited_total" => metrics::INGEST_RATE_LIMITED.get(),
+            "memory_query_requests_total" => metrics::QUERY_REQUESTS_TOTAL.get(),
+            "memory_query_errors_total" => metrics::QUERY_ERRORS_TOTAL.get(),
+            "memory_query_results_total" => metrics::QUERY_RESULTS_TOTAL.get(),
+            "memory_query_empty_results_total" => metrics::QUERY_EMPTY_RESULTS.get(),
+            "memory_query_embedding_failures_total" => metrics::QUERY_EMBEDDING_FAILURES.get(),
+            "memory_query_auth_failures_total" => metrics::QUERY_AUTH_FAILURES.get(),
+            "memory_query_rate_limited_total" => metrics::QUERY_RATE_LIMITED.get(),
+            "memory_query_cache_hits_total" => metrics::QUERY_CACHE_HITS.get(),
+            "memory_query_cache_misses_total" => metrics::QUERY_CACHE_MISSES.get(),
+            "memory_context_requests_total" => metrics::CONTEXT_REQUESTS_TOTAL.get(),
+            "memory_context_errors_total" => metrics::CONTEXT_ERRORS_TOTAL.get(),
+            "memory_context_semantic_hits_total" => metrics::CONTEXT_SEMANTIC_HITS.get(),
+            "memory_context_bm25_hits_total" => metrics::CONTEXT_BM25_HITS.get(),
+            "memory_context_graph_hits_total" => metrics::CONTEXT_GRAPH_HITS.get(),
+            "memory_context_empty_results_total" => metrics::CONTEXT_EMPTY_RESULTS.get(),
+            "memory_relevance_evals_total" => metrics::RELEVANCE_EVALS_TOTAL.get(),
+            "memory_relevance_errors_total" => metrics::RELEVANCE_ERRORS_TOTAL.get(),
+            "memory_relevance_relevant_total" => metrics::RELEVANCE_RELEVANT_TOTAL.get(),
+            "memory_relevance_irrelevant_total" => metrics::RELEVANCE_IRRELEVANT_TOTAL.get(),
+            "memory_write_entities_total" => metrics::WRITE_ENTITIES_TOTAL.get(),
+            "memory_write_edges_total" => metrics::WRITE_EDGES_TOTAL.get(),
+            "memory_write_chunks_total" => metrics::WRITE_CHUNKS_TOTAL.get(),
+            "memory_write_errors_total" => metrics::WRITE_ERRORS_TOTAL.get(),
+            "memory_write_bytes_total" => metrics::WRITE_BYTES_TOTAL.get(),
+            "memory_health_checks_total" => metrics::HEALTH_CHECKS_TOTAL.get(),
+            "memory_health_check_failures_total" => metrics::HEALTH_CHECK_FAILURES.get(),
+            "memory_ingest_dedup_total" => metrics::INGEST_DEDUP_TOTAL.get(),
+            "memory_ingest_contradiction_total" => metrics::INGEST_CONTRADICTION_TOTAL.get(),
+            "memory_db_queries_total" => metrics::DB_QUERY_TOTAL.get(),
+            "memory_db_query_errors_total" => metrics::DB_QUERY_ERRORS.get(),
+            _ => panic!("Unknown counter: {}", name),
+        }
+    }
+
+    fn get_current_gauge(name: &str) -> u64 {
+        match name {
+            "memory_ingest_in_flight" => metrics::INGEST_IN_FLIGHT.get(),
+            "memory_ingest_queue_size" => metrics::INGEST_QUEUE_SIZE.get(),
+            "memory_query_in_flight" => metrics::QUERY_IN_FLIGHT.get(),
+            "memory_dependency_db_up" => metrics::DEP_DB_UP.get(),
+            "memory_dependency_embedding_up" => metrics::DEP_EMBEDDING_UP.get(),
+            "memory_dependency_opensearch_up" => metrics::DEP_OPENSEARCH_UP.get(),
+            "memory_dependency_llm_up" => metrics::DEP_LLM_UP.get(),
+            "memory_app_uptime_seconds" => metrics::APP_UPTIME_SECONDS.get(),
+            "memory_db_pool_size" => metrics::DB_POOL_SIZE.get(),
+            "memory_db_pool_idle" => metrics::DB_POOL_IDLE.get(),
+            "memory_db_table_entity_rows" => metrics::DB_TABLE_ENTITY_ROWS.get(),
+            "memory_db_table_edge_rows" => metrics::DB_TABLE_EDGE_ROWS.get(),
+            "memory_db_table_chunk_rows" => metrics::DB_TABLE_CHUNK_ROWS.get(),
+            _ => panic!("Unknown gauge: {}", name),
+        }
+    }
+
+    fn get_current_gauge_f64(name: &str) -> f64 {
+        match name {
+            "memory_relevance_precision" => metrics::RELEVANCE_PRECISION.get(),
+            "memory_relevance_recall" => metrics::RELEVANCE_RECALL.get(),
+            "memory_relevance_f1_score" => metrics::RELEVANCE_F1.get(),
+            "memory_ingest_rate_1m" => metrics::INGEST_RATE_1M.get(),
+            "memory_ingest_rate_5m" => metrics::INGEST_RATE_5M.get(),
+            _ => panic!("Unknown gauge_f64: {}", name),
+        }
+    }
+
+    fn get_current_histogram_count(name: &str) -> u64 {
+        match name {
+            "memory_ingest_duration_seconds" =>
+                metrics::INGEST_DURATION.count.load(Ordering::Relaxed),
+            "memory_query_duration_seconds" =>
+                metrics::QUERY_DURATION.count.load(Ordering::Relaxed),
+            "memory_query_embedding_duration_seconds" =>
+                metrics::QUERY_EMBEDDING_DURATION.count.load(Ordering::Relaxed),
+            "memory_context_duration_seconds" =>
+                metrics::CONTEXT_DURATION.count.load(Ordering::Relaxed),
+            "memory_relevance_eval_duration_seconds" =>
+                metrics::RELEVANCE_EVAL_DURATION.count.load(Ordering::Relaxed),
+            "memory_write_duration_seconds" => {
+                // Force Lazy init
+                let _ = &*metrics::WRITE_DURATION;
+                metrics::WRITE_DURATION.count.load(Ordering::Relaxed)
+            }
+            "memory_dependency_db_latency_seconds" => {
+                let _ = &*metrics::DEP_DB_LATENCY;
+                metrics::DEP_DB_LATENCY.count.load(Ordering::Relaxed)
+            }
+            _ => panic!("Unknown histogram: {}", name),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::relevance_judge::RelevanceJudge;
+
+    #[test]
+    fn test_snapshot_captures_state() {
+        let snap = MetricsSnapshot::capture();
+        assert!(snap.counters.contains_key("memory_ingest_requests_total"));
+        assert!(snap.counters.contains_key("memory_query_requests_total"));
+        assert!(snap.gauges.contains_key("memory_ingest_in_flight"));
+        assert!(snap.histogram_counts.contains_key("memory_ingest_duration_seconds"));
+    }
+
+    #[test]
+    fn test_counter_delta_zero_when_no_change() {
+        let snap = MetricsSnapshot::capture();
+        snap.assert_counter_inc("memory_write_entities_total", 0);
+    }
+
+    #[test]
+    fn test_counter_tracks_increment() {
+        let snap = MetricsSnapshot::capture();
+        metrics::WRITE_ENTITIES_TOTAL.inc_by(3);
+        snap.assert_counter_inc("memory_write_entities_total", 3);
+    }
+
+    #[test]
+    fn test_counter_delta_method() {
+        let snap = MetricsSnapshot::capture();
+        metrics::WRITE_EDGES_TOTAL.inc_by(7);
+        assert_eq!(snap.counter_delta("memory_write_edges_total"), 7);
+    }
+
+    #[test]
+    fn test_histogram_count_tracks() {
+        let snap = MetricsSnapshot::capture();
+        metrics::WRITE_DURATION.observe(0.05);
+        metrics::WRITE_DURATION.observe(0.10);
+        snap.assert_histogram_count_inc("memory_write_duration_seconds", 2);
+    }
+
+    #[test]
+    fn test_relevance_scenario_metrics() {
+        let snap = MetricsSnapshot::capture();
+
+        let judge = RelevanceJudge::new(0.5);
+        let results = vec![
+            ("good result".to_string(), 0.9),
+            ("bad result".to_string(), 0.1),
+            ("ok result".to_string(), 0.6),
+        ];
+        let summary = judge.evaluate_batch("test query", &results);
+
+        // Verify metrics match scenario
+        snap.assert_counter_inc("memory_relevance_evals_total", 3);
+        snap.assert_counter_inc("memory_relevance_relevant_total", 2);  // 0.9 + 0.6
+        snap.assert_counter_inc("memory_relevance_irrelevant_total", 1); // 0.1
+
+        // Verify precision gauge
+        snap.assert_gauge_f64_approx("memory_relevance_precision", summary.precision, 0.01);
+
+        assert_eq!(summary.total, 3);
+        assert_eq!(summary.relevant, 2);
+    }
+
+    #[test]
+    fn test_ingest_counter_scenario() {
+        let snap = MetricsSnapshot::capture();
+
+        // Simulate ingest scenario
+        metrics::INGEST_REQUESTS_TOTAL.inc();
+        metrics::INGEST_RECORDS_TOTAL.inc_by(5);
+        metrics::INGEST_BYTES_TOTAL.inc_by(1024);
+        metrics::INGEST_ENTITIES_EXTRACTED.inc_by(3);
+        metrics::INGEST_EDGES_EXTRACTED.inc_by(2);
+
+        snap.assert_counter_inc("memory_ingest_requests_total", 1);
+        snap.assert_counter_inc("memory_ingest_records_total", 5);
+        snap.assert_counter_inc("memory_ingest_bytes_total", 1024);
+        snap.assert_counter_inc("memory_ingest_entities_extracted_total", 3);
+        snap.assert_counter_inc("memory_ingest_edges_extracted_total", 2);
+        snap.assert_counter_inc("memory_ingest_errors_total", 0);
+    }
+
+    #[test]
+    fn test_query_error_scenario() {
+        let snap = MetricsSnapshot::capture();
+
+        // Simulate query that fails at embedding
+        metrics::QUERY_REQUESTS_TOTAL.inc();
+        metrics::QUERY_IN_FLIGHT.inc();
+        metrics::QUERY_EMBEDDING_FAILURES.inc();
+        metrics::QUERY_ERRORS_TOTAL.inc();
+        metrics::QUERY_IN_FLIGHT.dec();
+
+        snap.assert_counter_inc("memory_query_requests_total", 1);
+        snap.assert_counter_inc("memory_query_embedding_failures_total", 1);
+        snap.assert_counter_inc("memory_query_errors_total", 1);
+        snap.assert_counter_inc("memory_query_results_total", 0);
+        snap.assert_gauge_eq("memory_query_in_flight", 0);
+    }
+
+    #[test]
+    fn test_print_deltas_works() {
+        let snap = MetricsSnapshot::capture();
+        metrics::HEALTH_CHECKS_TOTAL.inc();
+        snap.print_deltas(); // Should not panic
+    }
+}
@@ -0,0 +1,161 @@
+//! Relevance Judge (O4)
+//!
+//! Evaluates retrieval quality by scoring query-result relevance.
+//! Uses LLM (Qwen-7B or similar) to judge if retrieved results are relevant.
+//! Tracks precision, recall, F1 via Prometheus metrics.
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use tracing::{debug, error};
+
+use crate::metrics;
+
+/// Relevance evaluation result for a single query-result pair
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RelevanceResult {
+    pub query: String,
+    pub result_text: String,
+    pub score: f64,
+    pub relevant: bool,
+}
+
+/// Batch evaluation summary
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RelevanceSummary {
+    pub total: usize,
+    pub relevant: usize,
+    pub irrelevant: usize,
+    pub precision: f64,
+    pub recall: f64,
+    pub f1: f64,
+    pub avg_score: f64,
+}
+
+/// Simple relevance judge using cosine similarity threshold
+/// (LLM-based judge can be plugged in later via trait)
+pub struct RelevanceJudge {
+    threshold: f64,
+}
+
+impl RelevanceJudge {
+    pub fn new(threshold: f64) -> Self {
+        Self { threshold }
+    }
+
+    /// Evaluate a single query-result pair using similarity score
+    pub fn evaluate(&self, query: &str, result_text: &str, similarity: f64) -> RelevanceResult {
+        let start = std::time::Instant::now();
+        
+        metrics::RELEVANCE_EVALS_TOTAL.inc();
+        
+        let relevant = similarity >= self.threshold;
+        
+        if relevant {
+            metrics::RELEVANCE_RELEVANT_TOTAL.inc();
+        } else {
+            metrics::RELEVANCE_IRRELEVANT_TOTAL.inc();
+        }
+        
+        metrics::RELEVANCE_SCORE.observe(similarity);
+        metrics::RELEVANCE_EVAL_DURATION.observe(start.elapsed().as_secs_f64());
+        
+        debug!("Relevance eval: query='{}', score={:.3}, relevant={}",
+            &query[..query.len().min(50)], similarity, relevant);
+        
+        RelevanceResult {
+            query: query.to_string(),
+            result_text: result_text.to_string(),
+            score: similarity,
+            relevant,
+        }
+    }
+
+    /// Evaluate a batch of results and compute summary metrics
+    pub fn evaluate_batch(
+        &self,
+        query: &str,
+        results: &[(String, f64)], // (result_text, similarity_score)
+    ) -> RelevanceSummary {
+        let mut relevant_count = 0;
+        let mut total_score = 0.0;
+
+        for (text, score) in results {
+            let result = self.evaluate(query, text, *score);
+            if result.relevant {
+                relevant_count += 1;
+            }
+            total_score += score;
+        }
+
+        let total = results.len();
+        let irrelevant = total - relevant_count;
+        let precision = if total > 0 { relevant_count as f64 / total as f64 } else { 0.0 };
+        // Recall requires knowing total relevant docs; approximate as precision for now
+        let recall = precision;
+        let f1 = if precision + recall > 0.0 {
+            2.0 * precision * recall / (precision + recall)
+        } else {
+            0.0
+        };
+        let avg_score = if total > 0 { total_score / total as f64 } else { 0.0 };
+
+        // Update gauge metrics
+        metrics::RELEVANCE_PRECISION.set(precision);
+        metrics::RELEVANCE_RECALL.set(recall);
+        metrics::RELEVANCE_F1.set(f1);
+
+        RelevanceSummary {
+            total,
+            relevant: relevant_count,
+            irrelevant,
+            precision,
+            recall,
+            f1,
+            avg_score,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_relevance_judge_above_threshold() {
+        let judge = RelevanceJudge::new(0.5);
+        let result = judge.evaluate("test query", "test result", 0.8);
+        assert!(result.relevant);
+        assert!((result.score - 0.8).abs() < 0.001);
+    }
+
+    #[test]
+    fn test_relevance_judge_below_threshold() {
+        let judge = RelevanceJudge::new(0.5);
+        let result = judge.evaluate("test query", "test result", 0.3);
+        assert!(!result.relevant);
+    }
+
+    #[test]
+    fn test_relevance_batch() {
+        let judge = RelevanceJudge::new(0.5);
+        let results = vec![
+            ("relevant result".to_string(), 0.8),
+            ("somewhat relevant".to_string(), 0.6),
+            ("irrelevant".to_string(), 0.2),
+        ];
+        let summary = judge.evaluate_batch("test", &results);
+        assert_eq!(summary.total, 3);
+        assert_eq!(summary.relevant, 2);
+        assert_eq!(summary.irrelevant, 1);
+        assert!((summary.precision - 0.6667).abs() < 0.01);
+    }
+
+    #[test]
+    fn test_relevance_empty_batch() {
+        let judge = RelevanceJudge::new(0.5);
+        let summary = judge.evaluate_batch("test", &[]);
+        assert_eq!(summary.total, 0);
+        assert_eq!(summary.precision, 0.0);
+        assert_eq!(summary.f1, 0.0);
+    }
+}
@@ -166,8 +166,18 @@ impl EmbeddingsClient {
        }

        let resp = builder.json(&req).send().await?;
-        let _status = resp.status();
-        let body: EmbeddingResponse = resp.json().await?;
+        let status = resp.status();
+        let raw_body = resp.text().await?;
+        
+        if !status.is_success() {
+            tracing::error!("Embedding API returned {}: {}", status, &raw_body[..raw_body.len().min(500)]);
+            return Err(anyhow!("Embedding API returned {}: {}", status, &raw_body[..raw_body.len().min(200)]));
+        }
+        
+        let body: EmbeddingResponse = serde_json::from_str(&raw_body).map_err(|e| {
+            tracing::error!("Failed to parse embedding response: {}. Raw body: {}", e, &raw_body[..raw_body.len().min(500)]);
+            anyhow!("Failed to parse embedding response: {}. Raw: {}", e, &raw_body[..raw_body.len().min(200)])
+        })?;

        match body {
            EmbeddingResponse::Error { error } => {
@@ -202,4 +212,73 @@ mod tests {
        assert_eq!(BATCH_SIZE, 32);
        assert_eq!(EMBEDDINGS_DIM, 768);
    }
+
+    #[test]
+    fn test_parse_real_embedding_response() {
+        // Exact format returned by embeddings-predictor service
+        let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0}],"model":"nomic-ai/nomic-embed-text-v2-moe","usage":{"prompt_tokens":3,"total_tokens":3}}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data.len(), 1);
+                assert_eq!(data[0].embedding.len(), 3);
+                assert_eq!(data[0].index, 0);
+            }
+            EmbeddingResponse::Error { error } => panic!("parsed as error: {:?}", error),
+        }
+    }
+
+    #[test]
+    fn test_parse_embedding_error_response() {
+        let raw = r#"{"error":"model not found"}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Error { error } => {
+                assert_eq!(error.as_str().unwrap(), "model not found");
+            }
+            EmbeddingResponse::Success { .. } => panic!("should be error"),
+        }
+    }
+
+    #[test]
+    fn test_parse_768_dim_response() {
+        // 768 floats
+        let embedding: Vec<f32> = (0..768).map(|i| i as f32 * 0.001).collect();
+        let raw = format!(
+            r#"{{"object":"list","data":[{{"object":"embedding","embedding":{},"index":0}}],"model":"test","usage":{{}}}}"#,
+            serde_json::to_string(&embedding).unwrap()
+        );
+        let parsed: EmbeddingResponse = serde_json::from_str(&raw).expect("should parse 768-dim");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data[0].embedding.len(), 768);
+            }
+            _ => panic!("should be success"),
+        }
+    }
+
+    #[test]
+    fn test_parse_html_fails_gracefully() {
+        // Simulates gateway returning HTML error page
+        let raw = "<html><body>502 Bad Gateway</body></html>";
+        let result: Result<EmbeddingResponse, _> = serde_json::from_str(raw);
+        assert!(result.is_err(), "HTML should fail to parse as JSON");
+        let err_msg = result.unwrap_err().to_string();
+        assert!(err_msg.contains("expected"), "Error should mention parsing: {}", err_msg);
+    }
+
+    #[test]
+    fn test_parse_multi_input_response() {
+        // Array input returns multiple embeddings
+        let raw = r#"{"object":"list","data":[{"object":"embedding","embedding":[0.1,0.2,0.3],"index":0},{"object":"embedding","embedding":[0.4,0.5,0.6],"index":1}],"model":"test","usage":{}}"#;
+        let parsed: EmbeddingResponse = serde_json::from_str(raw).expect("should parse");
+        match parsed {
+            EmbeddingResponse::Success { data, .. } => {
+                assert_eq!(data.len(), 2);
+                assert_eq!(data[0].index, 0);
+                assert_eq!(data[1].index, 1);
+            }
+            _ => panic!("should be success"),
+        }
+    }
 }
@@ -0,0 +1,157 @@
+# Poimen Memory - Environment Configuration Guide
+
+All downstream service URIs are read from environment variables, sourced from ConfigMap.
+
+## How It Works
+
+1. **ConfigMap provides URIs**: `k8s/app/config.yaml` (production, SOPS-encrypted)
+2. **Deployment injects via envFrom**: `envFrom: configMapRef: poimen-memory-config`
+3. **Application reads from ENV**: Code parses `LLM_ENDPOINT`, `OPENSEARCH_HOST`, `AUTHENTIK_ISSUER`, etc.
+
+```yaml
+# deployment.yaml
+envFrom:
+  - configMapRef:
+      name: poimen-memory-config  # All vars injected as ENV
+```
+
+## Environment Variables
+
+### LLM Service (Entity & Fact Extraction)
+- `LLM_ENDPOINT` — full URL to chat/completions endpoint
+- `LLM_API_BASE` — base API URL (used for client initialization)
+- `LLM_MODEL` — model identifier (ornith:35b, qwen:7b, etc.)
+- `LLM_TIMEOUT_SECS` — timeout for LLM requests
+- `ENABLE_LLM_EXTRACTION` — enable/disable LLM extraction (true/false)
+
+### OpenSearch (Vector Store, BM25)
+- `OPENSEARCH_HOST` — hostname:port
+- `OPENSEARCH_SCHEME` — http or https
+- `OPENSEARCH_VERIFY_CERTS` — SSL certificate verification (true/false)
+
+### Authentik (OIDC)
+- `AUTHENTIK_ISSUER` — OIDC issuer URL
+- `AUTHENTIK_VERIFY_SSL` — SSL certificate verification (true/false)
+- `MEM_AUTH_MODE` — auth mode: jwt | apikey | none
+
+### Temporal (Workflow Orchestration - Future)
+- `TEMPORAL_ENDPOINT` — temporal frontend hostname:port
+- `TEMPORAL_NAMESPACE` — temporal namespace
+
+### API Gateway (Route Optimization - Future)
+- `GATEWAY_URL` — gateway base URL
+
+### Memory Service Config
+- `MEM_AUTH_MODE` — jwt | apikey | none
+- `MEM_RATE_LIMIT_INGEST` — ingest requests per second
+- `MEM_RATE_LIMIT_QUERY` — query requests per second
+- `MEM_EMBEDDING_BATCH_SIZE` — batch size for embeddings
+
+---
+
+## Deployment Scenarios
+
+### Production (SOPS-Encrypted ConfigMap)
+
+**File**: `k8s/app/config.yaml`
+
+Services use cluster-internal DNS:
+```yaml
+LLM_ENDPOINT: http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1/chat/completions
+OPENSEARCH_HOST: opensearch.poimen.svc.cluster.local:9200
+AUTHENTIK_ISSUER: https://authentik.auth.svc.cluster.local:9443/application/o/poimen/
+TEMPORAL_ENDPOINT: temporal-frontend.temporal.svc.cluster.local:7233
+GATEWAY_URL: http://api-gw.poimen.svc.cluster.local:8080
+MEM_AUTH_MODE: jwt
+```
+
+**Deploy**:
+```bash
+# SOPS auto-decrypts based on .sops.yaml age key
+kubectl apply -f k8s/app/config.yaml -k k8s/app/
+```
+
+### Local/Development (Plaintext ConfigMap)
+
+**File**: `k8s/app/config.local.yaml`
+
+Services via external URLs (ingress):
+```yaml
+LLM_ENDPOINT: https://api.riotpiao.com/v1/chat/completions
+OPENSEARCH_HOST: opensearch.riotpiao.com:443
+AUTHENTIK_ISSUER: https://authentik.riotpiao.com/application/o/poimen/
+TEMPORAL_ENDPOINT: temporal.riotpiao.com:443
+GATEWAY_URL: https://api.riotpiao.com
+MEM_AUTH_MODE: none
+```
+
+**Deploy** (override production config):
+```bash
+# Delete prod config, apply local
+kubectl delete configmap poimen-memory-config -n poimen
+kubectl apply -f k8s/app/config.local.yaml
+```
+
+---
+
+## Encrypting with SOPS
+
+Production `config.yaml` is encrypted with SOPS (Age-based).
+
+**Encrypt**:
+```bash
+sops -e k8s/app/config.yaml > k8s/app/config.yaml.enc
+mv k8s/app/config.yaml.enc k8s/app/config.yaml
+```
+
+**Decrypt for editing** (SOPS auto-handles with $EDITOR):
+```bash
+sops k8s/app/config.yaml
+```
+
+**View decrypted** (without editing):
+```bash
+sops -d k8s/app/config.yaml
+```
+
+**.sops.yaml** defines encryption key:
+```yaml
+creation_rules:
+  - path_regex: k8s/app/config.yaml
+    key_groups:
+      - age:
+          - <age-public-key>
+```
+
+---
+
+## Application Code Pattern
+
+Example: Application should read URIs from ENV at startup.
+
+```rust
+// Pseudocode
+let llm_endpoint = env::var("LLM_ENDPOINT")
+    .unwrap_or("http://localhost:11434/v1/chat/completions".to_string());
+let opensearch_host = env::var("OPENSEARCH_HOST")
+    .unwrap_or("localhost:9200".to_string());
+let auth_mode = env::var("MEM_AUTH_MODE")
+    .unwrap_or("none".to_string());
+
+// Initialize clients with these URIs
+let llm_client = LlmClient::new(llm_endpoint)?;
+let search_client = OpenSearchClient::new(opensearch_host)?;
+```
+
+---
+
+## Summary
+
+| Aspect | Production | Local |
+|--------|-----------|-------|
+| **Config File** | `config.yaml` | `config.local.yaml` |
+| **Encryption** | SOPS (Age) | Plaintext |
+| **Service URIs** | Cluster-internal DNS | External HTTPS |
+| **Auth Mode** | JWT (Authentik) | None (disabled) |
+| **Rate Limits** | 100/1000 | 1000/10000 |
+| **Deploy** | `kubectl apply -k k8s/app/` | `kubectl apply -f config.local.yaml` |
@@ -0,0 +1,47 @@
+# Local/Development configuration (plaintext, external URLs via ingress)
+# Use this instead of config.yaml for local testing
+# kubectl apply -f config.local.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: poimen-memory-config
+  namespace: poimen
+  labels:
+    app.kubernetes.io/name: poimen-memory
+    app.kubernetes.io/component: config
+data:
+  # Auth mode: jwt | apikey | none (disabled for local testing)
+  MEM_AUTH_MODE: "none"
+  
+  # Rate limiting (higher for testing)
+  MEM_RATE_LIMIT_INGEST: "1000"
+  MEM_RATE_LIMIT_QUERY: "10000"
+  MEM_IDEMPOTENCY_TTL_SECS: "86400"
+  
+  # Embeddings
+  MEM_EMBEDDING_BATCH_SIZE: "32"
+  
+  # Downstream services - external URLs via ingress
+  
+  # LLM Service (via api.riotpiao.com ingress)
+  LLM_ENDPOINT: "https://api.riotpiao.com/v1/chat/completions"
+  LLM_API_BASE: "https://api.riotpiao.com/v1"
+  LLM_MODEL: "qwen:7b"
+  LLM_TIMEOUT_SECS: "60"
+  ENABLE_LLM_EXTRACTION: "true"
+  
+  # OpenSearch (via ingress)
+  OPENSEARCH_HOST: "opensearch.riotpiao.com:443"
+  OPENSEARCH_SCHEME: "https"
+  OPENSEARCH_VERIFY_CERTS: "true"
+  
+  # Authentik (via ingress - optional for local)
+  AUTHENTIK_ISSUER: "https://authentik.riotpiao.com/application/o/poimen/"
+  AUTHENTIK_VERIFY_SSL: "true"
+  
+  # Temporal (via ingress)
+  TEMPORAL_ENDPOINT: "temporal.riotpiao.com:443"
+  TEMPORAL_NAMESPACE: "poimen"
+  
+  # API Gateway (via ingress)
+  GATEWAY_URL: "https://api.riotpiao.com"
@@ -1,5 +1,7 @@
-# Non-sensitive environment variables for poimen-memory
-# Change these without redeploying secrets.
+# Production environment configuration for poimen-memory
+# All services use cluster-internal DNS names
+# This file is encrypted with SOPS in production
+# For local dev, use plaintext version with external URLs
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -9,19 +11,39 @@ metadata:
    app.kubernetes.io/name: poimen-memory
    app.kubernetes.io/component: config
 data:
-  # Auth mode: jwt | apikey
-  MEM_AUTH_MODE: "none"
+  # Auth mode: jwt | apikey | none
+  MEM_AUTH_MODE: "jwt"
+  
  # Rate limiting
  MEM_RATE_LIMIT_INGEST: "100"
  MEM_RATE_LIMIT_QUERY: "1000"
  MEM_IDEMPOTENCY_TTL_SECS: "86400"
+  
  # Embeddings
  MEM_EMBEDDING_BATCH_SIZE: "32"
-  # OpenSearch
-  OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
-  # Obsidian
-  # LLM Configuration (for entity extraction)
-  LLM_ENDPOINT: "http://api-internal.riotpiao.com:8000/v1/chat/completions"
-  LLM_MODEL: "qwen:7b"
+  
+  # Downstream services - read by application from ENV
+  # Internal cluster DNS (prod) / external URLs (local)
+  
+  # LLM Service (entity extraction, fact extraction)
+  LLM_ENDPOINT: "http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1/chat/completions"
+  LLM_API_BASE: "http://reasoning-predictor.llm-serving.svc.cluster.local:8000/v1"
+  LLM_MODEL: "ornith:35b"
  LLM_TIMEOUT_SECS: "30"
  ENABLE_LLM_EXTRACTION: "true"
+  
+  # OpenSearch (vector store, BM25 retrieval)
+  OPENSEARCH_HOST: "opensearch.poimen.svc.cluster.local:9200"
+  OPENSEARCH_SCHEME: "http"
+  OPENSEARCH_VERIFY_CERTS: "false"
+  
+  # Authentik (OIDC provider)
+  AUTHENTIK_ISSUER: "https://authentik.auth.svc.cluster.local:9443/application/o/poimen/"
+  AUTHENTIK_VERIFY_SSL: "false"
+  
+  # Temporal (workflow orchestration - future)
+  TEMPORAL_ENDPOINT: "temporal-frontend.temporal.svc.cluster.local:7233"
+  TEMPORAL_NAMESPACE: "poimen"
+  
+  # API Gateway (external queue, route optimization - future)
+  GATEWAY_URL: "http://api-gw.poimen.svc.cluster.local:8080"
@@ -61,20 +61,12 @@ spec:
            - name: DATABASE_URL
              value: "postgresql://$(DATABASE_USER):$(DATABASE_PASSWORD)@$(DATABASE_HOST):$(DATABASE_PORT)/$(DATABASE_NAME)?sslmode=disable"

-            # LLM via api.riotpiao.com (Authentik JWT auth)
-            - name: LLM_ENDPOINT
-              value: "https://api.riotpiao.com/v1/chat/completions"
-            - name: LLM_API_BASE
-              value: "https://api.riotpiao.com/v1"
-            - name: LLM_MODEL
-              value: "ornith:35b"
-
+            # All downstream service URIs read from ConfigMap
+            # (LLM_ENDPOINT, LLM_API_BASE, LLM_MODEL, OPENSEARCH_HOST, etc.)
+            # These are injected via envFrom below
+            
            # Authentik service account (memory-agent-oidc secret)
-            - name: AUTHENTIK_ISSUER
-              valueFrom:
-                secretKeyRef:
-                  name: memory-agent-oidc
-                  key: ISSUER
+            # Only needed if MEM_AUTH_MODE=jwt in ConfigMap
            - name: AUTHENTIK_CLIENT_ID
              valueFrom:
                secretKeyRef:
@@ -102,6 +94,7 @@ spec:
            - name: MEM_HOME
              value: "/tmp"
          envFrom:
+            # ConfigMap with all service URIs (prod: encrypted, local: plaintext)
            - configMapRef:
                name: poimen-memory-config
          command: ["/app/mem"]
@@ -1,13 +1,11 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: poimen
+
 resources:
-  # vault-pvc.yaml removed — memory service uses pgvector, not local storage
  - deployment.yaml
  - service.yaml
-  - config.yaml
-  # obsidian.yaml retired — reference docs now via memory graph
-  # Legacy secret managed separately
-  # - secrets.yaml
+  - config.yaml        # Production config (SOPS-encrypted)
+
 generators:
  - secret-generator.yaml
@@ -0,0 +1,131 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "title": "Ingest Rate (req/s)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [
+        { "expr": "rate(memory_ingest_requests_total[5m])", "legendFormat": "ingest req/s" }
+      ]
+    },
+    {
+      "title": "Query Rate (req/s)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [
+        { "expr": "rate(memory_query_requests_total[5m])", "legendFormat": "query req/s" }
+      ]
+    },
+    {
+      "title": "Ingest Latency (p50/p95/p99)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [
+        { "expr": "histogram_quantile(0.5, rate(memory_ingest_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
+        { "expr": "histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m]))", "legendFormat": "p95" },
+        { "expr": "histogram_quantile(0.99, rate(memory_ingest_duration_seconds_bucket[5m]))", "legendFormat": "p99" }
+      ]
+    },
+    {
+      "title": "Query Latency (p50/p95/p99)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [
+        { "expr": "histogram_quantile(0.5, rate(memory_query_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
+        { "expr": "histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m]))", "legendFormat": "p95" },
+        { "expr": "histogram_quantile(0.99, rate(memory_query_duration_seconds_bucket[5m]))", "legendFormat": "p99" }
+      ]
+    },
+    {
+      "title": "Error Rates",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
+      "targets": [
+        { "expr": "rate(memory_ingest_errors_total[5m])", "legendFormat": "ingest errors" },
+        { "expr": "rate(memory_query_errors_total[5m])", "legendFormat": "query errors" },
+        { "expr": "rate(memory_query_embedding_failures_total[5m])", "legendFormat": "embedding failures" }
+      ]
+    },
+    {
+      "title": "Embedding Latency",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+      "targets": [
+        { "expr": "histogram_quantile(0.5, rate(memory_query_embedding_duration_seconds_bucket[5m]))", "legendFormat": "p50" },
+        { "expr": "histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m]))", "legendFormat": "p95" }
+      ]
+    },
+    {
+      "title": "DB Row Counts",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 12, "x": 0, "y": 24 },
+      "targets": [
+        { "expr": "memory_db_table_entity_rows", "legendFormat": "entities" },
+        { "expr": "memory_db_table_edge_rows", "legendFormat": "edges" },
+        { "expr": "memory_db_table_chunk_rows", "legendFormat": "chunks" }
+      ]
+    },
+    {
+      "title": "Dependency Health",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 12, "x": 12, "y": 24 },
+      "targets": [
+        { "expr": "memory_dependency_db_up", "legendFormat": "DB" },
+        { "expr": "memory_dependency_embedding_up", "legendFormat": "Embedding" },
+        { "expr": "memory_dependency_opensearch_up", "legendFormat": "OpenSearch" },
+        { "expr": "memory_dependency_llm_up", "legendFormat": "LLM" }
+      ]
+    },
+    {
+      "title": "DB Pool Stats",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
+      "targets": [
+        { "expr": "memory_db_pool_size", "legendFormat": "pool size" },
+        { "expr": "memory_db_pool_idle", "legendFormat": "idle" },
+        { "expr": "memory_db_pool_active", "legendFormat": "active" }
+      ]
+    },
+    {
+      "title": "Relevance Metrics",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
+      "targets": [
+        { "expr": "memory_relevance_precision", "legendFormat": "precision" },
+        { "expr": "memory_relevance_recall", "legendFormat": "recall" },
+        { "expr": "memory_relevance_f1_score", "legendFormat": "F1" }
+      ]
+    },
+    {
+      "title": "In-Flight Operations",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
+      "targets": [
+        { "expr": "memory_ingest_in_flight", "legendFormat": "ingest" },
+        { "expr": "memory_query_in_flight", "legendFormat": "query" }
+      ]
+    },
+    {
+      "title": "Write Volume",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
+      "targets": [
+        { "expr": "rate(memory_write_entities_total[5m])", "legendFormat": "entities/s" },
+        { "expr": "rate(memory_write_edges_total[5m])", "legendFormat": "edges/s" },
+        { "expr": "rate(memory_write_chunks_total[5m])", "legendFormat": "chunks/s" }
+      ]
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["poimen", "memory", "observability"],
+  "templating": { "list": [] },
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "Poimen Memory Observability",
+  "uid": "poimen-memory-obs"
+}
@@ -0,0 +1,130 @@
+# Prometheus alerting rules for Poimen Memory (O12)
+# Deploy: kubectl apply -f k8s/infra/prometheus-alerts.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: poimen-memory-alerts
+  namespace: poimen
+  labels:
+    app: poimen-memory
+    prometheus: k8s
+    role: alert-rules
+spec:
+  groups:
+    - name: poimen-memory.availability
+      rules:
+        - alert: MemoryServiceDown
+          expr: up{job="poimen-memory"} == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen memory service is down"
+            description: "Memory service has been unreachable for > 2 minutes"
+
+        - alert: MemoryDBDown
+          expr: memory_dependency_db_up == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Memory service cannot reach database"
+            description: "DB dependency health check failing for > 1 minute"
+
+        - alert: MemoryEmbeddingDown
+          expr: memory_dependency_embedding_up == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Embedding service unreachable"
+            description: "Embedding dependency health check failing for > 5 minutes"
+
+    - name: poimen-memory.latency
+      rules:
+        - alert: MemoryIngestLatencyHigh
+          expr: histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m])) > 5
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ingest p95 latency > 5s"
+            description: "95th percentile ingest latency is {{ $value }}s"
+
+        - alert: MemoryQueryLatencyHigh
+          expr: histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m])) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Query p95 latency > 2s"
+            description: "95th percentile query latency is {{ $value }}s"
+
+        - alert: MemoryEmbeddingLatencyHigh
+          expr: histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m])) > 10
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Embedding p95 latency > 10s"
+            description: "95th percentile embedding call latency is {{ $value }}s"
+
+    - name: poimen-memory.errors
+      rules:
+        - alert: MemoryIngestErrorRateHigh
+          expr: rate(memory_ingest_errors_total[5m]) / rate(memory_ingest_requests_total[5m]) > 0.1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ingest error rate > 10%"
+            description: "{{ $value | humanizePercentage }} of ingest requests are failing"
+
+        - alert: MemoryQueryErrorRateHigh
+          expr: rate(memory_query_errors_total[5m]) / rate(memory_query_requests_total[5m]) > 0.1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Query error rate > 10%"
+            description: "{{ $value | humanizePercentage }} of query requests are failing"
+
+        - alert: MemoryEmbeddingFailureRate
+          expr: rate(memory_query_embedding_failures_total[5m]) > 0.5
+          for: 3m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Embedding failures > 0.5/s"
+            description: "Embedding service failing at {{ $value }}/s — queries cannot embed"
+
+    - name: poimen-memory.storage
+      rules:
+        - alert: MemoryDBPoolExhausted
+          expr: memory_db_pool_idle == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "DB connection pool exhausted"
+            description: "No idle DB connections for > 5 minutes"
+
+        - alert: MemoryWriteErrorsHigh
+          expr: rate(memory_write_errors_total[5m]) > 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Write errors > 1/s"
+            description: "Database write errors at {{ $value }}/s"
+
+    - name: poimen-memory.quality
+      rules:
+        - alert: MemoryRelevanceLow
+          expr: memory_relevance_precision < 0.3
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Retrieval relevance precision < 30%"
+            description: "Relevance precision is {{ $value | humanizePercentage }}"
@@ -0,0 +1,94 @@
+# CronJob for periodic relevance evaluation (O13)
+# Runs sample queries against memory service and evaluates result relevance
+# Pushes metrics to Prometheus via pushgateway or direct scrape
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: memory-relevance-eval
+  namespace: poimen
+  labels:
+    app: memory-relevance-eval
+spec:
+  # Run every 6 hours
+  schedule: "0 */6 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: memory-relevance-eval
+        spec:
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: eval
+              image: curlimages/curl:8.13.0
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  MEMORY_URL="http://poimen-memory.poimen.svc.cluster.local:8080"
+                  
+                  echo "=== Relevance evaluation at $(date) ==="
+                  
+                  # Sample queries for evaluation
+                  QUERIES='[
+                    "kubernetes deployment",
+                    "database migration",
+                    "LLM entity extraction",
+                    "tea cli forgejo",
+                    "SOPS encryption secrets"
+                  ]'
+                  
+                  TOTAL=0
+                  RELEVANT=0
+                  
+                  for q in "kubernetes deployment" "database migration" "LLM entity extraction"; do
+                    echo "Testing query: $q"
+                    RESULT=$(curl -s --max-time 30 -X POST "$MEMORY_URL/memory/query" \
+                      -H "Content-Type: application/json" \
+                      -d "{\"query\": \"$q\", \"search_type\": \"entities\", \"top_k\": 5}")
+                    
+                    COUNT=$(echo "$RESULT" | grep -o '"total_count":[0-9]*' | cut -d: -f2)
+                    TOTAL=$((TOTAL + 1))
+                    
+                    if [ "${COUNT:-0}" -gt 0 ]; then
+                      RELEVANT=$((RELEVANT + 1))
+                      echo "  Result: $COUNT results (relevant)"
+                    else
+                      echo "  Result: 0 results (irrelevant)"
+                    fi
+                  done
+                  
+                  PRECISION=$(echo "scale=2; $RELEVANT / $TOTAL" | bc 2>/dev/null || echo "0")
+                  echo ""
+                  echo "=== Summary ==="
+                  echo "Total queries: $TOTAL"
+                  echo "Queries with results: $RELEVANT"
+                  echo "Precision: $PRECISION"
+                  echo ""
+                  echo "=== Health check ==="
+                  curl -s "$MEMORY_URL/health"
+                  echo ""
+                  echo "=== Metrics snapshot ==="
+                  curl -s "$MEMORY_URL/metrics" | grep -E "^memory_(query|relevance|ingest)_" | head -20
+              
+              resources:
+                requests:
+                  cpu: 10m
+                  memory: 16Mi
+                limits:
+                  cpu: 50m
+                  memory: 32Mi
+          
+          restartPolicy: OnFailure
@@ -0,0 +1,121 @@
+# CronJob to periodically clean Gitea Actions runner disk space
+# Prevents "no space left on device" errors during Docker builds
+# Deploy to: kubectl apply -f k8s/infra/runner-cleanup-cronjob.yaml
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: runner-disk-cleanup
+  namespace: ci  # Adjust to your runner namespace
+  labels:
+    app: runner-cleanup
+spec:
+  # Run daily at 2 AM
+  schedule: "0 2 * * *"
+  
+  # Keep last 3 successful jobs
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 1
+  
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: runner-cleanup
+        spec:
+          serviceAccountName: runner-cleanup
+          
+          # Run on node with Gitea Actions runner
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                  - matchExpressions:
+                      - key: kubernetes.io/hostname
+                        operator: In
+                        values:
+                          - runner-node  # Adjust to your runner node name
+          
+          containers:
+            - name: cleanup
+              image: docker:24
+              securityContext:
+                privileged: true  # Needed to access Docker daemon
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  echo "=== Runner disk cleanup at $(date) ==="
+                  
+                  df -h /
+                  echo ""
+                  
+                  echo "Cleaning Docker..."
+                  docker system prune -af --volumes 2>&1 | tail -5
+                  
+                  echo ""
+                  echo "Cleaning Cargo cache..."
+                  rm -rf /root/.cargo/registry/cache 2>/dev/null
+                  rm -rf /root/.cargo/registry/index 2>/dev/null
+                  rm -rf /root/.cargo/git 2>/dev/null
+                  
+                  echo ""
+                  echo "Cleaning /tmp..."
+                  rm -rf /tmp/* 2>/dev/null
+                  
+                  echo ""
+                  echo "Disk after cleanup:"
+                  df -h /
+              
+              volumeMounts:
+                - name: docker-sock
+                  mountPath: /var/run/docker.sock
+                - name: runner-home
+                  mountPath: /root
+          
+          volumes:
+            # Access Docker daemon on host
+            - name: docker-sock
+              hostPath:
+                path: /var/run/docker.sock
+            # Access runner home directory
+            - name: runner-home
+              hostPath:
+                path: /home/runner  # Adjust to your runner home path
+          
+          restartPolicy: OnFailure
+
+---
+# ServiceAccount for cleanup job
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: runner-cleanup
+  namespace: ci
+
+---
+# Role for cleanup job
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: runner-cleanup
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list"]
+
+---
+# RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: runner-cleanup
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: runner-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: runner-cleanup
+    namespace: ci