feat: add monitoring resources (alerts, dashboard, service-monitor)

- PrometheusRule with 16 alerts across 6 groups:
  availability, ingest, query, dependencies, resources, errors
- Grafana dashboard (ConfigMap, sidecar auto-discovery):
  8 rows covering overview, ingest pipeline, query perf,
  context retrieval, database, relevance, error breakdown,
  app resources, logs
- ServiceMonitor moved from k8s/app/ to k8s/monitoring/
- Dashboard generator script (Python, matches homelab pattern)
- Separate k8s/monitoring/ path for ArgoCD Application
- sqlx::migrate!() added to server startup for seamless migrations

crates/mem-cli/src/http_server.rs

@@ -200,12 +200,19 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res
     let pool = PgPool::connect(database_url).await?;
     tracing::info!("Connected to database");
 
-    // Initialize schema
+    // Initialize base schema (CREATE TABLE IF NOT EXISTS)
     match init_schema(&pool).await {
-        Ok(_) => tracing::info!("Schema initialized"),
+        Ok(_) => tracing::info!("Base schema initialized"),
         Err(e) => {
             tracing::warn!("Schema init error (may be non-fatal): {}", e);
-            // Continue anyway - tables might exist
+        }
+    }
+
+    // Run numbered migrations (001-009+) — idempotent, tracks state in _sqlx_migrations table
+    match sqlx::migrate!("./migrations").run(&pool).await {
+        Ok(_) => tracing::info!("Migrations applied"),
+        Err(e) => {
+            tracing::warn!("Migration error (may be non-fatal): {}", e);
         }
     }
 

k8s/monitoring/alerts.yaml

@@ -0,0 +1,205 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: poimen-memory-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # ================================================================
+    # SLA Targets:
+    #   Availability:  99.9%
+    #   Ingest p95:    < 10s (includes LLM extraction)
+    #   Query p95:     < 2s
+    #   Error rate:    < 5%
+    #   DB connection: always available
+    # ================================================================
+
+    - name: poimen-memory.availability
+      rules:
+        - alert: PoimenMemoryDown
+          expr: sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"}) == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen Memory has zero ready pods"
+
+        - alert: PoimenMemoryRestarted
+          expr: increase(kube_pod_container_status_restarts_total{namespace="poimen",pod=~"poimen-memory.*"}[15m]) > 0
+          for: 0m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen Memory pod {{ $labels.pod }} restarted"
+
+        - alert: PoimenMemoryHealthCheckFailing
+          expr: rate(memory_health_check_failures_total[5m]) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen Memory health checks failing"
+
+    - name: poimen-memory.ingest
+      rules:
+        - alert: PoimenIngestErrorRate
+          expr: |
+            rate(memory_ingest_errors_total[5m])
+            / (rate(memory_ingest_requests_total[5m]) > 0)
+            > 0.05
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen ingest error rate {{ $value | humanizePercentage }}"
+
+        - alert: PoimenIngestLatencyHigh
+          expr: |
+            histogram_quantile(0.95,
+              rate(memory_ingest_duration_seconds_bucket[5m])
+            ) > 10
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen ingest p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <10s)"
+
+        - alert: PoimenLLMExtractionSlow
+          expr: |
+            histogram_quantile(0.95,
+              rate(memory_ingest_llm_extract_duration_seconds_bucket[5m])
+            ) > 30
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LLM entity extraction p95 {{ $value | printf \"%.1f\" }}s"
+
+        - alert: PoimenIngestQueueBacklog
+          expr: memory_ingest_queue_size > 100
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen ingest queue size {{ $value }} (backlog)"
+
+    - name: poimen-memory.query
+      rules:
+        - alert: PoimenQueryErrorRate
+          expr: |
+            rate(memory_query_errors_total[5m])
+            / (rate(memory_query_requests_total[5m]) > 0)
+            > 0.05
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen query error rate {{ $value | humanizePercentage }}"
+
+        - alert: PoimenQueryLatencyHigh
+          expr: |
+            histogram_quantile(0.95,
+              rate(memory_query_duration_seconds_bucket[5m])
+            ) > 2
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen query p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
+
+        - alert: PoimenEmbeddingFailures
+          expr: rate(memory_query_embedding_failures_total[5m]) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen embedding failures — semantic search degraded"
+
+        - alert: PoimenQueryEmptyResultRate
+          expr: |
+            rate(memory_query_empty_results_total[5m])
+            / (rate(memory_query_requests_total[5m]) > 0)
+            > 0.5
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $value | humanizePercentage }} of queries returning empty results"
+
+    - name: poimen-memory.dependencies
+      rules:
+        - alert: PoimenDatabaseDown
+          expr: memory_dependency_db_up == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen Memory cannot reach database"
+
+        - alert: PoimenDatabaseLatencyHigh
+          expr: memory_dependency_db_latency_seconds > 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen DB latency {{ $value | printf \"%.1f\" }}s"
+
+        - alert: PoimenEmbeddingServiceDown
+          expr: memory_dependency_embedding_up == 0
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen embedding service unreachable — ingest + semantic query broken"
+
+        - alert: PoimenLLMServiceDown
+          expr: memory_dependency_llm_up == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen LLM service unreachable — entity extraction degraded"
+
+    - name: poimen-memory.resources
+      rules:
+        - alert: PoimenMemoryHighMemory
+          expr: |
+            sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)
+            / sum(kube_pod_container_resource_limits{namespace="poimen",container="poimen-memory",resource="memory"}) by (pod)
+            > 0.8
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen Memory pod {{ $labels.pod }} at {{ $value | humanizePercentage }} memory limit"
+
+        - alert: PoimenDBPoolExhausted
+          expr: memory_db_pool_active >= memory_db_pool_size
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Poimen DB connection pool exhausted: {{ $value }} active = pool size"
+
+    - name: poimen-memory.errors
+      rules:
+        - alert: PoimenAuthFailureSpike
+          expr: |
+            sum(rate(memory_ingest_auth_failures_total[5m]))
+            + sum(rate(memory_query_auth_failures_total[5m]))
+            > 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen auth failures: {{ $value | printf \"%.1f\" }}/sec"
+
+        - alert: PoimenUnexpectedErrors
+          expr: rate(memory_error_unexpected_total[5m]) > 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Poimen unexpected errors at {{ $value | printf \"%.2f\" }}/sec"

k8s/monitoring/dashboard.yaml

@@ -0,0 +1,80 @@
+apiVersion: v1
+data:
+  poimen-memory.json: '{"title":"Poimen Memory","uid":"poimen-memory","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["poimen","memory","knowledge-graph"],"panels":[{"id":1,"title":"Overview","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Pod
+    Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"poimen\",pod=~\"poimen-memory.*\",condition=\"true\"})"}]},{"id":3,"title":"Uptime","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_app_uptime_seconds"}]},{"id":4,"title":"DB","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_db_up"}]},{"id":5,"title":"Embeddings","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_embedding_up"}]},{"id":6,"title":"LLM","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_llm_up"}]},{"id":7,"title":"Connections","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_active_connections"}]}]},{"id":10,"title":"Ingest
+    Pipeline","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Ingest
+    Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_ingest_records_total[5m])","legendFormat":"records/s"}]},{"id":12,"title":"Ingest
+    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_ingest_auth_failures_total[5m])","legendFormat":"auth
+    failures/s"},{"expr":"rate(memory_ingest_rate_limited_total[5m])","legendFormat":"rate
+    limited/s"}]},{"id":13,"title":"Ingest Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":14,"title":"In
+    Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_in_flight"}]},{"id":15,"title":"Queue
+    Size","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_queue_size"}]},{"id":16,"title":"Entities
+    Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_entities_extracted_total[5m])"}]},{"id":17,"title":"Edges
+    Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_edges_extracted_total[5m])"}]},{"id":18,"title":"Duplicates","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_duplicates_total[5m])"}]},{"id":19,"title":"Bytes
+    Ingested","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"Bps"}},"targets":[{"expr":"rate(memory_ingest_bytes_total[5m])"}]},{"id":20,"title":"LLM
+    Extraction Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
+    p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
+    p95"},{"expr":"histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
+    p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
+    p95"}]},{"id":21,"title":"Contradiction & Dedup","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_contradiction_total[5m])","legendFormat":"contradictions/s"},{"expr":"rate(memory_ingest_dedup_total[5m])","legendFormat":"dedup/s"}]}]},{"id":30,"title":"Query
+    Performance","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":31,"title":"Query
+    Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_query_results_total[5m])","legendFormat":"results/s"}]},{"id":32,"title":"Query
+    Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_query_embedding_failures_total[5m])","legendFormat":"embedding
+    failures/s"},{"expr":"rate(memory_query_empty_results_total[5m])","legendFormat":"empty
+    results/s"}]},{"id":33,"title":"Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":34,"title":"In
+    Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_query_in_flight"}]},{"id":35,"title":"Embedding
+    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":36,"title":"Cache
+    Hit Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(memory_query_cache_hits_total[5m])
+    / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))","legendFormat":"hit
+    rate"}]}]},{"id":40,"title":"Context Retrieval (3-tier)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":41,"title":"Context
+    Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_context_errors_total[5m])","legendFormat":"errors/s"}]},{"id":42,"title":"Context
+    Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":43,"title":"Retrieval
+    Tier Hits","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_semantic_hits_total[5m])","legendFormat":"semantic"},{"expr":"rate(memory_context_graph_hits_total[5m])","legendFormat":"graph"}]},{"id":44,"title":"Tier
+    Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
+    rate(memory_context_tier_duration_seconds_bucket[5m]))","legendFormat":"p95"}]}]},{"id":50,"title":"Database","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":51,"title":"Entities","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_entity_count"}]},{"id":52,"title":"Edges","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_edge_count"}]},{"id":53,"title":"Chunks","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_chunk_count"}]},{"id":54,"title":"DB
+    Size","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_size_bytes"}]},{"id":55,"title":"Index
+    Size","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_index_size_bytes"}]},{"id":56,"title":"Pool
+    Active","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active"}]},{"id":57,"title":"DB
+    Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
+    rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
+    rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
+    rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":58,"title":"DB
+    Query Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_db_queries_total[5m])","legendFormat":"queries/s"},{"expr":"rate(memory_db_query_errors_total[5m])","legendFormat":"errors/s"}]},{"id":59,"title":"Connection
+    Pool","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active","legendFormat":"active"},{"expr":"memory_db_pool_idle","legendFormat":"idle"},{"expr":"memory_db_pool_size","legendFormat":"max"}]},{"id":60,"title":"Table
+    Row Counts","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_table_entity_rows","legendFormat":"entities"},{"expr":"memory_db_table_edge_rows","legendFormat":"edges"},{"expr":"memory_db_table_chunk_rows","legendFormat":"chunks"}]},{"id":61,"title":"Write
+    Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_write_entities_total[5m])","legendFormat":"entities/s"},{"expr":"rate(memory_write_edges_total[5m])","legendFormat":"edges/s"},{"expr":"rate(memory_write_chunks_total[5m])","legendFormat":"chunks/s"},{"expr":"rate(memory_write_errors_total[5m])","legendFormat":"errors/s"}]}]},{"id":70,"title":"Relevance
+    & Quality","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":71,"title":"Relevance
+    Score","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"memory_relevance_score","legendFormat":"score"},{"expr":"memory_relevance_precision","legendFormat":"precision"},{"expr":"memory_relevance_recall","legendFormat":"recall"}]},{"id":72,"title":"Relevance
+    Evals","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_relevance_relevant_total[5m])","legendFormat":"relevant/s"},{"expr":"rate(memory_relevance_irrelevant_total[5m])","legendFormat":"irrelevant/s"}]}]},{"id":80,"title":"Error
+    Breakdown","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":81,"title":"Ingest
+    Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_ingest_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_ingest_total[5m])","legendFormat":"bad
+    request"},{"expr":"rate(memory_error_db_error_ingest_total[5m])","legendFormat":"db
+    error"},{"expr":"rate(memory_error_forbidden_ingest_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_rate_limited_ingest_total[5m])","legendFormat":"rate
+    limited"},{"expr":"rate(memory_error_unexpected_ingest_total[5m])","legendFormat":"unexpected"}]},{"id":82,"title":"Query
+    Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_query_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_query_total[5m])","legendFormat":"bad
+    request"},{"expr":"rate(memory_error_embedding_failure_query_total[5m])","legendFormat":"embedding"},{"expr":"rate(memory_error_forbidden_query_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_search_failure_query_total[5m])","legendFormat":"search"},{"expr":"rate(memory_error_unexpected_query_total[5m])","legendFormat":"unexpected"}]}]},{"id":90,"title":"Application
+    Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":91,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"poimen\",container=\"poimen-memory\"}[5m]))
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":92,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"poimen\",container=\"poimen-memory\"})
+    by (pod)","legendFormat":"{{pod}}"}]},{"id":93,"title":"Heap Bytes","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_app_heap_bytes","legendFormat":"heap"}]},{"id":94,"title":"Tokio
+    Tasks","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_tokio_tasks","legendFormat":"active
+    tasks"}]},{"id":95,"title":"Dependency Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_dependency_db_latency_seconds","legendFormat":"db"},{"expr":"memory_dependency_embedding_latency_seconds","legendFormat":"embedding"}]}]},{"id":100,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":8},"panels":[{"id":101,"title":"Poimen
+    Memory Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":9},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"poimen\",container=\"poimen-memory\"}"}]}]}]}'
+kind: ConfigMap
+metadata:
+  annotations:
+    grafana_folder: Poimen
+  labels:
+    grafana_dashboard: '1'
+  name: poimen-memory-dashboard
+  namespace: logging

k8s/monitoring/generate-dashboard.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+"""Generate Poimen Memory Grafana dashboard as k8s ConfigMap YAML.
+
+Follows homelab pattern: ConfigMap with grafana_dashboard=1 label,
+auto-discovered by Grafana sidecar.
+
+Usage: python3 generate-dashboard.py
+"""
+
+import json
+import os
+import yaml
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+DS_PROM = {"type": "prometheus", "uid": "prometheus"}
+DS_LOKI = {"type": "loki", "uid": "loki"}
+
+
+def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
+    p = {
+        "id": id, "title": title, "type": "stat",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": [{"expr": expr}],
+    }
+    if mappings:
+        p["fieldConfig"]["defaults"]["mappings"] = mappings
+    if thresholds:
+        p["fieldConfig"]["defaults"]["thresholds"] = thresholds
+        p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
+    return p
+
+
+def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
+    targets = []
+    for e in exprs:
+        if isinstance(e, tuple):
+            targets.append({"expr": e[0], "legendFormat": e[1]})
+        else:
+            targets.append({"expr": e, "legendFormat": ""})
+    return {
+        "id": id, "title": title, "type": "timeseries",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_PROM,
+        "fieldConfig": {"defaults": {"unit": unit}},
+        "targets": targets,
+    }
+
+
+def log_panel(id, title, query, x, y, w=24, h=10):
+    return {
+        "id": id, "title": title, "type": "logs",
+        "gridPos": {"h": h, "w": w, "x": x, "y": y},
+        "datasource": DS_LOKI,
+        "targets": [{"expr": query}],
+    }
+
+
+def row(id, title, y, panels, collapsed=True):
+    return {
+        "id": id, "title": title, "type": "row",
+        "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Poimen Memory Dashboard
+# ============================================================================
+
+def build_dashboard():
+    up_down = [{"type": "value", "options": {
+        "0": {"text": "DOWN", "color": "red"},
+        "1": {"text": "UP", "color": "green"},
+    }}]
+
+    panels = [
+        # ── Row 0: Overview (always open) ──
+        row(1, "Overview", 0, [
+            stat_panel(2, "Pod Ready", 
+                'sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"})',
+                0, 1, thresholds={"mode": "absolute", "steps": [
+                    {"value": None, "color": "red"}, {"value": 1, "color": "green"}
+                ]}),
+            stat_panel(3, "Uptime",
+                'memory_app_uptime_seconds', 4, 1, unit="s"),
+            stat_panel(4, "DB", 'memory_dependency_db_up', 8, 1, mappings=up_down),
+            stat_panel(5, "Embeddings", 'memory_dependency_embedding_up', 12, 1, mappings=up_down),
+            stat_panel(6, "LLM", 'memory_dependency_llm_up', 16, 1, mappings=up_down),
+            stat_panel(7, "Connections", 'memory_app_active_connections', 20, 1),
+        ], collapsed=False),
+
+        # ── Row 1: Ingest Pipeline ──
+        row(10, "Ingest Pipeline", 1, [
+            ts_panel(11, "Ingest Rate", [
+                ('rate(memory_ingest_requests_total[5m])', "requests/s"),
+                ('rate(memory_ingest_records_total[5m])', "records/s"),
+            ], 0, 2, unit="ops"),
+            ts_panel(12, "Ingest Errors", [
+                ('rate(memory_ingest_errors_total[5m])', "errors/s"),
+                ('rate(memory_ingest_auth_failures_total[5m])', "auth failures/s"),
+                ('rate(memory_ingest_rate_limited_total[5m])', "rate limited/s"),
+            ], 8, 2, unit="ops"),
+            ts_panel(13, "Ingest Latency", [
+                ('histogram_quantile(0.50, rate(memory_ingest_duration_seconds_bucket[5m]))', "p50"),
+                ('histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m]))', "p95"),
+                ('histogram_quantile(0.99, rate(memory_ingest_duration_seconds_bucket[5m]))', "p99"),
+            ], 16, 2, unit="s"),
+            stat_panel(14, "In Flight", 'memory_ingest_in_flight', 0, 10),
+            stat_panel(15, "Queue Size", 'memory_ingest_queue_size', 4, 10),
+            stat_panel(16, "Entities Extracted", 'rate(memory_ingest_entities_extracted_total[5m])', 8, 10, unit="ops"),
+            stat_panel(17, "Edges Extracted", 'rate(memory_ingest_edges_extracted_total[5m])', 12, 10, unit="ops"),
+            stat_panel(18, "Duplicates", 'rate(memory_ingest_duplicates_total[5m])', 16, 10, unit="ops"),
+            stat_panel(19, "Bytes Ingested", 'rate(memory_ingest_bytes_total[5m])', 20, 10, unit="Bps"),
+            ts_panel(20, "LLM Extraction Latency", [
+                ('histogram_quantile(0.50, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p50"),
+                ('histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p95"),
+                ('histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p50"),
+                ('histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p95"),
+            ], 0, 14, w=12, unit="s"),
+            ts_panel(21, "Contradiction & Dedup", [
+                ('rate(memory_ingest_contradiction_total[5m])', "contradictions/s"),
+                ('rate(memory_ingest_dedup_total[5m])', "dedup/s"),
+            ], 12, 14, w=12, unit="ops"),
+        ]),
+
+        # ── Row 2: Query Performance ──
+        row(30, "Query Performance", 2, [
+            ts_panel(31, "Query Rate", [
+                ('rate(memory_query_requests_total[5m])', "requests/s"),
+                ('rate(memory_query_results_total[5m])', "results/s"),
+            ], 0, 3, unit="ops"),
+            ts_panel(32, "Query Errors", [
+                ('rate(memory_query_errors_total[5m])', "errors/s"),
+                ('rate(memory_query_embedding_failures_total[5m])', "embedding failures/s"),
+                ('rate(memory_query_empty_results_total[5m])', "empty results/s"),
+            ], 8, 3, unit="ops"),
+            ts_panel(33, "Query Latency", [
+                ('histogram_quantile(0.50, rate(memory_query_duration_seconds_bucket[5m]))', "p50"),
+                ('histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m]))', "p95"),
+                ('histogram_quantile(0.99, rate(memory_query_duration_seconds_bucket[5m]))', "p99"),
+            ], 16, 3, unit="s"),
+            stat_panel(34, "In Flight", 'memory_query_in_flight', 0, 11),
+            ts_panel(35, "Embedding Latency", [
+                ('histogram_quantile(0.50, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p50"),
+                ('histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p95"),
+            ], 4, 11, unit="s"),
+            ts_panel(36, "Cache Hit Rate", [
+                ('rate(memory_query_cache_hits_total[5m]) / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))', "hit rate"),
+            ], 12, 11, w=12, unit="percentunit"),
+        ]),
+
+        # ── Row 3: Context Retrieval ──
+        row(40, "Context Retrieval (3-tier)", 3, [
+            ts_panel(41, "Context Rate", [
+                ('rate(memory_context_requests_total[5m])', "requests/s"),
+                ('rate(memory_context_errors_total[5m])', "errors/s"),
+            ], 0, 4, unit="ops"),
+            ts_panel(42, "Context Latency", [
+                ('histogram_quantile(0.50, rate(memory_context_duration_seconds_bucket[5m]))', "p50"),
+                ('histogram_quantile(0.95, rate(memory_context_duration_seconds_bucket[5m]))', "p95"),
+            ], 8, 4, unit="s"),
+            ts_panel(43, "Retrieval Tier Hits", [
+                ('rate(memory_context_semantic_hits_total[5m])', "semantic"),
+                ('rate(memory_context_graph_hits_total[5m])', "graph"),
+            ], 16, 4, unit="ops"),
+            ts_panel(44, "Tier Latency", [
+                ('histogram_quantile(0.95, rate(memory_context_tier_duration_seconds_bucket[5m]))', "p95"),
+            ], 0, 12, w=12, unit="s"),
+        ]),
+
+        # ── Row 4: Database ──
+        row(50, "Database", 4, [
+            stat_panel(51, "Entities", 'memory_db_entity_count', 0, 5),
+            stat_panel(52, "Edges", 'memory_db_edge_count', 4, 5),
+            stat_panel(53, "Chunks", 'memory_db_chunk_count', 8, 5),
+            stat_panel(54, "DB Size", 'memory_db_size_bytes', 12, 5, unit="bytes"),
+            stat_panel(55, "Index Size", 'memory_db_index_size_bytes', 16, 5, unit="bytes"),
+            stat_panel(56, "Pool Active", 'memory_db_pool_active', 20, 5),
+            ts_panel(57, "DB Query Latency", [
+                ('histogram_quantile(0.50, rate(memory_db_query_duration_seconds_bucket[5m]))', "p50"),
+                ('histogram_quantile(0.95, rate(memory_db_query_duration_seconds_bucket[5m]))', "p95"),
+                ('histogram_quantile(0.99, rate(memory_db_query_duration_seconds_bucket[5m]))', "p99"),
+            ], 0, 9, unit="s"),
+            ts_panel(58, "DB Query Rate", [
+                ('rate(memory_db_queries_total[5m])', "queries/s"),
+                ('rate(memory_db_query_errors_total[5m])', "errors/s"),
+            ], 8, 9, unit="ops"),
+            ts_panel(59, "Connection Pool", [
+                ('memory_db_pool_active', "active"),
+                ('memory_db_pool_idle', "idle"),
+                ('memory_db_pool_size', "max"),
+            ], 16, 9),
+            ts_panel(60, "Table Row Counts", [
+                ('memory_db_table_entity_rows', "entities"),
+                ('memory_db_table_edge_rows', "edges"),
+                ('memory_db_table_chunk_rows', "chunks"),
+            ], 0, 17, w=12),
+            ts_panel(61, "Write Rate", [
+                ('rate(memory_write_entities_total[5m])', "entities/s"),
+                ('rate(memory_write_edges_total[5m])', "edges/s"),
+                ('rate(memory_write_chunks_total[5m])', "chunks/s"),
+                ('rate(memory_write_errors_total[5m])', "errors/s"),
+            ], 12, 17, w=12, unit="ops"),
+        ]),
+
+        # ── Row 5: Relevance / Quality ──
+        row(70, "Relevance & Quality", 5, [
+            ts_panel(71, "Relevance Score", [
+                ('memory_relevance_score', "score"),
+                ('memory_relevance_precision', "precision"),
+                ('memory_relevance_recall', "recall"),
+            ], 0, 6, w=12, unit="percentunit"),
+            ts_panel(72, "Relevance Evals", [
+                ('rate(memory_relevance_relevant_total[5m])', "relevant/s"),
+                ('rate(memory_relevance_irrelevant_total[5m])', "irrelevant/s"),
+            ], 12, 6, w=12, unit="ops"),
+        ]),
+
+        # ── Row 6: Errors Breakdown ──
+        row(80, "Error Breakdown", 6, [
+            ts_panel(81, "Ingest Errors by Type", [
+                ('rate(memory_error_auth_failure_ingest_total[5m])', "auth"),
+                ('rate(memory_error_bad_request_ingest_total[5m])', "bad request"),
+                ('rate(memory_error_db_error_ingest_total[5m])', "db error"),
+                ('rate(memory_error_forbidden_ingest_total[5m])', "forbidden"),
+                ('rate(memory_error_rate_limited_ingest_total[5m])', "rate limited"),
+                ('rate(memory_error_unexpected_ingest_total[5m])', "unexpected"),
+            ], 0, 7, w=12, unit="ops"),
+            ts_panel(82, "Query Errors by Type", [
+                ('rate(memory_error_auth_failure_query_total[5m])', "auth"),
+                ('rate(memory_error_bad_request_query_total[5m])', "bad request"),
+                ('rate(memory_error_embedding_failure_query_total[5m])', "embedding"),
+                ('rate(memory_error_forbidden_query_total[5m])', "forbidden"),
+                ('rate(memory_error_search_failure_query_total[5m])', "search"),
+                ('rate(memory_error_unexpected_query_total[5m])', "unexpected"),
+            ], 12, 7, w=12, unit="ops"),
+        ]),
+
+        # ── Row 7: Application Resources ──
+        row(90, "Application Resources", 7, [
+            ts_panel(91, "CPU", [
+                ('sum(rate(container_cpu_usage_seconds_total{namespace="poimen",container="poimen-memory"}[5m])) by (pod)', "{{pod}}"),
+            ], 0, 8),
+            ts_panel(92, "Memory", [
+                ('sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)', "{{pod}}"),
+            ], 8, 8, unit="bytes"),
+            ts_panel(93, "Heap Bytes", [
+                ('memory_app_heap_bytes', "heap"),
+            ], 16, 8, unit="bytes"),
+            ts_panel(94, "Tokio Tasks", [
+                ('memory_app_tokio_tasks', "active tasks"),
+            ], 0, 16),
+            ts_panel(95, "Dependency Latency", [
+                ('memory_dependency_db_latency_seconds', "db"),
+                ('memory_dependency_embedding_latency_seconds', "embedding"),
+            ], 8, 16, unit="s"),
+        ]),
+
+        # ── Row 8: Logs ──
+        row(100, "Logs", 8, [
+            log_panel(101, "Poimen Memory Logs", '{namespace="poimen",container="poimen-memory"}', 0, 9),
+        ]),
+    ]
+
+    return {
+        "title": "Poimen Memory",
+        "uid": "poimen-memory",
+        "schemaVersion": 39,
+        "timezone": "browser",
+        "time": {"from": "now-6h", "to": "now"},
+        "refresh": "30s",
+        "tags": ["poimen", "memory", "knowledge-graph"],
+        "panels": panels,
+    }
+
+
+# ============================================================================
+# Generate ConfigMap YAML
+# ============================================================================
+
+dashboard = build_dashboard()
+
+cm = {
+    "apiVersion": "v1",
+    "kind": "ConfigMap",
+    "metadata": {
+        "name": "poimen-memory-dashboard",
+        "namespace": "logging",
+        "labels": {"grafana_dashboard": "1"},
+        "annotations": {"grafana_folder": "Poimen"},
+    },
+    "data": {
+        "poimen-memory.json": json.dumps(dashboard, separators=(",", ":")),
+    },
+}
+
+out_path = os.path.join(SCRIPT_DIR, "dashboard.yaml")
+with open(out_path, "w") as f:
+    yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
+
+print(f"Generated {out_path}")

k8s/monitoring/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - service-monitor.yaml
+  - alerts.yaml
+  - dashboard.yaml

k8s/monitoring/service-monitor.yaml

@@ -2,10 +2,14 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: poimen-memory
-  namespace: poimen
+  namespace: monitoring
   labels:
+    release: kube-prometheus-stack
     app.kubernetes.io/name: poimen-memory
 spec:
+  namespaceSelector:
+    matchNames:
+      - poimen
   selector:
     matchLabels:
       app.kubernetes.io/name: poimen-memory
@@ -13,3 +17,4 @@ spec:
     - port: http
       path: /metrics
       interval: 30s
+      scrapeTimeout: 10s