From 8902e1b6fa158209d2099ad244c2502f2b0ccec6 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 16 Sep 2026 11:52:47 +0900 Subject: [PATCH] feat: add monitoring resources (alerts, dashboard, service-monitor) - PrometheusRule with 16 alerts across 6 groups: availability, ingest, query, dependencies, resources, errors - Grafana dashboard (ConfigMap, sidecar auto-discovery): 8 rows covering overview, ingest pipeline, query perf, context retrieval, database, relevance, error breakdown, app resources, logs - ServiceMonitor moved from k8s/app/ to k8s/monitoring/ - Dashboard generator script (Python, matches homelab pattern) - Separate k8s/monitoring/ path for ArgoCD Application - sqlx::migrate!() added to server startup for seamless migrations --- crates/mem-cli/src/http_server.rs | 13 +- k8s/monitoring/alerts.yaml | 205 +++++++++++++ k8s/monitoring/dashboard.yaml | 80 +++++ k8s/monitoring/generate-dashboard.py | 304 +++++++++++++++++++ k8s/monitoring/kustomization.yaml | 6 + k8s/{app => monitoring}/service-monitor.yaml | 13 +- 6 files changed, 614 insertions(+), 7 deletions(-) create mode 100644 k8s/monitoring/alerts.yaml create mode 100644 k8s/monitoring/dashboard.yaml create mode 100644 k8s/monitoring/generate-dashboard.py create mode 100644 k8s/monitoring/kustomization.yaml rename k8s/{app => monitoring}/service-monitor.yaml (54%) diff --git a/crates/mem-cli/src/http_server.rs b/crates/mem-cli/src/http_server.rs index 233a965..c4e1577 100644 --- a/crates/mem-cli/src/http_server.rs +++ b/crates/mem-cli/src/http_server.rs @@ -200,12 +200,19 @@ pub async fn start_server(port: u16, api_key: String, database_url: &str) -> Res let pool = PgPool::connect(database_url).await?; tracing::info!("Connected to database"); - // Initialize schema + // Initialize base schema (CREATE TABLE IF NOT EXISTS) match init_schema(&pool).await { - Ok(_) => tracing::info!("Schema initialized"), + Ok(_) => tracing::info!("Base schema initialized"), Err(e) => { tracing::warn!("Schema init error (may be non-fatal): {}", e); - // Continue anyway - tables might exist + } + } + + // Run numbered migrations (001-009+) — idempotent, tracks state in _sqlx_migrations table + match sqlx::migrate!("./migrations").run(&pool).await { + Ok(_) => tracing::info!("Migrations applied"), + Err(e) => { + tracing::warn!("Migration error (may be non-fatal): {}", e); } } diff --git a/k8s/monitoring/alerts.yaml b/k8s/monitoring/alerts.yaml new file mode 100644 index 0000000..d27c450 --- /dev/null +++ b/k8s/monitoring/alerts.yaml @@ -0,0 +1,205 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: poimen-memory-alerts + namespace: monitoring + labels: + release: prometheus +spec: + groups: + # ================================================================ + # SLA Targets: + # Availability: 99.9% + # Ingest p95: < 10s (includes LLM extraction) + # Query p95: < 2s + # Error rate: < 5% + # DB connection: always available + # ================================================================ + + - name: poimen-memory.availability + rules: + - alert: PoimenMemoryDown + expr: sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"}) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Poimen Memory has zero ready pods" + + - alert: PoimenMemoryRestarted + expr: increase(kube_pod_container_status_restarts_total{namespace="poimen",pod=~"poimen-memory.*"}[15m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: "Poimen Memory pod {{ $labels.pod }} restarted" + + - alert: PoimenMemoryHealthCheckFailing + expr: rate(memory_health_check_failures_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen Memory health checks failing" + + - name: poimen-memory.ingest + rules: + - alert: PoimenIngestErrorRate + expr: | + rate(memory_ingest_errors_total[5m]) + / (rate(memory_ingest_requests_total[5m]) > 0) + > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen ingest error rate {{ $value | humanizePercentage }}" + + - alert: PoimenIngestLatencyHigh + expr: | + histogram_quantile(0.95, + rate(memory_ingest_duration_seconds_bucket[5m]) + ) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen ingest p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <10s)" + + - alert: PoimenLLMExtractionSlow + expr: | + histogram_quantile(0.95, + rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]) + ) > 30 + for: 5m + labels: + severity: warning + annotations: + summary: "LLM entity extraction p95 {{ $value | printf \"%.1f\" }}s" + + - alert: PoimenIngestQueueBacklog + expr: memory_ingest_queue_size > 100 + for: 10m + labels: + severity: warning + annotations: + summary: "Poimen ingest queue size {{ $value }} (backlog)" + + - name: poimen-memory.query + rules: + - alert: PoimenQueryErrorRate + expr: | + rate(memory_query_errors_total[5m]) + / (rate(memory_query_requests_total[5m]) > 0) + > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen query error rate {{ $value | humanizePercentage }}" + + - alert: PoimenQueryLatencyHigh + expr: | + histogram_quantile(0.95, + rate(memory_query_duration_seconds_bucket[5m]) + ) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen query p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)" + + - alert: PoimenEmbeddingFailures + expr: rate(memory_query_embedding_failures_total[5m]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Poimen embedding failures — semantic search degraded" + + - alert: PoimenQueryEmptyResultRate + expr: | + rate(memory_query_empty_results_total[5m]) + / (rate(memory_query_requests_total[5m]) > 0) + > 0.5 + for: 15m + labels: + severity: warning + annotations: + summary: "{{ $value | humanizePercentage }} of queries returning empty results" + + - name: poimen-memory.dependencies + rules: + - alert: PoimenDatabaseDown + expr: memory_dependency_db_up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Poimen Memory cannot reach database" + + - alert: PoimenDatabaseLatencyHigh + expr: memory_dependency_db_latency_seconds > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen DB latency {{ $value | printf \"%.1f\" }}s" + + - alert: PoimenEmbeddingServiceDown + expr: memory_dependency_embedding_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Poimen embedding service unreachable — ingest + semantic query broken" + + - alert: PoimenLLMServiceDown + expr: memory_dependency_llm_up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen LLM service unreachable — entity extraction degraded" + + - name: poimen-memory.resources + rules: + - alert: PoimenMemoryHighMemory + expr: | + sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod) + / sum(kube_pod_container_resource_limits{namespace="poimen",container="poimen-memory",resource="memory"}) by (pod) + > 0.8 + for: 10m + labels: + severity: warning + annotations: + summary: "Poimen Memory pod {{ $labels.pod }} at {{ $value | humanizePercentage }} memory limit" + + - alert: PoimenDBPoolExhausted + expr: memory_db_pool_active >= memory_db_pool_size + for: 5m + labels: + severity: critical + annotations: + summary: "Poimen DB connection pool exhausted: {{ $value }} active = pool size" + + - name: poimen-memory.errors + rules: + - alert: PoimenAuthFailureSpike + expr: | + sum(rate(memory_ingest_auth_failures_total[5m])) + + sum(rate(memory_query_auth_failures_total[5m])) + > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen auth failures: {{ $value | printf \"%.1f\" }}/sec" + + - alert: PoimenUnexpectedErrors + expr: rate(memory_error_unexpected_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Poimen unexpected errors at {{ $value | printf \"%.2f\" }}/sec" diff --git a/k8s/monitoring/dashboard.yaml b/k8s/monitoring/dashboard.yaml new file mode 100644 index 0000000..fa19dc1 --- /dev/null +++ b/k8s/monitoring/dashboard.yaml @@ -0,0 +1,80 @@ +apiVersion: v1 +data: + poimen-memory.json: '{"title":"Poimen Memory","uid":"poimen-memory","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["poimen","memory","knowledge-graph"],"panels":[{"id":1,"title":"Overview","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Pod + Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"poimen\",pod=~\"poimen-memory.*\",condition=\"true\"})"}]},{"id":3,"title":"Uptime","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_app_uptime_seconds"}]},{"id":4,"title":"DB","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_db_up"}]},{"id":5,"title":"Embeddings","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_embedding_up"}]},{"id":6,"title":"LLM","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_llm_up"}]},{"id":7,"title":"Connections","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_active_connections"}]}]},{"id":10,"title":"Ingest + Pipeline","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Ingest + Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_ingest_records_total[5m])","legendFormat":"records/s"}]},{"id":12,"title":"Ingest + Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_ingest_auth_failures_total[5m])","legendFormat":"auth + failures/s"},{"expr":"rate(memory_ingest_rate_limited_total[5m])","legendFormat":"rate + limited/s"}]},{"id":13,"title":"Ingest Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":14,"title":"In + Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_in_flight"}]},{"id":15,"title":"Queue + Size","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_queue_size"}]},{"id":16,"title":"Entities + Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_entities_extracted_total[5m])"}]},{"id":17,"title":"Edges + Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_edges_extracted_total[5m])"}]},{"id":18,"title":"Duplicates","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_duplicates_total[5m])"}]},{"id":19,"title":"Bytes + Ingested","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"Bps"}},"targets":[{"expr":"rate(memory_ingest_bytes_total[5m])"}]},{"id":20,"title":"LLM + Extraction Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity + p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity + p95"},{"expr":"histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact + p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact + p95"}]},{"id":21,"title":"Contradiction & Dedup","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_contradiction_total[5m])","legendFormat":"contradictions/s"},{"expr":"rate(memory_ingest_dedup_total[5m])","legendFormat":"dedup/s"}]}]},{"id":30,"title":"Query + Performance","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":31,"title":"Query + Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_query_results_total[5m])","legendFormat":"results/s"}]},{"id":32,"title":"Query + Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_query_embedding_failures_total[5m])","legendFormat":"embedding + failures/s"},{"expr":"rate(memory_query_empty_results_total[5m])","legendFormat":"empty + results/s"}]},{"id":33,"title":"Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":34,"title":"In + Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_query_in_flight"}]},{"id":35,"title":"Embedding + Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":36,"title":"Cache + Hit Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(memory_query_cache_hits_total[5m]) + / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))","legendFormat":"hit + rate"}]}]},{"id":40,"title":"Context Retrieval (3-tier)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":41,"title":"Context + Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_context_errors_total[5m])","legendFormat":"errors/s"}]},{"id":42,"title":"Context + Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":43,"title":"Retrieval + Tier Hits","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_semantic_hits_total[5m])","legendFormat":"semantic"},{"expr":"rate(memory_context_graph_hits_total[5m])","legendFormat":"graph"}]},{"id":44,"title":"Tier + Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, + rate(memory_context_tier_duration_seconds_bucket[5m]))","legendFormat":"p95"}]}]},{"id":50,"title":"Database","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":51,"title":"Entities","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_entity_count"}]},{"id":52,"title":"Edges","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_edge_count"}]},{"id":53,"title":"Chunks","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_chunk_count"}]},{"id":54,"title":"DB + Size","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_size_bytes"}]},{"id":55,"title":"Index + Size","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_index_size_bytes"}]},{"id":56,"title":"Pool + Active","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active"}]},{"id":57,"title":"DB + Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":58,"title":"DB + Query Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_db_queries_total[5m])","legendFormat":"queries/s"},{"expr":"rate(memory_db_query_errors_total[5m])","legendFormat":"errors/s"}]},{"id":59,"title":"Connection + Pool","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active","legendFormat":"active"},{"expr":"memory_db_pool_idle","legendFormat":"idle"},{"expr":"memory_db_pool_size","legendFormat":"max"}]},{"id":60,"title":"Table + Row Counts","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_table_entity_rows","legendFormat":"entities"},{"expr":"memory_db_table_edge_rows","legendFormat":"edges"},{"expr":"memory_db_table_chunk_rows","legendFormat":"chunks"}]},{"id":61,"title":"Write + Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_write_entities_total[5m])","legendFormat":"entities/s"},{"expr":"rate(memory_write_edges_total[5m])","legendFormat":"edges/s"},{"expr":"rate(memory_write_chunks_total[5m])","legendFormat":"chunks/s"},{"expr":"rate(memory_write_errors_total[5m])","legendFormat":"errors/s"}]}]},{"id":70,"title":"Relevance + & Quality","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":71,"title":"Relevance + Score","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"memory_relevance_score","legendFormat":"score"},{"expr":"memory_relevance_precision","legendFormat":"precision"},{"expr":"memory_relevance_recall","legendFormat":"recall"}]},{"id":72,"title":"Relevance + Evals","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_relevance_relevant_total[5m])","legendFormat":"relevant/s"},{"expr":"rate(memory_relevance_irrelevant_total[5m])","legendFormat":"irrelevant/s"}]}]},{"id":80,"title":"Error + Breakdown","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":81,"title":"Ingest + Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_ingest_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_ingest_total[5m])","legendFormat":"bad + request"},{"expr":"rate(memory_error_db_error_ingest_total[5m])","legendFormat":"db + error"},{"expr":"rate(memory_error_forbidden_ingest_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_rate_limited_ingest_total[5m])","legendFormat":"rate + limited"},{"expr":"rate(memory_error_unexpected_ingest_total[5m])","legendFormat":"unexpected"}]},{"id":82,"title":"Query + Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_query_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_query_total[5m])","legendFormat":"bad + request"},{"expr":"rate(memory_error_embedding_failure_query_total[5m])","legendFormat":"embedding"},{"expr":"rate(memory_error_forbidden_query_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_search_failure_query_total[5m])","legendFormat":"search"},{"expr":"rate(memory_error_unexpected_query_total[5m])","legendFormat":"unexpected"}]}]},{"id":90,"title":"Application + Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":91,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"poimen\",container=\"poimen-memory\"}[5m])) + by (pod)","legendFormat":"{{pod}}"}]},{"id":92,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"poimen\",container=\"poimen-memory\"}) + by (pod)","legendFormat":"{{pod}}"}]},{"id":93,"title":"Heap Bytes","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_app_heap_bytes","legendFormat":"heap"}]},{"id":94,"title":"Tokio + Tasks","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_tokio_tasks","legendFormat":"active + tasks"}]},{"id":95,"title":"Dependency Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_dependency_db_latency_seconds","legendFormat":"db"},{"expr":"memory_dependency_embedding_latency_seconds","legendFormat":"embedding"}]}]},{"id":100,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":8},"panels":[{"id":101,"title":"Poimen + Memory Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":9},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"poimen\",container=\"poimen-memory\"}"}]}]}]}' +kind: ConfigMap +metadata: + annotations: + grafana_folder: Poimen + labels: + grafana_dashboard: '1' + name: poimen-memory-dashboard + namespace: logging diff --git a/k8s/monitoring/generate-dashboard.py b/k8s/monitoring/generate-dashboard.py new file mode 100644 index 0000000..00630cb --- /dev/null +++ b/k8s/monitoring/generate-dashboard.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Generate Poimen Memory Grafana dashboard as k8s ConfigMap YAML. + +Follows homelab pattern: ConfigMap with grafana_dashboard=1 label, +auto-discovered by Grafana sidecar. + +Usage: python3 generate-dashboard.py +""" + +import json +import os +import yaml + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +DS_PROM = {"type": "prometheus", "uid": "prometheus"} +DS_LOKI = {"type": "loki", "uid": "loki"} + + +def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None): + p = { + "id": id, "title": title, "type": "stat", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_PROM, + "fieldConfig": {"defaults": {"unit": unit}}, + "targets": [{"expr": expr}], + } + if mappings: + p["fieldConfig"]["defaults"]["mappings"] = mappings + if thresholds: + p["fieldConfig"]["defaults"]["thresholds"] = thresholds + p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"} + return p + + +def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"): + targets = [] + for e in exprs: + if isinstance(e, tuple): + targets.append({"expr": e[0], "legendFormat": e[1]}) + else: + targets.append({"expr": e, "legendFormat": ""}) + return { + "id": id, "title": title, "type": "timeseries", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_PROM, + "fieldConfig": {"defaults": {"unit": unit}}, + "targets": targets, + } + + +def log_panel(id, title, query, x, y, w=24, h=10): + return { + "id": id, "title": title, "type": "logs", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_LOKI, + "targets": [{"expr": query}], + } + + +def row(id, title, y, panels, collapsed=True): + return { + "id": id, "title": title, "type": "row", + "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, + "panels": panels, + } + + +# ============================================================================ +# Poimen Memory Dashboard +# ============================================================================ + +def build_dashboard(): + up_down = [{"type": "value", "options": { + "0": {"text": "DOWN", "color": "red"}, + "1": {"text": "UP", "color": "green"}, + }}] + + panels = [ + # ── Row 0: Overview (always open) ── + row(1, "Overview", 0, [ + stat_panel(2, "Pod Ready", + 'sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"})', + 0, 1, thresholds={"mode": "absolute", "steps": [ + {"value": None, "color": "red"}, {"value": 1, "color": "green"} + ]}), + stat_panel(3, "Uptime", + 'memory_app_uptime_seconds', 4, 1, unit="s"), + stat_panel(4, "DB", 'memory_dependency_db_up', 8, 1, mappings=up_down), + stat_panel(5, "Embeddings", 'memory_dependency_embedding_up', 12, 1, mappings=up_down), + stat_panel(6, "LLM", 'memory_dependency_llm_up', 16, 1, mappings=up_down), + stat_panel(7, "Connections", 'memory_app_active_connections', 20, 1), + ], collapsed=False), + + # ── Row 1: Ingest Pipeline ── + row(10, "Ingest Pipeline", 1, [ + ts_panel(11, "Ingest Rate", [ + ('rate(memory_ingest_requests_total[5m])', "requests/s"), + ('rate(memory_ingest_records_total[5m])', "records/s"), + ], 0, 2, unit="ops"), + ts_panel(12, "Ingest Errors", [ + ('rate(memory_ingest_errors_total[5m])', "errors/s"), + ('rate(memory_ingest_auth_failures_total[5m])', "auth failures/s"), + ('rate(memory_ingest_rate_limited_total[5m])', "rate limited/s"), + ], 8, 2, unit="ops"), + ts_panel(13, "Ingest Latency", [ + ('histogram_quantile(0.50, rate(memory_ingest_duration_seconds_bucket[5m]))', "p50"), + ('histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m]))', "p95"), + ('histogram_quantile(0.99, rate(memory_ingest_duration_seconds_bucket[5m]))', "p99"), + ], 16, 2, unit="s"), + stat_panel(14, "In Flight", 'memory_ingest_in_flight', 0, 10), + stat_panel(15, "Queue Size", 'memory_ingest_queue_size', 4, 10), + stat_panel(16, "Entities Extracted", 'rate(memory_ingest_entities_extracted_total[5m])', 8, 10, unit="ops"), + stat_panel(17, "Edges Extracted", 'rate(memory_ingest_edges_extracted_total[5m])', 12, 10, unit="ops"), + stat_panel(18, "Duplicates", 'rate(memory_ingest_duplicates_total[5m])', 16, 10, unit="ops"), + stat_panel(19, "Bytes Ingested", 'rate(memory_ingest_bytes_total[5m])', 20, 10, unit="Bps"), + ts_panel(20, "LLM Extraction Latency", [ + ('histogram_quantile(0.50, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p50"), + ('histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p95"), + ('histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p50"), + ('histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p95"), + ], 0, 14, w=12, unit="s"), + ts_panel(21, "Contradiction & Dedup", [ + ('rate(memory_ingest_contradiction_total[5m])', "contradictions/s"), + ('rate(memory_ingest_dedup_total[5m])', "dedup/s"), + ], 12, 14, w=12, unit="ops"), + ]), + + # ── Row 2: Query Performance ── + row(30, "Query Performance", 2, [ + ts_panel(31, "Query Rate", [ + ('rate(memory_query_requests_total[5m])', "requests/s"), + ('rate(memory_query_results_total[5m])', "results/s"), + ], 0, 3, unit="ops"), + ts_panel(32, "Query Errors", [ + ('rate(memory_query_errors_total[5m])', "errors/s"), + ('rate(memory_query_embedding_failures_total[5m])', "embedding failures/s"), + ('rate(memory_query_empty_results_total[5m])', "empty results/s"), + ], 8, 3, unit="ops"), + ts_panel(33, "Query Latency", [ + ('histogram_quantile(0.50, rate(memory_query_duration_seconds_bucket[5m]))', "p50"), + ('histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m]))', "p95"), + ('histogram_quantile(0.99, rate(memory_query_duration_seconds_bucket[5m]))', "p99"), + ], 16, 3, unit="s"), + stat_panel(34, "In Flight", 'memory_query_in_flight', 0, 11), + ts_panel(35, "Embedding Latency", [ + ('histogram_quantile(0.50, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p50"), + ('histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p95"), + ], 4, 11, unit="s"), + ts_panel(36, "Cache Hit Rate", [ + ('rate(memory_query_cache_hits_total[5m]) / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))', "hit rate"), + ], 12, 11, w=12, unit="percentunit"), + ]), + + # ── Row 3: Context Retrieval ── + row(40, "Context Retrieval (3-tier)", 3, [ + ts_panel(41, "Context Rate", [ + ('rate(memory_context_requests_total[5m])', "requests/s"), + ('rate(memory_context_errors_total[5m])', "errors/s"), + ], 0, 4, unit="ops"), + ts_panel(42, "Context Latency", [ + ('histogram_quantile(0.50, rate(memory_context_duration_seconds_bucket[5m]))', "p50"), + ('histogram_quantile(0.95, rate(memory_context_duration_seconds_bucket[5m]))', "p95"), + ], 8, 4, unit="s"), + ts_panel(43, "Retrieval Tier Hits", [ + ('rate(memory_context_semantic_hits_total[5m])', "semantic"), + ('rate(memory_context_graph_hits_total[5m])', "graph"), + ], 16, 4, unit="ops"), + ts_panel(44, "Tier Latency", [ + ('histogram_quantile(0.95, rate(memory_context_tier_duration_seconds_bucket[5m]))', "p95"), + ], 0, 12, w=12, unit="s"), + ]), + + # ── Row 4: Database ── + row(50, "Database", 4, [ + stat_panel(51, "Entities", 'memory_db_entity_count', 0, 5), + stat_panel(52, "Edges", 'memory_db_edge_count', 4, 5), + stat_panel(53, "Chunks", 'memory_db_chunk_count', 8, 5), + stat_panel(54, "DB Size", 'memory_db_size_bytes', 12, 5, unit="bytes"), + stat_panel(55, "Index Size", 'memory_db_index_size_bytes', 16, 5, unit="bytes"), + stat_panel(56, "Pool Active", 'memory_db_pool_active', 20, 5), + ts_panel(57, "DB Query Latency", [ + ('histogram_quantile(0.50, rate(memory_db_query_duration_seconds_bucket[5m]))', "p50"), + ('histogram_quantile(0.95, rate(memory_db_query_duration_seconds_bucket[5m]))', "p95"), + ('histogram_quantile(0.99, rate(memory_db_query_duration_seconds_bucket[5m]))', "p99"), + ], 0, 9, unit="s"), + ts_panel(58, "DB Query Rate", [ + ('rate(memory_db_queries_total[5m])', "queries/s"), + ('rate(memory_db_query_errors_total[5m])', "errors/s"), + ], 8, 9, unit="ops"), + ts_panel(59, "Connection Pool", [ + ('memory_db_pool_active', "active"), + ('memory_db_pool_idle', "idle"), + ('memory_db_pool_size', "max"), + ], 16, 9), + ts_panel(60, "Table Row Counts", [ + ('memory_db_table_entity_rows', "entities"), + ('memory_db_table_edge_rows', "edges"), + ('memory_db_table_chunk_rows', "chunks"), + ], 0, 17, w=12), + ts_panel(61, "Write Rate", [ + ('rate(memory_write_entities_total[5m])', "entities/s"), + ('rate(memory_write_edges_total[5m])', "edges/s"), + ('rate(memory_write_chunks_total[5m])', "chunks/s"), + ('rate(memory_write_errors_total[5m])', "errors/s"), + ], 12, 17, w=12, unit="ops"), + ]), + + # ── Row 5: Relevance / Quality ── + row(70, "Relevance & Quality", 5, [ + ts_panel(71, "Relevance Score", [ + ('memory_relevance_score', "score"), + ('memory_relevance_precision', "precision"), + ('memory_relevance_recall', "recall"), + ], 0, 6, w=12, unit="percentunit"), + ts_panel(72, "Relevance Evals", [ + ('rate(memory_relevance_relevant_total[5m])', "relevant/s"), + ('rate(memory_relevance_irrelevant_total[5m])', "irrelevant/s"), + ], 12, 6, w=12, unit="ops"), + ]), + + # ── Row 6: Errors Breakdown ── + row(80, "Error Breakdown", 6, [ + ts_panel(81, "Ingest Errors by Type", [ + ('rate(memory_error_auth_failure_ingest_total[5m])', "auth"), + ('rate(memory_error_bad_request_ingest_total[5m])', "bad request"), + ('rate(memory_error_db_error_ingest_total[5m])', "db error"), + ('rate(memory_error_forbidden_ingest_total[5m])', "forbidden"), + ('rate(memory_error_rate_limited_ingest_total[5m])', "rate limited"), + ('rate(memory_error_unexpected_ingest_total[5m])', "unexpected"), + ], 0, 7, w=12, unit="ops"), + ts_panel(82, "Query Errors by Type", [ + ('rate(memory_error_auth_failure_query_total[5m])', "auth"), + ('rate(memory_error_bad_request_query_total[5m])', "bad request"), + ('rate(memory_error_embedding_failure_query_total[5m])', "embedding"), + ('rate(memory_error_forbidden_query_total[5m])', "forbidden"), + ('rate(memory_error_search_failure_query_total[5m])', "search"), + ('rate(memory_error_unexpected_query_total[5m])', "unexpected"), + ], 12, 7, w=12, unit="ops"), + ]), + + # ── Row 7: Application Resources ── + row(90, "Application Resources", 7, [ + ts_panel(91, "CPU", [ + ('sum(rate(container_cpu_usage_seconds_total{namespace="poimen",container="poimen-memory"}[5m])) by (pod)', "{{pod}}"), + ], 0, 8), + ts_panel(92, "Memory", [ + ('sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)', "{{pod}}"), + ], 8, 8, unit="bytes"), + ts_panel(93, "Heap Bytes", [ + ('memory_app_heap_bytes', "heap"), + ], 16, 8, unit="bytes"), + ts_panel(94, "Tokio Tasks", [ + ('memory_app_tokio_tasks', "active tasks"), + ], 0, 16), + ts_panel(95, "Dependency Latency", [ + ('memory_dependency_db_latency_seconds', "db"), + ('memory_dependency_embedding_latency_seconds', "embedding"), + ], 8, 16, unit="s"), + ]), + + # ── Row 8: Logs ── + row(100, "Logs", 8, [ + log_panel(101, "Poimen Memory Logs", '{namespace="poimen",container="poimen-memory"}', 0, 9), + ]), + ] + + return { + "title": "Poimen Memory", + "uid": "poimen-memory", + "schemaVersion": 39, + "timezone": "browser", + "time": {"from": "now-6h", "to": "now"}, + "refresh": "30s", + "tags": ["poimen", "memory", "knowledge-graph"], + "panels": panels, + } + + +# ============================================================================ +# Generate ConfigMap YAML +# ============================================================================ + +dashboard = build_dashboard() + +cm = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "poimen-memory-dashboard", + "namespace": "logging", + "labels": {"grafana_dashboard": "1"}, + "annotations": {"grafana_folder": "Poimen"}, + }, + "data": { + "poimen-memory.json": json.dumps(dashboard, separators=(",", ":")), + }, +} + +out_path = os.path.join(SCRIPT_DIR, "dashboard.yaml") +with open(out_path, "w") as f: + yaml.dump(cm, f, default_flow_style=False, allow_unicode=True) + +print(f"Generated {out_path}") diff --git a/k8s/monitoring/kustomization.yaml b/k8s/monitoring/kustomization.yaml new file mode 100644 index 0000000..d9786b9 --- /dev/null +++ b/k8s/monitoring/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - service-monitor.yaml + - alerts.yaml + - dashboard.yaml diff --git a/k8s/app/service-monitor.yaml b/k8s/monitoring/service-monitor.yaml similarity index 54% rename from k8s/app/service-monitor.yaml rename to k8s/monitoring/service-monitor.yaml index caf141c..feb27bd 100644 --- a/k8s/app/service-monitor.yaml +++ b/k8s/monitoring/service-monitor.yaml @@ -2,14 +2,19 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: poimen-memory - namespace: poimen + namespace: monitoring labels: + release: kube-prometheus-stack app.kubernetes.io/name: poimen-memory spec: + namespaceSelector: + matchNames: + - poimen selector: matchLabels: app.kubernetes.io/name: poimen-memory endpoints: - - port: http - path: /metrics - interval: 30s \ No newline at end of file + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s