riotpiao-poimen/poimen-memory
2
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
8902e1b6fa158209d2099ad244c2502f2b0ccec6
poimen-memory/k8s/monitoring/dashboard.yaml
T
rock 8902e1b6fa feat: add monitoring resources (alerts, dashboard, service-monitor) 
- PrometheusRule with 16 alerts across 6 groups:
  availability, ingest, query, dependencies, resources, errors
- Grafana dashboard (ConfigMap, sidecar auto-discovery):
  8 rows covering overview, ingest pipeline, query perf,
  context retrieval, database, relevance, error breakdown,
  app resources, logs
- ServiceMonitor moved from k8s/app/ to k8s/monitoring/
- Dashboard generator script (Python, matches homelab pattern)
- Separate k8s/monitoring/ path for ArgoCD Application
- sqlx::migrate!() added to server startup for seamless migrations
2026-09-16 11:52:47 +09:00

81 lines
18 KiB
YAML
Raw Blame History

apiVersion: v1
data:
poimen-memory.json: '{"title":"Poimen Memory","uid":"poimen-memory","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["poimen","memory","knowledge-graph"],"panels":[{"id":1,"title":"Overview","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Pod
Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"poimen\",pod=~\"poimen-memory.*\",condition=\"true\"})"}]},{"id":3,"title":"Uptime","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_app_uptime_seconds"}]},{"id":4,"title":"DB","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_db_up"}]},{"id":5,"title":"Embeddings","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_embedding_up"}]},{"id":6,"title":"LLM","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_llm_up"}]},{"id":7,"title":"Connections","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_active_connections"}]}]},{"id":10,"title":"Ingest
Pipeline","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Ingest
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_ingest_records_total[5m])","legendFormat":"records/s"}]},{"id":12,"title":"Ingest
Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_ingest_auth_failures_total[5m])","legendFormat":"auth
failures/s"},{"expr":"rate(memory_ingest_rate_limited_total[5m])","legendFormat":"rate
limited/s"}]},{"id":13,"title":"Ingest Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":14,"title":"In
Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_in_flight"}]},{"id":15,"title":"Queue
Size","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_queue_size"}]},{"id":16,"title":"Entities
Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_entities_extracted_total[5m])"}]},{"id":17,"title":"Edges
Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_edges_extracted_total[5m])"}]},{"id":18,"title":"Duplicates","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_duplicates_total[5m])"}]},{"id":19,"title":"Bytes
Ingested","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"Bps"}},"targets":[{"expr":"rate(memory_ingest_bytes_total[5m])"}]},{"id":20,"title":"LLM
Extraction Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
p95"},{"expr":"histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
p95"}]},{"id":21,"title":"Contradiction & Dedup","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_contradiction_total[5m])","legendFormat":"contradictions/s"},{"expr":"rate(memory_ingest_dedup_total[5m])","legendFormat":"dedup/s"}]}]},{"id":30,"title":"Query
Performance","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":31,"title":"Query
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_query_results_total[5m])","legendFormat":"results/s"}]},{"id":32,"title":"Query
Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_query_embedding_failures_total[5m])","legendFormat":"embedding
failures/s"},{"expr":"rate(memory_query_empty_results_total[5m])","legendFormat":"empty
results/s"}]},{"id":33,"title":"Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":34,"title":"In
Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_query_in_flight"}]},{"id":35,"title":"Embedding
Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":36,"title":"Cache
Hit Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(memory_query_cache_hits_total[5m])
/ (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))","legendFormat":"hit
rate"}]}]},{"id":40,"title":"Context Retrieval (3-tier)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":41,"title":"Context
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_context_errors_total[5m])","legendFormat":"errors/s"}]},{"id":42,"title":"Context
Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":43,"title":"Retrieval
Tier Hits","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_semantic_hits_total[5m])","legendFormat":"semantic"},{"expr":"rate(memory_context_graph_hits_total[5m])","legendFormat":"graph"}]},{"id":44,"title":"Tier
Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
rate(memory_context_tier_duration_seconds_bucket[5m]))","legendFormat":"p95"}]}]},{"id":50,"title":"Database","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":51,"title":"Entities","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_entity_count"}]},{"id":52,"title":"Edges","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_edge_count"}]},{"id":53,"title":"Chunks","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_chunk_count"}]},{"id":54,"title":"DB
Size","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_size_bytes"}]},{"id":55,"title":"Index
Size","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_index_size_bytes"}]},{"id":56,"title":"Pool
Active","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active"}]},{"id":57,"title":"DB
Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":58,"title":"DB
Query Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_db_queries_total[5m])","legendFormat":"queries/s"},{"expr":"rate(memory_db_query_errors_total[5m])","legendFormat":"errors/s"}]},{"id":59,"title":"Connection
Pool","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active","legendFormat":"active"},{"expr":"memory_db_pool_idle","legendFormat":"idle"},{"expr":"memory_db_pool_size","legendFormat":"max"}]},{"id":60,"title":"Table
Row Counts","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_table_entity_rows","legendFormat":"entities"},{"expr":"memory_db_table_edge_rows","legendFormat":"edges"},{"expr":"memory_db_table_chunk_rows","legendFormat":"chunks"}]},{"id":61,"title":"Write
Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_write_entities_total[5m])","legendFormat":"entities/s"},{"expr":"rate(memory_write_edges_total[5m])","legendFormat":"edges/s"},{"expr":"rate(memory_write_chunks_total[5m])","legendFormat":"chunks/s"},{"expr":"rate(memory_write_errors_total[5m])","legendFormat":"errors/s"}]}]},{"id":70,"title":"Relevance
& Quality","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":71,"title":"Relevance
Score","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"memory_relevance_score","legendFormat":"score"},{"expr":"memory_relevance_precision","legendFormat":"precision"},{"expr":"memory_relevance_recall","legendFormat":"recall"}]},{"id":72,"title":"Relevance
Evals","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_relevance_relevant_total[5m])","legendFormat":"relevant/s"},{"expr":"rate(memory_relevance_irrelevant_total[5m])","legendFormat":"irrelevant/s"}]}]},{"id":80,"title":"Error
Breakdown","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":81,"title":"Ingest
Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_ingest_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_ingest_total[5m])","legendFormat":"bad
request"},{"expr":"rate(memory_error_db_error_ingest_total[5m])","legendFormat":"db
error"},{"expr":"rate(memory_error_forbidden_ingest_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_rate_limited_ingest_total[5m])","legendFormat":"rate
limited"},{"expr":"rate(memory_error_unexpected_ingest_total[5m])","legendFormat":"unexpected"}]},{"id":82,"title":"Query
Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_query_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_query_total[5m])","legendFormat":"bad
request"},{"expr":"rate(memory_error_embedding_failure_query_total[5m])","legendFormat":"embedding"},{"expr":"rate(memory_error_forbidden_query_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_search_failure_query_total[5m])","legendFormat":"search"},{"expr":"rate(memory_error_unexpected_query_total[5m])","legendFormat":"unexpected"}]}]},{"id":90,"title":"Application
Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":91,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"poimen\",container=\"poimen-memory\"}[5m]))
by (pod)","legendFormat":"{{pod}}"}]},{"id":92,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"poimen\",container=\"poimen-memory\"})
by (pod)","legendFormat":"{{pod}}"}]},{"id":93,"title":"Heap Bytes","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_app_heap_bytes","legendFormat":"heap"}]},{"id":94,"title":"Tokio
Tasks","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_tokio_tasks","legendFormat":"active
tasks"}]},{"id":95,"title":"Dependency Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_dependency_db_latency_seconds","legendFormat":"db"},{"expr":"memory_dependency_embedding_latency_seconds","legendFormat":"embedding"}]}]},{"id":100,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":8},"panels":[{"id":101,"title":"Poimen
Memory Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":9},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"poimen\",container=\"poimen-memory\"}"}]}]}]}'
kind: ConfigMap
metadata:
annotations:
grafana_folder: Poimen
labels:
grafana_dashboard: '1'
name: poimen-memory-dashboard
namespace: logging
Reference in New Issue View Git Blame Copy Permalink