feat: add monitoring resources (alerts, dashboard, service-monitor)
- PrometheusRule with 16 alerts across 6 groups: availability, ingest, query, dependencies, resources, errors - Grafana dashboard (ConfigMap, sidecar auto-discovery): 8 rows covering overview, ingest pipeline, query perf, context retrieval, database, relevance, error breakdown, app resources, logs - ServiceMonitor moved from k8s/app/ to k8s/monitoring/ - Dashboard generator script (Python, matches homelab pattern) - Separate k8s/monitoring/ path for ArgoCD Application - sqlx::migrate!() added to server startup for seamless migrations
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: poimen-memory-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
groups:
|
||||
# ================================================================
|
||||
# SLA Targets:
|
||||
# Availability: 99.9%
|
||||
# Ingest p95: < 10s (includes LLM extraction)
|
||||
# Query p95: < 2s
|
||||
# Error rate: < 5%
|
||||
# DB connection: always available
|
||||
# ================================================================
|
||||
|
||||
- name: poimen-memory.availability
|
||||
rules:
|
||||
- alert: PoimenMemoryDown
|
||||
expr: sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"}) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen Memory has zero ready pods"
|
||||
|
||||
- alert: PoimenMemoryRestarted
|
||||
expr: increase(kube_pod_container_status_restarts_total{namespace="poimen",pod=~"poimen-memory.*"}[15m]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory pod {{ $labels.pod }} restarted"
|
||||
|
||||
- alert: PoimenMemoryHealthCheckFailing
|
||||
expr: rate(memory_health_check_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory health checks failing"
|
||||
|
||||
- name: poimen-memory.ingest
|
||||
rules:
|
||||
- alert: PoimenIngestErrorRate
|
||||
expr: |
|
||||
rate(memory_ingest_errors_total[5m])
|
||||
/ (rate(memory_ingest_requests_total[5m]) > 0)
|
||||
> 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest error rate {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: PoimenIngestLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_ingest_duration_seconds_bucket[5m])
|
||||
) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <10s)"
|
||||
|
||||
- alert: PoimenLLMExtractionSlow
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_ingest_llm_extract_duration_seconds_bucket[5m])
|
||||
) > 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "LLM entity extraction p95 {{ $value | printf \"%.1f\" }}s"
|
||||
|
||||
- alert: PoimenIngestQueueBacklog
|
||||
expr: memory_ingest_queue_size > 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest queue size {{ $value }} (backlog)"
|
||||
|
||||
- name: poimen-memory.query
|
||||
rules:
|
||||
- alert: PoimenQueryErrorRate
|
||||
expr: |
|
||||
rate(memory_query_errors_total[5m])
|
||||
/ (rate(memory_query_requests_total[5m]) > 0)
|
||||
> 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen query error rate {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: PoimenQueryLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_query_duration_seconds_bucket[5m])
|
||||
) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen query p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
|
||||
|
||||
- alert: PoimenEmbeddingFailures
|
||||
expr: rate(memory_query_embedding_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen embedding failures — semantic search degraded"
|
||||
|
||||
- alert: PoimenQueryEmptyResultRate
|
||||
expr: |
|
||||
rate(memory_query_empty_results_total[5m])
|
||||
/ (rate(memory_query_requests_total[5m]) > 0)
|
||||
> 0.5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $value | humanizePercentage }} of queries returning empty results"
|
||||
|
||||
- name: poimen-memory.dependencies
|
||||
rules:
|
||||
- alert: PoimenDatabaseDown
|
||||
expr: memory_dependency_db_up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen Memory cannot reach database"
|
||||
|
||||
- alert: PoimenDatabaseLatencyHigh
|
||||
expr: memory_dependency_db_latency_seconds > 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen DB latency {{ $value | printf \"%.1f\" }}s"
|
||||
|
||||
- alert: PoimenEmbeddingServiceDown
|
||||
expr: memory_dependency_embedding_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen embedding service unreachable — ingest + semantic query broken"
|
||||
|
||||
- alert: PoimenLLMServiceDown
|
||||
expr: memory_dependency_llm_up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen LLM service unreachable — entity extraction degraded"
|
||||
|
||||
- name: poimen-memory.resources
|
||||
rules:
|
||||
- alert: PoimenMemoryHighMemory
|
||||
expr: |
|
||||
sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)
|
||||
/ sum(kube_pod_container_resource_limits{namespace="poimen",container="poimen-memory",resource="memory"}) by (pod)
|
||||
> 0.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory pod {{ $labels.pod }} at {{ $value | humanizePercentage }} memory limit"
|
||||
|
||||
- alert: PoimenDBPoolExhausted
|
||||
expr: memory_db_pool_active >= memory_db_pool_size
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen DB connection pool exhausted: {{ $value }} active = pool size"
|
||||
|
||||
- name: poimen-memory.errors
|
||||
rules:
|
||||
- alert: PoimenAuthFailureSpike
|
||||
expr: |
|
||||
sum(rate(memory_ingest_auth_failures_total[5m]))
|
||||
+ sum(rate(memory_query_auth_failures_total[5m]))
|
||||
> 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen auth failures: {{ $value | printf \"%.1f\" }}/sec"
|
||||
|
||||
- alert: PoimenUnexpectedErrors
|
||||
expr: rate(memory_error_unexpected_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen unexpected errors at {{ $value | printf \"%.2f\" }}/sec"
|
||||
@@ -0,0 +1,80 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
poimen-memory.json: '{"title":"Poimen Memory","uid":"poimen-memory","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["poimen","memory","knowledge-graph"],"panels":[{"id":1,"title":"Overview","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Pod
|
||||
Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"poimen\",pod=~\"poimen-memory.*\",condition=\"true\"})"}]},{"id":3,"title":"Uptime","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_app_uptime_seconds"}]},{"id":4,"title":"DB","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_db_up"}]},{"id":5,"title":"Embeddings","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_embedding_up"}]},{"id":6,"title":"LLM","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"memory_dependency_llm_up"}]},{"id":7,"title":"Connections","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_active_connections"}]}]},{"id":10,"title":"Ingest
|
||||
Pipeline","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Ingest
|
||||
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_ingest_records_total[5m])","legendFormat":"records/s"}]},{"id":12,"title":"Ingest
|
||||
Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_ingest_auth_failures_total[5m])","legendFormat":"auth
|
||||
failures/s"},{"expr":"rate(memory_ingest_rate_limited_total[5m])","legendFormat":"rate
|
||||
limited/s"}]},{"id":13,"title":"Ingest Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
|
||||
rate(memory_ingest_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":14,"title":"In
|
||||
Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_in_flight"}]},{"id":15,"title":"Queue
|
||||
Size","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_ingest_queue_size"}]},{"id":16,"title":"Entities
|
||||
Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_entities_extracted_total[5m])"}]},{"id":17,"title":"Edges
|
||||
Extracted","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_edges_extracted_total[5m])"}]},{"id":18,"title":"Duplicates","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_duplicates_total[5m])"}]},{"id":19,"title":"Bytes
|
||||
Ingested","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"Bps"}},"targets":[{"expr":"rate(memory_ingest_bytes_total[5m])"}]},{"id":20,"title":"LLM
|
||||
Extraction Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
|
||||
p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))","legendFormat":"entity
|
||||
p95"},{"expr":"histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
|
||||
p50"},{"expr":"histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))","legendFormat":"fact
|
||||
p95"}]},{"id":21,"title":"Contradiction & Dedup","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_ingest_contradiction_total[5m])","legendFormat":"contradictions/s"},{"expr":"rate(memory_ingest_dedup_total[5m])","legendFormat":"dedup/s"}]}]},{"id":30,"title":"Query
|
||||
Performance","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":31,"title":"Query
|
||||
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_query_results_total[5m])","legendFormat":"results/s"}]},{"id":32,"title":"Query
|
||||
Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_query_errors_total[5m])","legendFormat":"errors/s"},{"expr":"rate(memory_query_embedding_failures_total[5m])","legendFormat":"embedding
|
||||
failures/s"},{"expr":"rate(memory_query_empty_results_total[5m])","legendFormat":"empty
|
||||
results/s"}]},{"id":33,"title":"Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
|
||||
rate(memory_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":34,"title":"In
|
||||
Flight","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_query_in_flight"}]},{"id":35,"title":"Embedding
|
||||
Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_query_embedding_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":36,"title":"Cache
|
||||
Hit Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(memory_query_cache_hits_total[5m])
|
||||
/ (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))","legendFormat":"hit
|
||||
rate"}]}]},{"id":40,"title":"Context Retrieval (3-tier)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":41,"title":"Context
|
||||
Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_requests_total[5m])","legendFormat":"requests/s"},{"expr":"rate(memory_context_errors_total[5m])","legendFormat":"errors/s"}]},{"id":42,"title":"Context
|
||||
Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_context_duration_seconds_bucket[5m]))","legendFormat":"p95"}]},{"id":43,"title":"Retrieval
|
||||
Tier Hits","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_context_semantic_hits_total[5m])","legendFormat":"semantic"},{"expr":"rate(memory_context_graph_hits_total[5m])","legendFormat":"graph"}]},{"id":44,"title":"Tier
|
||||
Latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_context_tier_duration_seconds_bucket[5m]))","legendFormat":"p95"}]}]},{"id":50,"title":"Database","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":51,"title":"Entities","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_entity_count"}]},{"id":52,"title":"Edges","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_edge_count"}]},{"id":53,"title":"Chunks","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_chunk_count"}]},{"id":54,"title":"DB
|
||||
Size","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_size_bytes"}]},{"id":55,"title":"Index
|
||||
Size","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_db_index_size_bytes"}]},{"id":56,"title":"Pool
|
||||
Active","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active"}]},{"id":57,"title":"DB
|
||||
Query Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50,
|
||||
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
|
||||
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
|
||||
rate(memory_db_query_duration_seconds_bucket[5m]))","legendFormat":"p99"}]},{"id":58,"title":"DB
|
||||
Query Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_db_queries_total[5m])","legendFormat":"queries/s"},{"expr":"rate(memory_db_query_errors_total[5m])","legendFormat":"errors/s"}]},{"id":59,"title":"Connection
|
||||
Pool","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_pool_active","legendFormat":"active"},{"expr":"memory_db_pool_idle","legendFormat":"idle"},{"expr":"memory_db_pool_size","legendFormat":"max"}]},{"id":60,"title":"Table
|
||||
Row Counts","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_db_table_entity_rows","legendFormat":"entities"},{"expr":"memory_db_table_edge_rows","legendFormat":"edges"},{"expr":"memory_db_table_chunk_rows","legendFormat":"chunks"}]},{"id":61,"title":"Write
|
||||
Rate","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_write_entities_total[5m])","legendFormat":"entities/s"},{"expr":"rate(memory_write_edges_total[5m])","legendFormat":"edges/s"},{"expr":"rate(memory_write_chunks_total[5m])","legendFormat":"chunks/s"},{"expr":"rate(memory_write_errors_total[5m])","legendFormat":"errors/s"}]}]},{"id":70,"title":"Relevance
|
||||
& Quality","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":71,"title":"Relevance
|
||||
Score","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"memory_relevance_score","legendFormat":"score"},{"expr":"memory_relevance_precision","legendFormat":"precision"},{"expr":"memory_relevance_recall","legendFormat":"recall"}]},{"id":72,"title":"Relevance
|
||||
Evals","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_relevance_relevant_total[5m])","legendFormat":"relevant/s"},{"expr":"rate(memory_relevance_irrelevant_total[5m])","legendFormat":"irrelevant/s"}]}]},{"id":80,"title":"Error
|
||||
Breakdown","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":81,"title":"Ingest
|
||||
Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_ingest_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_ingest_total[5m])","legendFormat":"bad
|
||||
request"},{"expr":"rate(memory_error_db_error_ingest_total[5m])","legendFormat":"db
|
||||
error"},{"expr":"rate(memory_error_forbidden_ingest_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_rate_limited_ingest_total[5m])","legendFormat":"rate
|
||||
limited"},{"expr":"rate(memory_error_unexpected_ingest_total[5m])","legendFormat":"unexpected"}]},{"id":82,"title":"Query
|
||||
Errors by Type","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"ops"}},"targets":[{"expr":"rate(memory_error_auth_failure_query_total[5m])","legendFormat":"auth"},{"expr":"rate(memory_error_bad_request_query_total[5m])","legendFormat":"bad
|
||||
request"},{"expr":"rate(memory_error_embedding_failure_query_total[5m])","legendFormat":"embedding"},{"expr":"rate(memory_error_forbidden_query_total[5m])","legendFormat":"forbidden"},{"expr":"rate(memory_error_search_failure_query_total[5m])","legendFormat":"search"},{"expr":"rate(memory_error_unexpected_query_total[5m])","legendFormat":"unexpected"}]}]},{"id":90,"title":"Application
|
||||
Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":91,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"poimen\",container=\"poimen-memory\"}[5m]))
|
||||
by (pod)","legendFormat":"{{pod}}"}]},{"id":92,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"poimen\",container=\"poimen-memory\"})
|
||||
by (pod)","legendFormat":"{{pod}}"}]},{"id":93,"title":"Heap Bytes","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"memory_app_heap_bytes","legendFormat":"heap"}]},{"id":94,"title":"Tokio
|
||||
Tasks","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"memory_app_tokio_tasks","legendFormat":"active
|
||||
tasks"}]},{"id":95,"title":"Dependency Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":16},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"memory_dependency_db_latency_seconds","legendFormat":"db"},{"expr":"memory_dependency_embedding_latency_seconds","legendFormat":"embedding"}]}]},{"id":100,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":8},"panels":[{"id":101,"title":"Poimen
|
||||
Memory Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":9},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"poimen\",container=\"poimen-memory\"}"}]}]}]}'
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
annotations:
|
||||
grafana_folder: Poimen
|
||||
labels:
|
||||
grafana_dashboard: '1'
|
||||
name: poimen-memory-dashboard
|
||||
namespace: logging
|
||||
@@ -0,0 +1,304 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate Poimen Memory Grafana dashboard as k8s ConfigMap YAML.
|
||||
|
||||
Follows homelab pattern: ConfigMap with grafana_dashboard=1 label,
|
||||
auto-discovered by Grafana sidecar.
|
||||
|
||||
Usage: python3 generate-dashboard.py
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import yaml
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
DS_PROM = {"type": "prometheus", "uid": "prometheus"}
|
||||
DS_LOKI = {"type": "loki", "uid": "loki"}
|
||||
|
||||
|
||||
def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
|
||||
p = {
|
||||
"id": id, "title": title, "type": "stat",
|
||||
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
||||
"datasource": DS_PROM,
|
||||
"fieldConfig": {"defaults": {"unit": unit}},
|
||||
"targets": [{"expr": expr}],
|
||||
}
|
||||
if mappings:
|
||||
p["fieldConfig"]["defaults"]["mappings"] = mappings
|
||||
if thresholds:
|
||||
p["fieldConfig"]["defaults"]["thresholds"] = thresholds
|
||||
p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
|
||||
return p
|
||||
|
||||
|
||||
def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
|
||||
targets = []
|
||||
for e in exprs:
|
||||
if isinstance(e, tuple):
|
||||
targets.append({"expr": e[0], "legendFormat": e[1]})
|
||||
else:
|
||||
targets.append({"expr": e, "legendFormat": ""})
|
||||
return {
|
||||
"id": id, "title": title, "type": "timeseries",
|
||||
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
||||
"datasource": DS_PROM,
|
||||
"fieldConfig": {"defaults": {"unit": unit}},
|
||||
"targets": targets,
|
||||
}
|
||||
|
||||
|
||||
def log_panel(id, title, query, x, y, w=24, h=10):
|
||||
return {
|
||||
"id": id, "title": title, "type": "logs",
|
||||
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
||||
"datasource": DS_LOKI,
|
||||
"targets": [{"expr": query}],
|
||||
}
|
||||
|
||||
|
||||
def row(id, title, y, panels, collapsed=True):
|
||||
return {
|
||||
"id": id, "title": title, "type": "row",
|
||||
"collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
|
||||
"panels": panels,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Poimen Memory Dashboard
|
||||
# ============================================================================
|
||||
|
||||
def build_dashboard():
|
||||
up_down = [{"type": "value", "options": {
|
||||
"0": {"text": "DOWN", "color": "red"},
|
||||
"1": {"text": "UP", "color": "green"},
|
||||
}}]
|
||||
|
||||
panels = [
|
||||
# ── Row 0: Overview (always open) ──
|
||||
row(1, "Overview", 0, [
|
||||
stat_panel(2, "Pod Ready",
|
||||
'sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"})',
|
||||
0, 1, thresholds={"mode": "absolute", "steps": [
|
||||
{"value": None, "color": "red"}, {"value": 1, "color": "green"}
|
||||
]}),
|
||||
stat_panel(3, "Uptime",
|
||||
'memory_app_uptime_seconds', 4, 1, unit="s"),
|
||||
stat_panel(4, "DB", 'memory_dependency_db_up', 8, 1, mappings=up_down),
|
||||
stat_panel(5, "Embeddings", 'memory_dependency_embedding_up', 12, 1, mappings=up_down),
|
||||
stat_panel(6, "LLM", 'memory_dependency_llm_up', 16, 1, mappings=up_down),
|
||||
stat_panel(7, "Connections", 'memory_app_active_connections', 20, 1),
|
||||
], collapsed=False),
|
||||
|
||||
# ── Row 1: Ingest Pipeline ──
|
||||
row(10, "Ingest Pipeline", 1, [
|
||||
ts_panel(11, "Ingest Rate", [
|
||||
('rate(memory_ingest_requests_total[5m])', "requests/s"),
|
||||
('rate(memory_ingest_records_total[5m])', "records/s"),
|
||||
], 0, 2, unit="ops"),
|
||||
ts_panel(12, "Ingest Errors", [
|
||||
('rate(memory_ingest_errors_total[5m])', "errors/s"),
|
||||
('rate(memory_ingest_auth_failures_total[5m])', "auth failures/s"),
|
||||
('rate(memory_ingest_rate_limited_total[5m])', "rate limited/s"),
|
||||
], 8, 2, unit="ops"),
|
||||
ts_panel(13, "Ingest Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_ingest_duration_seconds_bucket[5m]))', "p50"),
|
||||
('histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m]))', "p95"),
|
||||
('histogram_quantile(0.99, rate(memory_ingest_duration_seconds_bucket[5m]))', "p99"),
|
||||
], 16, 2, unit="s"),
|
||||
stat_panel(14, "In Flight", 'memory_ingest_in_flight', 0, 10),
|
||||
stat_panel(15, "Queue Size", 'memory_ingest_queue_size', 4, 10),
|
||||
stat_panel(16, "Entities Extracted", 'rate(memory_ingest_entities_extracted_total[5m])', 8, 10, unit="ops"),
|
||||
stat_panel(17, "Edges Extracted", 'rate(memory_ingest_edges_extracted_total[5m])', 12, 10, unit="ops"),
|
||||
stat_panel(18, "Duplicates", 'rate(memory_ingest_duplicates_total[5m])', 16, 10, unit="ops"),
|
||||
stat_panel(19, "Bytes Ingested", 'rate(memory_ingest_bytes_total[5m])', 20, 10, unit="Bps"),
|
||||
ts_panel(20, "LLM Extraction Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p50"),
|
||||
('histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p95"),
|
||||
('histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p50"),
|
||||
('histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p95"),
|
||||
], 0, 14, w=12, unit="s"),
|
||||
ts_panel(21, "Contradiction & Dedup", [
|
||||
('rate(memory_ingest_contradiction_total[5m])', "contradictions/s"),
|
||||
('rate(memory_ingest_dedup_total[5m])', "dedup/s"),
|
||||
], 12, 14, w=12, unit="ops"),
|
||||
]),
|
||||
|
||||
# ── Row 2: Query Performance ──
|
||||
row(30, "Query Performance", 2, [
|
||||
ts_panel(31, "Query Rate", [
|
||||
('rate(memory_query_requests_total[5m])', "requests/s"),
|
||||
('rate(memory_query_results_total[5m])', "results/s"),
|
||||
], 0, 3, unit="ops"),
|
||||
ts_panel(32, "Query Errors", [
|
||||
('rate(memory_query_errors_total[5m])', "errors/s"),
|
||||
('rate(memory_query_embedding_failures_total[5m])', "embedding failures/s"),
|
||||
('rate(memory_query_empty_results_total[5m])', "empty results/s"),
|
||||
], 8, 3, unit="ops"),
|
||||
ts_panel(33, "Query Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_query_duration_seconds_bucket[5m]))', "p50"),
|
||||
('histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m]))', "p95"),
|
||||
('histogram_quantile(0.99, rate(memory_query_duration_seconds_bucket[5m]))', "p99"),
|
||||
], 16, 3, unit="s"),
|
||||
stat_panel(34, "In Flight", 'memory_query_in_flight', 0, 11),
|
||||
ts_panel(35, "Embedding Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p50"),
|
||||
('histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p95"),
|
||||
], 4, 11, unit="s"),
|
||||
ts_panel(36, "Cache Hit Rate", [
|
||||
('rate(memory_query_cache_hits_total[5m]) / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))', "hit rate"),
|
||||
], 12, 11, w=12, unit="percentunit"),
|
||||
]),
|
||||
|
||||
# ── Row 3: Context Retrieval ──
|
||||
row(40, "Context Retrieval (3-tier)", 3, [
|
||||
ts_panel(41, "Context Rate", [
|
||||
('rate(memory_context_requests_total[5m])', "requests/s"),
|
||||
('rate(memory_context_errors_total[5m])', "errors/s"),
|
||||
], 0, 4, unit="ops"),
|
||||
ts_panel(42, "Context Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_context_duration_seconds_bucket[5m]))', "p50"),
|
||||
('histogram_quantile(0.95, rate(memory_context_duration_seconds_bucket[5m]))', "p95"),
|
||||
], 8, 4, unit="s"),
|
||||
ts_panel(43, "Retrieval Tier Hits", [
|
||||
('rate(memory_context_semantic_hits_total[5m])', "semantic"),
|
||||
('rate(memory_context_graph_hits_total[5m])', "graph"),
|
||||
], 16, 4, unit="ops"),
|
||||
ts_panel(44, "Tier Latency", [
|
||||
('histogram_quantile(0.95, rate(memory_context_tier_duration_seconds_bucket[5m]))', "p95"),
|
||||
], 0, 12, w=12, unit="s"),
|
||||
]),
|
||||
|
||||
# ── Row 4: Database ──
|
||||
row(50, "Database", 4, [
|
||||
stat_panel(51, "Entities", 'memory_db_entity_count', 0, 5),
|
||||
stat_panel(52, "Edges", 'memory_db_edge_count', 4, 5),
|
||||
stat_panel(53, "Chunks", 'memory_db_chunk_count', 8, 5),
|
||||
stat_panel(54, "DB Size", 'memory_db_size_bytes', 12, 5, unit="bytes"),
|
||||
stat_panel(55, "Index Size", 'memory_db_index_size_bytes', 16, 5, unit="bytes"),
|
||||
stat_panel(56, "Pool Active", 'memory_db_pool_active', 20, 5),
|
||||
ts_panel(57, "DB Query Latency", [
|
||||
('histogram_quantile(0.50, rate(memory_db_query_duration_seconds_bucket[5m]))', "p50"),
|
||||
('histogram_quantile(0.95, rate(memory_db_query_duration_seconds_bucket[5m]))', "p95"),
|
||||
('histogram_quantile(0.99, rate(memory_db_query_duration_seconds_bucket[5m]))', "p99"),
|
||||
], 0, 9, unit="s"),
|
||||
ts_panel(58, "DB Query Rate", [
|
||||
('rate(memory_db_queries_total[5m])', "queries/s"),
|
||||
('rate(memory_db_query_errors_total[5m])', "errors/s"),
|
||||
], 8, 9, unit="ops"),
|
||||
ts_panel(59, "Connection Pool", [
|
||||
('memory_db_pool_active', "active"),
|
||||
('memory_db_pool_idle', "idle"),
|
||||
('memory_db_pool_size', "max"),
|
||||
], 16, 9),
|
||||
ts_panel(60, "Table Row Counts", [
|
||||
('memory_db_table_entity_rows', "entities"),
|
||||
('memory_db_table_edge_rows', "edges"),
|
||||
('memory_db_table_chunk_rows', "chunks"),
|
||||
], 0, 17, w=12),
|
||||
ts_panel(61, "Write Rate", [
|
||||
('rate(memory_write_entities_total[5m])', "entities/s"),
|
||||
('rate(memory_write_edges_total[5m])', "edges/s"),
|
||||
('rate(memory_write_chunks_total[5m])', "chunks/s"),
|
||||
('rate(memory_write_errors_total[5m])', "errors/s"),
|
||||
], 12, 17, w=12, unit="ops"),
|
||||
]),
|
||||
|
||||
# ── Row 5: Relevance / Quality ──
|
||||
row(70, "Relevance & Quality", 5, [
|
||||
ts_panel(71, "Relevance Score", [
|
||||
('memory_relevance_score', "score"),
|
||||
('memory_relevance_precision', "precision"),
|
||||
('memory_relevance_recall', "recall"),
|
||||
], 0, 6, w=12, unit="percentunit"),
|
||||
ts_panel(72, "Relevance Evals", [
|
||||
('rate(memory_relevance_relevant_total[5m])', "relevant/s"),
|
||||
('rate(memory_relevance_irrelevant_total[5m])', "irrelevant/s"),
|
||||
], 12, 6, w=12, unit="ops"),
|
||||
]),
|
||||
|
||||
# ── Row 6: Errors Breakdown ──
|
||||
row(80, "Error Breakdown", 6, [
|
||||
ts_panel(81, "Ingest Errors by Type", [
|
||||
('rate(memory_error_auth_failure_ingest_total[5m])', "auth"),
|
||||
('rate(memory_error_bad_request_ingest_total[5m])', "bad request"),
|
||||
('rate(memory_error_db_error_ingest_total[5m])', "db error"),
|
||||
('rate(memory_error_forbidden_ingest_total[5m])', "forbidden"),
|
||||
('rate(memory_error_rate_limited_ingest_total[5m])', "rate limited"),
|
||||
('rate(memory_error_unexpected_ingest_total[5m])', "unexpected"),
|
||||
], 0, 7, w=12, unit="ops"),
|
||||
ts_panel(82, "Query Errors by Type", [
|
||||
('rate(memory_error_auth_failure_query_total[5m])', "auth"),
|
||||
('rate(memory_error_bad_request_query_total[5m])', "bad request"),
|
||||
('rate(memory_error_embedding_failure_query_total[5m])', "embedding"),
|
||||
('rate(memory_error_forbidden_query_total[5m])', "forbidden"),
|
||||
('rate(memory_error_search_failure_query_total[5m])', "search"),
|
||||
('rate(memory_error_unexpected_query_total[5m])', "unexpected"),
|
||||
], 12, 7, w=12, unit="ops"),
|
||||
]),
|
||||
|
||||
# ── Row 7: Application Resources ──
|
||||
row(90, "Application Resources", 7, [
|
||||
ts_panel(91, "CPU", [
|
||||
('sum(rate(container_cpu_usage_seconds_total{namespace="poimen",container="poimen-memory"}[5m])) by (pod)', "{{pod}}"),
|
||||
], 0, 8),
|
||||
ts_panel(92, "Memory", [
|
||||
('sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)', "{{pod}}"),
|
||||
], 8, 8, unit="bytes"),
|
||||
ts_panel(93, "Heap Bytes", [
|
||||
('memory_app_heap_bytes', "heap"),
|
||||
], 16, 8, unit="bytes"),
|
||||
ts_panel(94, "Tokio Tasks", [
|
||||
('memory_app_tokio_tasks', "active tasks"),
|
||||
], 0, 16),
|
||||
ts_panel(95, "Dependency Latency", [
|
||||
('memory_dependency_db_latency_seconds', "db"),
|
||||
('memory_dependency_embedding_latency_seconds', "embedding"),
|
||||
], 8, 16, unit="s"),
|
||||
]),
|
||||
|
||||
# ── Row 8: Logs ──
|
||||
row(100, "Logs", 8, [
|
||||
log_panel(101, "Poimen Memory Logs", '{namespace="poimen",container="poimen-memory"}', 0, 9),
|
||||
]),
|
||||
]
|
||||
|
||||
return {
|
||||
"title": "Poimen Memory",
|
||||
"uid": "poimen-memory",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"refresh": "30s",
|
||||
"tags": ["poimen", "memory", "knowledge-graph"],
|
||||
"panels": panels,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Generate ConfigMap YAML
|
||||
# ============================================================================
|
||||
|
||||
dashboard = build_dashboard()
|
||||
|
||||
cm = {
|
||||
"apiVersion": "v1",
|
||||
"kind": "ConfigMap",
|
||||
"metadata": {
|
||||
"name": "poimen-memory-dashboard",
|
||||
"namespace": "logging",
|
||||
"labels": {"grafana_dashboard": "1"},
|
||||
"annotations": {"grafana_folder": "Poimen"},
|
||||
},
|
||||
"data": {
|
||||
"poimen-memory.json": json.dumps(dashboard, separators=(",", ":")),
|
||||
},
|
||||
}
|
||||
|
||||
out_path = os.path.join(SCRIPT_DIR, "dashboard.yaml")
|
||||
with open(out_path, "w") as f:
|
||||
yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
print(f"Generated {out_path}")
|
||||
@@ -0,0 +1,6 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- service-monitor.yaml
|
||||
- alerts.yaml
|
||||
- dashboard.yaml
|
||||
@@ -0,0 +1,20 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: poimen-memory
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
app.kubernetes.io/name: poimen-memory
|
||||
spec:
|
||||
namespaceSelector:
|
||||
matchNames:
|
||||
- poimen
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: poimen-memory
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 10s
|
||||
Reference in New Issue
Block a user