rock
8902e1b6fa
- PrometheusRule with 16 alerts across 6 groups: availability, ingest, query, dependencies, resources, errors - Grafana dashboard (ConfigMap, sidecar auto-discovery): 8 rows covering overview, ingest pipeline, query perf, context retrieval, database, relevance, error breakdown, app resources, logs - ServiceMonitor moved from k8s/app/ to k8s/monitoring/ - Dashboard generator script (Python, matches homelab pattern) - Separate k8s/monitoring/ path for ArgoCD Application - sqlx::migrate!() added to server startup for seamless migrations
305 lines
14 KiB
Python
305 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate Poimen Memory Grafana dashboard as k8s ConfigMap YAML.
|
|
|
|
Follows homelab pattern: ConfigMap with grafana_dashboard=1 label,
|
|
auto-discovered by Grafana sidecar.
|
|
|
|
Usage: python3 generate-dashboard.py
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import yaml
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
DS_PROM = {"type": "prometheus", "uid": "prometheus"}
|
|
DS_LOKI = {"type": "loki", "uid": "loki"}
|
|
|
|
|
|
def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
|
|
p = {
|
|
"id": id, "title": title, "type": "stat",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": [{"expr": expr}],
|
|
}
|
|
if mappings:
|
|
p["fieldConfig"]["defaults"]["mappings"] = mappings
|
|
if thresholds:
|
|
p["fieldConfig"]["defaults"]["thresholds"] = thresholds
|
|
p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
|
|
return p
|
|
|
|
|
|
def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
|
|
targets = []
|
|
for e in exprs:
|
|
if isinstance(e, tuple):
|
|
targets.append({"expr": e[0], "legendFormat": e[1]})
|
|
else:
|
|
targets.append({"expr": e, "legendFormat": ""})
|
|
return {
|
|
"id": id, "title": title, "type": "timeseries",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": targets,
|
|
}
|
|
|
|
|
|
def log_panel(id, title, query, x, y, w=24, h=10):
|
|
return {
|
|
"id": id, "title": title, "type": "logs",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_LOKI,
|
|
"targets": [{"expr": query}],
|
|
}
|
|
|
|
|
|
def row(id, title, y, panels, collapsed=True):
|
|
return {
|
|
"id": id, "title": title, "type": "row",
|
|
"collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Poimen Memory Dashboard
|
|
# ============================================================================
|
|
|
|
def build_dashboard():
|
|
up_down = [{"type": "value", "options": {
|
|
"0": {"text": "DOWN", "color": "red"},
|
|
"1": {"text": "UP", "color": "green"},
|
|
}}]
|
|
|
|
panels = [
|
|
# ── Row 0: Overview (always open) ──
|
|
row(1, "Overview", 0, [
|
|
stat_panel(2, "Pod Ready",
|
|
'sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"})',
|
|
0, 1, thresholds={"mode": "absolute", "steps": [
|
|
{"value": None, "color": "red"}, {"value": 1, "color": "green"}
|
|
]}),
|
|
stat_panel(3, "Uptime",
|
|
'memory_app_uptime_seconds', 4, 1, unit="s"),
|
|
stat_panel(4, "DB", 'memory_dependency_db_up', 8, 1, mappings=up_down),
|
|
stat_panel(5, "Embeddings", 'memory_dependency_embedding_up', 12, 1, mappings=up_down),
|
|
stat_panel(6, "LLM", 'memory_dependency_llm_up', 16, 1, mappings=up_down),
|
|
stat_panel(7, "Connections", 'memory_app_active_connections', 20, 1),
|
|
], collapsed=False),
|
|
|
|
# ── Row 1: Ingest Pipeline ──
|
|
row(10, "Ingest Pipeline", 1, [
|
|
ts_panel(11, "Ingest Rate", [
|
|
('rate(memory_ingest_requests_total[5m])', "requests/s"),
|
|
('rate(memory_ingest_records_total[5m])', "records/s"),
|
|
], 0, 2, unit="ops"),
|
|
ts_panel(12, "Ingest Errors", [
|
|
('rate(memory_ingest_errors_total[5m])', "errors/s"),
|
|
('rate(memory_ingest_auth_failures_total[5m])', "auth failures/s"),
|
|
('rate(memory_ingest_rate_limited_total[5m])', "rate limited/s"),
|
|
], 8, 2, unit="ops"),
|
|
ts_panel(13, "Ingest Latency", [
|
|
('histogram_quantile(0.50, rate(memory_ingest_duration_seconds_bucket[5m]))', "p50"),
|
|
('histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m]))', "p95"),
|
|
('histogram_quantile(0.99, rate(memory_ingest_duration_seconds_bucket[5m]))', "p99"),
|
|
], 16, 2, unit="s"),
|
|
stat_panel(14, "In Flight", 'memory_ingest_in_flight', 0, 10),
|
|
stat_panel(15, "Queue Size", 'memory_ingest_queue_size', 4, 10),
|
|
stat_panel(16, "Entities Extracted", 'rate(memory_ingest_entities_extracted_total[5m])', 8, 10, unit="ops"),
|
|
stat_panel(17, "Edges Extracted", 'rate(memory_ingest_edges_extracted_total[5m])', 12, 10, unit="ops"),
|
|
stat_panel(18, "Duplicates", 'rate(memory_ingest_duplicates_total[5m])', 16, 10, unit="ops"),
|
|
stat_panel(19, "Bytes Ingested", 'rate(memory_ingest_bytes_total[5m])', 20, 10, unit="Bps"),
|
|
ts_panel(20, "LLM Extraction Latency", [
|
|
('histogram_quantile(0.50, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p50"),
|
|
('histogram_quantile(0.95, rate(memory_ingest_llm_extract_duration_seconds_bucket[5m]))', "entity p95"),
|
|
('histogram_quantile(0.50, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p50"),
|
|
('histogram_quantile(0.95, rate(memory_ingest_fact_extract_duration_seconds_bucket[5m]))', "fact p95"),
|
|
], 0, 14, w=12, unit="s"),
|
|
ts_panel(21, "Contradiction & Dedup", [
|
|
('rate(memory_ingest_contradiction_total[5m])', "contradictions/s"),
|
|
('rate(memory_ingest_dedup_total[5m])', "dedup/s"),
|
|
], 12, 14, w=12, unit="ops"),
|
|
]),
|
|
|
|
# ── Row 2: Query Performance ──
|
|
row(30, "Query Performance", 2, [
|
|
ts_panel(31, "Query Rate", [
|
|
('rate(memory_query_requests_total[5m])', "requests/s"),
|
|
('rate(memory_query_results_total[5m])', "results/s"),
|
|
], 0, 3, unit="ops"),
|
|
ts_panel(32, "Query Errors", [
|
|
('rate(memory_query_errors_total[5m])', "errors/s"),
|
|
('rate(memory_query_embedding_failures_total[5m])', "embedding failures/s"),
|
|
('rate(memory_query_empty_results_total[5m])', "empty results/s"),
|
|
], 8, 3, unit="ops"),
|
|
ts_panel(33, "Query Latency", [
|
|
('histogram_quantile(0.50, rate(memory_query_duration_seconds_bucket[5m]))', "p50"),
|
|
('histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m]))', "p95"),
|
|
('histogram_quantile(0.99, rate(memory_query_duration_seconds_bucket[5m]))', "p99"),
|
|
], 16, 3, unit="s"),
|
|
stat_panel(34, "In Flight", 'memory_query_in_flight', 0, 11),
|
|
ts_panel(35, "Embedding Latency", [
|
|
('histogram_quantile(0.50, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p50"),
|
|
('histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m]))', "p95"),
|
|
], 4, 11, unit="s"),
|
|
ts_panel(36, "Cache Hit Rate", [
|
|
('rate(memory_query_cache_hits_total[5m]) / (rate(memory_query_cache_hits_total[5m]) + rate(memory_query_cache_misses_total[5m]))', "hit rate"),
|
|
], 12, 11, w=12, unit="percentunit"),
|
|
]),
|
|
|
|
# ── Row 3: Context Retrieval ──
|
|
row(40, "Context Retrieval (3-tier)", 3, [
|
|
ts_panel(41, "Context Rate", [
|
|
('rate(memory_context_requests_total[5m])', "requests/s"),
|
|
('rate(memory_context_errors_total[5m])', "errors/s"),
|
|
], 0, 4, unit="ops"),
|
|
ts_panel(42, "Context Latency", [
|
|
('histogram_quantile(0.50, rate(memory_context_duration_seconds_bucket[5m]))', "p50"),
|
|
('histogram_quantile(0.95, rate(memory_context_duration_seconds_bucket[5m]))', "p95"),
|
|
], 8, 4, unit="s"),
|
|
ts_panel(43, "Retrieval Tier Hits", [
|
|
('rate(memory_context_semantic_hits_total[5m])', "semantic"),
|
|
('rate(memory_context_graph_hits_total[5m])', "graph"),
|
|
], 16, 4, unit="ops"),
|
|
ts_panel(44, "Tier Latency", [
|
|
('histogram_quantile(0.95, rate(memory_context_tier_duration_seconds_bucket[5m]))', "p95"),
|
|
], 0, 12, w=12, unit="s"),
|
|
]),
|
|
|
|
# ── Row 4: Database ──
|
|
row(50, "Database", 4, [
|
|
stat_panel(51, "Entities", 'memory_db_entity_count', 0, 5),
|
|
stat_panel(52, "Edges", 'memory_db_edge_count', 4, 5),
|
|
stat_panel(53, "Chunks", 'memory_db_chunk_count', 8, 5),
|
|
stat_panel(54, "DB Size", 'memory_db_size_bytes', 12, 5, unit="bytes"),
|
|
stat_panel(55, "Index Size", 'memory_db_index_size_bytes', 16, 5, unit="bytes"),
|
|
stat_panel(56, "Pool Active", 'memory_db_pool_active', 20, 5),
|
|
ts_panel(57, "DB Query Latency", [
|
|
('histogram_quantile(0.50, rate(memory_db_query_duration_seconds_bucket[5m]))', "p50"),
|
|
('histogram_quantile(0.95, rate(memory_db_query_duration_seconds_bucket[5m]))', "p95"),
|
|
('histogram_quantile(0.99, rate(memory_db_query_duration_seconds_bucket[5m]))', "p99"),
|
|
], 0, 9, unit="s"),
|
|
ts_panel(58, "DB Query Rate", [
|
|
('rate(memory_db_queries_total[5m])', "queries/s"),
|
|
('rate(memory_db_query_errors_total[5m])', "errors/s"),
|
|
], 8, 9, unit="ops"),
|
|
ts_panel(59, "Connection Pool", [
|
|
('memory_db_pool_active', "active"),
|
|
('memory_db_pool_idle', "idle"),
|
|
('memory_db_pool_size', "max"),
|
|
], 16, 9),
|
|
ts_panel(60, "Table Row Counts", [
|
|
('memory_db_table_entity_rows', "entities"),
|
|
('memory_db_table_edge_rows', "edges"),
|
|
('memory_db_table_chunk_rows', "chunks"),
|
|
], 0, 17, w=12),
|
|
ts_panel(61, "Write Rate", [
|
|
('rate(memory_write_entities_total[5m])', "entities/s"),
|
|
('rate(memory_write_edges_total[5m])', "edges/s"),
|
|
('rate(memory_write_chunks_total[5m])', "chunks/s"),
|
|
('rate(memory_write_errors_total[5m])', "errors/s"),
|
|
], 12, 17, w=12, unit="ops"),
|
|
]),
|
|
|
|
# ── Row 5: Relevance / Quality ──
|
|
row(70, "Relevance & Quality", 5, [
|
|
ts_panel(71, "Relevance Score", [
|
|
('memory_relevance_score', "score"),
|
|
('memory_relevance_precision', "precision"),
|
|
('memory_relevance_recall', "recall"),
|
|
], 0, 6, w=12, unit="percentunit"),
|
|
ts_panel(72, "Relevance Evals", [
|
|
('rate(memory_relevance_relevant_total[5m])', "relevant/s"),
|
|
('rate(memory_relevance_irrelevant_total[5m])', "irrelevant/s"),
|
|
], 12, 6, w=12, unit="ops"),
|
|
]),
|
|
|
|
# ── Row 6: Errors Breakdown ──
|
|
row(80, "Error Breakdown", 6, [
|
|
ts_panel(81, "Ingest Errors by Type", [
|
|
('rate(memory_error_auth_failure_ingest_total[5m])', "auth"),
|
|
('rate(memory_error_bad_request_ingest_total[5m])', "bad request"),
|
|
('rate(memory_error_db_error_ingest_total[5m])', "db error"),
|
|
('rate(memory_error_forbidden_ingest_total[5m])', "forbidden"),
|
|
('rate(memory_error_rate_limited_ingest_total[5m])', "rate limited"),
|
|
('rate(memory_error_unexpected_ingest_total[5m])', "unexpected"),
|
|
], 0, 7, w=12, unit="ops"),
|
|
ts_panel(82, "Query Errors by Type", [
|
|
('rate(memory_error_auth_failure_query_total[5m])', "auth"),
|
|
('rate(memory_error_bad_request_query_total[5m])', "bad request"),
|
|
('rate(memory_error_embedding_failure_query_total[5m])', "embedding"),
|
|
('rate(memory_error_forbidden_query_total[5m])', "forbidden"),
|
|
('rate(memory_error_search_failure_query_total[5m])', "search"),
|
|
('rate(memory_error_unexpected_query_total[5m])', "unexpected"),
|
|
], 12, 7, w=12, unit="ops"),
|
|
]),
|
|
|
|
# ── Row 7: Application Resources ──
|
|
row(90, "Application Resources", 7, [
|
|
ts_panel(91, "CPU", [
|
|
('sum(rate(container_cpu_usage_seconds_total{namespace="poimen",container="poimen-memory"}[5m])) by (pod)', "{{pod}}"),
|
|
], 0, 8),
|
|
ts_panel(92, "Memory", [
|
|
('sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)', "{{pod}}"),
|
|
], 8, 8, unit="bytes"),
|
|
ts_panel(93, "Heap Bytes", [
|
|
('memory_app_heap_bytes', "heap"),
|
|
], 16, 8, unit="bytes"),
|
|
ts_panel(94, "Tokio Tasks", [
|
|
('memory_app_tokio_tasks', "active tasks"),
|
|
], 0, 16),
|
|
ts_panel(95, "Dependency Latency", [
|
|
('memory_dependency_db_latency_seconds', "db"),
|
|
('memory_dependency_embedding_latency_seconds', "embedding"),
|
|
], 8, 16, unit="s"),
|
|
]),
|
|
|
|
# ── Row 8: Logs ──
|
|
row(100, "Logs", 8, [
|
|
log_panel(101, "Poimen Memory Logs", '{namespace="poimen",container="poimen-memory"}', 0, 9),
|
|
]),
|
|
]
|
|
|
|
return {
|
|
"title": "Poimen Memory",
|
|
"uid": "poimen-memory",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["poimen", "memory", "knowledge-graph"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Generate ConfigMap YAML
|
|
# ============================================================================
|
|
|
|
dashboard = build_dashboard()
|
|
|
|
cm = {
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {
|
|
"name": "poimen-memory-dashboard",
|
|
"namespace": "logging",
|
|
"labels": {"grafana_dashboard": "1"},
|
|
"annotations": {"grafana_folder": "Poimen"},
|
|
},
|
|
"data": {
|
|
"poimen-memory.json": json.dumps(dashboard, separators=(",", ":")),
|
|
},
|
|
}
|
|
|
|
out_path = os.path.join(SCRIPT_DIR, "dashboard.yaml")
|
|
with open(out_path, "w") as f:
|
|
yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
|
|
|
|
print(f"Generated {out_path}")
|