#!/usr/bin/env python3 """Generate consolidated Grafana dashboards as k8s ConfigMap YAML files.""" import json import os DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards") DS_PROM = {"type": "prometheus", "uid": "prometheus"} DS_LOKI = {"type": "loki", "uid": "loki"} def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None): p = { "id": id, "title": title, "type": "stat", "gridPos": {"h": h, "w": w, "x": x, "y": y}, "datasource": DS_PROM, "fieldConfig": {"defaults": {"unit": unit}}, "targets": [{"expr": expr}], } if mappings: p["fieldConfig"]["defaults"]["mappings"] = mappings if thresholds: p["fieldConfig"]["defaults"]["thresholds"] = thresholds p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"} return p def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"): targets = [] for e in exprs: if isinstance(e, tuple): targets.append({"expr": e[0], "legendFormat": e[1]}) else: targets.append({"expr": e, "legendFormat": "{{pod}}"}) return { "id": id, "title": title, "type": "timeseries", "gridPos": {"h": h, "w": w, "x": x, "y": y}, "datasource": DS_PROM, "fieldConfig": {"defaults": {"unit": unit}}, "targets": targets, } def table_panel(id, title, expr, x, y, w=12, h=8): return { "id": id, "title": title, "type": "table", "gridPos": {"h": h, "w": w, "x": x, "y": y}, "datasource": DS_PROM, "targets": [{"expr": expr, "format": "table", "instant": True}], } def log_panel(id, title, query, x, y, w=24, h=10): return { "id": id, "title": title, "type": "logs", "gridPos": {"h": h, "w": w, "x": x, "y": y}, "datasource": DS_LOKI, "targets": [{"expr": query}], } def row(id, title, y, panels, collapsed=True): return { "id": id, "title": title, "type": "row", "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, "panels": panels, } def write_dashboard(filename, dashboard, folder): cm = { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": filename.replace(".yaml", "-dashboard"), "namespace": "logging", "labels": {"grafana_dashboard": "1"}, "annotations": {"grafana_folder": folder}, }, "data": { filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":")) }, } import yaml path = os.path.join(DASHBOARD_DIR, filename) with open(path, "w") as f: yaml.dump(cm, f, default_flow_style=False, allow_unicode=True) print(f" wrote {path}") # ============================================================================ # Dashboard 1: Cluster Infrastructure # ============================================================================ def build_cluster_infrastructure(): zero_thresholds = {"mode": "absolute", "steps": [ {"value": None, "color": "green"}, {"value": 1, "color": "red"} ]} panels = [ row(1, "Cluster Health", 0, [ stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}), stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds), stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds), stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds), stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds), stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds), ]), row(10, "Jobs & CronJobs", 1, [ stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds), table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10), table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10), ts_panel(14, "CronJob Last Success", [ ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}") ], 0, 10, w=12, unit="dateTimeFromNow"), ts_panel(15, "Container Restart Storm (top 10)", [ ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}") ], 12, 10, w=12), ]), row(20, "Node Resources", 2, [ ts_panel(21, "CPU % by Node", [ ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}") ], 0, 3, unit="percent"), ts_panel(22, "Memory % by Node", [ ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}") ], 8, 3, unit="percent"), ts_panel(23, "Disk % by Node", [ ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}") ], 16, 3, unit="percent"), ts_panel(24, "Load Average", [ ("node_load1", "1m {{instance}}"), ("node_load5", "5m {{instance}}"), ], 0, 11), ts_panel(25, "Network Errors & Drops", [ ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"), ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"), ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"), ], 8, 11), ]), row(30, "Control Plane", 3, [ stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[ {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}} ]), ts_panel(32, "API Server Request Rate", [ ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}") ], 4, 4, w=10), ts_panel(33, "API Server Error Rate %", [ ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %") ], 14, 4, w=10, unit="percent"), ts_panel(34, "API Server Latency", [ ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"), ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"), ], 0, 12, unit="s"), ts_panel(35, "etcd Request Duration", [ ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"), ], 8, 12, unit="s"), ]), row(40, "Storage", 4, [ ts_panel(41, "Longhorn Disk Capacity", [ ("longhorn_disk_capacity_bytes", "capacity {{node}}"), ("longhorn_disk_reservation_bytes", "reserved {{node}}"), ], 0, 5, unit="bytes"), ts_panel(42, "PVC Phase", [ ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}") ], 8, 5), ]), row(50, "DNS & Networking", 5, [ ts_panel(51, "CoreDNS Cache Hit Rate", [ ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}") ], 0, 6, unit="percentunit"), ts_panel(52, "CoreDNS Errors", [ ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}") ], 8, 6), ]), row(60, "Logs", 6, [ ts_panel(61, "Error Rate by Namespace", [ ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}") ], 0, 7), log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15), log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25), ]), ] return { "title": "Cluster Infrastructure", "uid": "cluster-infra", "schemaVersion": 39, "timezone": "browser", "time": {"from": "now-6h", "to": "now"}, "refresh": "30s", "tags": ["infrastructure", "k8s"], "panels": panels, } # ============================================================================ # Dashboard 3: API Gateway # ============================================================================ def build_api_gateway(): panels = [ row(1, "Gateway Health", 0, [ stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}), stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[ {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}} ]), ts_panel(4, "Probe Latency", [ ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}") ], 8, 1, unit="s"), ], collapsed=False), row(10, "Ingress Traffic (nginx)", 1, [ ts_panel(11, "Request Rate by Status", [ ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}") ], 0, 2), ts_panel(12, "Error Rate %", [ ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"), ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"), ], 8, 2, unit="percent"), ts_panel(13, "Latency p50/p95/p99", [ ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"), ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"), ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"), ], 16, 2, unit="s"), ]), row(20, "LLM Serving", 2, [ stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3), ts_panel(22, "CPU by Predictor", [ ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}") ], 4, 3), ts_panel(23, "Memory by Predictor", [ ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}") ], 12, 3, unit="bytes"), ts_panel(24, "Predictor Restarts", [ ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}") ], 20, 3, w=4), ]), row(30, "Gateway Resources", 3, [ ts_panel(31, "CPU by Gateway Pod", [ ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}") ], 0, 4), ts_panel(32, "Memory by Gateway Pod", [ ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}") ], 8, 4, unit="bytes"), ts_panel(33, "Gateway Restarts", [ ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}") ], 16, 4), ]), row(40, "Logs", 4, [ log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5), log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15), ]), ] return { "title": "API Gateway", "uid": "api-gateway", "schemaVersion": 39, "timezone": "browser", "time": {"from": "now-6h", "to": "now"}, "refresh": "30s", "tags": ["api", "gateway", "llm"], "panels": panels, } # ============================================================================ # Generate # ============================================================================ print("Generating dashboards...") # Dashboard 1 write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure") # Dashboard 3 write_dashboard("api-gateway.yaml", build_api_gateway(), "API") print("Done.")