diff --git a/k8s/infra/monitoring/dashboards/api-gateway.yaml b/k8s/infra/monitoring/dashboards/api-gateway.yaml new file mode 100644 index 0000000..735dce6 --- /dev/null +++ b/k8s/infra/monitoring/dashboards/api-gateway.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +data: + api-gateway.json: '{"title":"API Gateway","uid":"api-gateway","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["api","gateway","llm"],"panels":[{"id":1,"title":"Gateway + Health","type":"row","collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Gateway + Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]},{"id":3,"title":"Probe: + healthz","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"probe_success{instance=~\".*api.riotpiao.com/healthz\"}"}]},{"id":4,"title":"Probe + Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"probe_duration_seconds{instance=~\".*api.riotpiao.com.*\"}","legendFormat":"{{instance}}"}]}]},{"id":10,"title":"Ingress + Traffic (nginx)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Request + Rate by Status","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) + by (status)","legendFormat":"{{status}}"}]},{"id":12,"title":"Error Rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"5..\"}[5m])) + / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"5xx"},{"expr":"sum(rate(nginx_ingress_controller_requests{ingress=\"api\",status=~\"4..\"}[5m])) + / sum(rate(nginx_ingress_controller_requests{ingress=\"api\"}[5m])) * 100","legendFormat":"4xx"}]},{"id":13,"title":"Latency + p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.50, + sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m])) + by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m])) + by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"api\"}[5m])) + by (le))","legendFormat":"p99"}]}]},{"id":20,"title":"LLM Serving","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"LLM + Pods Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":22,"title":"CPU + by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":4,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"llm-serving\"}[5m])) + by (pod)","legendFormat":"{{pod}}"}]},{"id":23,"title":"Memory by Predictor","type":"timeseries","gridPos":{"h":8,"w":8,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"llm-serving\"}) + by (pod)","legendFormat":"{{pod}}"}]},{"id":24,"title":"Predictor Restarts","type":"timeseries","gridPos":{"h":8,"w":4,"x":20,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) + by (pod)","legendFormat":"{{pod}}"}]}]},{"id":30,"title":"Gateway Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"CPU + by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"api\"}[5m])) + by (pod)","legendFormat":"{{pod}}"}]},{"id":32,"title":"Memory by Gateway Pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"api\"}) + by (pod)","legendFormat":"{{pod}}"}]},{"id":33,"title":"Gateway Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"api\"}[15m])) + by (pod)","legendFormat":"{{pod}}"}]}]},{"id":40,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Gateway + Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":5},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\",container=\"gateway\"}"}]},{"id":42,"title":"LLM + Serving Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]}]}]}' +kind: ConfigMap +metadata: + annotations: + grafana_folder: API + labels: + grafana_dashboard: '1' + name: api-gateway-dashboard + namespace: logging diff --git a/k8s/infra/monitoring/dashboards/cluster-infrastructure.yaml b/k8s/infra/monitoring/dashboards/cluster-infrastructure.yaml new file mode 100644 index 0000000..666c281 --- /dev/null +++ b/k8s/infra/monitoring/dashboards/cluster-infrastructure.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +data: + cluster-infrastructure.json: '{"title":"Cluster Infrastructure","uid":"cluster-infra","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["infrastructure","k8s"],"panels":[{"id":1,"title":"Cluster + Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Nodes + Ready","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":3,"color":"green"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_node_status_condition{condition=\"Ready\",status=\"true\"} + == 1)"}]},{"id":3,"title":"Pods Pending","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_status_phase{phase=\"Pending\"}) + OR on() vector(0)"}]},{"id":4,"title":"CrashLoopBackOff","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) + OR on() vector(0)"}]},{"id":5,"title":"OOMKilled (1h)","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"sum(increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[1h])) + OR on() vector(0)"}]},{"id":6,"title":"Deploys Unavailable","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_deployment_status_replicas_unavailable + > 0) OR on() vector(0)"}]},{"id":7,"title":"Services Down","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(probe_success + == 0) OR on() vector(0)"}]}]},{"id":10,"title":"Jobs & CronJobs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"Failed + Jobs","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]},"color":{"mode":"thresholds"}}},"targets":[{"expr":"count(kube_job_status_failed + > 0) OR on() vector(0)"}]},{"id":12,"title":"Failed Jobs Detail","type":"table","gridPos":{"h":8,"w":10,"x":4,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_failed + > 0","format":"table","instant":true}]},{"id":13,"title":"Stuck Jobs (>1h)","type":"table","gridPos":{"h":8,"w":10,"x":14,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"kube_job_status_active + == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600","format":"table","instant":true}]},{"id":14,"title":"CronJob + Last Success","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"dateTimeFromNow"}},"targets":[{"expr":"kube_cronjob_status_last_successful_time{namespace=~\"cicd|kube-system|paperless\"}","legendFormat":"{{namespace}}/{{cronjob}}"}]},{"id":15,"title":"Container + Restart Storm (top 10)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"topk(10, + sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))","legendFormat":"{{namespace}}/{{pod}}"}]}]},{"id":20,"title":"Node + Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"CPU + % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1 + - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100","legendFormat":"{{instance}}"}]},{"id":22,"title":"Memory + % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1 + - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100","legendFormat":"{{instance}}"}]},{"id":23,"title":"Disk + % by Node","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"(1 + - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) + * 100","legendFormat":"{{instance}}"}]},{"id":24,"title":"Load Average","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"node_load1","legendFormat":"1m + {{instance}}"},{"expr":"node_load5","legendFormat":"5m {{instance}}"}]},{"id":25,"title":"Network + Errors & Drops","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(node_network_receive_errs_total[5m])","legendFormat":"rx-err + {{instance}}"},{"expr":"rate(node_network_transmit_errs_total[5m])","legendFormat":"tx-err + {{instance}}"},{"expr":"rate(node_network_receive_drop_total[5m])","legendFormat":"rx-drop + {{instance}}"}]}]},{"id":30,"title":"Control Plane","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"API + Server Up","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short","mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"apiserver\"})"}]},{"id":32,"title":"API + Server Request Rate","type":"timeseries","gridPos":{"h":8,"w":10,"x":4,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(apiserver_request_total[5m])) + by (verb, code)","legendFormat":"{{verb}} {{code}}"}]},{"id":33,"title":"API Server + Error Rate %","type":"timeseries","gridPos":{"h":8,"w":10,"x":14,"y":4},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percent"}},"targets":[{"expr":"sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) + / sum(rate(apiserver_request_total[5m])) * 100","legendFormat":"5xx %"}]},{"id":34,"title":"API + Server Latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, + sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]},{"id":35,"title":"etcd + Request Duration","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.99, + sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":40,"title":"Storage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":41,"title":"Longhorn + Disk Capacity","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"bytes"}},"targets":[{"expr":"longhorn_disk_capacity_bytes","legendFormat":"capacity + {{node}}"},{"expr":"longhorn_disk_reservation_bytes","legendFormat":"reserved + {{node}}"}]},{"id":42,"title":"PVC Phase","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"kube_persistentvolumeclaim_status_phase","legendFormat":"{{namespace}}/{{persistentvolumeclaim}} + {{phase}}"}]}]},{"id":50,"title":"DNS & Networking","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":5},"panels":[{"id":51,"title":"CoreDNS + Cache Hit Rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"rate(coredns_cache_hits_total[5m]) + / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))","legendFormat":"{{server}}"}]},{"id":52,"title":"CoreDNS + Errors","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(coredns_dns_responses_total{rcode=~\"SERVFAIL|NXDOMAIN\"}[5m])) + by (rcode)","legendFormat":"{{rcode}}"}]}]},{"id":60,"title":"Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":6},"panels":[{"id":61,"title":"Error + Rate by Namespace","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":7},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum + by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} + |= \"error\" [5m]))","legendFormat":"{{namespace}}"}]},{"id":62,"title":"Control + Plane Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":15},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"kube-system\"}"}]},{"id":63,"title":"Cluster + Addon Logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":25},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}"}]}]}]}' +kind: ConfigMap +metadata: + annotations: + grafana_folder: Infrastructure + labels: + grafana_dashboard: '1' + name: cluster-infrastructure-dashboard + namespace: logging diff --git a/k8s/infra/monitoring/dashboards/generate-dashboards.py b/k8s/infra/monitoring/dashboards/generate-dashboards.py new file mode 100755 index 0000000..de018ef --- /dev/null +++ b/k8s/infra/monitoring/dashboards/generate-dashboards.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files.""" + +import json +import os + +DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards") + +DS_PROM = {"type": "prometheus", "uid": "prometheus"} +DS_LOKI = {"type": "loki", "uid": "loki"} + + +def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None): + p = { + "id": id, "title": title, "type": "stat", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_PROM, + "fieldConfig": {"defaults": {"unit": unit}}, + "targets": [{"expr": expr}], + } + if mappings: + p["fieldConfig"]["defaults"]["mappings"] = mappings + if thresholds: + p["fieldConfig"]["defaults"]["thresholds"] = thresholds + p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"} + return p + + +def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"): + targets = [] + for e in exprs: + if isinstance(e, tuple): + targets.append({"expr": e[0], "legendFormat": e[1]}) + else: + targets.append({"expr": e, "legendFormat": "{{pod}}"}) + return { + "id": id, "title": title, "type": "timeseries", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_PROM, + "fieldConfig": {"defaults": {"unit": unit}}, + "targets": targets, + } + + +def table_panel(id, title, expr, x, y, w=12, h=8): + return { + "id": id, "title": title, "type": "table", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_PROM, + "targets": [{"expr": expr, "format": "table", "instant": True}], + } + + +def log_panel(id, title, query, x, y, w=24, h=10): + return { + "id": id, "title": title, "type": "logs", + "gridPos": {"h": h, "w": w, "x": x, "y": y}, + "datasource": DS_LOKI, + "targets": [{"expr": query}], + } + + +def row(id, title, y, panels, collapsed=True): + return { + "id": id, "title": title, "type": "row", + "collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y}, + "panels": panels, + } + + +def write_dashboard(filename, dashboard, folder): + cm = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": filename.replace(".yaml", "-dashboard"), + "namespace": "logging", + "labels": {"grafana_dashboard": "1"}, + "annotations": {"grafana_folder": folder}, + }, + "data": { + filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":")) + }, + } + + import yaml + path = os.path.join(DASHBOARD_DIR, filename) + with open(path, "w") as f: + yaml.dump(cm, f, default_flow_style=False, allow_unicode=True) + print(f" wrote {path}") + + +# ============================================================================ +# Dashboard 1: Cluster Infrastructure +# ============================================================================ + +def build_cluster_infrastructure(): + zero_thresholds = {"mode": "absolute", "steps": [ + {"value": None, "color": "green"}, {"value": 1, "color": "red"} + ]} + + panels = [ + row(1, "Cluster Health", 0, [ + stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}), + stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds), + stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds), + stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds), + stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds), + stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds), + ]), + row(10, "Jobs & CronJobs", 1, [ + stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds), + table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10), + table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10), + ts_panel(14, "CronJob Last Success", [ + ('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}") + ], 0, 10, w=12, unit="dateTimeFromNow"), + ts_panel(15, "Container Restart Storm (top 10)", [ + ('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}") + ], 12, 10, w=12), + ]), + row(20, "Node Resources", 2, [ + ts_panel(21, "CPU % by Node", [ + ('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}") + ], 0, 3, unit="percent"), + ts_panel(22, "Memory % by Node", [ + ('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}") + ], 8, 3, unit="percent"), + ts_panel(23, "Disk % by Node", [ + ('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}") + ], 16, 3, unit="percent"), + ts_panel(24, "Load Average", [ + ("node_load1", "1m {{instance}}"), + ("node_load5", "5m {{instance}}"), + ], 0, 11), + ts_panel(25, "Network Errors & Drops", [ + ("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"), + ("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"), + ("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"), + ], 8, 11), + ]), + row(30, "Control Plane", 3, [ + stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[ + {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}} + ]), + ts_panel(32, "API Server Request Rate", [ + ('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}") + ], 4, 4, w=10), + ts_panel(33, "API Server Error Rate %", [ + ('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %") + ], 14, 4, w=10, unit="percent"), + ts_panel(34, "API Server Latency", [ + ('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"), + ('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"), + ], 0, 12, unit="s"), + ts_panel(35, "etcd Request Duration", [ + ('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"), + ], 8, 12, unit="s"), + ]), + row(40, "Storage", 4, [ + ts_panel(41, "Longhorn Disk Capacity", [ + ("longhorn_disk_capacity_bytes", "capacity {{node}}"), + ("longhorn_disk_reservation_bytes", "reserved {{node}}"), + ], 0, 5, unit="bytes"), + ts_panel(42, "PVC Phase", [ + ('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}") + ], 8, 5), + ]), + row(50, "DNS & Networking", 5, [ + ts_panel(51, "CoreDNS Cache Hit Rate", [ + ('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}") + ], 0, 6, unit="percentunit"), + ts_panel(52, "CoreDNS Errors", [ + ('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}") + ], 8, 6), + ]), + row(60, "Logs", 6, [ + ts_panel(61, "Error Rate by Namespace", [ + ('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}") + ], 0, 7), + log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15), + log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25), + ]), + ] + + return { + "title": "Cluster Infrastructure", + "uid": "cluster-infra", + "schemaVersion": 39, + "timezone": "browser", + "time": {"from": "now-6h", "to": "now"}, + "refresh": "30s", + "tags": ["infrastructure", "k8s"], + "panels": panels, + } + + +# ============================================================================ +# Dashboard 3: API Gateway +# ============================================================================ + +def build_api_gateway(): + panels = [ + row(1, "Gateway Health", 0, [ + stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}), + stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[ + {"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}} + ]), + ts_panel(4, "Probe Latency", [ + ('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}") + ], 8, 1, unit="s"), + ], collapsed=False), + row(10, "Ingress Traffic (nginx)", 1, [ + ts_panel(11, "Request Rate by Status", [ + ('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}") + ], 0, 2), + ts_panel(12, "Error Rate %", [ + ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"), + ('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"), + ], 8, 2, unit="percent"), + ts_panel(13, "Latency p50/p95/p99", [ + ('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"), + ('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"), + ('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"), + ], 16, 2, unit="s"), + ]), + row(20, "LLM Serving", 2, [ + stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3), + ts_panel(22, "CPU by Predictor", [ + ('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}") + ], 4, 3), + ts_panel(23, "Memory by Predictor", [ + ('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}") + ], 12, 3, unit="bytes"), + ts_panel(24, "Predictor Restarts", [ + ('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}") + ], 20, 3, w=4), + ]), + row(30, "Gateway Resources", 3, [ + ts_panel(31, "CPU by Gateway Pod", [ + ('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}") + ], 0, 4), + ts_panel(32, "Memory by Gateway Pod", [ + ('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}") + ], 8, 4, unit="bytes"), + ts_panel(33, "Gateway Restarts", [ + ('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}") + ], 16, 4), + ]), + row(40, "Logs", 4, [ + log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5), + log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15), + ]), + ] + + return { + "title": "API Gateway", + "uid": "api-gateway", + "schemaVersion": 39, + "timezone": "browser", + "time": {"from": "now-6h", "to": "now"}, + "refresh": "30s", + "tags": ["api", "gateway", "llm"], + "panels": panels, + } + + +# ============================================================================ +# Generate +# ============================================================================ + +print("Generating dashboards...") + +# Dashboard 1 +write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure") + +# Dashboard 3 +write_dashboard("api-gateway.yaml", build_api_gateway(), "API") + +print("Done.") diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index b328de0..0dffaad 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -29,3 +29,5 @@ resources: - dashboards/svc-grafana.yaml - dashboards/svc-minio.yaml - dashboards/svc-vault.yaml + - dashboards/cluster-infrastructure.yaml + - dashboards/api-gateway.yaml