diff --git a/k8s/infra/monitoring/dashboards/control-plane-logs.yaml b/k8s/infra/monitoring/dashboards/control-plane-logs.yaml deleted file mode 100644 index bd5458b..0000000 --- a/k8s/infra/monitoring/dashboards/control-plane-logs.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# k8s/monitoring/dashboards/control-plane-logs.yaml -# Surfaces controller/control-plane logs that are already in Loki today -# (Promtail scrapes every namespace with no filter) — this dashboard is the -# "make it visible" piece, not new log collection. -apiVersion: v1 -kind: ConfigMap -metadata: - name: control-plane-logs-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - control-plane-logs.json: | - { - "title": "Cluster Control Plane & Controllers (Logs)", - "uid": "control-plane-logs", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-1h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, - "title": "Error rate by namespace", - "type": "timeseries", - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { - "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))" - } - ] - }, - { - "id": 2, - "title": "Control plane (kube-apiserver, controller-manager, scheduler)", - "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { "expr": "{namespace=\"kube-system\"}" } - ] - }, - { - "id": 3, - "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)", - "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [ - { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/hardware-overview.yaml b/k8s/infra/monitoring/dashboards/hardware-overview.yaml deleted file mode 100644 index 93307d9..0000000 --- a/k8s/infra/monitoring/dashboards/hardware-overview.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# k8s/monitoring/dashboards/hardware-overview.yaml -# Trimmed operator at-a-glance view across all nodes — node-exporter already -# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml), -# this is the quick health-check version, not a replacement for it. -apiVersion: v1 -kind: ConfigMap -metadata: - name: hardware-overview-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - hardware-overview.json: | - { - "title": "Hardware Statistics (Operator Overview)", - "uid": "hardware-overview", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, - "title": "Nodes up / down", - "type": "stat", - "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, - { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } - ] - } - }, - "targets": [ - { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" } - ] - }, - { - "id": 2, - "title": "CPU usage % by node", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, - "targets": [ - { - "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100", - "legendFormat": "{{instance}}" - } - ] - }, - { - "id": 3, - "title": "Memory usage % by node", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, - "targets": [ - { - "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", - "legendFormat": "{{instance}}" - } - ] - }, - { - "id": 4, - "title": "Root filesystem usage % by node", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, - "targets": [ - { - "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100", - "legendFormat": "{{instance}}" - } - ] - }, - { - "id": 5, - "title": "Root filesystem space remaining", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "targets": [ - { - "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}", - "legendFormat": "{{instance}}" - } - ] - }, - { - "id": 6, - "title": "Network errors/drops by node", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" }, - { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" }, - { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" }, - { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" } - ] - }, - { - "id": 7, - "title": "Load average (1m / 5m) by node", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "node_load1", "legendFormat": "{{instance}} load1" }, - { "expr": "node_load5", "legendFormat": "{{instance}} load5" } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/kube-controller-health.yaml b/k8s/infra/monitoring/dashboards/kube-controller-health.yaml deleted file mode 100644 index 4cf8cbe..0000000 --- a/k8s/infra/monitoring/dashboards/kube-controller-health.yaml +++ /dev/null @@ -1,185 +0,0 @@ -# k8s/monitoring/dashboards/kube-controller-health.yaml -# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus -# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is -# the one control-plane component that's still reachable (its ServiceMonitor -# targets the in-cluster `kubernetes` service, not localhost) — paired with -# kube-state-metrics signals as a proxy for controller/scheduler health. -apiVersion: v1 -kind: ConfigMap -metadata: - name: kube-controller-health-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - kube-controller-health.json: | - { - "title": "Kube-Controller Health", - "uid": "kube-controller-health", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, - "title": "API server — up", - "type": "stat", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, - { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } - ] - } - }, - "targets": [ - { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" } - ] - }, - { - "id": 2, - "title": "API server — request rate by verb/code", - "type": "timeseries", - "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)", - "legendFormat": "{{verb}} {{code}}" - } - ] - }, - { - "id": 3, - "title": "API server — error rate % (5xx)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "targets": [ - { - "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100", - "legendFormat": "5xx %" - } - ] - }, - { - "id": 4, - "title": "API server — latency p95 / p99", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p95" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p99" - } - ] - }, - { - "id": 5, - "title": "Pods stuck Pending", - "type": "stat", - "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "yellow" }, - { "value": 5, "color": "red" } - ] - } - } - }, - "targets": [ - { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" } - ] - }, - { - "id": 6, - "title": "CrashLoopBackOff containers", - "type": "stat", - "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "red" } - ] - } - } - }, - "targets": [ - { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" } - ] - }, - { - "id": 7, - "title": "Nodes NotReady", - "type": "stat", - "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "red" } - ] - } - } - }, - "targets": [ - { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" } - ] - }, - { - "id": 8, - "title": "Failed Jobs", - "type": "table", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true } - ] - }, - { - "id": 9, - "title": "Deployments with unavailable replicas", - "type": "table", - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true } - ] - }, - { - "id": 10, - "title": "Container restart rate by pod", - "type": "timeseries", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)", - "legendFormat": "{{namespace}}/{{pod}}" - } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/llm-frontend.yaml b/k8s/infra/monitoring/dashboards/llm-frontend.yaml deleted file mode 100644 index ed5b4e4..0000000 --- a/k8s/infra/monitoring/dashboards/llm-frontend.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: llm-frontend-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "LLM" - # No request-level panels. The rate/error/latency/bandwidth row used to run - # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go - # gateway that replaced it does not expose /metrics yet, so those panels - # were removed rather than left querying series that no longer exist. - # What is left is pod-level: readiness, CPU/memory, restarts, logs. - # - # Restoring request-level and per-model observability means wiring three - # sources, none of which are in place: gateway metrics (RED plus token - # counts and TTFT, which the gateway can measure because it sees the - # response stream), vLLM's own /metrics on reasoning-predictor (rich -- - # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds, - # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's - # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at - # all (verified: /metrics returns 404), so ornith can only ever be observed - # from the gateway side. No ServiceMonitor exists for the llm-serving - # namespace today, so none of the engine metrics are being scraped. -data: - llm-frontend.json: | - {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]} diff --git a/k8s/infra/monitoring/dashboards/service-availability.yaml b/k8s/infra/monitoring/dashboards/service-availability.yaml deleted file mode 100644 index 172e552..0000000 --- a/k8s/infra/monitoring/dashboards/service-availability.yaml +++ /dev/null @@ -1,172 +0,0 @@ -# k8s/monitoring/dashboards/service-availability.yaml -# Active uptime/availability from blackbox-exporter probes — the signal that -# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics -# alone can't distinguish "idle" from "down". -apiVersion: v1 -kind: ConfigMap -metadata: - name: service-availability-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - service-availability.json: | - { - "title": "Service Availability & Certificate Expiration", - "uid": "svc-availability", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-24h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, - "title": "Up / Down — all probed services", - "type": "stat", - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, - { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "red" }, - { "value": 1, "color": "green" } - ] - } - } - }, - "targets": [ - { "expr": "probe_success", "legendFormat": "{{instance}}" } - ] - }, - { - "id": 2, - "title": "Uptime % trend", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, - "targets": [ - { - "expr": "avg_over_time(probe_success[$__rate_interval]) * 100", - "legendFormat": "{{instance}}" - } - ] - }, - { - "id": 3, - "title": "Probe latency", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "targets": [ - { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" } - ] - }, - { - "id": 4, - "title": "7-day SLO (% successful probes)", - "type": "table", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "red" }, - { "value": 99, "color": "yellow" }, - { "value": 99.9, "color": "green" } - ] - } - } - }, - "targets": [ - { - "expr": "avg_over_time(probe_success[7d]) * 100", - "format": "table", - "instant": true - } - ] - }, - { - "id": 5, - "title": "Services DOWN right now", - "type": "stat", - "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "red" } - ] - } - } - }, - "targets": [ - { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" } - ] - }, - { - "id": 6, - "title": "Certs expiring in < 14 days", - "type": "stat", - "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "red" } - ] - } - } - }, - "targets": [ - { - "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)", - "legendFormat": "expiring" - } - ] - }, - { - "id": 7, - "title": "Certificate expiry — days remaining", - "type": "table", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "red" }, - { "value": 14, "color": "yellow" }, - { "value": 30, "color": "green" } - ] - } - } - }, - "targets": [ - { - "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", - "legendFormat": "{{name}}", - "format": "table", - "instant": true - } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/service-golden-signals.yaml b/k8s/infra/monitoring/dashboards/service-golden-signals.yaml deleted file mode 100644 index 9a0093a..0000000 --- a/k8s/infra/monitoring/dashboards/service-golden-signals.yaml +++ /dev/null @@ -1,141 +0,0 @@ -# k8s/monitoring/dashboards/service-golden-signals.yaml -# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx. -# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see -# sidecar.dashboards in k8s/logging/grafana-values.yaml. -apiVersion: v1 -kind: ConfigMap -metadata: - name: service-golden-signals-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - service-golden-signals.json: | - { - "title": "Latency & Golden Signals (Ingress RED)", - "uid": "svc-golden-signals", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "templating": { - "list": [ - { - "name": "ingress", - "type": "query", - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "query": "label_values(nginx_ingress_controller_requests, ingress)", - "refresh": 2, - "includeAll": false - } - ] - }, - "panels": [ - { - "id": 1, - "title": "Request rate by status — $ingress", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)", - "legendFormat": "{{status}}" - } - ] - }, - { - "id": 2, - "title": "Error rate % (4xx / 5xx) — $ingress", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "targets": [ - { - "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100", - "legendFormat": "5xx" - }, - { - "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100", - "legendFormat": "4xx" - } - ] - }, - { - "id": 3, - "title": "Latency p50 / p95 / p99 — $ingress", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", - "legendFormat": "p50" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", - "legendFormat": "p95" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", - "legendFormat": "p99" - } - ] - }, - { - "id": 4, - "title": "All services — traffic overview", - "type": "table", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))", - "format": "table", - "instant": true - } - ] - }, - { - "id": 5, - "title": "Customer-facing failures (5xx count, window total)", - "type": "stat", - "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "green" }, - { "value": 1, "color": "yellow" }, - { "value": 50, "color": "red" } - ] - } - } - }, - "targets": [ - { - "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)", - "legendFormat": "5xx total" - } - ] - }, - { - "id": 6, - "title": "Top 5 error-contributing services", - "type": "table", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { - "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))", - "format": "table", - "instant": true - } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/service-internals.yaml b/k8s/infra/monitoring/dashboards/service-internals.yaml deleted file mode 100644 index 17d115f..0000000 --- a/k8s/infra/monitoring/dashboards/service-internals.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# k8s/monitoring/dashboards/service-internals.yaml -# Native per-service metrics — the "why" layer behind the ingress RED/uptime -# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline). -apiVersion: v1 -kind: ConfigMap -metadata: - name: service-internals-dashboard - namespace: logging - labels: - grafana_dashboard: "1" -data: - service-internals.json: | - { - "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)", - "uid": "svc-internals", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" }, - { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" } - ] - }, - { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" } - ] - }, - { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "targets": [ - { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" } - ] - }, - { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "gitea_repositories", "legendFormat": "repos" }, - { "expr": "gitea_organizations", "legendFormat": "orgs" } - ] - }, - { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" }, - { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" } - ] - }, - { "id": 6, "title": "Argo CD — app sync/health status", "type": "table", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "argocd_app_info", "format": "table", "instant": true } - ] - }, - { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "value": 0, "color": "red" }, - { "value": 14, "color": "yellow" }, - { "value": 30, "color": "green" } - ] - } - } - }, - "targets": [ - { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" } - ] - }, - { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat", - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "mappings": [ - { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } }, - { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } } - ] - } - }, - "targets": [ - { "expr": "vault_core_unsealed", "legendFormat": "vault" } - ] - }, - { "id": 9, "title": "Longhorn — volume robustness", "type": "table", - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "longhorn_volume_robustness", "format": "table", "instant": true } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/svc-argocd.yaml b/k8s/infra/monitoring/dashboards/svc-argocd.yaml deleted file mode 100644 index 8d62d1b..0000000 --- a/k8s/infra/monitoring/dashboards/svc-argocd.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-argocd-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Argo CD" -data: - svc-argocd.json: | - {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]} diff --git a/k8s/infra/monitoring/dashboards/svc-authentik.yaml b/k8s/infra/monitoring/dashboards/svc-authentik.yaml deleted file mode 100644 index e9e6371..0000000 --- a/k8s/infra/monitoring/dashboards/svc-authentik.yaml +++ /dev/null @@ -1,143 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-authentik-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Authentik" -data: - svc-authentik.json: | - { - "title": "Authentik — Service Overview", - "uid": "svc-authentik", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "panels": [ - { - "id": 2, "title": "Up", "type": "stat", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } - ], - "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } - } - }, - "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }] - }, - { - "id": 3, "title": "HTTP request rate by status", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }] - }, - { - "id": 4, "title": "Error rate % (5xx)", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }] - }, - { - "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" }, - { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" }, - { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" } - ] - } - ] - }, - { - "id": 10, "title": "Row: Resource Usage", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, - "panels": [ - { - "id": 11, "title": "CPU by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 12, "title": "Memory by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 13, "title": "Restart rate by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] - } - ] - }, - { - "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, - "panels": [ - { - "id": 21, "title": "Outpost connections", "type": "stat", - "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "authentik_outposts_connected" }] - }, - { - "id": 22, "title": "Flows cached", "type": "stat", - "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "authentik_flows_cached" }] - }, - { - "id": 23, "title": "Policies cached", "type": "stat", - "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "authentik_policies_cached" }] - }, - { - "id": 24, "title": "Queued tasks", "type": "stat", - "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 }, - "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "authentik_tasks_queued" }] - }, - { - "id": 25, "title": "Admin workers", "type": "timeseries", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "authentik_admin_workers" }] - } - ] - }, - { - "id": 30, "title": "Row: Logs", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, - "panels": [ - { - "id": 31, "title": "Recent logs", "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [{ "expr": "{namespace=\"iam\"}" }] - } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/svc-forgejo.yaml b/k8s/infra/monitoring/dashboards/svc-forgejo.yaml deleted file mode 100644 index 2f17c86..0000000 --- a/k8s/infra/monitoring/dashboards/svc-forgejo.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-forgejo-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Forgejo" -data: - svc-forgejo.json: | - {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]} diff --git a/k8s/infra/monitoring/dashboards/svc-grafana.yaml b/k8s/infra/monitoring/dashboards/svc-grafana.yaml deleted file mode 100644 index ed28281..0000000 --- a/k8s/infra/monitoring/dashboards/svc-grafana.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-grafana-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Grafana" -data: - svc-grafana.json: | - {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]} diff --git a/k8s/infra/monitoring/dashboards/svc-minio.yaml b/k8s/infra/monitoring/dashboards/svc-minio.yaml deleted file mode 100644 index 4c6ec94..0000000 --- a/k8s/infra/monitoring/dashboards/svc-minio.yaml +++ /dev/null @@ -1,143 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-minio-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "MinIO" -data: - svc-minio.json: | - { - "title": "MinIO — Service Overview", - "uid": "svc-minio", - "schemaVersion": 39, - "timezone": "browser", - "time": { "from": "now-6h", "to": "now" }, - "refresh": "30s", - "panels": [ - { - "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "panels": [ - { - "id": 2, "title": "Up", "type": "stat", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } - ], - "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } - } - }, - "targets": [{ "expr": "min(up{job=\"minio\"})" }] - }, - { - "id": 3, "title": "S3 request rate by method", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }] - }, - { - "id": 4, "title": "Error rate %", "type": "timeseries", - "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, - "fieldConfig": { "defaults": { "unit": "percent" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }] - }, - { - "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "fieldConfig": { "defaults": { "unit": "s" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" }, - { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" }, - { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" } - ] - } - ] - }, - { - "id": 10, "title": "Row: Resource Usage", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, - "panels": [ - { - "id": 11, "title": "CPU by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 12, "title": "Memory by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }] - }, - { - "id": 13, "title": "Restart rate by pod", "type": "timeseries", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] - } - ] - }, - { - "id": 20, "title": "Row: Storage & Replication", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, - "panels": [ - { - "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" }, - { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" } - ] - }, - { - "id": 22, "title": "Drive health (online/offline)", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 }, - "fieldConfig": { "defaults": { "unit": "short" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [ - { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" }, - { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" } - ] - }, - { - "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }] - }, - { - "id": 24, "title": "Replication failures (bytes)", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, - "fieldConfig": { "defaults": { "unit": "bytes" } }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }] - } - ] - }, - { - "id": 30, "title": "Row: Logs", "type": "row", - "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, - "panels": [ - { - "id": 31, "title": "Recent logs", "type": "logs", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, - "datasource": { "type": "loki", "uid": "loki" }, - "targets": [{ "expr": "{namespace=\"storage\"}" }] - } - ] - } - ] - } diff --git a/k8s/infra/monitoring/dashboards/svc-vault.yaml b/k8s/infra/monitoring/dashboards/svc-vault.yaml deleted file mode 100644 index 2ac8663..0000000 --- a/k8s/infra/monitoring/dashboards/svc-vault.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: svc-vault-dashboard - namespace: logging - labels: - grafana_dashboard: "1" - annotations: - grafana_folder: "Vault" -data: - svc-vault.json: | - {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]} diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 549ee07..187010a 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -18,18 +18,5 @@ resources: - servicemonitors/authentik.yaml - servicemonitors/forgejo.yaml - servicemonitors/minio.yaml - - dashboards/control-plane-logs.yaml - - dashboards/hardware-overview.yaml - - dashboards/kube-controller-health.yaml - - dashboards/llm-frontend.yaml - - dashboards/service-availability.yaml - - dashboards/service-golden-signals.yaml - - dashboards/service-internals.yaml - - dashboards/svc-argocd.yaml - - dashboards/svc-authentik.yaml - - dashboards/svc-forgejo.yaml - - dashboards/svc-grafana.yaml - - dashboards/svc-minio.yaml - - dashboards/svc-vault.yaml - dashboards/cluster-infrastructure.yaml - dashboards/api-gateway.yaml