142 lines
5.1 KiB
YAML
142 lines
5.1 KiB
YAML
# k8s/monitoring/dashboards/service-golden-signals.yaml
|
|
# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
|
|
# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
|
|
# sidecar.dashboards in k8s/logging/grafana-values.yaml.
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: service-golden-signals-dashboard
|
|
namespace: logging
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
service-golden-signals.json: |
|
|
{
|
|
"title": "Latency & Golden Signals (Ingress RED)",
|
|
"uid": "svc-golden-signals",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"refresh": "30s",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "ingress",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"query": "label_values(nginx_ingress_controller_requests, ingress)",
|
|
"refresh": 2,
|
|
"includeAll": false
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Request rate by status — $ingress",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
|
|
"legendFormat": "{{status}}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "Error rate % (4xx / 5xx) — $ingress",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "percent" } },
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
|
"legendFormat": "5xx"
|
|
},
|
|
{
|
|
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
|
"legendFormat": "4xx"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Latency p50 / p95 / p99 — $ingress",
|
|
"type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
|
"legendFormat": "p50"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
|
"legendFormat": "p99"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "All services — traffic overview",
|
|
"type": "table",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{
|
|
"expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
|
|
"format": "table",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Customer-facing failures (5xx count, window total)",
|
|
"type": "stat",
|
|
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "value": 0, "color": "green" },
|
|
{ "value": 1, "color": "yellow" },
|
|
{ "value": 50, "color": "red" }
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
|
|
"legendFormat": "5xx total"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "Top 5 error-contributing services",
|
|
"type": "table",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{
|
|
"expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
|
|
"format": "table",
|
|
"instant": true
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|