k8s/monitoring: add prometheus grafana loki observability
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,55 @@
|
||||
# k8s/monitoring/dashboards/control-plane-logs.yaml
|
||||
# Surfaces controller/control-plane logs that are already in Loki today
|
||||
# (Promtail scrapes every namespace with no filter) — this dashboard is the
|
||||
# "make it visible" piece, not new log collection.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: control-plane-logs-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
control-plane-logs.json: |
|
||||
{
|
||||
"title": "Cluster Control Plane & Controllers (Logs)",
|
||||
"uid": "control-plane-logs",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Error rate by namespace",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Control plane (kube-apiserver, controller-manager, scheduler)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{ "expr": "{namespace=\"kube-system\"}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
|
||||
"type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [
|
||||
{ "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
# k8s/monitoring/dashboards/hardware-overview.yaml
|
||||
# Trimmed operator at-a-glance view across all nodes — node-exporter already
|
||||
# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
|
||||
# this is the quick health-check version, not a replacement for it.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: hardware-overview-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
hardware-overview.json: |
|
||||
{
|
||||
"title": "Hardware Statistics (Operator Overview)",
|
||||
"uid": "hardware-overview",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Nodes up / down",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "CPU usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Memory usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Root filesystem usage % by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Root filesystem space remaining",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Network errors/drops by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
|
||||
{ "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
|
||||
{ "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
|
||||
{ "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Load average (1m / 5m) by node",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "node_load1", "legendFormat": "{{instance}} load1" },
|
||||
{ "expr": "node_load5", "legendFormat": "{{instance}} load5" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
# k8s/monitoring/dashboards/kube-controller-health.yaml
|
||||
# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
|
||||
# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
|
||||
# the one control-plane component that's still reachable (its ServiceMonitor
|
||||
# targets the in-cluster `kubernetes` service, not localhost) — paired with
|
||||
# kube-state-metrics signals as a proxy for controller/scheduler health.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: kube-controller-health-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
kube-controller-health.json: |
|
||||
{
|
||||
"title": "Kube-Controller Health",
|
||||
"uid": "kube-controller-health",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API server — up",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API server — request rate by verb/code",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
|
||||
"legendFormat": "{{verb}} {{code}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API server — error rate % (5xx)",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
|
||||
"legendFormat": "5xx %"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "API server — latency p95 / p99",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Pods stuck Pending",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "yellow" },
|
||||
{ "value": 5, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "CrashLoopBackOff containers",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Nodes NotReady",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Failed Jobs",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Deployments with unavailable replicas",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Container restart rate by pod",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
|
||||
"legendFormat": "{{namespace}}/{{pod}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
# k8s/monitoring/dashboards/service-availability.yaml
|
||||
# Active uptime/availability from blackbox-exporter probes — the signal that
|
||||
# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
|
||||
# alone can't distinguish "idle" from "down".
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-availability-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-availability.json: |
|
||||
{
|
||||
"title": "Service Availability & Certificate Expiration",
|
||||
"uid": "svc-availability",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-24h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Up / Down — all probed services",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 1, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "probe_success", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Uptime % trend",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
|
||||
"legendFormat": "{{instance}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Probe latency",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{ "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "7-day SLO (% successful probes)",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 99, "color": "yellow" },
|
||||
{ "value": 99.9, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success[7d]) * 100",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Services DOWN right now",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Certs expiring in < 14 days",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
|
||||
"legendFormat": "expiring"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Certificate expiry — days remaining",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 14, "color": "yellow" },
|
||||
{ "value": 30, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
|
||||
"legendFormat": "{{name}}",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
# k8s/monitoring/dashboards/service-golden-signals.yaml
|
||||
# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
|
||||
# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
|
||||
# sidecar.dashboards in k8s/logging/grafana-values.yaml.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-golden-signals-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-golden-signals.json: |
|
||||
{
|
||||
"title": "Latency & Golden Signals (Ingress RED)",
|
||||
"uid": "svc-golden-signals",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "ingress",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": "label_values(nginx_ingress_controller_requests, ingress)",
|
||||
"refresh": 2,
|
||||
"includeAll": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Request rate by status — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
|
||||
"legendFormat": "{{status}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Error rate % (4xx / 5xx) — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
||||
"legendFormat": "5xx"
|
||||
},
|
||||
{
|
||||
"expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
|
||||
"legendFormat": "4xx"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "Latency p50 / p95 / p99 — $ingress",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "All services — traffic overview",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Customer-facing failures (5xx count, window total)",
|
||||
"type": "stat",
|
||||
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "green" },
|
||||
{ "value": 1, "color": "yellow" },
|
||||
{ "value": 50, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
|
||||
"legendFormat": "5xx total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Top 5 error-contributing services",
|
||||
"type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
|
||||
"format": "table",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
# k8s/monitoring/dashboards/service-internals.yaml
|
||||
# Native per-service metrics — the "why" layer behind the ingress RED/uptime
|
||||
# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: service-internals-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
service-internals.json: |
|
||||
{
|
||||
"title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
|
||||
"uid": "svc-internals",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{ "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
|
||||
{ "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
|
||||
]
|
||||
},
|
||||
{ "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"targets": [
|
||||
{ "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "gitea_repositories", "legendFormat": "repos" },
|
||||
{ "expr": "gitea_organizations", "legendFormat": "orgs" }
|
||||
]
|
||||
},
|
||||
{ "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
|
||||
{ "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
|
||||
]
|
||||
},
|
||||
{ "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "argocd_app_info", "format": "table", "instant": true }
|
||||
]
|
||||
},
|
||||
{ "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": 0, "color": "red" },
|
||||
{ "value": 14, "color": "yellow" },
|
||||
{ "value": 30, "color": "green" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
|
||||
]
|
||||
},
|
||||
{ "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{ "expr": "vault_core_unsealed", "legendFormat": "vault" }
|
||||
]
|
||||
},
|
||||
{ "id": 9, "title": "Longhorn — volume robustness", "type": "table",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
service: argocd
|
||||
display_name: "Argo CD"
|
||||
namespace: argocd
|
||||
service_type: web-app
|
||||
folder: "Argo CD"
|
||||
jobs: [argocd-server, argocd-repo-server]
|
||||
metrics:
|
||||
prefix: argocd
|
||||
up_selector: 'job=~"argocd-.*"'
|
||||
rate:
|
||||
metric: argocd_http_request_total
|
||||
labels: [method, path, status]
|
||||
duration:
|
||||
metric: argocd_http_request_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
error:
|
||||
metric: argocd_http_request_total
|
||||
filter: 'status=~"5.."'
|
||||
domain:
|
||||
- metric: argocd_app_total
|
||||
kind: gauge
|
||||
- metric: argocd_app_sync_total
|
||||
kind: counter
|
||||
labels: [sync_status]
|
||||
- metric: argocd_app_health_degraded_total
|
||||
kind: counter
|
||||
- metric: argocd_git_sync_total
|
||||
kind: counter
|
||||
labels: [git_operation, git_status]
|
||||
- metric: argocd_reconcile_total
|
||||
kind: counter
|
||||
labels: [app_name]
|
||||
correlation_ids: []
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="argocd"'
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: AppSyncFailure
|
||||
expr_template: gauge_above_threshold
|
||||
metric: argocd_app_health_degraded_total
|
||||
threshold: 1
|
||||
for: 10m
|
||||
severity: warning
|
||||
@@ -0,0 +1,64 @@
|
||||
# authentik.descriptor.yaml
|
||||
# Build input for generating svc-authentik.yaml dashboard ConfigMap
|
||||
# and svc-authentik-rules.yaml PrometheusRule.
|
||||
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
|
||||
|
||||
service: authentik
|
||||
display_name: "Authentik"
|
||||
namespace: iam
|
||||
service_type: identity-provider
|
||||
folder: "Authentik"
|
||||
|
||||
jobs:
|
||||
- authentik-server
|
||||
- authentik-worker
|
||||
|
||||
metrics:
|
||||
prefix: authentik
|
||||
up_selector: 'job="authentik-server"'
|
||||
|
||||
rate:
|
||||
metric: authentik_flows_execution_stage_time_count
|
||||
labels: [flow_slug, stage_name]
|
||||
|
||||
duration:
|
||||
metric: authentik_main_request_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
|
||||
error:
|
||||
metric: authentik_flows_cached
|
||||
filter: null
|
||||
|
||||
domain:
|
||||
- metric: authentik_outpost_connection
|
||||
kind: gauge
|
||||
group_by: [outpost_name, outpost_type]
|
||||
- metric: authentik_outposts_connected
|
||||
kind: gauge
|
||||
- metric: authentik_flows_cached
|
||||
kind: gauge
|
||||
- metric: authentik_policies_cached
|
||||
kind: gauge
|
||||
- metric: authentik_tasks_queued
|
||||
kind: gauge
|
||||
- metric: authentik_admin_workers
|
||||
kind: gauge
|
||||
|
||||
correlation_ids: []
|
||||
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="iam"'
|
||||
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: OutpostDown
|
||||
expr_template: gauge_below_threshold
|
||||
metric: authentik_outpost_total_up
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: warning
|
||||
@@ -0,0 +1,39 @@
|
||||
service: forgejo
|
||||
display_name: "Forgejo"
|
||||
namespace: forgejo
|
||||
service_type: web-app
|
||||
folder: "Forgejo"
|
||||
jobs: [forgejo]
|
||||
metrics:
|
||||
prefix: forgejo
|
||||
up_selector: 'job="forgejo"'
|
||||
rate:
|
||||
metric: forgejo_http_request_total
|
||||
labels: [method, status]
|
||||
duration:
|
||||
metric: forgejo_http_request_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
error:
|
||||
metric: forgejo_http_request_total
|
||||
filter: 'status=~"5.."'
|
||||
domain:
|
||||
- metric: forgejo_repositories_total
|
||||
kind: gauge
|
||||
- metric: forgejo_users_total
|
||||
kind: gauge
|
||||
- metric: forgejo_git_operations_total
|
||||
kind: counter
|
||||
labels: [operation_type]
|
||||
- metric: forgejo_runner_tasks_total
|
||||
kind: counter
|
||||
labels: [status]
|
||||
correlation_ids: []
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="forgejo"'
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
@@ -0,0 +1,35 @@
|
||||
service: grafana
|
||||
display_name: "Grafana"
|
||||
namespace: logging
|
||||
service_type: web-app
|
||||
folder: "Grafana"
|
||||
jobs: [grafana]
|
||||
metrics:
|
||||
prefix: grafana
|
||||
up_selector: 'job="grafana"'
|
||||
rate:
|
||||
metric: grafana_http_request_total
|
||||
labels: [handler, status]
|
||||
duration:
|
||||
metric: grafana_http_request_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
error:
|
||||
metric: grafana_http_request_total
|
||||
filter: 'status=~"5.."'
|
||||
domain:
|
||||
- metric: grafana_dashboard_total
|
||||
kind: gauge
|
||||
- metric: grafana_user_total
|
||||
kind: gauge
|
||||
- metric: grafana_alerts_total
|
||||
kind: gauge
|
||||
correlation_ids: []
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="logging"'
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
@@ -0,0 +1,68 @@
|
||||
# minio.descriptor.yaml
|
||||
# Build input for generating svc-minio.yaml dashboard ConfigMap
|
||||
# and svc-minio-rules.yaml PrometheusRule.
|
||||
|
||||
service: minio
|
||||
display_name: "MinIO"
|
||||
namespace: storage
|
||||
service_type: stateful-store
|
||||
folder: "MinIO"
|
||||
|
||||
jobs:
|
||||
- minio
|
||||
|
||||
metrics:
|
||||
prefix: minio
|
||||
up_selector: 'job="minio"'
|
||||
|
||||
rate:
|
||||
metric: minio_s3_requests_total
|
||||
labels: [method, bucket]
|
||||
|
||||
duration:
|
||||
metric: minio_s3_requests_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
|
||||
error:
|
||||
metric: minio_s3_requests_total
|
||||
filter: 'error="true"'
|
||||
|
||||
domain:
|
||||
- metric: minio_cluster_capacity_usable_bytes
|
||||
kind: gauge
|
||||
- metric: minio_cluster_capacity_raw_total_bytes
|
||||
kind: gauge
|
||||
- metric: minio_replication_metrics_failed_byte_count
|
||||
kind: gauge
|
||||
- metric: minio_replication_metrics_replicating_byte_count
|
||||
kind: gauge
|
||||
- metric: minio_cluster_health_drives_online
|
||||
kind: gauge
|
||||
- metric: minio_cluster_health_drives_offline
|
||||
kind: gauge
|
||||
|
||||
correlation_ids: []
|
||||
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="storage"'
|
||||
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: DiskSpaceLow
|
||||
expr_template: gauge_below_percentage
|
||||
metric: minio_cluster_capacity_usable_bytes
|
||||
threshold: 0.1
|
||||
base_metric: minio_cluster_capacity_raw_total_bytes
|
||||
for: 5m
|
||||
severity: critical
|
||||
- name: ReplicationLag
|
||||
expr_template: gauge_above_threshold
|
||||
metric: minio_replication_metrics_replicating_byte_count
|
||||
threshold: 1073741824
|
||||
for: 15m
|
||||
severity: warning
|
||||
@@ -0,0 +1,74 @@
|
||||
# story-crater-backend.descriptor.yaml
|
||||
# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap
|
||||
# and svc-story-crater-backend-rules.yaml PrometheusRule.
|
||||
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
|
||||
|
||||
service: story-crater-backend
|
||||
display_name: "Story Crater Backend"
|
||||
namespace: story-crater-backend
|
||||
service_type: message-worker
|
||||
folder: "Story Crater Backend"
|
||||
|
||||
jobs:
|
||||
- agent-worker
|
||||
- check
|
||||
- edit-collab
|
||||
- canary
|
||||
|
||||
metrics:
|
||||
prefix: story_crater
|
||||
up_selector: 'job=~"agent-worker|check|edit-collab|canary"'
|
||||
|
||||
rate:
|
||||
metric: story_crater_messages_handled_total
|
||||
labels: [agent, queue, status] # status: succeeded | failed
|
||||
|
||||
duration:
|
||||
metric: story_crater_message_handle_duration_ms
|
||||
type: histogram
|
||||
unit: ms
|
||||
|
||||
error:
|
||||
metric: story_crater_app_metric_total
|
||||
filter: 'severity="error"'
|
||||
|
||||
domain:
|
||||
- metric: story_crater_queue_depth
|
||||
kind: gauge
|
||||
group_by: [queue]
|
||||
- metric: story_crater_dedup_redeliveries_total
|
||||
kind: counter
|
||||
group_by: [agent, queue]
|
||||
- metric: story_crater_outbox_publish_lag_ms
|
||||
kind: histogram
|
||||
- metric: story_crater_llm_tokens_used_total
|
||||
kind: counter
|
||||
group_by: [agent, kind]
|
||||
- metric: story_crater_llm_call_duration_seconds
|
||||
kind: histogram
|
||||
group_by: [agent, model_provider]
|
||||
- metric: story_crater_check_latency_ms
|
||||
kind: histogram
|
||||
slo_ms: 1200
|
||||
- metric: story_crater_degrade_state
|
||||
kind: gauge
|
||||
group_by: [service]
|
||||
|
||||
correlation_ids: [session_id, trace_id, tenant]
|
||||
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="story-crater-backend"'
|
||||
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: CheckLatencySLOBreach
|
||||
expr_template: histogram_quantile_over_threshold
|
||||
metric: story_crater_check_latency_ms
|
||||
quantile: 0.95
|
||||
threshold: 1200
|
||||
for: 10m
|
||||
severity: critical
|
||||
@@ -0,0 +1,63 @@
|
||||
# story-crater-frontend.descriptor.yaml
|
||||
# Build input for generating svc-story-crater-frontend.yaml dashboard ConfigMap
|
||||
# and svc-story-crater-frontend-rules.yaml PrometheusRule.
|
||||
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
|
||||
|
||||
service: story-crater-frontend
|
||||
display_name: "Story Crater Frontend"
|
||||
namespace: story-crater-frontend
|
||||
service_type: web-app
|
||||
folder: "Story Crater Frontend"
|
||||
|
||||
jobs:
|
||||
- story-crater-frontend
|
||||
|
||||
metrics:
|
||||
prefix: story_crater_frontend
|
||||
up_selector: 'job="story-crater-frontend"'
|
||||
|
||||
rate:
|
||||
metric: story_crater_frontend_http_requests_total
|
||||
labels: [method, route, status]
|
||||
|
||||
duration:
|
||||
metric: story_crater_frontend_http_request_duration_ms
|
||||
type: histogram
|
||||
unit: ms
|
||||
|
||||
error:
|
||||
metric: story_crater_frontend_http_requests_total
|
||||
filter: 'status=~"5.."'
|
||||
|
||||
domain:
|
||||
- metric: story_crater_frontend_web_vitals_lcp_ms
|
||||
kind: gauge
|
||||
group_by: [page]
|
||||
- metric: story_crater_frontend_web_vitals_fid_ms
|
||||
kind: gauge
|
||||
group_by: [page]
|
||||
- metric: story_crater_frontend_web_vitals_cls
|
||||
kind: gauge
|
||||
group_by: [page]
|
||||
- metric: story_crater_frontend_web_vitals_inp_ms
|
||||
kind: gauge
|
||||
group_by: [page]
|
||||
|
||||
correlation_ids: [] # Frontend doesn't have session_id/trace_id in metrics
|
||||
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="story-crater-frontend"'
|
||||
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: WebVitalsLCPRegression
|
||||
expr_template: histogram_quantile_over_threshold
|
||||
metric: story_crater_frontend_web_vitals_lcp_ms
|
||||
quantile: 0.75
|
||||
threshold: 2500
|
||||
for: 10m
|
||||
severity: warning
|
||||
@@ -0,0 +1,45 @@
|
||||
service: vault
|
||||
display_name: "Vault"
|
||||
namespace: storage
|
||||
service_type: stateful-store
|
||||
folder: "Vault"
|
||||
jobs: [vault]
|
||||
metrics:
|
||||
prefix: vault
|
||||
up_selector: 'job="vault"'
|
||||
rate:
|
||||
metric: vault_core_handle_request_total
|
||||
labels: [method, path]
|
||||
duration:
|
||||
metric: vault_core_handle_request_duration_seconds
|
||||
type: histogram
|
||||
unit: s
|
||||
error:
|
||||
metric: vault_core_handle_request_total
|
||||
filter: 'error="true"'
|
||||
domain:
|
||||
- metric: vault_core_unsealed
|
||||
kind: gauge
|
||||
- metric: vault_core_active
|
||||
kind: gauge
|
||||
- metric: vault_core_replication_primary
|
||||
kind: gauge
|
||||
- metric: vault_token_total
|
||||
kind: gauge
|
||||
- metric: vault_database_connection_close_total
|
||||
kind: counter
|
||||
correlation_ids: []
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="iam"'
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: VaultSealed
|
||||
expr_template: gauge_below_threshold
|
||||
metric: vault_core_unsealed
|
||||
threshold: 1
|
||||
for: 1m
|
||||
severity: critical
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-argocd-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Argo CD"
|
||||
data:
|
||||
svc-argocd.json: |
|
||||
{"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}
|
||||
@@ -0,0 +1,143 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-authentik-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Authentik"
|
||||
data:
|
||||
svc-authentik.json: |
|
||||
{
|
||||
"title": "Authentik — Service Overview",
|
||||
"uid": "svc-authentik",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "HTTP request rate by status", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "Outpost connections", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_outposts_connected" }]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "Flows cached", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_flows_cached" }]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "Policies cached", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_policies_cached" }]
|
||||
},
|
||||
{
|
||||
"id": 24, "title": "Queued tasks", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_tasks_queued" }]
|
||||
},
|
||||
{
|
||||
"id": 25, "title": "Admin workers", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "authentik_admin_workers" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Recent logs", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"iam\"}" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-forgejo-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Forgejo"
|
||||
data:
|
||||
svc-forgejo.json: |
|
||||
{"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-grafana-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Grafana"
|
||||
data:
|
||||
svc-grafana.json: |
|
||||
{"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}
|
||||
@@ -0,0 +1,143 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-minio-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "MinIO"
|
||||
data:
|
||||
svc-minio.json: |
|
||||
{
|
||||
"title": "MinIO — Service Overview",
|
||||
"uid": "svc-minio",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=\"minio\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "S3 request rate by method", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate %", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Storage & Replication", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
|
||||
{ "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "short" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
|
||||
{ "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
|
||||
},
|
||||
{
|
||||
"id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Recent logs", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"storage\"}" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-story-crater-backend-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Story Crater Backend"
|
||||
data:
|
||||
svc-story-crater-backend.json: |
|
||||
{
|
||||
"title": "Story Crater Backend — Service Overview",
|
||||
"uid": "svc-story-crater-backend",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "Message rate by status", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate %", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "ms" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "Queue depth", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "Outbox publish lag p95", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 },
|
||||
"fieldConfig": { "defaults": { "unit": "ms" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "Dedup redeliveries rate", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: LLM / Inference", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Token usage rate", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }]
|
||||
},
|
||||
{
|
||||
"id": 32, "title": "LLM call duration p95", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 },
|
||||
"fieldConfig": { "defaults": { "unit": "s" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }]
|
||||
},
|
||||
{
|
||||
"id": 33, "title": "LLM call errors rate", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 40, "title": "Row: SLOs / Degradation", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": null, "color": "green" },
|
||||
{ "value": 1200, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }]
|
||||
},
|
||||
{
|
||||
"id": 42, "title": "Degrade state by service", "type": "stat",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } },
|
||||
{ "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 50, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-story-crater-frontend-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Story Crater Frontend"
|
||||
data:
|
||||
svc-story-crater-frontend.json: |
|
||||
{
|
||||
"title": "Story Crater Frontend — Service Overview",
|
||||
"uid": "svc-story-crater-frontend",
|
||||
"schemaVersion": 39,
|
||||
"timezone": "browser",
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"refresh": "30s",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 2, "title": "Up", "type": "stat",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": { "mode": "thresholds" },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
||||
}
|
||||
},
|
||||
"targets": [{ "expr": "min(up{job=\"story-crater-frontend\"})" }]
|
||||
},
|
||||
{
|
||||
"id": 3, "title": "HTTP request rate by status", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total[5m])) by (status)", "legendFormat": "{{status}}" }]
|
||||
},
|
||||
{
|
||||
"id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
||||
"fieldConfig": { "defaults": { "unit": "percent" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(story_crater_frontend_http_requests_total[5m])) * 100" }]
|
||||
},
|
||||
{
|
||||
"id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
||||
"fieldConfig": { "defaults": { "unit": "ms" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "expr": "histogram_quantile(0.50, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" },
|
||||
{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" },
|
||||
{ "expr": "histogram_quantile(0.99, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-frontend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-frontend\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
},
|
||||
{
|
||||
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-frontend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20, "title": "Row: Web Vitals", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 21, "title": "LCP p75 (Largest Contentful Paint)", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": null, "color": "green" },
|
||||
{ "value": 2500, "color": "orange" },
|
||||
{ "value": 4000, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms)", "legendFormat": "LCP p75" }]
|
||||
},
|
||||
{
|
||||
"id": 22, "title": "FID p75 (First Input Delay)", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": null, "color": "green" },
|
||||
{ "value": 100, "color": "orange" },
|
||||
{ "value": 300, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_fid_ms)", "legendFormat": "FID p75" }]
|
||||
},
|
||||
{
|
||||
"id": 23, "title": "CLS (Cumulative Layout Shift)", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": null, "color": "green" },
|
||||
{ "value": 0.1, "color": "orange" },
|
||||
{ "value": 0.25, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "max(story_crater_frontend_web_vitals_cls)", "legendFormat": "CLS" }]
|
||||
},
|
||||
{
|
||||
"id": 24, "title": "INP p75 (Interaction to Next Paint)", "type": "timeseries",
|
||||
"gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ms",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "value": null, "color": "green" },
|
||||
{ "value": 200, "color": "orange" },
|
||||
{ "value": 500, "color": "red" }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_inp_ms)", "legendFormat": "INP p75" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30, "title": "Row: Logs", "type": "row",
|
||||
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
||||
"panels": [
|
||||
{
|
||||
"id": 31, "title": "Recent logs", "type": "logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "loki", "uid": "loki" },
|
||||
"targets": [{ "expr": "{namespace=\"story-crater-frontend\"}" }]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: svc-vault-dashboard
|
||||
namespace: logging
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
annotations:
|
||||
grafana_folder: "Vault"
|
||||
data:
|
||||
svc-vault.json: |
|
||||
{"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}
|
||||
Reference in New Issue
Block a user