homelab/k8s/monitoring/dashboards/kube-controller-health.yaml

# k8s/monitoring/dashboards/kube-controller-health.yaml
# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
# the one control-plane component that's still reachable (its ServiceMonitor
# targets the in-cluster `kubernetes` service, not localhost) — paired with
# kube-state-metrics signals as a proxy for controller/scheduler health.
apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-controller-health-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
data:
  kube-controller-health.json: |
    {
      "title": "Kube-Controller Health",
      "uid": "kube-controller-health",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": { "from": "now-6h", "to": "now" },
      "refresh": "30s",
      "panels": [
        {
          "id": 1,
          "title": "API server — up",
          "type": "stat",
          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "mappings": [
                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
              ]
            }
          },
          "targets": [
            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
          ]
        },
        {
          "id": 2,
          "title": "API server — request rate by verb/code",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
              "legendFormat": "{{verb}} {{code}}"
            }
          ]
        },
        {
          "id": 3,
          "title": "API server — error rate % (5xx)",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "percent" } },
          "targets": [
            {
              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
              "legendFormat": "5xx %"
            }
          ]
        },
        {
          "id": 4,
          "title": "API server — latency p95 / p99",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": { "defaults": { "unit": "s" } },
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p95"
            },
            {
              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
              "legendFormat": "p99"
            }
          ]
        },
        {
          "id": 5,
          "title": "Pods stuck Pending",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "yellow" },
                  { "value": 5, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
          ]
        },
        {
          "id": 6,
          "title": "CrashLoopBackOff containers",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
          ]
        },
        {
          "id": 7,
          "title": "Nodes NotReady",
          "type": "stat",
          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "fieldConfig": {
            "defaults": {
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  { "value": 0, "color": "green" },
                  { "value": 1, "color": "red" }
                ]
              }
            }
          },
          "targets": [
            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
          ]
        },
        {
          "id": 8,
          "title": "Failed Jobs",
          "type": "table",
          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
          ]
        },
        {
          "id": 9,
          "title": "Deployments with unavailable replicas",
          "type": "table",
          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
          ]
        },
        {
          "id": 10,
          "title": "Container restart rate by pod",
          "type": "timeseries",
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [
            {
              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
              "legendFormat": "{{namespace}}/{{pod}}"
            }
          ]
        }
      ]
    }