homelab/k8s/infra/tracing/api-latency-dashboard.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: api-latency-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Tracing"
data:
  api-latency.json: |
    {
      "title": "API Latency & Traces",
      "uid": "api-latency-traces",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": {"from": "now-1h", "to": "now"},
      "refresh": "10s",
      "panels": [
        {
          "id": 1,
          "title": "Request Rate by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)",
              "legendFormat": "{{service}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "reqps",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 2,
          "title": "P95 Latency by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))",
              "legendFormat": "{{service}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "s",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 3,
          "title": "Error Rate by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
              "legendFormat": "{{service}} errors"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "reqps",
              "custom": {"fillOpacity": 10},
              "color": {"mode": "palette-classic"}
            }
          }
        },
        {
          "id": 4,
          "title": "Service Dependency Graph",
          "type": "nodeGraph",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
          "datasource": {"type": "tempo", "uid": "tempo"},
          "targets": [
            {
              "queryType": "serviceMap"
            }
          ]
        },
        {
          "id": 5,
          "title": "Trace Search",
          "type": "traces",
          "gridPos": {"h": 12, "w": 24, "x": 0, "y": 16},
          "datasource": {"type": "tempo", "uid": "tempo"},
          "targets": [
            {
              "queryType": "traceqlSearch",
              "serviceName": "api-gateway",
              "limit": 20
            }
          ]
        },
        {
          "id": 6,
          "title": "LLM Inference Latency",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
              "legendFormat": "p50 {{service}} {{span_name}}"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
              "legendFormat": "p95 {{service}} {{span_name}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "s",
              "custom": {"fillOpacity": 10}
            }
          }
        }
      ]
    }