apiVersion: v1 kind: ConfigMap metadata: name: api-latency-dashboard namespace: logging labels: grafana_dashboard: "1" annotations: grafana_folder: "Tracing" data: api-latency.json: | { "title": "API Latency & Traces", "uid": "api-latency-traces", "schemaVersion": 39, "timezone": "browser", "time": {"from": "now-1h", "to": "now"}, "refresh": "10s", "panels": [ { "id": 1, "title": "Request Rate by Service", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ { "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)", "legendFormat": "{{service}}" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": {"fillOpacity": 10} } } }, { "id": 2, "title": "P95 Latency by Service", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))", "legendFormat": "{{service}}" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": {"fillOpacity": 10} } } }, { "id": 3, "title": "Error Rate by Service", "type": "timeseries", "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ { "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)", "legendFormat": "{{service}} errors" } ], "fieldConfig": { "defaults": { "unit": "reqps", "custom": {"fillOpacity": 10}, "color": {"mode": "palette-classic"} } } }, { "id": 4, "title": "Service Dependency Graph", "type": "nodeGraph", "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, "datasource": {"type": "tempo", "uid": "tempo"}, "targets": [ { "queryType": "serviceMap" } ] }, { "id": 5, "title": "Trace Search", "type": "traces", "gridPos": {"h": 12, "w": 24, "x": 0, "y": 16}, "datasource": {"type": "tempo", "uid": "tempo"}, "targets": [ { "queryType": "traceqlSearch", "serviceName": "api-gateway", "limit": 20 } ] }, { "id": 6, "title": "LLM Inference Latency", "type": "timeseries", "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28}, "datasource": {"type": "prometheus", "uid": "prometheus"}, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))", "legendFormat": "p50 {{service}} {{span_name}}" }, { "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))", "legendFormat": "p95 {{service}} {{span_name}}" } ], "fieldConfig": { "defaults": { "unit": "s", "custom": {"fillOpacity": 10} } } } ] }