apiVersion: v1
kind: ConfigMap
metadata:
  name: api-latency-dashboard
  namespace: logging
  labels:
    grafana_dashboard: "1"
  annotations:
    grafana_folder: "Tracing"
data:
  api-latency.json: |
    {
      "title": "API Latency & Traces",
      "uid": "api-latency-traces",
      "schemaVersion": 39,
      "timezone": "browser",
      "time": {"from": "now-1h", "to": "now"},
      "refresh": "10s",
      "panels": [
        {
          "id": 1,
          "title": "Request Rate by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)",
              "legendFormat": "{{service}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "reqps",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 2,
          "title": "P95 Latency by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))",
              "legendFormat": "{{service}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "s",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 3,
          "title": "Error Rate by Service",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
              "legendFormat": "{{service}} errors"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "reqps",
              "custom": {"fillOpacity": 10},
              "color": {"mode": "palette-classic"}
            }
          }
        },
        {
          "id": 4,
          "title": "Service Dependency Graph",
          "type": "nodeGraph",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
          "datasource": {"type": "tempo", "uid": "tempo"},
          "targets": [
            {
              "queryType": "serviceMap"
            }
          ]
        },
        {
          "id": 5,
          "title": "Trace Search",
          "type": "traces",
          "gridPos": {"h": 12, "w": 24, "x": 0, "y": 16},
          "datasource": {"type": "tempo", "uid": "tempo"},
          "targets": [
            {
              "queryType": "traceqlSearch",
              "serviceName": "api-gateway",
              "limit": 20
            }
          ]
        },
        {
          "id": 6,
          "title": "LLM Inference Latency",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
              "legendFormat": "p50 {{service}} {{span_name}}"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
              "legendFormat": "p95 {{service}} {{span_name}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "s",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 7,
          "title": "vLLM Prefix Cache Hit Rate",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "vllm:prefix_cache_hit_rate",
              "legendFormat": "prefix cache hit rate"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percentunit",
              "min": 0,
              "max": 1,
              "custom": {"fillOpacity": 15},
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {"color": "red", "value": 0},
                  {"color": "yellow", "value": 0.7},
                  {"color": "green", "value": 0.85}
                ]
              }
            }
          }
        },
        {
          "id": 8,
          "title": "KV Warmup Hit vs Miss",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
              "legendFormat": "warmup hit"
            },
            {
              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
              "legendFormat": "warmup miss"
            },
            {
              "expr": "rate(memory_kv_cache_warmup_error_total[5m])",
              "legendFormat": "warmup error"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ops",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 9,
          "title": "KV Warmup Latency p50/p95 (ms)",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
              "legendFormat": "p50 warmup latency"
            },
            {
              "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
              "legendFormat": "p95 warmup latency"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ms",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 10,
          "title": "Standing Context Cache Tier Hits",
          "type": "timeseries",
          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
              "legendFormat": "Redis L1 hit"
            },
            {
              "expr": "rate(memory_standing_ctx_hit_db_total[5m])",
              "legendFormat": "DB L2 hit"
            },
            {
              "expr": "rate(memory_standing_ctx_recompute_total[5m])",
              "legendFormat": "recompute (L3)"
            },
            {
              "expr": "rate(memory_standing_ctx_304_total[5m])",
              "legendFormat": "304 Not Modified"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ops",
              "custom": {"fillOpacity": 10}
            }
          }
        },
        {
          "id": 11,
          "title": "Standing Context Alert: Stale > 10min post-ingest",
          "type": "timeseries",
          "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
          "datasource": {"type": "prometheus", "uid": "prometheus"},
          "targets": [
            {
              "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
              "legendFormat": "recompute rate (non-zero = fresh)"
            },
            {
              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
              "legendFormat": "warmup miss rate (alert if > 3/5min)"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "ops",
              "custom": {"fillOpacity": 10},
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {"color": "green", "value": 0},
                  {"color": "red", "value": 0.01}
                ]
              }
            },
            "overrides": [
              {
                "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
                "properties": [
                  {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
                  {"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
                ]
              }
            ]
          }
        }
      ]
    }