rock
605bc706e0
- cag-kv-cache-rules.yaml: recording rule vllm:prefix_cache_hit_rate (from cache_query_hit/miss counters), 3 alert rules: KVWarmupFailureHigh, KVWarmupMissHigh, VLLMPrefixCacheHitRateLow - api-latency-dashboard.yaml: 5 new panels (id 7-11): prefix cache hit rate, warmup hit/miss, warmup latency p50/p95, standing context tier hits, alert panel with fixed threshold 0.01/s
279 lines
9.2 KiB
YAML
279 lines
9.2 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: api-latency-dashboard
|
|
namespace: logging
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
annotations:
|
|
grafana_folder: "Tracing"
|
|
data:
|
|
api-latency.json: |
|
|
{
|
|
"title": "API Latency & Traces",
|
|
"uid": "api-latency-traces",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"refresh": "10s",
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"title": "Request Rate by Service",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\"}[5m])) by (service)",
|
|
"legendFormat": "{{service}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"title": "P95 Latency by Service",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\"api-gateway|poimen.*\"}[5m])) by (le, service))",
|
|
"legendFormat": "{{service}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"title": "Error Rate by Service",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(traces_spanmetrics_calls_total{service=~\"api-gateway|poimen.*\", status_code=\"STATUS_CODE_ERROR\"}[5m])) by (service)",
|
|
"legendFormat": "{{service}} errors"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "reqps",
|
|
"custom": {"fillOpacity": 10},
|
|
"color": {"mode": "palette-classic"}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"title": "Service Dependency Graph",
|
|
"type": "nodeGraph",
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
|
|
"datasource": {"type": "tempo", "uid": "tempo"},
|
|
"targets": [
|
|
{
|
|
"queryType": "serviceMap"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"title": "Trace Search",
|
|
"type": "traces",
|
|
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 16},
|
|
"datasource": {"type": "tempo", "uid": "tempo"},
|
|
"targets": [
|
|
{
|
|
"queryType": "traceqlSearch",
|
|
"serviceName": "api-gateway",
|
|
"limit": 20
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"title": "LLM Inference Latency",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
|
|
"legendFormat": "p50 {{service}} {{span_name}}"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=~\".*llm.*|.*reasoning.*\", span_name=~\".*inference.*|.*generate.*\"}[5m])) by (le, service, span_name))",
|
|
"legendFormat": "p95 {{service}} {{span_name}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "s",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 7,
|
|
"title": "vLLM Prefix Cache Hit Rate",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "vllm:prefix_cache_hit_rate",
|
|
"legendFormat": "prefix cache hit rate"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"custom": {"fillOpacity": 15},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "red", "value": 0},
|
|
{"color": "yellow", "value": 0.7},
|
|
{"color": "green", "value": 0.85}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 8,
|
|
"title": "KV Warmup Hit vs Miss",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
|
|
"legendFormat": "warmup hit"
|
|
},
|
|
{
|
|
"expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
|
|
"legendFormat": "warmup miss"
|
|
},
|
|
{
|
|
"expr": "rate(memory_kv_cache_warmup_error_total[5m])",
|
|
"legendFormat": "warmup error"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ops",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 9,
|
|
"title": "KV Warmup Latency p50/p95 (ms)",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
|
|
"legendFormat": "p50 warmup latency"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
|
|
"legendFormat": "p95 warmup latency"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 10,
|
|
"title": "Standing Context Cache Tier Hits",
|
|
"type": "timeseries",
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
|
|
"legendFormat": "Redis L1 hit"
|
|
},
|
|
{
|
|
"expr": "rate(memory_standing_ctx_hit_db_total[5m])",
|
|
"legendFormat": "DB L2 hit"
|
|
},
|
|
{
|
|
"expr": "rate(memory_standing_ctx_recompute_total[5m])",
|
|
"legendFormat": "recompute (L3)"
|
|
},
|
|
{
|
|
"expr": "rate(memory_standing_ctx_304_total[5m])",
|
|
"legendFormat": "304 Not Modified"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ops",
|
|
"custom": {"fillOpacity": 10}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"title": "Standing Context Alert: Stale > 10min post-ingest",
|
|
"type": "timeseries",
|
|
"description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
|
|
"legendFormat": "recompute rate (non-zero = fresh)"
|
|
},
|
|
{
|
|
"expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
|
|
"legendFormat": "warmup miss rate (alert if > 3/5min)"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ops",
|
|
"custom": {"fillOpacity": 10},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{"color": "green", "value": 0},
|
|
{"color": "red", "value": 0.01}
|
|
]
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
|
|
"properties": [
|
|
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
|
|
{"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|