From 605bc706e01fed2a8481bbadab0fd8534591f9f0 Mon Sep 17 00:00:00 2001 From: rock Date: Sun, 27 Sep 2026 18:37:03 +0800 Subject: [PATCH] feat(cag): PrometheusRule + Grafana panels for KV cache observability - cag-kv-cache-rules.yaml: recording rule vllm:prefix_cache_hit_rate (from cache_query_hit/miss counters), 3 alert rules: KVWarmupFailureHigh, KVWarmupMissHigh, VLLMPrefixCacheHitRateLow - api-latency-dashboard.yaml: 5 new panels (id 7-11): prefix cache hit rate, warmup hit/miss, warmup latency p50/p95, standing context tier hits, alert panel with fixed threshold 0.01/s --- .../monitoring/alerts/cag-kv-cache-rules.yaml | 71 +++++++++ k8s/infra/monitoring/kustomization.yaml | 1 + k8s/infra/tracing/api-latency-dashboard.yaml | 150 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml diff --git a/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml b/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml new file mode 100644 index 0000000..3d69fae --- /dev/null +++ b/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml @@ -0,0 +1,71 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: cag-kv-cache-rules + namespace: monitoring + labels: + release: prometheus +spec: + groups: + # Recording rules: derive vLLM prefix cache hit rate from raw counters + - name: vllm.recording + rules: + - record: vllm:prefix_cache_hit_rate + expr: | + rate(vllm:cache_query_hit_total[5m]) + / + clamp_min( + rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]), + 0.001 + ) + labels: + job: "vllm" + + # Alert rules: CAG pipeline health + - name: cag.alerts + rules: + # KV warmup consistently failing — cache never gets warm + - alert: KVWarmupFailureHigh + expr: | + rate(memory_kv_cache_warmup_error_total[5m]) > 0.01 + for: 5m + labels: + severity: warning + team: poimen + annotations: + summary: "KV warmup errors elevated ({{ $value | humanize }}/s)" + description: | + The KV cache warmer is failing to reach vLLM at a rate > 0.01/s + for 5 consecutive minutes. The prefix KV cache is not being warmed; + agents will see full prefill latency on every call. + Check reasoning-predictor health and poimen-memory NetworkPolicy. + + # Warmup misses: vLLM reachable but not caching + - alert: KVWarmupMissHigh + expr: | + rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01 + for: 5m + labels: + severity: warning + team: poimen + annotations: + summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)" + description: | + vLLM is reachable but cached_tokens=0 on warmup responses. + Possible causes: prefix caching disabled in vLLM, max-num-seqs + saturated during warmup, or context format changed. + + # Prefix cache hit rate low after warmup should be established + - alert: VLLMPrefixCacheHitRateLow + expr: | + vllm:prefix_cache_hit_rate < 0.70 + for: 10m + labels: + severity: warning + team: poimen + annotations: + summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})" + description: | + vLLM prefix cache hit rate below 70% for 10 minutes. + Expected >= 85% in steady state with CAG warming active. + Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause. diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 42a1112..6d7e629 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -14,6 +14,7 @@ resources: - alerts/svc-vault-rules.yaml - alerts/cluster-alerts.yaml - alerts/api-gateway-alerts.yaml + - alerts/cag-kv-cache-rules.yaml - servicemonitors/argocd.yaml - servicemonitors/authentik.yaml - servicemonitors/forgejo.yaml diff --git a/k8s/infra/tracing/api-latency-dashboard.yaml b/k8s/infra/tracing/api-latency-dashboard.yaml index b876e94..2a16749 100644 --- a/k8s/infra/tracing/api-latency-dashboard.yaml +++ b/k8s/infra/tracing/api-latency-dashboard.yaml @@ -123,6 +123,156 @@ data: "custom": {"fillOpacity": 10} } } + }, + { + "id": 7, + "title": "vLLM Prefix Cache Hit Rate", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "vllm:prefix_cache_hit_rate", + "legendFormat": "prefix cache hit rate" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": {"fillOpacity": 15}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": 0}, + {"color": "yellow", "value": 0.7}, + {"color": "green", "value": 0.85} + ] + } + } + } + }, + { + "id": 8, + "title": "KV Warmup Hit vs Miss", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "rate(memory_kv_cache_warmup_hit_total[5m])", + "legendFormat": "warmup hit" + }, + { + "expr": "rate(memory_kv_cache_warmup_miss_total[5m])", + "legendFormat": "warmup miss" + }, + { + "expr": "rate(memory_kv_cache_warmup_error_total[5m])", + "legendFormat": "warmup error" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {"fillOpacity": 10} + } + } + }, + { + "id": 9, + "title": "KV Warmup Latency p50/p95 (ms)", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))", + "legendFormat": "p50 warmup latency" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))", + "legendFormat": "p95 warmup latency" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": {"fillOpacity": 10} + } + } + }, + { + "id": 10, + "title": "Standing Context Cache Tier Hits", + "type": "timeseries", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "rate(memory_standing_ctx_hit_redis_total[5m])", + "legendFormat": "Redis L1 hit" + }, + { + "expr": "rate(memory_standing_ctx_hit_db_total[5m])", + "legendFormat": "DB L2 hit" + }, + { + "expr": "rate(memory_standing_ctx_recompute_total[5m])", + "legendFormat": "recompute (L3)" + }, + { + "expr": "rate(memory_standing_ctx_304_total[5m])", + "legendFormat": "304 Not Modified" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {"fillOpacity": 10} + } + } + }, + { + "id": 11, + "title": "Standing Context Alert: Stale > 10min post-ingest", + "type": "timeseries", + "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.", + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [ + { + "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0", + "legendFormat": "recompute rate (non-zero = fresh)" + }, + { + "expr": "rate(memory_kv_cache_warmup_miss_total[5m])", + "legendFormat": "warmup miss rate (alert if > 3/5min)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": {"fillOpacity": 10}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": 0}, + {"color": "red", "value": 0.01} + ] + } + }, + "overrides": [ + { + "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"}, + "properties": [ + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}}, + {"id": "custom.thresholdsStyle", "value": {"mode": "area"}} + ] + } + ] + } } ] }