72 lines
2.6 KiB
YAML
72 lines
2.6 KiB
YAML
|
apiVersion: monitoring.coreos.com/v1
|
||
|
|
kind: PrometheusRule
|
||
|
|
metadata:
|
||
|
|
name: cag-kv-cache-rules
|
||
|
|
namespace: monitoring
|
||
|
|
labels:
|
||
|
|
release: prometheus
|
||
|
|
spec:
|
||
|
|
groups:
|
||
|
|
# Recording rules: derive vLLM prefix cache hit rate from raw counters
|
||
|
|
- name: vllm.recording
|
||
|
|
rules:
|
||
|
|
- record: vllm:prefix_cache_hit_rate
|
||
|
|
expr: |
|
||
|
|
rate(vllm:cache_query_hit_total[5m])
|
||
|
|
/
|
||
|
|
clamp_min(
|
||
|
|
rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]),
|
||
|
|
0.001
|
||
|
|
)
|
||
|
|
labels:
|
||
|
|
job: "vllm"
|
||
|
|
|
||
|
|
# Alert rules: CAG pipeline health
|
||
|
|
- name: cag.alerts
|
||
|
|
rules:
|
||
|
|
# KV warmup consistently failing — cache never gets warm
|
||
|
|
- alert: KVWarmupFailureHigh
|
||
|
|
expr: |
|
||
|
|
rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
team: poimen
|
||
|
|
annotations:
|
||
|
|
summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
|
||
|
|
description: |
|
||
|
|
The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
|
||
|
|
for 5 consecutive minutes. The prefix KV cache is not being warmed;
|
||
|
|
agents will see full prefill latency on every call.
|
||
|
|
Check reasoning-predictor health and poimen-memory NetworkPolicy.
|
||
|
|
|
||
|
|
# Warmup misses: vLLM reachable but not caching
|
||
|
|
- alert: KVWarmupMissHigh
|
||
|
|
expr: |
|
||
|
|
rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
|
||
|
|
for: 5m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
team: poimen
|
||
|
|
annotations:
|
||
|
|
summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
|
||
|
|
description: |
|
||
|
|
vLLM is reachable but cached_tokens=0 on warmup responses.
|
||
|
|
Possible causes: prefix caching disabled in vLLM, max-num-seqs
|
||
|
|
saturated during warmup, or context format changed.
|
||
|
|
|
||
|
|
# Prefix cache hit rate low after warmup should be established
|
||
|
|
- alert: VLLMPrefixCacheHitRateLow
|
||
|
|
expr: |
|
||
|
|
vllm:prefix_cache_hit_rate < 0.70
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
severity: warning
|
||
|
|
team: poimen
|
||
|
|
annotations:
|
||
|
|
summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
|
||
|
|
description: |
|
||
|
|
vLLM prefix cache hit rate below 70% for 10 minutes.
|
||
|
|
Expected >= 85% in steady state with CAG warming active.
|
||
|
|
Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.
|