homelab/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cag-kv-cache-rules
  namespace: monitoring
  labels:
    release: prometheus
spec:
  groups:
    # Recording rules: derive vLLM prefix cache hit rate from raw counters.
    # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29).
    # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total)
    # do not exist in this version — vLLM exposes prefix_cache_hits_total and
    # prefix_cache_queries_total — so this rule produced no data and
    # VLLMPrefixCacheHitRateLow could never fire.
    - name: vllm.recording
      rules:
        - record: vllm:prefix_cache_hit_rate
          expr: |
            rate(vllm:prefix_cache_hits_total[5m])
            /
            clamp_min(
              rate(vllm:prefix_cache_queries_total[5m]),
              0.001
            )
          labels:
            job: "reasoning-predictor"

    # Alert rules: CAG pipeline health
    - name: cag.alerts
      rules:
        # KV warmup consistently failing — cache never gets warm
        - alert: KVWarmupFailureHigh
          expr: |
            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
          for: 5m
          labels:
            severity: warning
            team: poimen
          annotations:
            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
            description: |
              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
              for 5 consecutive minutes. The prefix KV cache is not being warmed;
              agents will see full prefill latency on every call.
              Check reasoning-predictor health and poimen-memory NetworkPolicy.

        # Warmup misses: vLLM reachable but not caching
        - alert: KVWarmupMissHigh
          expr: |
            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
          for: 5m
          labels:
            severity: warning
            team: poimen
          annotations:
            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
            description: |
              vLLM is reachable but cached_tokens=0 on warmup responses.
              Possible causes: prefix caching disabled in vLLM, max-num-seqs
              saturated during warmup, or context format changed.

        # Prefix cache hit rate low after warmup should be established
        - alert: VLLMPrefixCacheHitRateLow
          expr: |
            vllm:prefix_cache_hit_rate < 0.70
          for: 10m
          labels:
            severity: warning
            team: poimen
          annotations:
            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
            description: |
              vLLM prefix cache hit rate below 70% for 10 minutes.
              Expected >= 85% in steady state with CAG warming active.
              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.