feat(cag): PrometheusRule + Grafana panels for KV cache observability

- cag-kv-cache-rules.yaml: recording rule vllm:prefix_cache_hit_rate
  (from cache_query_hit/miss counters), 3 alert rules:
  KVWarmupFailureHigh, KVWarmupMissHigh, VLLMPrefixCacheHitRateLow
- api-latency-dashboard.yaml: 5 new panels (id 7-11):
  prefix cache hit rate, warmup hit/miss, warmup latency p50/p95,
  standing context tier hits, alert panel with fixed threshold 0.01/s

k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml

@@ -0,0 +1,71 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cag-kv-cache-rules
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    # Recording rules: derive vLLM prefix cache hit rate from raw counters
+    - name: vllm.recording
+      rules:
+        - record: vllm:prefix_cache_hit_rate
+          expr: |
+            rate(vllm:cache_query_hit_total[5m])
+            /
+            clamp_min(
+              rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]),
+              0.001
+            )
+          labels:
+            job: "vllm"
+
+    # Alert rules: CAG pipeline health
+    - name: cag.alerts
+      rules:
+        # KV warmup consistently failing — cache never gets warm
+        - alert: KVWarmupFailureHigh
+          expr: |
+            rate(memory_kv_cache_warmup_error_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup errors elevated ({{ $value | humanize }}/s)"
+            description: |
+              The KV cache warmer is failing to reach vLLM at a rate > 0.01/s
+              for 5 consecutive minutes. The prefix KV cache is not being warmed;
+              agents will see full prefill latency on every call.
+              Check reasoning-predictor health and poimen-memory NetworkPolicy.
+
+        # Warmup misses: vLLM reachable but not caching
+        - alert: KVWarmupMissHigh
+          expr: |
+            rate(memory_kv_cache_warmup_miss_total[5m]) > 0.01
+          for: 5m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "KV warmup cache misses elevated ({{ $value | humanize }}/s)"
+            description: |
+              vLLM is reachable but cached_tokens=0 on warmup responses.
+              Possible causes: prefix caching disabled in vLLM, max-num-seqs
+              saturated during warmup, or context format changed.
+
+        # Prefix cache hit rate low after warmup should be established
+        - alert: VLLMPrefixCacheHitRateLow
+          expr: |
+            vllm:prefix_cache_hit_rate < 0.70
+          for: 10m
+          labels:
+            severity: warning
+            team: poimen
+          annotations:
+            summary: "vLLM prefix cache hit rate low ({{ $value | humanizePercentage }})"
+            description: |
+              vLLM prefix cache hit rate below 70% for 10 minutes.
+              Expected >= 85% in steady state with CAG warming active.
+              Check KVWarmupFailureHigh / KVWarmupMissHigh alerts for root cause.

k8s/infra/monitoring/kustomization.yaml

@@ -14,6 +14,7 @@ resources:
   - alerts/svc-vault-rules.yaml
   - alerts/cluster-alerts.yaml
   - alerts/api-gateway-alerts.yaml
+  - alerts/cag-kv-cache-rules.yaml
   - servicemonitors/argocd.yaml
   - servicemonitors/authentik.yaml
   - servicemonitors/forgejo.yaml

k8s/infra/tracing/api-latency-dashboard.yaml

@@ -123,6 +123,156 @@ data:
               "custom": {"fillOpacity": 10}
             }
           }
+        },
+        {
+          "id": 7,
+          "title": "vLLM Prefix Cache Hit Rate",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 36},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "vllm:prefix_cache_hit_rate",
+              "legendFormat": "prefix cache hit rate"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percentunit",
+              "min": 0,
+              "max": 1,
+              "custom": {"fillOpacity": 15},
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {"color": "red", "value": 0},
+                  {"color": "yellow", "value": 0.7},
+                  {"color": "green", "value": 0.85}
+                ]
+              }
+            }
+          }
+        },
+        {
+          "id": 8,
+          "title": "KV Warmup Hit vs Miss",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 36},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_kv_cache_warmup_hit_total[5m])",
+              "legendFormat": "warmup hit"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
+              "legendFormat": "warmup miss"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_error_total[5m])",
+              "legendFormat": "warmup error"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 9,
+          "title": "KV Warmup Latency p50/p95 (ms)",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 0, "y": 44},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
+              "legendFormat": "p50 warmup latency"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum(rate(memory_kv_cache_warmup_latency_ms_bucket[5m])) by (le))",
+              "legendFormat": "p95 warmup latency"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ms",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 10,
+          "title": "Standing Context Cache Tier Hits",
+          "type": "timeseries",
+          "gridPos": {"h": 8, "w": 12, "x": 12, "y": 44},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_standing_ctx_hit_redis_total[5m])",
+              "legendFormat": "Redis L1 hit"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_hit_db_total[5m])",
+              "legendFormat": "DB L2 hit"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_recompute_total[5m])",
+              "legendFormat": "recompute (L3)"
+            },
+            {
+              "expr": "rate(memory_standing_ctx_304_total[5m])",
+              "legendFormat": "304 Not Modified"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10}
+            }
+          }
+        },
+        {
+          "id": 11,
+          "title": "Standing Context Alert: Stale > 10min post-ingest",
+          "type": "timeseries",
+          "description": "Alert if standing context was not refreshed within 10min of an ingest event. Indicates staleness propagation failure.",
+          "gridPos": {"h": 8, "w": 24, "x": 0, "y": 52},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {
+              "expr": "rate(memory_standing_ctx_recompute_total[5m]) > 0",
+              "legendFormat": "recompute rate (non-zero = fresh)"
+            },
+            {
+              "expr": "rate(memory_kv_cache_warmup_miss_total[5m])",
+              "legendFormat": "warmup miss rate (alert if > 3/5min)"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "ops",
+              "custom": {"fillOpacity": 10},
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {"color": "green", "value": 0},
+                  {"color": "red", "value": 0.01}
+                ]
+              }
+            },
+            "overrides": [
+              {
+                "matcher": {"id": "byName", "options": "warmup miss rate (alert if > 3/5min)"},
+                "properties": [
+                  {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": 0}, {"color": "red", "value": 0.01}]}},
+                  {"id": "custom.thresholdsStyle", "value": {"mode": "area"}}
+                ]
+              }
+            ]
+          }
         }
       ]
     }