From b7fc6ed60b083b1475eb6de20b83cfe8408f29c4 Mon Sep 17 00:00:00 2001 From: rock Date: Sat, 24 Oct 2026 21:43:01 -0700 Subject: [PATCH] fix(monitoring): cap Prometheus retention below the PVC size retentionSize was 95GB against a 25Gi volume, so the limit never fired. The volume filled, and Prometheus crashlooped with: write /prometheus/queries.active: no space left on device failed to initialize active query tracker 18GB keeps roughly 7GiB of headroom for the WAL and block overhead. --- k8s/infra/monitoring/prometheus-values.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/k8s/infra/monitoring/prometheus-values.yaml b/k8s/infra/monitoring/prometheus-values.yaml index 71efaae..c340397 100644 --- a/k8s/infra/monitoring/prometheus-values.yaml +++ b/k8s/infra/monitoring/prometheus-values.yaml @@ -55,7 +55,10 @@ alertmanager: prometheus: prometheusSpec: retention: 30d - retentionSize: "95GB" + # Must stay under the PVC size below (25Gi). At 95GB the limit never + # triggered, the volume filled, and Prometheus crashlooped with + # "no space left on device" writing /prometheus/queries.active. + retentionSize: "18GB" # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no