From 795cef5a85e9621885b764b86804039d849cf911 Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 15 Sep 2026 08:16:01 +0900 Subject: [PATCH] feat(prometheus): scale up storage to 100Gi and move to worker-1 - Increase PVC from 20Gi to 100Gi (talos-cp-1 disk full) - Move from az-a (talos-cp-1) to worker-1 (648GB available) - Increase retention from 15d to 30d (95GB limit) - Fixes Prometheus pod restarts due to disk pressure --- k8s/infra/monitoring/prometheus-values.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/k8s/infra/monitoring/prometheus-values.yaml b/k8s/infra/monitoring/prometheus-values.yaml index a7360e6..0796c63 100644 --- a/k8s/infra/monitoring/prometheus-values.yaml +++ b/k8s/infra/monitoring/prometheus-values.yaml @@ -54,8 +54,8 @@ alertmanager: # ── Prometheus ──────────────────────────────────────────────────────────────── prometheus: prometheusSpec: - retention: 15d - retentionSize: "18GB" + retention: 30d + retentionSize: "95GB" # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no @@ -64,10 +64,9 @@ prometheus: scrapeTimeout: 60s evaluationInterval: 30s - # Pin to az-a (talos-cp-1) — sole Longhorn node; else the RWO PVC can't - # attach on cp-2/cp-3 (CSINode lacks driver.longhorn.io). + # Pin to worker-1 for more storage capacity nodeSelector: - topology.kubernetes.io/zone: az-a + kubernetes.io/hostname: worker-1 # Persistent storage — metrics survive node reboots and pod restarts. # Uses the default `longhorn` StorageClass; nodeSelector above already pins @@ -79,7 +78,7 @@ prometheus: accessModes: ["ReadWriteOnce"] resources: requests: - storage: 20Gi + storage: 100Gi resources: requests: