refactor(k8s): Reorganize into 5-layer structure with production kustomizations

2026-07-16 14:28:19 -07:00
parent a81b9b6169
commit 2d7330798b
214 changed files with 405 additions and 885 deletions
@@ -0,0 +1,75 @@
+# k8s/monitoring/blackbox-exporter-values.yaml
+# Active black-box HTTP probing of every ingress-exposed service — gives an
+# uptime/availability signal independent of real traffic. Homelab services
+# like Vault/MinIO/Longhorn UI get almost no organic requests, so ingress RED
+# metrics alone can't tell "idle" from "down"; this closes that gap.
+
+config:
+  modules:
+    http_2xx:
+      prober: http
+      timeout: 5s
+      http:
+        valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+        valid_status_codes: []   # any 2xx
+        follow_redirects: true
+        preferred_ip_protocol: "ip4"
+        tls_config:
+          insecure_skip_verify: true   # homelab-ca is a private CA; skip verify for simplicity
+
+resources:
+  requests:
+    cpu: 20m
+    memory: 32Mi
+  limits:
+    cpu: 100m
+    memory: 64Mi
+
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+serviceMonitor:
+  enabled: true
+  selfMonitor:
+    enabled: true
+  defaults:
+    interval: 30s
+    scrapeTimeout: 10s
+    module: http_2xx
+  targets:
+    - name: grafana
+      url: https://grafana.riotpiao.homelab.com/
+    - name: loki
+      url: https://loki.riotpiao.homelab.com/ready
+    - name: authentik
+      url: https://authentik.riotpiao.homelab.com/-/health/ready/
+    - name: vault
+      url: https://vault.riotpiao.homelab.com/v1/sys/health
+    - name: minio-console
+      url: https://minio.riotpiao.homelab.com/
+    - name: minio-api
+      url: https://minio-api.riotpiao.homelab.com/minio/health/live
+    - name: prometheus
+      url: https://prometheus.riotpiao.homelab.com/-/healthy
+    - name: portainer
+      url: https://portainer.riotpiao.homelab.com/
+    - name: forgejo
+      url: https://forgejo.riotpiao.homelab.com/api/healthz
+    - name: argocd
+      url: https://argocd.riotpiao.homelab.com/healthz
+    - name: longhorn
+      url: https://longhorn.riotpiao.homelab.com/
+
+prometheusRule:
+  enabled: true
+  rules:
+    - alert: ServiceProbeDown
+      expr: probe_success == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Probe failing for {{ $labels.instance }}"
+        description: "Blackbox probe for {{ $labels.instance }} has failed for more than 5 minutes."