refactor(k8s): Reorganize into 5-layer structure with production kustomizations

2026-07-16 14:28:19 -07:00
parent a81b9b6169
commit 2d7330798b
214 changed files with 405 additions and 885 deletions
@@ -0,0 +1,34 @@
+# k8s/monitoring/ingress-alerts.yaml
+# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
+# a standalone CRD instance. Applied via the prometheus release's postsync hook
+# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ingress-nginx-rules
+  namespace: ingress-nginx
+spec:
+  groups:
+    - name: ingress-nginx.rules
+      rules:
+        - alert: IngressHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
+            / sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High 5xx rate on {{ $labels.ingress }}"
+            description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
+        - alert: IngressHighLatencyP95
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
+            ) > 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "p95 latency > 1s on {{ $labels.ingress }}"
+            description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."