Story Crater Bot
0af06b1239
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
35 lines
1.4 KiB
YAML
35 lines
1.4 KiB
YAML
# k8s/monitoring/ingress-alerts.yaml
|
|
# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
|
|
# a standalone CRD instance. Applied via the prometheus release's postsync hook
|
|
# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: ingress-nginx-rules
|
|
namespace: ingress-nginx
|
|
spec:
|
|
groups:
|
|
- name: ingress-nginx.rules
|
|
rules:
|
|
- alert: IngressHighErrorRate
|
|
expr: |
|
|
sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
|
|
/ sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High 5xx rate on {{ $labels.ingress }}"
|
|
description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
|
|
- alert: IngressHighLatencyP95
|
|
expr: |
|
|
histogram_quantile(0.95,
|
|
sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
|
|
) > 1
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "p95 latency > 1s on {{ $labels.ingress }}"
|
|
description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."
|