Story Crater Bot
0af06b1239
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
58 lines
1.8 KiB
YAML
58 lines
1.8 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: minio-rules
|
|
namespace: storage
|
|
spec:
|
|
groups:
|
|
- name: minio.rules
|
|
interval: 15s
|
|
rules:
|
|
- alert: MinIOHighErrorRate
|
|
expr: |
|
|
(
|
|
sum(rate(minio_s3_requests_total{error="true"}[5m]))
|
|
/
|
|
sum(rate(minio_s3_requests_total[5m]))
|
|
) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High error rate on MinIO"
|
|
description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
|
|
|
- alert: MinIODiskSpaceLow
|
|
expr: |
|
|
(
|
|
minio_cluster_capacity_usable_bytes
|
|
/
|
|
minio_cluster_capacity_raw_total_bytes
|
|
) < 0.1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "MinIO disk space critically low"
|
|
description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
|
|
|
|
- alert: MinIOReplicationLag
|
|
expr: |
|
|
minio_replication_metrics_replicating_byte_count > 1073741824
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "MinIO replication lag detected"
|
|
description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
|
|
|
|
- alert: MinIODriveOffline
|
|
expr: |
|
|
minio_cluster_health_drives_offline > 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "MinIO drive offline"
|
|
description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"
|