k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,57 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: minio-rules
+  namespace: storage
+spec:
+  groups:
+    - name: minio.rules
+      interval: 15s
+      rules:
+        - alert: MinIOHighErrorRate
+          expr: |
+            (
+              sum(rate(minio_s3_requests_total{error="true"}[5m]))
+              /
+              sum(rate(minio_s3_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on MinIO"
+            description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: MinIODiskSpaceLow
+          expr: |
+            (
+              minio_cluster_capacity_usable_bytes
+              /
+              minio_cluster_capacity_raw_total_bytes
+            ) < 0.1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO disk space critically low"
+            description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
+
+        - alert: MinIOReplicationLag
+          expr: |
+            minio_replication_metrics_replicating_byte_count > 1073741824
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "MinIO replication lag detected"
+            description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
+
+        - alert: MinIODriveOffline
+          expr: |
+            minio_cluster_health_drives_offline > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO drive offline"
+            description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"