k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,23 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: grafana-rules
namespace: logging
spec:
groups:
- name: grafana.rules
interval: 15s
rules:
- alert: GrafanaHighErrorRate
expr: |
(
sum(rate(grafana_http_request_total{status=~"5.."}[5m]))
/
sum(rate(grafana_http_request_total[5m]))
) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: "High error rate on Grafana"
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"