k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,33 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: authentik-rules
namespace: iam
spec:
groups:
- name: authentik.rules
interval: 15s
rules:
- alert: AuthentikHighErrorRate
expr: |
(
sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
/
sum(rate(authentik_http_requests_total[5m]))
) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: "High error rate on Authentik"
description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
- alert: AuthentikOutpostDown
expr: |
authentik_outpost_total_up == 0
for: 5m
labels:
severity: warning
annotations:
summary: "Authentik outpost is down"
description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"