k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,33 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: argocd-rules
namespace: argocd
spec:
groups:
- name: argocd.rules
interval: 15s
rules:
- alert: ArgoCDHighErrorRate
expr: |
(
sum(rate(argocd_http_request_total{status=~"5.."}[5m]))
/
sum(rate(argocd_http_request_total[5m]))
) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: "High error rate on Argo CD"
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
- alert: ArgoCDApplicationSyncFailure
expr: |
argocd_app_health_degraded_total > 0
for: 10m
labels:
severity: warning
annotations:
summary: "Argo CD applications in degraded state"
description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes"