k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,33 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: vault-rules
namespace: storage
spec:
groups:
- name: vault.rules
interval: 15s
rules:
- alert: VaultHighErrorRate
expr: |
(
sum(rate(vault_core_handle_request_total{error="true"}[5m]))
/
sum(rate(vault_core_handle_request_total[5m]))
) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: "High error rate on Vault"
description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
- alert: VaultSealed
expr: |
vault_core_unsealed == 0
for: 1m
labels:
severity: critical
annotations:
summary: "Vault is sealed"
description: "Vault has been sealed for 1 minute. Immediate attention required."