k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,33 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: story-crater-frontend-rules
namespace: story-crater-frontend
spec:
groups:
- name: story-crater-frontend.rules
interval: 15s
rules:
- alert: StoryCraterFrontendHighErrorRate
expr: |
(
sum(rate(story_crater_frontend_http_requests_total{status=~"5.."}[5m]))
/
sum(rate(story_crater_frontend_http_requests_total[5m]))
) > 0.05
for: 10m
labels:
severity: warning
annotations:
summary: "High error rate on story-crater-frontend"
description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
- alert: StoryCraterFrontendWebVitalsLCPRegression
expr: |
histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms) > 2500
for: 10m
labels:
severity: warning
annotations:
summary: "LCP (Largest Contentful Paint) regression on story-crater-frontend"
description: "LCP p75 exceeded 2500ms for 10 minutes. Value: {{ $value }}ms"