k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: story-crater-backend-rules
+  namespace: story-crater-backend
+spec:
+  groups:
+    - name: story-crater-backend.rules
+      interval: 15s
+      rules:
+        - alert: StoryCraterBackendHighErrorRate
+          expr: |
+            (
+              sum(rate(story_crater_app_metric_total{severity="error"}[5m]))
+              /
+              sum(rate(story_crater_messages_handled_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High application error rate on story-crater-backend"
+            description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: StoryCraterBackendCheckLatencySLOBreach
+          expr: |
+            histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)"
+            description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms"