Story Crater Bot
63d7256b9e
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
34 lines
1.2 KiB
YAML
34 lines
1.2 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: story-crater-backend-rules
|
|
namespace: story-crater-backend
|
|
spec:
|
|
groups:
|
|
- name: story-crater-backend.rules
|
|
interval: 15s
|
|
rules:
|
|
- alert: StoryCraterBackendHighErrorRate
|
|
expr: |
|
|
(
|
|
sum(rate(story_crater_app_metric_total{severity="error"}[5m]))
|
|
/
|
|
sum(rate(story_crater_messages_handled_total[5m]))
|
|
) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High application error rate on story-crater-backend"
|
|
description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
|
|
|
- alert: StoryCraterBackendCheckLatencySLOBreach
|
|
expr: |
|
|
histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)"
|
|
description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms"
|