k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:00 -07:00
parent 831dd50805
commit 0af06b1239
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,63 @@
# story-crater-frontend.descriptor.yaml
# Build input for generating svc-story-crater-frontend.yaml dashboard ConfigMap
# and svc-story-crater-frontend-rules.yaml PrometheusRule.
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
service: story-crater-frontend
display_name: "Story Crater Frontend"
namespace: story-crater-frontend
service_type: web-app
folder: "Story Crater Frontend"
jobs:
- story-crater-frontend
metrics:
prefix: story_crater_frontend
up_selector: 'job="story-crater-frontend"'
rate:
metric: story_crater_frontend_http_requests_total
labels: [method, route, status]
duration:
metric: story_crater_frontend_http_request_duration_ms
type: histogram
unit: ms
error:
metric: story_crater_frontend_http_requests_total
filter: 'status=~"5.."'
domain:
- metric: story_crater_frontend_web_vitals_lcp_ms
kind: gauge
group_by: [page]
- metric: story_crater_frontend_web_vitals_fid_ms
kind: gauge
group_by: [page]
- metric: story_crater_frontend_web_vitals_cls
kind: gauge
group_by: [page]
- metric: story_crater_frontend_web_vitals_inp_ms
kind: gauge
group_by: [page]
correlation_ids: [] # Frontend doesn't have session_id/trace_id in metrics
logs:
loki_namespace_selector: 'namespace="story-crater-frontend"'
alerts:
- name: HighErrorRate
expr_template: rate_error_ratio
threshold: 0.05
for: 10m
severity: warning
- name: WebVitalsLCPRegression
expr_template: histogram_quantile_over_threshold
metric: story_crater_frontend_web_vitals_lcp_ms
quantile: 0.75
threshold: 2500
for: 10m
severity: warning