k8s/monitoring: add prometheus grafana loki observability
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
# story-crater-backend.descriptor.yaml
|
||||
# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap
|
||||
# and svc-story-crater-backend-rules.yaml PrometheusRule.
|
||||
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
|
||||
|
||||
service: story-crater-backend
|
||||
display_name: "Story Crater Backend"
|
||||
namespace: story-crater-backend
|
||||
service_type: message-worker
|
||||
folder: "Story Crater Backend"
|
||||
|
||||
jobs:
|
||||
- agent-worker
|
||||
- check
|
||||
- edit-collab
|
||||
- canary
|
||||
|
||||
metrics:
|
||||
prefix: story_crater
|
||||
up_selector: 'job=~"agent-worker|check|edit-collab|canary"'
|
||||
|
||||
rate:
|
||||
metric: story_crater_messages_handled_total
|
||||
labels: [agent, queue, status] # status: succeeded | failed
|
||||
|
||||
duration:
|
||||
metric: story_crater_message_handle_duration_ms
|
||||
type: histogram
|
||||
unit: ms
|
||||
|
||||
error:
|
||||
metric: story_crater_app_metric_total
|
||||
filter: 'severity="error"'
|
||||
|
||||
domain:
|
||||
- metric: story_crater_queue_depth
|
||||
kind: gauge
|
||||
group_by: [queue]
|
||||
- metric: story_crater_dedup_redeliveries_total
|
||||
kind: counter
|
||||
group_by: [agent, queue]
|
||||
- metric: story_crater_outbox_publish_lag_ms
|
||||
kind: histogram
|
||||
- metric: story_crater_llm_tokens_used_total
|
||||
kind: counter
|
||||
group_by: [agent, kind]
|
||||
- metric: story_crater_llm_call_duration_seconds
|
||||
kind: histogram
|
||||
group_by: [agent, model_provider]
|
||||
- metric: story_crater_check_latency_ms
|
||||
kind: histogram
|
||||
slo_ms: 1200
|
||||
- metric: story_crater_degrade_state
|
||||
kind: gauge
|
||||
group_by: [service]
|
||||
|
||||
correlation_ids: [session_id, trace_id, tenant]
|
||||
|
||||
logs:
|
||||
loki_namespace_selector: 'namespace="story-crater-backend"'
|
||||
|
||||
alerts:
|
||||
- name: HighErrorRate
|
||||
expr_template: rate_error_ratio
|
||||
threshold: 0.05
|
||||
for: 10m
|
||||
severity: warning
|
||||
- name: CheckLatencySLOBreach
|
||||
expr_template: histogram_quantile_over_threshold
|
||||
metric: story_crater_check_latency_ms
|
||||
quantile: 0.95
|
||||
threshold: 1200
|
||||
for: 10m
|
||||
severity: critical
|
||||
Reference in New Issue
Block a user