Story Crater Bot
0af06b1239
- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
75 lines
1.9 KiB
YAML
75 lines
1.9 KiB
YAML
# story-crater-backend.descriptor.yaml
|
|
# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap
|
|
# and svc-story-crater-backend-rules.yaml PrometheusRule.
|
|
# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
|
|
|
|
service: story-crater-backend
|
|
display_name: "Story Crater Backend"
|
|
namespace: story-crater-backend
|
|
service_type: message-worker
|
|
folder: "Story Crater Backend"
|
|
|
|
jobs:
|
|
- agent-worker
|
|
- check
|
|
- edit-collab
|
|
- canary
|
|
|
|
metrics:
|
|
prefix: story_crater
|
|
up_selector: 'job=~"agent-worker|check|edit-collab|canary"'
|
|
|
|
rate:
|
|
metric: story_crater_messages_handled_total
|
|
labels: [agent, queue, status] # status: succeeded | failed
|
|
|
|
duration:
|
|
metric: story_crater_message_handle_duration_ms
|
|
type: histogram
|
|
unit: ms
|
|
|
|
error:
|
|
metric: story_crater_app_metric_total
|
|
filter: 'severity="error"'
|
|
|
|
domain:
|
|
- metric: story_crater_queue_depth
|
|
kind: gauge
|
|
group_by: [queue]
|
|
- metric: story_crater_dedup_redeliveries_total
|
|
kind: counter
|
|
group_by: [agent, queue]
|
|
- metric: story_crater_outbox_publish_lag_ms
|
|
kind: histogram
|
|
- metric: story_crater_llm_tokens_used_total
|
|
kind: counter
|
|
group_by: [agent, kind]
|
|
- metric: story_crater_llm_call_duration_seconds
|
|
kind: histogram
|
|
group_by: [agent, model_provider]
|
|
- metric: story_crater_check_latency_ms
|
|
kind: histogram
|
|
slo_ms: 1200
|
|
- metric: story_crater_degrade_state
|
|
kind: gauge
|
|
group_by: [service]
|
|
|
|
correlation_ids: [session_id, trace_id, tenant]
|
|
|
|
logs:
|
|
loki_namespace_selector: 'namespace="story-crater-backend"'
|
|
|
|
alerts:
|
|
- name: HighErrorRate
|
|
expr_template: rate_error_ratio
|
|
threshold: 0.05
|
|
for: 10m
|
|
severity: warning
|
|
- name: CheckLatencySLOBreach
|
|
expr_template: histogram_quantile_over_threshold
|
|
metric: story_crater_check_latency_ms
|
|
quantile: 0.95
|
|
threshold: 1200
|
|
for: 10m
|
|
severity: critical
|