apiVersion: v1 kind: ConfigMap metadata: name: svc-story-crater-backend-dashboard namespace: logging labels: grafana_dashboard: "1" annotations: grafana_folder: "Story Crater Backend" data: svc-story-crater-backend.json: | { "title": "Story Crater Backend — Service Overview", "uid": "svc-story-crater-backend", "schemaVersion": 39, "timezone": "browser", "time": { "from": "now-6h", "to": "now" }, "refresh": "30s", "panels": [ { "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [ { "id": 2, "title": "Up", "type": "stat", "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } ], "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } } }, "targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }] }, { "id": 3, "title": "Message rate by status", "type": "timeseries", "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }] }, { "id": 4, "title": "Error rate %", "type": "timeseries", "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, "fieldConfig": { "defaults": { "unit": "percent" } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }] }, { "id": 5, "title": "Duration p50/p95/p99", "type": "timeseries", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "fieldConfig": { "defaults": { "unit": "ms" } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" }, { "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" }, { "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" } ] } ] }, { "id": 10, "title": "Row: Resource Usage", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "panels": [ { "id": 11, "title": "CPU by pod", "type": "timeseries", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] }, { "id": 12, "title": "Memory by pod", "type": "timeseries", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, "fieldConfig": { "defaults": { "unit": "bytes" } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }] }, { "id": 13, "title": "Restart rate by pod", "type": "timeseries", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] } ] }, { "id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "panels": [ { "id": 21, "title": "Queue depth", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }] }, { "id": 22, "title": "Outbox publish lag p95", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 }, "fieldConfig": { "defaults": { "unit": "ms" } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }] }, { "id": 23, "title": "Dedup redeliveries rate", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }] } ] }, { "id": 30, "title": "Row: LLM / Inference", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, "panels": [ { "id": 31, "title": "Token usage rate", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }] }, { "id": 32, "title": "LLM call duration p95", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, "fieldConfig": { "defaults": { "unit": "s" } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }] }, { "id": 33, "title": "LLM call errors rate", "type": "timeseries", "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }] } ] }, { "id": 40, "title": "Row: SLOs / Degradation", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "panels": [ { "id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 }, "fieldConfig": { "defaults": { "unit": "ms", "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "green" }, { "value": 1200, "color": "red" } ] } } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }] }, { "id": 42, "title": "Degrade state by service", "type": "stat", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 }, "fieldConfig": { "defaults": { "mappings": [ { "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } }, { "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } } ] } }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }] } ] }, { "id": 50, "title": "Row: Logs", "type": "row", "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "panels": [ { "id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, "datasource": { "type": "loki", "uid": "loki" }, "targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }] } ] } ] }