192 lines
9.4 KiB
YAML
192 lines
9.4 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: svc-story-crater-backend-dashboard
|
|
namespace: logging
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
annotations:
|
|
grafana_folder: "Story Crater Backend"
|
|
data:
|
|
svc-story-crater-backend.json: |
|
|
{
|
|
"title": "Story Crater Backend — Service Overview",
|
|
"uid": "svc-story-crater-backend",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"refresh": "30s",
|
|
"panels": [
|
|
{
|
|
"id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"panels": [
|
|
{
|
|
"id": 2, "title": "Up", "type": "stat",
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"mappings": [
|
|
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
|
],
|
|
"thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
|
|
}
|
|
},
|
|
"targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }]
|
|
},
|
|
{
|
|
"id": 3, "title": "Message rate by status", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }]
|
|
},
|
|
{
|
|
"id": 4, "title": "Error rate %", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
|
|
"fieldConfig": { "defaults": { "unit": "percent" } },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }]
|
|
},
|
|
{
|
|
"id": 5, "title": "Duration p50/p95/p99", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
|
"fieldConfig": { "defaults": { "unit": "ms" } },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [
|
|
{ "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" },
|
|
{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" },
|
|
{ "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" }
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 10, "title": "Row: Resource Usage", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
|
|
"panels": [
|
|
{
|
|
"id": 11, "title": "CPU by pod", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
|
|
},
|
|
{
|
|
"id": 12, "title": "Memory by pod", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
|
|
"fieldConfig": { "defaults": { "unit": "bytes" } },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }]
|
|
},
|
|
{
|
|
"id": 13, "title": "Restart rate by pod", "type": "timeseries",
|
|
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
|
|
"panels": [
|
|
{
|
|
"id": 21, "title": "Queue depth", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }]
|
|
},
|
|
{
|
|
"id": 22, "title": "Outbox publish lag p95", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 },
|
|
"fieldConfig": { "defaults": { "unit": "ms" } },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }]
|
|
},
|
|
{
|
|
"id": 23, "title": "Dedup redeliveries rate", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 30, "title": "Row: LLM / Inference", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
|
|
"panels": [
|
|
{
|
|
"id": 31, "title": "Token usage rate", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }]
|
|
},
|
|
{
|
|
"id": 32, "title": "LLM call duration p95", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 },
|
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }]
|
|
},
|
|
{
|
|
"id": 33, "title": "LLM call errors rate", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 },
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 40, "title": "Row: SLOs / Degradation", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 },
|
|
"panels": [
|
|
{
|
|
"id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries",
|
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "ms",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "value": null, "color": "green" },
|
|
{ "value": 1200, "color": "red" }
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }]
|
|
},
|
|
{
|
|
"id": 42, "title": "Degrade state by service", "type": "stat",
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"mappings": [
|
|
{ "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } },
|
|
{ "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } }
|
|
]
|
|
}
|
|
},
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 50, "title": "Row: Logs", "type": "row",
|
|
"collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
|
"panels": [
|
|
{
|
|
"id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs",
|
|
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|