rock
3c3b243790
Previous comments described llm_ttft_seconds / llm_itl_seconds / llm_tokens_total as "streaming-only". That was wrong, and optimistic. They are dead code. internal/proxy/llm_metrics.go defines LLMMetricsCapture, which is the only thing that calls RecordTTFT / RecordITL / RecordTokens — but NewLLMMetricsCapture() has zero call sites anywhere in the repo, verified by grep. The exporter skips keys with no samples, so those three series render as HELP-only stubs and can never be graphed, streaming or not. No behaviour change here; only the comments and the dashboard generator's header are corrected so the next person does not build on a false assumption. The gateway latency panels are left in place so the gap stays visible — they populate unchanged once the capture wrapper is wired into the streaming proxy path. Series that DO work from the gateway, confirmed live with 3 series (one per replica): queue_full_requests_total, warmup_requests_delayed_total, vllm_prefix_cache_hit_tokens_total.
467 lines
23 KiB
Python
Executable File
467 lines
23 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
|
|
|
|
import json
|
|
import os
|
|
|
|
DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
|
|
|
|
DS_PROM = {"type": "prometheus", "uid": "prometheus"}
|
|
DS_LOKI = {"type": "loki", "uid": "loki"}
|
|
|
|
|
|
def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
|
|
p = {
|
|
"id": id, "title": title, "type": "stat",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": [{"expr": expr}],
|
|
}
|
|
if mappings:
|
|
p["fieldConfig"]["defaults"]["mappings"] = mappings
|
|
if thresholds:
|
|
p["fieldConfig"]["defaults"]["thresholds"] = thresholds
|
|
p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
|
|
return p
|
|
|
|
|
|
def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"):
|
|
targets = []
|
|
for e in exprs:
|
|
if isinstance(e, tuple):
|
|
targets.append({"expr": e[0], "legendFormat": e[1]})
|
|
else:
|
|
targets.append({"expr": e, "legendFormat": "{{pod}}"})
|
|
return {
|
|
"id": id, "title": title, "type": "timeseries",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": targets,
|
|
}
|
|
|
|
|
|
def table_panel(id, title, expr, x, y, w=12, h=8):
|
|
return {
|
|
"id": id, "title": title, "type": "table",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"targets": [{"expr": expr, "format": "table", "instant": True}],
|
|
}
|
|
|
|
|
|
def log_panel(id, title, query, x, y, w=24, h=10):
|
|
return {
|
|
"id": id, "title": title, "type": "logs",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_LOKI,
|
|
"targets": [{"expr": query}],
|
|
}
|
|
|
|
|
|
def row(id, title, y, panels, collapsed=True):
|
|
return {
|
|
"id": id, "title": title, "type": "row",
|
|
"collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
def write_dashboard(filename, dashboard, folder):
|
|
cm = {
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {
|
|
"name": filename.replace(".yaml", "-dashboard"),
|
|
"namespace": "logging",
|
|
"labels": {"grafana_dashboard": "1"},
|
|
"annotations": {"grafana_folder": folder},
|
|
},
|
|
"data": {
|
|
filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
|
|
},
|
|
}
|
|
|
|
import yaml
|
|
path = os.path.join(DASHBOARD_DIR, filename)
|
|
with open(path, "w") as f:
|
|
yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
|
|
print(f" wrote {path}")
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 1: Cluster Infrastructure
|
|
# ============================================================================
|
|
|
|
def build_cluster_infrastructure():
|
|
zero_thresholds = {"mode": "absolute", "steps": [
|
|
{"value": None, "color": "green"}, {"value": 1, "color": "red"}
|
|
]}
|
|
|
|
panels = [
|
|
row(1, "Cluster Health", 0, [
|
|
stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
|
|
stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
|
|
stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
|
|
stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
|
|
stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
|
|
stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
|
|
]),
|
|
row(10, "Jobs & CronJobs", 1, [
|
|
stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
|
|
table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
|
|
table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
|
|
ts_panel(14, "CronJob Last Success", [
|
|
('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
|
|
], 0, 10, w=12, unit="dateTimeFromNow"),
|
|
ts_panel(15, "Container Restart Storm (top 10)", [
|
|
('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
|
|
], 12, 10, w=12),
|
|
]),
|
|
row(20, "Node Resources", 2, [
|
|
ts_panel(21, "CPU % by Node", [
|
|
('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
|
|
], 0, 3, unit="percent"),
|
|
ts_panel(22, "Memory % by Node", [
|
|
('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
|
|
], 8, 3, unit="percent"),
|
|
ts_panel(23, "Disk % by Node", [
|
|
('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
|
|
], 16, 3, unit="percent"),
|
|
ts_panel(24, "Load Average", [
|
|
("node_load1", "1m {{instance}}"),
|
|
("node_load5", "5m {{instance}}"),
|
|
], 0, 11),
|
|
ts_panel(25, "Network Errors & Drops", [
|
|
("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
|
|
("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
|
|
("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
|
|
], 8, 11),
|
|
]),
|
|
row(30, "Control Plane", 3, [
|
|
stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
|
|
{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
|
|
]),
|
|
ts_panel(32, "API Server Request Rate", [
|
|
('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
|
|
], 4, 4, w=10),
|
|
ts_panel(33, "API Server Error Rate %", [
|
|
('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
|
|
], 14, 4, w=10, unit="percent"),
|
|
ts_panel(34, "API Server Latency", [
|
|
('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
|
|
('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
|
|
], 0, 12, unit="s"),
|
|
ts_panel(35, "etcd Request Duration", [
|
|
('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
|
|
], 8, 12, unit="s"),
|
|
]),
|
|
row(40, "Storage", 4, [
|
|
ts_panel(41, "Longhorn Disk Capacity", [
|
|
("longhorn_disk_capacity_bytes", "capacity {{node}}"),
|
|
("longhorn_disk_reservation_bytes", "reserved {{node}}"),
|
|
], 0, 5, unit="bytes"),
|
|
ts_panel(42, "PVC Phase", [
|
|
('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
|
|
], 8, 5),
|
|
]),
|
|
row(50, "DNS & Networking", 5, [
|
|
ts_panel(51, "CoreDNS Cache Hit Rate", [
|
|
('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
|
|
], 0, 6, unit="percentunit"),
|
|
ts_panel(52, "CoreDNS Errors", [
|
|
('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
|
|
], 8, 6),
|
|
]),
|
|
row(60, "Logs", 6, [
|
|
ts_panel(61, "Error Rate by Namespace", [
|
|
('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
|
|
], 0, 7),
|
|
log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
|
|
log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
|
|
]),
|
|
]
|
|
|
|
return {
|
|
"title": "Cluster Infrastructure",
|
|
"uid": "cluster-infra",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["infrastructure", "k8s"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 3: API Gateway
|
|
# ============================================================================
|
|
|
|
def build_api_gateway():
|
|
panels = [
|
|
row(1, "Gateway Health", 0, [
|
|
stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
|
|
stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
|
|
{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
|
|
]),
|
|
ts_panel(4, "Probe Latency", [
|
|
('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
|
|
], 8, 1, unit="s"),
|
|
], collapsed=False),
|
|
row(10, "Ingress Traffic (nginx)", 1, [
|
|
ts_panel(11, "Request Rate by Status", [
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
|
|
], 0, 2),
|
|
ts_panel(12, "Error Rate %", [
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
|
|
], 8, 2, unit="percent"),
|
|
ts_panel(13, "Latency p50/p95/p99", [
|
|
('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
|
|
('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
|
|
('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
|
|
], 16, 2, unit="s"),
|
|
]),
|
|
row(20, "LLM Serving", 2, [
|
|
stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
|
|
ts_panel(22, "CPU by Predictor", [
|
|
('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
|
|
], 4, 3),
|
|
ts_panel(23, "Memory by Predictor", [
|
|
('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
|
|
], 12, 3, unit="bytes"),
|
|
ts_panel(24, "Predictor Restarts", [
|
|
('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
|
|
], 20, 3, w=4),
|
|
]),
|
|
row(30, "Gateway Resources", 3, [
|
|
ts_panel(31, "CPU by Gateway Pod", [
|
|
('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
|
|
], 0, 4),
|
|
ts_panel(32, "Memory by Gateway Pod", [
|
|
('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
|
|
], 8, 4, unit="bytes"),
|
|
ts_panel(33, "Gateway Restarts", [
|
|
('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
|
|
], 16, 4),
|
|
]),
|
|
row(40, "Logs", 4, [
|
|
log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
|
|
log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
|
|
]),
|
|
]
|
|
|
|
return {
|
|
"title": "API Gateway",
|
|
"uid": "api-gateway",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["api", "gateway", "llm"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
|
|
# ============================================================================
|
|
#
|
|
# Two independent metric sources, deliberately both shown:
|
|
#
|
|
# vLLM + TEI (llm-serving) — engine-truth, but only for the three engines
|
|
# that expose /metrics. Ollama (ornith:35b,
|
|
# qwen2.5:3b-instruct) exposes nothing at all.
|
|
# api-gateway (/metrics) — intended to cover EVERY model including the
|
|
# Ollama ones, but see the dead-metrics note
|
|
# below: its latency series are not recorded.
|
|
#
|
|
# NOTE — llm_ttft_seconds, llm_itl_seconds and llm_tokens_total are DEAD as of
|
|
# 2026-09-29. The only recorder is LLMMetricsCapture
|
|
# (internal/proxy/llm_metrics.go), and NewLLMMetricsCapture() has ZERO call
|
|
# sites in the whole repo, so the exporter emits HELP-only stubs and any panel
|
|
# querying them stays empty forever. Verified by grep, not assumed.
|
|
#
|
|
# The gateway latency panels are kept deliberately so the gap is visible — they
|
|
# light up unchanged once the capture wrapper is wired into the streaming proxy
|
|
# path. Until then vLLM's own histograms are the latency source for `reasoning`,
|
|
# and the two Ollama models have no latency visibility from anywhere.
|
|
#
|
|
# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
|
|
# vllm_prefix_cache_hit_tokens_total DO work today (3 series, one per replica).
|
|
#
|
|
# All metric names below were verified against a live scrape, not assumed:
|
|
# vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
|
|
# NOT cache_query_hit_total / cache_query_miss_total.
|
|
|
|
def build_llm_observability():
|
|
def hq(metric, q, by="le"):
|
|
return (
|
|
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
|
|
f"p{int(q * 100)}",
|
|
)
|
|
|
|
def hq_gw(metric, q):
|
|
return (
|
|
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
|
|
f"p{int(q * 100)} · {{{{model}}}}",
|
|
)
|
|
|
|
def avg_of(metric):
|
|
# mean = rate(_sum) / rate(_count) — valid for cumulative histograms
|
|
return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
|
|
|
|
panels = []
|
|
|
|
# ── TTFT ────────────────────────────────────────────────────────────────
|
|
panels.append(row(100, "Latency — Time To First Token", 0, [
|
|
stat_panel(101, "TTFT p95 (gateway)",
|
|
"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
|
|
0, 1, w=4, h=4, unit="s"),
|
|
stat_panel(102, "TTFT p95 (vLLM)",
|
|
"histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
|
|
4, 1, w=4, h=4, unit="s"),
|
|
stat_panel(103, "KV cache in use",
|
|
"max(vllm:kv_cache_usage_perc)",
|
|
8, 1, w=4, h=4, unit="percentunit"),
|
|
stat_panel(104, "Requests running",
|
|
"sum(vllm:num_requests_running)",
|
|
12, 1, w=4, h=4, unit="short"),
|
|
stat_panel(105, "Requests waiting",
|
|
"sum(vllm:num_requests_waiting)",
|
|
16, 1, w=4, h=4, unit="short"),
|
|
stat_panel(106, "Preemptions (5m)",
|
|
"increase(vllm:num_preemptions_total[5m])",
|
|
20, 1, w=4, h=4, unit="short"),
|
|
ts_panel(110, "TTFT percentile by model (gateway)",
|
|
[hq_gw("llm_ttft_seconds", 0.50),
|
|
hq_gw("llm_ttft_seconds", 0.95),
|
|
hq_gw("llm_ttft_seconds", 0.99)],
|
|
0, 2, w=12, h=8, unit="s"),
|
|
ts_panel(111, "TTFT percentile — vLLM reasoning",
|
|
[hq("vllm:time_to_first_token_seconds", 0.50),
|
|
hq("vllm:time_to_first_token_seconds", 0.95),
|
|
hq("vllm:time_to_first_token_seconds", 0.99)],
|
|
12, 2, w=12, h=8, unit="s"),
|
|
ts_panel(112, "Prefill vs decode (mean)",
|
|
[(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
|
|
(avg_of("vllm:request_decode_time_seconds"), "decode")],
|
|
0, 3, w=12, h=8, unit="s"),
|
|
ts_panel(113, "Queue wait before scheduling (mean)",
|
|
[(avg_of("vllm:request_queue_time_seconds"), "queue")],
|
|
12, 3, w=12, h=8, unit="s"),
|
|
]))
|
|
|
|
# ── Inter-token latency ─────────────────────────────────────────────────
|
|
panels.append(row(120, "Latency — Inter-Token", 4, [
|
|
ts_panel(121, "Inter-token latency by model (gateway)",
|
|
[hq_gw("llm_itl_seconds", 0.50),
|
|
hq_gw("llm_itl_seconds", 0.95),
|
|
hq_gw("llm_itl_seconds", 0.99)],
|
|
0, 5, w=12, h=8, unit="s"),
|
|
ts_panel(122, "Inter-token latency — vLLM reasoning",
|
|
[hq("vllm:inter_token_latency_seconds", 0.50),
|
|
hq("vllm:inter_token_latency_seconds", 0.95),
|
|
hq("vllm:inter_token_latency_seconds", 0.99)],
|
|
12, 5, w=12, h=8, unit="s"),
|
|
ts_panel(123, "Decode throughput (tokens/s per request)",
|
|
[(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
|
|
0, 6, w=12, h=8, unit="short"),
|
|
ts_panel(124, "End-to-end request latency p50/p95/p99",
|
|
[hq("vllm:e2e_request_latency_seconds", 0.50),
|
|
hq("vllm:e2e_request_latency_seconds", 0.95),
|
|
hq("vllm:e2e_request_latency_seconds", 0.99)],
|
|
12, 6, w=12, h=8, unit="s"),
|
|
]))
|
|
|
|
# ── Batching / scheduling ───────────────────────────────────────────────
|
|
panels.append(row(130, "Batching & Scheduling", 7, [
|
|
ts_panel(131, "vLLM running vs waiting requests (batch depth)",
|
|
[("sum(vllm:num_requests_running)", "running (in batch)"),
|
|
("sum(vllm:num_requests_waiting)", "waiting (queued)")],
|
|
0, 8, w=12, h=8, unit="short"),
|
|
ts_panel(132, "Preemption rate (KV cache eviction pressure)",
|
|
[("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
|
|
12, 8, w=12, h=8, unit="short"),
|
|
ts_panel(133, "TEI batch size (embeddings / reranker)",
|
|
[("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
|
|
"{{app}}")],
|
|
0, 9, w=12, h=8, unit="short"),
|
|
ts_panel(134, "TEI queue depth",
|
|
[("max(te_queue_size) by (app)", "{{app}}")],
|
|
12, 9, w=12, h=8, unit="short"),
|
|
]))
|
|
|
|
# ── KV cache ────────────────────────────────────────────────────────────
|
|
panels.append(row(140, "KV Cache", 10, [
|
|
ts_panel(141, "KV cache utilisation (GPU memory pressure)",
|
|
[("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
|
|
0, 11, w=12, h=8, unit="percentunit"),
|
|
ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
|
|
[("vllm:prefix_cache_hit_rate", "{{model_name}}")],
|
|
12, 11, w=12, h=8, unit="percentunit"),
|
|
ts_panel(143, "Prefix cache hits vs queries",
|
|
[("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
|
|
("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
|
|
0, 12, w=12, h=8, unit="short"),
|
|
ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
|
|
[("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
|
|
12, 12, w=12, h=8, unit="short"),
|
|
]))
|
|
|
|
# ── Throughput & health ─────────────────────────────────────────────────
|
|
panels.append(row(150, "Throughput & Health", 13, [
|
|
ts_panel(151, "Token throughput",
|
|
[("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
|
|
("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
|
|
("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
|
|
0, 14, w=12, h=8, unit="short"),
|
|
ts_panel(152, "Successful requests/s",
|
|
[("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
|
|
("sum(rate(te_request_count[5m]))", "TEI")],
|
|
12, 14, w=12, h=8, unit="short"),
|
|
ts_panel(153, "Backpressure — queue-full rejects",
|
|
[("rate(queue_full_requests_total[5m])", "rejected/s")],
|
|
0, 15, w=8, h=8, unit="short"),
|
|
ts_panel(154, "KV warmup requests delayed",
|
|
[("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
|
|
8, 15, w=8, h=8, unit="short"),
|
|
ts_panel(155, "Embedding / rerank latency p95",
|
|
[("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
|
|
"{{app}} embed"),
|
|
("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
|
|
"{{app}} queue")],
|
|
16, 15, w=8, h=8, unit="s"),
|
|
]))
|
|
|
|
return {
|
|
"title": "LLM Observability",
|
|
"uid": "llm-observability",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["llm", "vllm", "tei", "gpu", "observability"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Generate
|
|
# ============================================================================
|
|
|
|
print("Generating dashboards...")
|
|
|
|
# Dashboard 1
|
|
write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
|
|
|
|
# Dashboard 3
|
|
write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
|
|
|
|
# Dashboard 4
|
|
write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
|
|
|
|
print("Done.")
|