rock
c7d52b5971
The reasoning engine starves every other request while one long prompt
prefills. Measured: 262 tok/s contended prefill, ~250s for a 65k prompt,
with Avg generation throughput 0.0-0.1 tok/s at Running: 2-4 reqs.
Root cause is NOT what the issue first diagnosed. The issue proposed
--max-num-partial-prefills / --max-long-partial-prefills, which are V0-only
knobs. Verified in the live container: vLLM 0.11.0 ships V1 only (vllm/core/
is absent, so there is no V0 to fall back to), and EngineArgs.
_is_v1_supported_oracle returns False whenever either flag differs from its
default, calling _raise_or_fallback("Concurrent Partial Prefill") ->
"Falling back to V0" -> AssertionError. With strategy Recreate that is an
outage, not a slow rollout. Both flags are therefore NOT set, and a comment
records why so they are not reintroduced.
V1 needs no such flag: it has no prefill/decode phase at all. Every step it
walks self.running and hands out tokens from one shared budget
(vllm/v1/core/sched/scheduler.py). The real defect is that the per-request
cap is disabled by default (long_prefill_token_threshold=0), so one long
prompt takes the WHOLE budget, token_budget hits 0, the `while ... and
token_budget > 0` loop exits, and every other request is skipped.
The cap is applied before the budget clamp, so threshold >= budget never
binds. Replaying the loop: 8192/8192 serves 1 request per step (still
starved); 8192/2048 serves 4, and two long prompts advance together, which
is the issue's actual acceptance criterion.
--max-num-batched-tokens=8192 (was unset -> default 2048)
--long-prefill-token-threshold=2048 (was unset -> 0, cap disabled)
--max-num-seqs=5, --gpu-memory-utilization=0.95
The ~166k KV figure is marked as a projection; the live reading is
num_gpu_blocks=9527 x block_size=16 = 152,432 tokens and is authoritative
post-boot. A preemption alert covers the accept-and-watch risk.
Observability for the companion admission gate (homelab-frontend#64, now
deployed at 64f2bfc1):
- four dashboard panels: parked, rejected by reason, engine state stale,
and gateway-view vs engine-truth. Each carries a description, because
an unexplained blank panel reads as a broken dashboard.
- alerts on observer_degraded and engine_state_stale_total, both with an
absent() guard: the gate fails OPEN, so a missing or renamed metric must
alert rather than look healthy. Verified live that the guards do not
false-fire (both series present, 3 each).
- LLMPreemptions and LLMPrefillStarvation: the acceptance metric and the
regression direction this change introduces.
- max(vllm_requests_running), not sum(): one engine-global gauge is read
by all three gateway replicas, so sum() read 3x and could never agree.
Also corrects comments that were factually wrong. An earlier revision of
this branch claimed llm_ttft_seconds/llm_itl_seconds/llm_tokens_total were
dead code; homelab-frontend#63 wired them, verified live as 3 series each.
request_duration_seconds is the genuinely dead one (0 samples, zero
non-test call sites) and is now marked as such instead of being listed as
working.
Closes #51
518 lines
27 KiB
Python
Executable File
518 lines
27 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Generate consolidated Grafana dashboards as k8s ConfigMap YAML files."""
|
|
|
|
import json
|
|
import os
|
|
|
|
DASHBOARD_DIR = os.path.expanduser("~/workplace/homelab/k8s/infra/monitoring/dashboards")
|
|
|
|
DS_PROM = {"type": "prometheus", "uid": "prometheus"}
|
|
DS_LOKI = {"type": "loki", "uid": "loki"}
|
|
|
|
|
|
def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thresholds=None):
|
|
p = {
|
|
"id": id, "title": title, "type": "stat",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": [{"expr": expr}],
|
|
}
|
|
if mappings:
|
|
p["fieldConfig"]["defaults"]["mappings"] = mappings
|
|
if thresholds:
|
|
p["fieldConfig"]["defaults"]["thresholds"] = thresholds
|
|
p["fieldConfig"]["defaults"]["color"] = {"mode": "thresholds"}
|
|
return p
|
|
|
|
|
|
def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None):
|
|
targets = []
|
|
for e in exprs:
|
|
if isinstance(e, tuple):
|
|
targets.append({"expr": e[0], "legendFormat": e[1]})
|
|
else:
|
|
targets.append({"expr": e, "legendFormat": "{{pod}}"})
|
|
p = {
|
|
"id": id, "title": title, "type": "timeseries",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"fieldConfig": {"defaults": {"unit": unit}},
|
|
"targets": targets,
|
|
}
|
|
if description:
|
|
p["description"] = description
|
|
return p
|
|
|
|
|
|
def table_panel(id, title, expr, x, y, w=12, h=8):
|
|
return {
|
|
"id": id, "title": title, "type": "table",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_PROM,
|
|
"targets": [{"expr": expr, "format": "table", "instant": True}],
|
|
}
|
|
|
|
|
|
def log_panel(id, title, query, x, y, w=24, h=10):
|
|
return {
|
|
"id": id, "title": title, "type": "logs",
|
|
"gridPos": {"h": h, "w": w, "x": x, "y": y},
|
|
"datasource": DS_LOKI,
|
|
"targets": [{"expr": query}],
|
|
}
|
|
|
|
|
|
def row(id, title, y, panels, collapsed=True):
|
|
return {
|
|
"id": id, "title": title, "type": "row",
|
|
"collapsed": collapsed, "gridPos": {"h": 1, "w": 24, "x": 0, "y": y},
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
def write_dashboard(filename, dashboard, folder):
|
|
cm = {
|
|
"apiVersion": "v1",
|
|
"kind": "ConfigMap",
|
|
"metadata": {
|
|
"name": filename.replace(".yaml", "-dashboard"),
|
|
"namespace": "logging",
|
|
"labels": {"grafana_dashboard": "1"},
|
|
"annotations": {"grafana_folder": folder},
|
|
},
|
|
"data": {
|
|
filename.replace(".yaml", ".json"): json.dumps(dashboard, separators=(",", ":"))
|
|
},
|
|
}
|
|
|
|
import yaml
|
|
path = os.path.join(DASHBOARD_DIR, filename)
|
|
with open(path, "w") as f:
|
|
yaml.dump(cm, f, default_flow_style=False, allow_unicode=True)
|
|
print(f" wrote {path}")
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 1: Cluster Infrastructure
|
|
# ============================================================================
|
|
|
|
def build_cluster_infrastructure():
|
|
zero_thresholds = {"mode": "absolute", "steps": [
|
|
{"value": None, "color": "green"}, {"value": 1, "color": "red"}
|
|
]}
|
|
|
|
panels = [
|
|
row(1, "Cluster Health", 0, [
|
|
stat_panel(2, "Nodes Ready", 'count(kube_node_status_condition{condition="Ready",status="true"} == 1)', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
|
|
stat_panel(3, "Pods Pending", 'sum(kube_pod_status_phase{phase="Pending"}) OR on() vector(0)', 4, 1, thresholds=zero_thresholds),
|
|
stat_panel(4, "CrashLoopBackOff", 'sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) OR on() vector(0)', 8, 1, thresholds=zero_thresholds),
|
|
stat_panel(5, "OOMKilled (1h)", 'sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) OR on() vector(0)', 12, 1, thresholds=zero_thresholds),
|
|
stat_panel(6, "Deploys Unavailable", 'count(kube_deployment_status_replicas_unavailable > 0) OR on() vector(0)', 16, 1, thresholds=zero_thresholds),
|
|
stat_panel(7, "Services Down", 'count(probe_success == 0) OR on() vector(0)', 20, 1, thresholds=zero_thresholds),
|
|
]),
|
|
row(10, "Jobs & CronJobs", 1, [
|
|
stat_panel(11, "Failed Jobs", 'count(kube_job_status_failed > 0) OR on() vector(0)', 0, 2, thresholds=zero_thresholds),
|
|
table_panel(12, "Failed Jobs Detail", 'kube_job_status_failed > 0', 4, 2, w=10),
|
|
table_panel(13, "Stuck Jobs (>1h)", 'kube_job_status_active == 1 and on(job_name,namespace) (time() - kube_job_status_start_time) > 3600', 14, 2, w=10),
|
|
ts_panel(14, "CronJob Last Success", [
|
|
('kube_cronjob_status_last_successful_time{namespace=~"cicd|kube-system|paperless"}', "{{namespace}}/{{cronjob}}")
|
|
], 0, 10, w=12, unit="dateTimeFromNow"),
|
|
ts_panel(15, "Container Restart Storm (top 10)", [
|
|
('topk(10, sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod))', "{{namespace}}/{{pod}}")
|
|
], 12, 10, w=12),
|
|
]),
|
|
row(20, "Node Resources", 2, [
|
|
ts_panel(21, "CPU % by Node", [
|
|
('(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100', "{{instance}}")
|
|
], 0, 3, unit="percent"),
|
|
ts_panel(22, "Memory % by Node", [
|
|
('(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100', "{{instance}}")
|
|
], 8, 3, unit="percent"),
|
|
ts_panel(23, "Disk % by Node", [
|
|
('(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100', "{{instance}}")
|
|
], 16, 3, unit="percent"),
|
|
ts_panel(24, "Load Average", [
|
|
("node_load1", "1m {{instance}}"),
|
|
("node_load5", "5m {{instance}}"),
|
|
], 0, 11),
|
|
ts_panel(25, "Network Errors & Drops", [
|
|
("rate(node_network_receive_errs_total[5m])", "rx-err {{instance}}"),
|
|
("rate(node_network_transmit_errs_total[5m])", "tx-err {{instance}}"),
|
|
("rate(node_network_receive_drop_total[5m])", "rx-drop {{instance}}"),
|
|
], 8, 11),
|
|
]),
|
|
row(30, "Control Plane", 3, [
|
|
stat_panel(31, "API Server Up", 'min(up{job="apiserver"})', 0, 4, mappings=[
|
|
{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
|
|
]),
|
|
ts_panel(32, "API Server Request Rate", [
|
|
('sum(rate(apiserver_request_total[5m])) by (verb, code)', "{{verb}} {{code}}")
|
|
], 4, 4, w=10),
|
|
ts_panel(33, "API Server Error Rate %", [
|
|
('sum(rate(apiserver_request_total{code=~"5.."}[5m])) / sum(rate(apiserver_request_total[5m])) * 100', "5xx %")
|
|
], 14, 4, w=10, unit="percent"),
|
|
ts_panel(34, "API Server Latency", [
|
|
('histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p95"),
|
|
('histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))', "p99"),
|
|
], 0, 12, unit="s"),
|
|
ts_panel(35, "etcd Request Duration", [
|
|
('histogram_quantile(0.99, sum(rate(etcd_request_duration_seconds_bucket[5m])) by (le))', "p99"),
|
|
], 8, 12, unit="s"),
|
|
]),
|
|
row(40, "Storage", 4, [
|
|
ts_panel(41, "Longhorn Disk Capacity", [
|
|
("longhorn_disk_capacity_bytes", "capacity {{node}}"),
|
|
("longhorn_disk_reservation_bytes", "reserved {{node}}"),
|
|
], 0, 5, unit="bytes"),
|
|
ts_panel(42, "PVC Phase", [
|
|
('kube_persistentvolumeclaim_status_phase', "{{namespace}}/{{persistentvolumeclaim}} {{phase}}")
|
|
], 8, 5),
|
|
]),
|
|
row(50, "DNS & Networking", 5, [
|
|
ts_panel(51, "CoreDNS Cache Hit Rate", [
|
|
('rate(coredns_cache_hits_total[5m]) / (rate(coredns_cache_hits_total[5m]) + rate(coredns_cache_misses_total[5m]))', "{{server}}")
|
|
], 0, 6, unit="percentunit"),
|
|
ts_panel(52, "CoreDNS Errors", [
|
|
('sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL|NXDOMAIN"}[5m])) by (rcode)', "{{rcode}}")
|
|
], 8, 6),
|
|
]),
|
|
row(60, "Logs", 6, [
|
|
ts_panel(61, "Error Rate by Namespace", [
|
|
('sum by (namespace) (count_over_time({namespace=~"kube-system|cert-manager|ingress-nginx|longhorn-system"} |= "error" [5m]))', "{{namespace}}")
|
|
], 0, 7),
|
|
log_panel(62, "Control Plane Logs", '{namespace="kube-system"}', 0, 15),
|
|
log_panel(63, "Cluster Addon Logs", '{namespace=~"cert-manager|ingress-nginx|longhorn-system"}', 0, 25),
|
|
]),
|
|
]
|
|
|
|
return {
|
|
"title": "Cluster Infrastructure",
|
|
"uid": "cluster-infra",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["infrastructure", "k8s"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 3: API Gateway
|
|
# ============================================================================
|
|
|
|
def build_api_gateway():
|
|
panels = [
|
|
row(1, "Gateway Health", 0, [
|
|
stat_panel(2, "Gateway Pods Ready", 'sum(kube_pod_status_ready{namespace="api",condition="true"})', 0, 1, thresholds={"mode":"absolute","steps":[{"value":None,"color":"red"},{"value":3,"color":"green"}]}),
|
|
stat_panel(3, "Probe: healthz", 'probe_success{instance=~".*api.riotpiao.com/healthz"}', 4, 1, mappings=[
|
|
{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}
|
|
]),
|
|
ts_panel(4, "Probe Latency", [
|
|
('probe_duration_seconds{instance=~".*api.riotpiao.com.*"}', "{{instance}}")
|
|
], 8, 1, unit="s"),
|
|
], collapsed=False),
|
|
row(10, "Ingress Traffic (nginx)", 1, [
|
|
ts_panel(11, "Request Rate by Status", [
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) by (status)', "{{status}}")
|
|
], 0, 2),
|
|
ts_panel(12, "Error Rate %", [
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "5xx"),
|
|
('sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"4.."}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) * 100', "4xx"),
|
|
], 8, 2, unit="percent"),
|
|
ts_panel(13, "Latency p50/p95/p99", [
|
|
('histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p50"),
|
|
('histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p95"),
|
|
('histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le))', "p99"),
|
|
], 16, 2, unit="s"),
|
|
]),
|
|
row(20, "LLM Serving", 2, [
|
|
stat_panel(21, "LLM Pods Ready", 'sum(kube_pod_status_ready{namespace="llm-serving",condition="true"})', 0, 3),
|
|
ts_panel(22, "CPU by Predictor", [
|
|
('sum(rate(container_cpu_usage_seconds_total{namespace="llm-serving"}[5m])) by (pod)', "{{pod}}")
|
|
], 4, 3),
|
|
ts_panel(23, "Memory by Predictor", [
|
|
('sum(container_memory_working_set_bytes{namespace="llm-serving"}) by (pod)', "{{pod}}")
|
|
], 12, 3, unit="bytes"),
|
|
ts_panel(24, "Predictor Restarts", [
|
|
('sum(rate(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m])) by (pod)', "{{pod}}")
|
|
], 20, 3, w=4),
|
|
]),
|
|
row(30, "Gateway Resources", 3, [
|
|
ts_panel(31, "CPU by Gateway Pod", [
|
|
('sum(rate(container_cpu_usage_seconds_total{namespace="api"}[5m])) by (pod)', "{{pod}}")
|
|
], 0, 4),
|
|
ts_panel(32, "Memory by Gateway Pod", [
|
|
('sum(container_memory_working_set_bytes{namespace="api"}) by (pod)', "{{pod}}")
|
|
], 8, 4, unit="bytes"),
|
|
ts_panel(33, "Gateway Restarts", [
|
|
('sum(rate(kube_pod_container_status_restarts_total{namespace="api"}[15m])) by (pod)', "{{pod}}")
|
|
], 16, 4),
|
|
]),
|
|
row(40, "Logs", 4, [
|
|
log_panel(41, "Gateway Logs", '{namespace="api",container="gateway"}', 0, 5),
|
|
log_panel(42, "LLM Serving Logs", '{namespace="llm-serving"}', 0, 15),
|
|
]),
|
|
]
|
|
|
|
return {
|
|
"title": "API Gateway",
|
|
"uid": "api-gateway",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["api", "gateway", "llm"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
|
|
# ============================================================================
|
|
#
|
|
# Two independent metric sources, deliberately both shown:
|
|
#
|
|
# vLLM + TEI (llm-serving) — engine-truth, but only for the three engines
|
|
# that expose /metrics. Ollama (ornith:35b,
|
|
# qwen2.5:3b-instruct) exposes nothing at all.
|
|
# api-gateway (/metrics) — covers EVERY model including the two Ollama
|
|
# ones, derived from the SSE byte stream: TTFT is
|
|
# request-start to first byte, ITL is the gap
|
|
# between SSE writes carrying "data: ", so it
|
|
# counts SSE chunks rather than true tokens.
|
|
# vLLM's own histograms remain authoritative for
|
|
# `reasoning`.
|
|
#
|
|
# These three series ARE live as of 2026-09-29 — 3 series each, one per replica,
|
|
# verified against the running gateway and Prometheus:
|
|
# llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
|
|
#
|
|
# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these
|
|
# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture
|
|
# wrapper that was never constructed, so the exporter emitted HELP-only stubs.
|
|
# homelab-frontend#63 fixed exactly that — the wrapper is now
|
|
# ResponseWriterWrapper, constructed by NewResponseWriterWrapper
|
|
# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path.
|
|
# The "dead code" note that used to live in this header is superseded.
|
|
#
|
|
# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and
|
|
# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica).
|
|
#
|
|
#
|
|
# All metric names below were verified against a live scrape, not assumed:
|
|
# vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
|
|
# NOT cache_query_hit_total / cache_query_miss_total.
|
|
|
|
def build_llm_observability():
|
|
def hq(metric, q, by="le"):
|
|
return (
|
|
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
|
|
f"p{int(q * 100)}",
|
|
)
|
|
|
|
def hq_gw(metric, q):
|
|
return (
|
|
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
|
|
f"p{int(q * 100)} · {{{{model}}}}",
|
|
)
|
|
|
|
def avg_of(metric):
|
|
# mean = rate(_sum) / rate(_count) — valid for cumulative histograms
|
|
return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
|
|
|
|
panels = []
|
|
|
|
# ── TTFT ────────────────────────────────────────────────────────────────
|
|
panels.append(row(100, "Latency — Time To First Token", 0, [
|
|
stat_panel(101, "TTFT p95 (gateway)",
|
|
"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
|
|
0, 1, w=4, h=4, unit="s"),
|
|
stat_panel(102, "TTFT p95 (vLLM)",
|
|
"histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
|
|
4, 1, w=4, h=4, unit="s"),
|
|
stat_panel(103, "KV cache in use",
|
|
"max(vllm:kv_cache_usage_perc)",
|
|
8, 1, w=4, h=4, unit="percentunit"),
|
|
stat_panel(104, "Requests running",
|
|
"sum(vllm:num_requests_running)",
|
|
12, 1, w=4, h=4, unit="short"),
|
|
stat_panel(105, "Requests waiting",
|
|
"sum(vllm:num_requests_waiting)",
|
|
16, 1, w=4, h=4, unit="short"),
|
|
stat_panel(106, "Preemptions (5m)",
|
|
"increase(vllm:num_preemptions_total[5m])",
|
|
20, 1, w=4, h=4, unit="short"),
|
|
ts_panel(110, "TTFT percentile by model (gateway)",
|
|
[hq_gw("llm_ttft_seconds", 0.50),
|
|
hq_gw("llm_ttft_seconds", 0.95),
|
|
hq_gw("llm_ttft_seconds", 0.99)],
|
|
0, 2, w=12, h=8, unit="s"),
|
|
ts_panel(111, "TTFT percentile — vLLM reasoning",
|
|
[hq("vllm:time_to_first_token_seconds", 0.50),
|
|
hq("vllm:time_to_first_token_seconds", 0.95),
|
|
hq("vllm:time_to_first_token_seconds", 0.99)],
|
|
12, 2, w=12, h=8, unit="s"),
|
|
ts_panel(112, "Prefill vs decode (mean)",
|
|
[(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
|
|
(avg_of("vllm:request_decode_time_seconds"), "decode")],
|
|
0, 3, w=12, h=8, unit="s"),
|
|
ts_panel(113, "Queue wait before scheduling (mean)",
|
|
[(avg_of("vllm:request_queue_time_seconds"), "queue")],
|
|
12, 3, w=12, h=8, unit="s"),
|
|
]))
|
|
|
|
# ── Inter-token latency ─────────────────────────────────────────────────
|
|
panels.append(row(120, "Latency — Inter-Token", 4, [
|
|
ts_panel(121, "Inter-token latency by model (gateway)",
|
|
[hq_gw("llm_itl_seconds", 0.50),
|
|
hq_gw("llm_itl_seconds", 0.95),
|
|
hq_gw("llm_itl_seconds", 0.99)],
|
|
0, 5, w=12, h=8, unit="s"),
|
|
ts_panel(122, "Inter-token latency — vLLM reasoning",
|
|
[hq("vllm:inter_token_latency_seconds", 0.50),
|
|
hq("vllm:inter_token_latency_seconds", 0.95),
|
|
hq("vllm:inter_token_latency_seconds", 0.99)],
|
|
12, 5, w=12, h=8, unit="s"),
|
|
ts_panel(123, "Decode throughput (tokens/s per request)",
|
|
[(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
|
|
0, 6, w=12, h=8, unit="short"),
|
|
ts_panel(124, "End-to-end request latency p50/p95/p99",
|
|
[hq("vllm:e2e_request_latency_seconds", 0.50),
|
|
hq("vllm:e2e_request_latency_seconds", 0.95),
|
|
hq("vllm:e2e_request_latency_seconds", 0.99)],
|
|
12, 6, w=12, h=8, unit="s"),
|
|
]))
|
|
|
|
# ── Batching / scheduling ───────────────────────────────────────────────
|
|
panels.append(row(130, "Batching & Scheduling", 7, [
|
|
ts_panel(131, "vLLM running vs waiting requests (batch depth)",
|
|
[("sum(vllm:num_requests_running)", "running (in batch)"),
|
|
("sum(vllm:num_requests_waiting)", "waiting (queued)")],
|
|
0, 8, w=12, h=8, unit="short"),
|
|
ts_panel(132, "Preemption rate (KV cache eviction pressure)",
|
|
[("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
|
|
12, 8, w=12, h=8, unit="short"),
|
|
ts_panel(133, "TEI batch size (embeddings / reranker)",
|
|
[("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
|
|
"{{app}}")],
|
|
0, 9, w=12, h=8, unit="short"),
|
|
ts_panel(134, "TEI queue depth",
|
|
[("max(te_queue_size) by (app)", "{{app}}")],
|
|
12, 9, w=12, h=8, unit="short"),
|
|
]))
|
|
|
|
# ── KV cache ────────────────────────────────────────────────────────────
|
|
panels.append(row(140, "KV Cache", 10, [
|
|
ts_panel(141, "KV cache utilisation (GPU memory pressure)",
|
|
[("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
|
|
0, 11, w=12, h=8, unit="percentunit"),
|
|
ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
|
|
[("vllm:prefix_cache_hit_rate", "{{model_name}}")],
|
|
12, 11, w=12, h=8, unit="percentunit"),
|
|
ts_panel(143, "Prefix cache hits vs queries",
|
|
[("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
|
|
("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
|
|
0, 12, w=12, h=8, unit="short"),
|
|
ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
|
|
[("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
|
|
12, 12, w=12, h=8, unit="short"),
|
|
]))
|
|
|
|
# ── Throughput & health ─────────────────────────────────────────────────
|
|
panels.append(row(150, "Throughput & Health", 13, [
|
|
ts_panel(151, "Token throughput",
|
|
[("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
|
|
("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
|
|
("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
|
|
0, 14, w=12, h=8, unit="short"),
|
|
ts_panel(152, "Successful requests/s",
|
|
[("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
|
|
("sum(rate(te_request_count[5m]))", "TEI")],
|
|
12, 14, w=12, h=8, unit="short"),
|
|
ts_panel(153, "Backpressure — queue-full rejects",
|
|
[("rate(queue_full_requests_total[5m])", "rejected/s")],
|
|
0, 15, w=8, h=8, unit="short"),
|
|
ts_panel(154, "KV warmup requests delayed",
|
|
[("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
|
|
8, 15, w=8, h=8, unit="short"),
|
|
ts_panel(155, "Embedding / rerank latency p95",
|
|
[("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
|
|
"{{app}} embed"),
|
|
("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
|
|
"{{app}} queue")],
|
|
16, 15, w=8, h=8, unit="s"),
|
|
]))
|
|
|
|
# ── Admission control ───────────────────────────────────────────────────
|
|
# These four panels read the gateway's admission gate, which is the
|
|
# companion change in homelab-frontend#64. That emitter is NOT deployed
|
|
# yet (verified against the live /metrics endpoint: 0 series matching
|
|
# ^(admission_|observer_|engine_state_)), so they render "No data" until
|
|
# #64 ships. That is expected, and each panel carries a description
|
|
# saying so — an unexplained blank panel would read as a broken
|
|
# dashboard, which is exactly the failure mode this row exists to catch.
|
|
panels.append(row(160, "Admission Control", 16, [
|
|
ts_panel(161, "Admission parked (5m)",
|
|
[("increase(admission_parked_total[5m])", "parked")],
|
|
0, 17, w=8, h=8, unit="short",
|
|
description="Requests the gateway held back rather than admitting. "
|
|
"Emitted by the admission gate (homelab-frontend#64); "
|
|
"empty until that change is deployed."),
|
|
ts_panel(162, "Admission rejected by reason",
|
|
[("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")],
|
|
8, 17, w=8, h=8, unit="short",
|
|
description="Admission rejections split by the gate's own reason label. "
|
|
"Emitted by homelab-frontend#64; empty until deployed."),
|
|
ts_panel(163, "Engine state stale",
|
|
[("increase(engine_state_stale_total[5m])", "stale")],
|
|
16, 17, w=8, h=8, unit="short",
|
|
description="Times the gateway's view of the engine gauges went stale. "
|
|
"The gate fails OPEN when this happens, so any non-zero rate "
|
|
"means admission is silently degraded to admit-everything. "
|
|
"Emitted by homelab-frontend#64; empty until deployed."),
|
|
ts_panel(164, "Gateway view vs engine truth",
|
|
[("max(vllm_requests_running)", "gateway: running"),
|
|
("sum(vllm:num_requests_running)", "engine: running"),
|
|
("max(vllm_kv_usage_perc)", "gateway: kv (fraction) [same as engine if healthy]"),
|
|
("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")],
|
|
0, 18, w=24, h=8, unit="short",
|
|
description="The gateway's own reading of the engine against the engine's "
|
|
"self-reported value. A disagreement between a pair is how a "
|
|
"stale observer or a mistuned admission target shows itself, "
|
|
"and nothing else in this dashboard reveals it. The two "
|
|
"gateway series (vllm_requests_running, vllm_kv_usage_perc) "
|
|
"ship with homelab-frontend#64 and are absent until it is "
|
|
"deployed — until then this panel shows the engine's pair only."),
|
|
]))
|
|
|
|
return {
|
|
"title": "LLM Observability",
|
|
"uid": "llm-observability",
|
|
"schemaVersion": 39,
|
|
"timezone": "browser",
|
|
"time": {"from": "now-6h", "to": "now"},
|
|
"refresh": "30s",
|
|
"tags": ["llm", "vllm", "tei", "gpu", "observability"],
|
|
"panels": panels,
|
|
}
|
|
|
|
|
|
# ============================================================================
|
|
# Generate
|
|
# ============================================================================
|
|
|
|
print("Generating dashboards...")
|
|
|
|
# Dashboard 1
|
|
write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "Infrastructure")
|
|
|
|
# Dashboard 3
|
|
write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
|
|
|
|
# Dashboard 4
|
|
write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
|
|
|
|
print("Done.")
|