riotpiao-poimen/homelab
2
0
Fork 0
Code Issues 23 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

feat(monitoring): LLM observability dashboard + fix dead CAG rule

Browse Source 
New Grafana dashboard (llm-observability) with 27 panels across 5 rows:
TTFT, inter-token latency, batching/scheduling, KV cache, throughput.

Both metric sources are shown side by side because they answer different
questions and neither is complete alone:

  vLLM 0.11.0 (reasoning) — engine truth: kv_cache_usage_perc,
    num_requests_running/waiting, num_preemptions_total, prefill vs
    decode time, prefix cache, per-request histogram quantiles.
  TEI 1.8.2 (embeddings, reranker) — te_batch_next_size, te_queue_size,
    te_embed_duration, te_embed_queue_duration.
  api-gateway — the ONLY per-request latency source covering Ollama
    models (ornith:35b, qwen2.5:3b-instruct), which expose no metrics.
    It is derived from the SSE byte stream, so it is streaming-only and
    counts SSE chunks rather than true tokens. vLLM's own histograms are
    authoritative for reasoning.

All 44 panel queries were executed against live Prometheus before commit —
all return success. vLLM/TEI panels have live series; gateway panels show
no data until the netpol fix in homelab-frontend PR #62 merges.

Also fixes a dead alert. alerts/cag-kv-cache-rules.yaml derived
vllm:prefix_cache_hit_rate from vllm:cache_query_hit_total and
vllm:cache_query_miss_total. Those metric names do not exist in vLLM
0.11.0 — the real series are vllm:prefix_cache_hits_total and
vllm:prefix_cache_queries_total. The recording rule produced no data, so
VLLMPrefixCacheHitRateLow could never fire.

Not included: real GPU hardware metrics (SM utilisation, framebuffer,
power, temperature). Only nvidia-device-plugin is deployed, which exposes
none of them; DCGM exporter is required. vllm:kv_cache_usage_perc and
preemption rate are the only GPU-pressure proxies available today.
This commit is contained in:
rock
2026-09-29 22:11:28 +08:00
parent 6ae36981bb
commit 1f3439c9e0
4 changed files with 261 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/infra/monitoring/dashboards/generate-dashboards.py
+176
View File
@@ -265,6 +265,179 @@ def build_api_gateway():
}
# ============================================================================
# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway)
# ============================================================================
#
# Two independent metric sources, deliberately both shown:
#
# vLLM + TEI (llm-serving) — engine-truth, but only for the three engines
# that expose /metrics. Ollama (ornith:35b,
# qwen2.5:3b-instruct) exposes nothing at all.
# api-gateway (/metrics) — proxy-side, covers EVERY model including the
# Ollama ones, but is derived from the SSE byte
# stream: TTFT = request-start to first byte,
# ITL = gap between SSE writes carrying "data: ".
# It is therefore streaming-only and counts SSE
# chunks, not true tokens. Treat vLLM's own
# histograms as authoritative for `reasoning`.
#
# All metric names below were verified against a live scrape, not assumed:
# vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total,
# NOT cache_query_hit_total / cache_query_miss_total.
def build_llm_observability():
def hq(metric, q, by="le"):
return (
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))",
f"p{int(q * 100)}",
)
def hq_gw(metric, q):
return (
f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))",
f"p{int(q * 100)} · {{{{model}}}}",
)
def avg_of(metric):
# mean = rate(_sum) / rate(_count) — valid for cumulative histograms
return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)"
panels = []
# ── TTFT ────────────────────────────────────────────────────────────────
panels.append(row(100, "Latency — Time To First Token", 0, [
stat_panel(101, "TTFT p95 (gateway)",
"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))",
0, 1, w=4, h=4, unit="s"),
stat_panel(102, "TTFT p95 (vLLM)",
"histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))",
4, 1, w=4, h=4, unit="s"),
stat_panel(103, "KV cache in use",
"max(vllm:kv_cache_usage_perc)",
8, 1, w=4, h=4, unit="percentunit"),
stat_panel(104, "Requests running",
"sum(vllm:num_requests_running)",
12, 1, w=4, h=4, unit="short"),
stat_panel(105, "Requests waiting",
"sum(vllm:num_requests_waiting)",
16, 1, w=4, h=4, unit="short"),
stat_panel(106, "Preemptions (5m)",
"increase(vllm:num_preemptions_total[5m])",
20, 1, w=4, h=4, unit="short"),
ts_panel(110, "TTFT percentile by model (gateway)",
[hq_gw("llm_ttft_seconds", 0.50),
hq_gw("llm_ttft_seconds", 0.95),
hq_gw("llm_ttft_seconds", 0.99)],
0, 2, w=12, h=8, unit="s"),
ts_panel(111, "TTFT percentile — vLLM reasoning",
[hq("vllm:time_to_first_token_seconds", 0.50),
hq("vllm:time_to_first_token_seconds", 0.95),
hq("vllm:time_to_first_token_seconds", 0.99)],
12, 2, w=12, h=8, unit="s"),
ts_panel(112, "Prefill vs decode (mean)",
[(avg_of("vllm:request_prefill_time_seconds"), "prefill"),
(avg_of("vllm:request_decode_time_seconds"), "decode")],
0, 3, w=12, h=8, unit="s"),
ts_panel(113, "Queue wait before scheduling (mean)",
[(avg_of("vllm:request_queue_time_seconds"), "queue")],
12, 3, w=12, h=8, unit="s"),
]))
# ── Inter-token latency ─────────────────────────────────────────────────
panels.append(row(120, "Latency — Inter-Token", 4, [
ts_panel(121, "Inter-token latency by model (gateway)",
[hq_gw("llm_itl_seconds", 0.50),
hq_gw("llm_itl_seconds", 0.95),
hq_gw("llm_itl_seconds", 0.99)],
0, 5, w=12, h=8, unit="s"),
ts_panel(122, "Inter-token latency — vLLM reasoning",
[hq("vllm:inter_token_latency_seconds", 0.50),
hq("vllm:inter_token_latency_seconds", 0.95),
hq("vllm:inter_token_latency_seconds", 0.99)],
12, 5, w=12, h=8, unit="s"),
ts_panel(123, "Decode throughput (tokens/s per request)",
[(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")],
0, 6, w=12, h=8, unit="short"),
ts_panel(124, "End-to-end request latency p50/p95/p99",
[hq("vllm:e2e_request_latency_seconds", 0.50),
hq("vllm:e2e_request_latency_seconds", 0.95),
hq("vllm:e2e_request_latency_seconds", 0.99)],
12, 6, w=12, h=8, unit="s"),
]))
# ── Batching / scheduling ───────────────────────────────────────────────
panels.append(row(130, "Batching & Scheduling", 7, [
ts_panel(131, "vLLM running vs waiting requests (batch depth)",
[("sum(vllm:num_requests_running)", "running (in batch)"),
("sum(vllm:num_requests_waiting)", "waiting (queued)")],
0, 8, w=12, h=8, unit="short"),
ts_panel(132, "Preemption rate (KV cache eviction pressure)",
[("rate(vllm:num_preemptions_total[5m])", "preemptions/s")],
12, 8, w=12, h=8, unit="short"),
ts_panel(133, "TEI batch size (embeddings / reranker)",
[("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)",
"{{app}}")],
0, 9, w=12, h=8, unit="short"),
ts_panel(134, "TEI queue depth",
[("max(te_queue_size) by (app)", "{{app}}")],
12, 9, w=12, h=8, unit="short"),
]))
# ── KV cache ────────────────────────────────────────────────────────────
panels.append(row(140, "KV Cache", 10, [
ts_panel(141, "KV cache utilisation (GPU memory pressure)",
[("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")],
0, 11, w=12, h=8, unit="percentunit"),
ts_panel(142, "Prefix cache hit rate (CAG effectiveness)",
[("vllm:prefix_cache_hit_rate", "{{model_name}}")],
12, 11, w=12, h=8, unit="percentunit"),
ts_panel(143, "Prefix cache hits vs queries",
[("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"),
("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")],
0, 12, w=12, h=8, unit="short"),
ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)",
[("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")],
12, 12, w=12, h=8, unit="short"),
]))
# ── Throughput & health ─────────────────────────────────────────────────
panels.append(row(150, "Throughput & Health", 13, [
ts_panel(151, "Token throughput",
[("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"),
("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"),
("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")],
0, 14, w=12, h=8, unit="short"),
ts_panel(152, "Successful requests/s",
[("sum(rate(vllm:request_success_total[5m]))", "vLLM"),
("sum(rate(te_request_count[5m]))", "TEI")],
12, 14, w=12, h=8, unit="short"),
ts_panel(153, "Backpressure — queue-full rejects",
[("rate(queue_full_requests_total[5m])", "rejected/s")],
0, 15, w=8, h=8, unit="short"),
ts_panel(154, "KV warmup requests delayed",
[("rate(warmup_requests_delayed_total[5m])", "delayed/s")],
8, 15, w=8, h=8, unit="short"),
ts_panel(155, "Embedding / rerank latency p95",
[("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))",
"{{app}} embed"),
("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))",
"{{app}} queue")],
16, 15, w=8, h=8, unit="s"),
]))
return {
"title": "LLM Observability",
"uid": "llm-observability",
"schemaVersion": 39,
"timezone": "browser",
"time": {"from": "now-6h", "to": "now"},
"refresh": "30s",
"tags": ["llm", "vllm", "tei", "gpu", "observability"],
"panels": panels,
}
# ============================================================================
# Generate
# ============================================================================
@@ -277,4 +450,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), "
# Dashboard 3
write_dashboard("api-gateway.yaml", build_api_gateway(), "API")
# Dashboard 4
write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM")
print("Done.")