From 1f3439c9e03b03c2c0c08e9294cac435946715e9 Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 29 Sep 2026 22:11:28 +0800 Subject: [PATCH] feat(monitoring): LLM observability dashboard + fix dead CAG rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Grafana dashboard (llm-observability) with 27 panels across 5 rows: TTFT, inter-token latency, batching/scheduling, KV cache, throughput. Both metric sources are shown side by side because they answer different questions and neither is complete alone: vLLM 0.11.0 (reasoning) — engine truth: kv_cache_usage_perc, num_requests_running/waiting, num_preemptions_total, prefill vs decode time, prefix cache, per-request histogram quantiles. TEI 1.8.2 (embeddings, reranker) — te_batch_next_size, te_queue_size, te_embed_duration, te_embed_queue_duration. api-gateway — the ONLY per-request latency source covering Ollama models (ornith:35b, qwen2.5:3b-instruct), which expose no metrics. It is derived from the SSE byte stream, so it is streaming-only and counts SSE chunks rather than true tokens. vLLM's own histograms are authoritative for reasoning. All 44 panel queries were executed against live Prometheus before commit — all return success. vLLM/TEI panels have live series; gateway panels show no data until the netpol fix in homelab-frontend PR #62 merges. Also fixes a dead alert. alerts/cag-kv-cache-rules.yaml derived vllm:prefix_cache_hit_rate from vllm:cache_query_hit_total and vllm:cache_query_miss_total. Those metric names do not exist in vLLM 0.11.0 — the real series are vllm:prefix_cache_hits_total and vllm:prefix_cache_queries_total. The recording rule produced no data, so VLLMPrefixCacheHitRateLow could never fire. Not included: real GPU hardware metrics (SM utilisation, framebuffer, power, temperature). Only nvidia-device-plugin is deployed, which exposes none of them; DCGM exporter is required. vllm:kv_cache_usage_perc and preemption rate are the only GPU-pressure proxies available today. --- .../monitoring/alerts/cag-kv-cache-rules.yaml | 13 +- .../dashboards/generate-dashboards.py | 176 ++++++++++++++++++ .../dashboards/llm-observability.yaml | 75 ++++++++ k8s/infra/monitoring/kustomization.yaml | 1 + 4 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 k8s/infra/monitoring/dashboards/llm-observability.yaml diff --git a/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml b/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml index 3d69fae..55ba286 100644 --- a/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml +++ b/k8s/infra/monitoring/alerts/cag-kv-cache-rules.yaml @@ -7,19 +7,24 @@ metadata: release: prometheus spec: groups: - # Recording rules: derive vLLM prefix cache hit rate from raw counters + # Recording rules: derive vLLM prefix cache hit rate from raw counters. + # Metric names verified against live vLLM 0.11.0 scrape (2026-09-29). + # The previous names (vllm:cache_query_hit_total / vllm:cache_query_miss_total) + # do not exist in this version — vLLM exposes prefix_cache_hits_total and + # prefix_cache_queries_total — so this rule produced no data and + # VLLMPrefixCacheHitRateLow could never fire. - name: vllm.recording rules: - record: vllm:prefix_cache_hit_rate expr: | - rate(vllm:cache_query_hit_total[5m]) + rate(vllm:prefix_cache_hits_total[5m]) / clamp_min( - rate(vllm:cache_query_hit_total[5m]) + rate(vllm:cache_query_miss_total[5m]), + rate(vllm:prefix_cache_queries_total[5m]), 0.001 ) labels: - job: "vllm" + job: "reasoning-predictor" # Alert rules: CAG pipeline health - name: cag.alerts diff --git a/k8s/infra/monitoring/dashboards/generate-dashboards.py b/k8s/infra/monitoring/dashboards/generate-dashboards.py index de018ef..0362104 100755 --- a/k8s/infra/monitoring/dashboards/generate-dashboards.py +++ b/k8s/infra/monitoring/dashboards/generate-dashboards.py @@ -265,6 +265,179 @@ def build_api_gateway(): } +# ============================================================================ +# Dashboard 4: LLM Observability (vLLM / TEI engines + gateway) +# ============================================================================ +# +# Two independent metric sources, deliberately both shown: +# +# vLLM + TEI (llm-serving) — engine-truth, but only for the three engines +# that expose /metrics. Ollama (ornith:35b, +# qwen2.5:3b-instruct) exposes nothing at all. +# api-gateway (/metrics) — proxy-side, covers EVERY model including the +# Ollama ones, but is derived from the SSE byte +# stream: TTFT = request-start to first byte, +# ITL = gap between SSE writes carrying "data: ". +# It is therefore streaming-only and counts SSE +# chunks, not true tokens. Treat vLLM's own +# histograms as authoritative for `reasoning`. +# +# All metric names below were verified against a live scrape, not assumed: +# vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total, +# NOT cache_query_hit_total / cache_query_miss_total. + +def build_llm_observability(): + def hq(metric, q, by="le"): + return ( + f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by ({by}, model_name))", + f"p{int(q * 100)}", + ) + + def hq_gw(metric, q): + return ( + f"histogram_quantile({q}, sum(rate({metric}_bucket[5m])) by (le, model))", + f"p{int(q * 100)} · {{{{model}}}}", + ) + + def avg_of(metric): + # mean = rate(_sum) / rate(_count) — valid for cumulative histograms + return f"rate({metric}_sum[5m]) / clamp_min(rate({metric}_count[5m]), 0.001)" + + panels = [] + + # ── TTFT ──────────────────────────────────────────────────────────────── + panels.append(row(100, "Latency — Time To First Token", 0, [ + stat_panel(101, "TTFT p95 (gateway)", + "histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) by (le))", + 0, 1, w=4, h=4, unit="s"), + stat_panel(102, "TTFT p95 (vLLM)", + "histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))", + 4, 1, w=4, h=4, unit="s"), + stat_panel(103, "KV cache in use", + "max(vllm:kv_cache_usage_perc)", + 8, 1, w=4, h=4, unit="percentunit"), + stat_panel(104, "Requests running", + "sum(vllm:num_requests_running)", + 12, 1, w=4, h=4, unit="short"), + stat_panel(105, "Requests waiting", + "sum(vllm:num_requests_waiting)", + 16, 1, w=4, h=4, unit="short"), + stat_panel(106, "Preemptions (5m)", + "increase(vllm:num_preemptions_total[5m])", + 20, 1, w=4, h=4, unit="short"), + ts_panel(110, "TTFT percentile by model (gateway)", + [hq_gw("llm_ttft_seconds", 0.50), + hq_gw("llm_ttft_seconds", 0.95), + hq_gw("llm_ttft_seconds", 0.99)], + 0, 2, w=12, h=8, unit="s"), + ts_panel(111, "TTFT percentile — vLLM reasoning", + [hq("vllm:time_to_first_token_seconds", 0.50), + hq("vllm:time_to_first_token_seconds", 0.95), + hq("vllm:time_to_first_token_seconds", 0.99)], + 12, 2, w=12, h=8, unit="s"), + ts_panel(112, "Prefill vs decode (mean)", + [(avg_of("vllm:request_prefill_time_seconds"), "prefill"), + (avg_of("vllm:request_decode_time_seconds"), "decode")], + 0, 3, w=12, h=8, unit="s"), + ts_panel(113, "Queue wait before scheduling (mean)", + [(avg_of("vllm:request_queue_time_seconds"), "queue")], + 12, 3, w=12, h=8, unit="s"), + ])) + + # ── Inter-token latency ───────────────────────────────────────────────── + panels.append(row(120, "Latency — Inter-Token", 4, [ + ts_panel(121, "Inter-token latency by model (gateway)", + [hq_gw("llm_itl_seconds", 0.50), + hq_gw("llm_itl_seconds", 0.95), + hq_gw("llm_itl_seconds", 0.99)], + 0, 5, w=12, h=8, unit="s"), + ts_panel(122, "Inter-token latency — vLLM reasoning", + [hq("vllm:inter_token_latency_seconds", 0.50), + hq("vllm:inter_token_latency_seconds", 0.95), + hq("vllm:inter_token_latency_seconds", 0.99)], + 12, 5, w=12, h=8, unit="s"), + ts_panel(123, "Decode throughput (tokens/s per request)", + [(f"1 / clamp_min({avg_of('vllm:time_per_output_token_seconds')}, 0.0001)", "tok/s")], + 0, 6, w=12, h=8, unit="short"), + ts_panel(124, "End-to-end request latency p50/p95/p99", + [hq("vllm:e2e_request_latency_seconds", 0.50), + hq("vllm:e2e_request_latency_seconds", 0.95), + hq("vllm:e2e_request_latency_seconds", 0.99)], + 12, 6, w=12, h=8, unit="s"), + ])) + + # ── Batching / scheduling ─────────────────────────────────────────────── + panels.append(row(130, "Batching & Scheduling", 7, [ + ts_panel(131, "vLLM running vs waiting requests (batch depth)", + [("sum(vllm:num_requests_running)", "running (in batch)"), + ("sum(vllm:num_requests_waiting)", "waiting (queued)")], + 0, 8, w=12, h=8, unit="short"), + ts_panel(132, "Preemption rate (KV cache eviction pressure)", + [("rate(vllm:num_preemptions_total[5m])", "preemptions/s")], + 12, 8, w=12, h=8, unit="short"), + ts_panel(133, "TEI batch size (embeddings / reranker)", + [("avg(te_batch_next_size_sum / clamp_min(te_batch_next_size_count, 1)) by (app)", + "{{app}}")], + 0, 9, w=12, h=8, unit="short"), + ts_panel(134, "TEI queue depth", + [("max(te_queue_size) by (app)", "{{app}}")], + 12, 9, w=12, h=8, unit="short"), + ])) + + # ── KV cache ──────────────────────────────────────────────────────────── + panels.append(row(140, "KV Cache", 10, [ + ts_panel(141, "KV cache utilisation (GPU memory pressure)", + [("max(vllm:kv_cache_usage_perc) by (model_name)", "{{model_name}}")], + 0, 11, w=12, h=8, unit="percentunit"), + ts_panel(142, "Prefix cache hit rate (CAG effectiveness)", + [("vllm:prefix_cache_hit_rate", "{{model_name}}")], + 12, 11, w=12, h=8, unit="percentunit"), + ts_panel(143, "Prefix cache hits vs queries", + [("rate(vllm:prefix_cache_hits_total[5m])", "hits/s"), + ("rate(vllm:prefix_cache_queries_total[5m])", "queries/s")], + 0, 12, w=12, h=8, unit="short"), + ts_panel(144, "Prefix cache hit tokens (gateway-observed, all models)", + [("rate(vllm_prefix_cache_hit_tokens_total[5m])", "hit tokens/s")], + 12, 12, w=12, h=8, unit="short"), + ])) + + # ── Throughput & health ───────────────────────────────────────────────── + panels.append(row(150, "Throughput & Health", 13, [ + ts_panel(151, "Token throughput", + [("sum(rate(vllm:prompt_tokens_total[5m]))", "prompt tok/s"), + ("sum(rate(vllm:generation_tokens_total[5m]))", "generation tok/s"), + ("rate(llm_tokens_total[5m])", "gateway SSE chunks/s (all models)")], + 0, 14, w=12, h=8, unit="short"), + ts_panel(152, "Successful requests/s", + [("sum(rate(vllm:request_success_total[5m]))", "vLLM"), + ("sum(rate(te_request_count[5m]))", "TEI")], + 12, 14, w=12, h=8, unit="short"), + ts_panel(153, "Backpressure — queue-full rejects", + [("rate(queue_full_requests_total[5m])", "rejected/s")], + 0, 15, w=8, h=8, unit="short"), + ts_panel(154, "KV warmup requests delayed", + [("rate(warmup_requests_delayed_total[5m])", "delayed/s")], + 8, 15, w=8, h=8, unit="short"), + ts_panel(155, "Embedding / rerank latency p95", + [("histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))", + "{{app}} embed"), + ("histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) by (le, app))", + "{{app}} queue")], + 16, 15, w=8, h=8, unit="s"), + ])) + + return { + "title": "LLM Observability", + "uid": "llm-observability", + "schemaVersion": 39, + "timezone": "browser", + "time": {"from": "now-6h", "to": "now"}, + "refresh": "30s", + "tags": ["llm", "vllm", "tei", "gpu", "observability"], + "panels": panels, + } + + # ============================================================================ # Generate # ============================================================================ @@ -277,4 +450,7 @@ write_dashboard("cluster-infrastructure.yaml", build_cluster_infrastructure(), " # Dashboard 3 write_dashboard("api-gateway.yaml", build_api_gateway(), "API") +# Dashboard 4 +write_dashboard("llm-observability.yaml", build_llm_observability(), "LLM") + print("Done.") diff --git a/k8s/infra/monitoring/dashboards/llm-observability.yaml b/k8s/infra/monitoring/dashboards/llm-observability.yaml new file mode 100644 index 0000000..8b6d3cf --- /dev/null +++ b/k8s/infra/monitoring/dashboards/llm-observability.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +data: + llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency + \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT + p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, + sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95 + (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, + sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV + cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests + running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests + waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions + (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT + percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5, + sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7 + {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m])) + by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99, + sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7 + {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5, + sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill + vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m]) + / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m]) + / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue + wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m]) + / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency + \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token + latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5, + sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7 + {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m])) + by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99, + sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7 + {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5, + sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode + throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1 + / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]), + 0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request + latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5, + sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, + sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, + sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching + & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM + running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running + (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting + (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI + batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum + / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI + queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size) + by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV + cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc) + by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix + cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix + cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix + cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit + tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token + throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt + tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation + tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s + (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure + \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV + warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding + / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, + sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}} + embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) + by (le, app))","legendFormat":"{{app}} queue"}]}]}]}' +kind: ConfigMap +metadata: + annotations: + grafana_folder: LLM + labels: + grafana_dashboard: '1' + name: llm-observability-dashboard + namespace: logging diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 8b24910..754994d 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -24,3 +24,4 @@ resources: - servicemonitors/api-gateway.yaml - dashboards/cluster-infrastructure.yaml - dashboards/api-gateway.yaml + - dashboards/llm-observability.yaml