homelab/k8s/infra/monitoring/dashboards/llm-observability.yaml

apiVersion: v1
data:
  llm-observability.json: '{"title":"LLM Observability","uid":"llm-observability","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","tags":["llm","vllm","tei","gpu","observability"],"panels":[{"id":100,"title":"Latency
    \u2014 Time To First Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":101,"title":"TTFT
    p95 (gateway)","type":"stat","gridPos":{"h":4,"w":4,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
    sum(rate(llm_ttft_seconds_bucket[5m])) by (le))"}]},{"id":102,"title":"TTFT p95
    (vLLM)","type":"stat","gridPos":{"h":4,"w":4,"x":4,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le))"}]},{"id":103,"title":"KV
    cache in use","type":"stat","gridPos":{"h":4,"w":4,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)"}]},{"id":104,"title":"Requests
    running","type":"stat","gridPos":{"h":4,"w":4,"x":12,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)"}]},{"id":105,"title":"Requests
    waiting","type":"stat","gridPos":{"h":4,"w":4,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_waiting)"}]},{"id":106,"title":"Preemptions
    (5m)","type":"stat","gridPos":{"h":4,"w":4,"x":20,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(vllm:num_preemptions_total[5m])"}]},{"id":110,"title":"TTFT
    percentile by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_ttft_seconds_bucket[5m]))
    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
    sum(rate(llm_ttft_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
    {{model}}"}]},{"id":111,"title":"TTFT percentile \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
    sum(rate(vllm:time_to_first_token_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":112,"title":"Prefill
    vs decode (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_prefill_time_seconds_sum[5m])
    / clamp_min(rate(vllm:request_prefill_time_seconds_count[5m]), 0.001)","legendFormat":"prefill"},{"expr":"rate(vllm:request_decode_time_seconds_sum[5m])
    / clamp_min(rate(vllm:request_decode_time_seconds_count[5m]), 0.001)","legendFormat":"decode"}]},{"id":113,"title":"Queue
    wait before scheduling (mean)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"rate(vllm:request_queue_time_seconds_sum[5m])
    / clamp_min(rate(vllm:request_queue_time_seconds_count[5m]), 0.001)","legendFormat":"queue"}]}]},{"id":120,"title":"Latency
    \u2014 Inter-Token","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":4},"panels":[{"id":121,"title":"Inter-token
    latency by model (gateway)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p50 \u00b7
    {{model}}"},{"expr":"histogram_quantile(0.95, sum(rate(llm_itl_seconds_bucket[5m]))
    by (le, model))","legendFormat":"p95 \u00b7 {{model}}"},{"expr":"histogram_quantile(0.99,
    sum(rate(llm_itl_seconds_bucket[5m])) by (le, model))","legendFormat":"p99 \u00b7
    {{model}}"}]},{"id":122,"title":"Inter-token latency \u2014 vLLM reasoning","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":5},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
    sum(rate(vllm:inter_token_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]},{"id":123,"title":"Decode
    throughput (tokens/s per request)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"1
    / clamp_min(rate(vllm:time_per_output_token_seconds_sum[5m]) / clamp_min(rate(vllm:time_per_output_token_seconds_count[5m]),
    0.001), 0.0001)","legendFormat":"tok/s"}]},{"id":124,"title":"End-to-end request
    latency p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":6},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.5,
    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95,
    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99,
    sum(rate(vllm:e2e_request_latency_seconds_bucket[5m])) by (le, model_name))","legendFormat":"p99"}]}]},{"id":130,"title":"Batching
    & Scheduling","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":7},"panels":[{"id":131,"title":"vLLM
    running vs waiting requests (batch depth)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(vllm:num_requests_running)","legendFormat":"running
    (in batch)"},{"expr":"sum(vllm:num_requests_waiting)","legendFormat":"waiting
    (queued)"}]},{"id":132,"title":"Preemption rate (KV cache eviction pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":8},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:num_preemptions_total[5m])","legendFormat":"preemptions/s"}]},{"id":133,"title":"TEI
    batch size (embeddings / reranker)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"avg(te_batch_next_size_sum
    / clamp_min(te_batch_next_size_count, 1)) by (app)","legendFormat":"{{app}}"}]},{"id":134,"title":"TEI
    queue depth","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(te_queue_size)
    by (app)","legendFormat":"{{app}}"}]}]},{"id":140,"title":"KV Cache","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"panels":[{"id":141,"title":"KV
    cache utilisation (GPU memory pressure)","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"max(vllm:kv_cache_usage_perc)
    by (model_name)","legendFormat":"{{model_name}}"}]},{"id":142,"title":"Prefix
    cache hit rate (CAG effectiveness)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":11},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"percentunit"}},"targets":[{"expr":"vllm:prefix_cache_hit_rate","legendFormat":"{{model_name}}"}]},{"id":143,"title":"Prefix
    cache hits vs queries","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm:prefix_cache_hits_total[5m])","legendFormat":"hits/s"},{"expr":"rate(vllm:prefix_cache_queries_total[5m])","legendFormat":"queries/s"}]},{"id":144,"title":"Prefix
    cache hit tokens (gateway-observed, all models)","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":12},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(vllm_prefix_cache_hit_tokens_total[5m])","legendFormat":"hit
    tokens/s"}]}]},{"id":150,"title":"Throughput & Health","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":13},"panels":[{"id":151,"title":"Token
    throughput","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:prompt_tokens_total[5m]))","legendFormat":"prompt
    tok/s"},{"expr":"sum(rate(vllm:generation_tokens_total[5m]))","legendFormat":"generation
    tok/s"},{"expr":"rate(llm_tokens_total[5m])","legendFormat":"gateway SSE chunks/s
    (all models)"}]},{"id":152,"title":"Successful requests/s","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":14},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum(rate(vllm:request_success_total[5m]))","legendFormat":"vLLM"},{"expr":"sum(rate(te_request_count[5m]))","legendFormat":"TEI"}]},{"id":153,"title":"Backpressure
    \u2014 queue-full rejects","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(queue_full_requests_total[5m])","legendFormat":"rejected/s"}]},{"id":154,"title":"KV
    warmup requests delayed","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"rate(warmup_requests_delayed_total[5m])","legendFormat":"delayed/s"}]},{"id":155,"title":"Embedding
    / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95,
    sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}}
    embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m]))
    by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission
    Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission
    parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests
    the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64);
    empty until that change is deployed."},{"id":162,"title":"Admission rejected by
    reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum
    by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission
    rejections split by the gate''s own reason label. Emitted by homelab-frontend#64;
    empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times
    the gateway''s view of the engine gauges went stale. The gate fails OPEN when
    this happens, so any non-zero rate means admission is silently degraded to admit-everything.
    Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway
    view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway:
    running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway:
    kv (fraction)  [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine:
    kv (fraction)"}],"description":"The gateway''s own reading of the engine against
    the engine''s self-reported value. A disagreement between a pair is how a stale
    observer or a mistuned admission target shows itself, and nothing else in this
    dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc)
    ship with homelab-frontend#64 and are absent until it is deployed \u2014 until
    then this panel shows the engine''s pair only."}]}]}'
kind: ConfigMap
metadata:
  annotations:
    grafana_folder: LLM
  labels:
    grafana_dashboard: '1'
  name: llm-observability-dashboard
  namespace: logging