diff --git a/k8s/infra/monitoring/dashboards/generate-dashboards.py b/k8s/infra/monitoring/dashboards/generate-dashboards.py index 0362104..2123342 100755 --- a/k8s/infra/monitoring/dashboards/generate-dashboards.py +++ b/k8s/infra/monitoring/dashboards/generate-dashboards.py @@ -274,13 +274,23 @@ def build_api_gateway(): # vLLM + TEI (llm-serving) — engine-truth, but only for the three engines # that expose /metrics. Ollama (ornith:35b, # qwen2.5:3b-instruct) exposes nothing at all. -# api-gateway (/metrics) — proxy-side, covers EVERY model including the -# Ollama ones, but is derived from the SSE byte -# stream: TTFT = request-start to first byte, -# ITL = gap between SSE writes carrying "data: ". -# It is therefore streaming-only and counts SSE -# chunks, not true tokens. Treat vLLM's own -# histograms as authoritative for `reasoning`. +# api-gateway (/metrics) — intended to cover EVERY model including the +# Ollama ones, but see the dead-metrics note +# below: its latency series are not recorded. +# +# NOTE — llm_ttft_seconds, llm_itl_seconds and llm_tokens_total are DEAD as of +# 2026-09-29. The only recorder is LLMMetricsCapture +# (internal/proxy/llm_metrics.go), and NewLLMMetricsCapture() has ZERO call +# sites in the whole repo, so the exporter emits HELP-only stubs and any panel +# querying them stays empty forever. Verified by grep, not assumed. +# +# The gateway latency panels are kept deliberately so the gap is visible — they +# light up unchanged once the capture wrapper is wired into the streaming proxy +# path. Until then vLLM's own histograms are the latency source for `reasoning`, +# and the two Ollama models have no latency visibility from anywhere. +# +# Panels backed by queue_full_requests_total, warmup_requests_delayed_total and +# vllm_prefix_cache_hit_tokens_total DO work today (3 series, one per replica). # # All metric names below were verified against a live scrape, not assumed: # vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total, diff --git a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml index 1a7bbfa..216c5fd 100644 --- a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml +++ b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml @@ -1,25 +1,30 @@ -# The gateway is the only source of per-request LLM latency covering EVERY -# model, including the two Ollama ones (ornith:35b, qwen2.5:3b-instruct) that -# expose no metrics endpoint at all. Series it emits, per -# internal/observability/prometheus.go: -# llm_ttft_seconds histogram time-to-first-token -# llm_itl_seconds histogram inter-token latency -# llm_tokens_total counter SSE chunks generated +# WARNING — three of the series documented below are DEAD CODE as of +# 2026-09-29 and will never produce data: +# +# llm_ttft_seconds, llm_itl_seconds, llm_tokens_total +# +# They are declared and exported in internal/observability/prometheus.go, but +# the only thing that would ever record samples is LLMMetricsCapture +# (internal/proxy/llm_metrics.go) — and NewLLMMetricsCapture() has ZERO call +# sites. Verified: `grep -rn NewLLMMetricsCapture` returns only the definition, +# and RecordTTFT/RecordITL/RecordTokens are called from nowhere else. The +# exporter skips keys with no samples, so these render as HELP-only stubs. +# +# The gateway latency panels on the llm-observability dashboard depend on that +# wiring. Until it lands, use vLLM's own histograms as the latency source for +# `reasoning`; the two Ollama models (ornith:35b, qwen2.5:3b-instruct) have no +# latency visibility at all, because Ollama exposes no metrics endpoint either. +# +# These three ARE recorded and do work (3 series, one per replica): # queue_full_requests_total counter backpressure rejects # warmup_requests_delayed_total counter vLLM slot contention # vllm_prefix_cache_hit_tokens_total counter prefix-cache hits # request_duration_seconds histogram full request # -# Caveat: these are derived from the SSE byte stream, not from the engine. -# TTFT is request-start to first byte, ITL is the gap between SSE writes -# carrying "data: ", and llm_tokens_total counts SSE chunks rather than true -# tokens. They are also streaming-only — non-streaming requests record -# nothing. Treat vLLM's own histograms as authoritative for `reasoning`. -# -# The Deployment already carried prometheus.io/scrape annotations, but -# kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs, -# serviceMonitorSelectorNilUsesHelmValues: false), so none of the above was -# ever collected. This ServiceMonitor is what actually wires it up. +# Also note the Deployment already carried prometheus.io/scrape annotations, +# but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs, +# serviceMonitorSelectorNilUsesHelmValues: false) — so nothing at all was being +# collected before this ServiceMonitor existed. apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: