diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index f936a78..5585ad2 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -28,9 +28,14 @@ spec: # KV cache math: 256KB/token total (64 layers, 8 KV heads, 128 # head_dim, fp16), PP=2 splits both weights and KV load ~evenly, so # each GPU carries ~9.67GB weights + ~128KB/token KV. At - # gpu-memory-utilization=0.90 (28.8GB/GPU usable), that leaves - # ~19.1GB/GPU for KV cache -> ~156k tokens/GPU capacity, comfortably - # above the 131072 target with room to spare -- the old + # gpu-memory-utilization=0.95 (30.4GB/GPU usable), that leaves + # ~20.7GB/GPU for KV cache. NOTE this is a projection, not a + # measurement: the live reading at 0.90 is num_gpu_blocks=9527 x + # block_size=16 = 152,432 tokens (vllm:cache_config_info). Re-read that + # metric after the rollout -- max_num_batched_tokens=8192 also raises + # activation memory out of the same budget, so the post-boot + # num_gpu_blocks is the authoritative number, and --max-num-seqs=5 is + # only valid if it still covers 2x65k + 3x8k. The old # OffloadingConnector CPU-DRAM spillover (tuned for the previous # model's much smaller 16384 context) is no longer needed and is # dropped. Staying on PP=2 and vLLM 0.11.0 (no version bump needed, @@ -54,10 +59,37 @@ spec: - --tensor-parallel-size=1 - --pipeline-parallel-size=2 - --max-model-len=131072 - - --gpu-memory-utilization=0.90 - - --max-num-seqs=4 + - --gpu-memory-utilization=0.95 + - --max-num-seqs=5 + # Prefill scheduling -- see the measured starvation this fixes. + # V1's scheduler has no prefill/decode phase: it walks self.running and + # hands out tokens from one shared budget (vllm/v1/core/sched/scheduler.py: + # "There's no decoding phase nor prefill phase in the scheduler"). The cap + # per request is applied BEFORE the budget clamp: + # if 0 < long_prefill_token_threshold < num_new_tokens: take = threshold + # take = min(take, token_budget) + # The loop is `while ... and token_budget > 0`. With the threshold at its + # default of 0 the cap is disabled, so one long prompt takes the ENTIRE + # budget, token_budget hits 0, the loop exits and every other request is + # skipped for that step -- measured as 262 tok/s contended prefill and + # 0.0-0.1 tok/s generation for everything else, Running: 2-4 reqs. + # Hence threshold must be strictly BELOW the budget or it never binds: + # at 8192/8192 a single request is served per step (still starved), while + # 8192/2048 serves four. Verified by replaying the loop. + - --max-num-batched-tokens=8192 + - --long-prefill-token-threshold=2048 - --enable-chunked-prefill - --enable-prefix-caching + # + # NOT set, deliberately: --max-num-partial-prefills and + # --max-long-partial-prefills. Those are V0-only knobs. vLLM 0.11.0 ships + # V1 only (vllm/core/ absent), and EngineArgs._is_v1_supported_oracle + # rejects any value other than the default for either one, calling + # _raise_or_fallback("Concurrent Partial Prefill") -> "Falling back to V0" + # -> AssertionError, because there is no V0 engine to fall back to. + # The Deployment uses strategy Recreate, so setting them is an outage. + # V1 does concurrent chunked prefill natively; it needs no flag, only the + # budget/threshold ratio above. # qwen3 is vLLM's dedicated reasoning parser for this family's # blocks. - --reasoning-parser=qwen3 diff --git a/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml b/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml index 49e904b..82402de 100644 --- a/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml +++ b/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml @@ -158,3 +158,66 @@ spec: severity: warning annotations: summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit" + + - name: api-gateway.admission + # The admission gate fails OPEN when its view of the engine gauges goes + # stale, so a broken observer degrades silently to the pre-#64 behaviour + # of admitting everything. These two series are the only detection — + # the gate has no automated tests. + rules: + # The gate fails OPEN when it cannot read the engine, so "no data" must + # alert: otherwise a missing emitter, a renamed metric, or a broken + # scrape is indistinguishable from a healthy observer. Both expressions + # therefore OR in an absent() guard on the metric itself. + - alert: APIGatewayObserverDegraded + expr: observer_degraded == 1 or absent(observer_degraded) + for: 2m + labels: + severity: warning + annotations: + summary: "Gateway admission observer degraded or not reporting — gate is failing open" + description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case." + + # Engine gauges went stale, or are not being reported at all + - alert: GatewayEngineStateStale + expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total) + for: 2m + labels: + severity: warning + annotations: + summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges" + description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all." + + # The acceptance metric for reasoning.yaml's prefill change. Preemption + # of a 65k prompt costs a full re-prefill (~250s at the measured + # 262 tok/s), so any sustained rate is a regression, not noise. + - alert: LLMPreemptions + # Guarded absent() is deliberately NOT used bare here: the engine + # restarts (strategy Recreate) and the series legitimately vanishes + # for the length of a boot, so a bare absent() would false-fire on + # every rollout. It only fires when the metric is missing WHILE the + # predictor is up, which is the real "renamed or unscraped" case. + expr: | + increase(vllm:num_preemptions_total[15m]) > 0 + or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1) + for: 5m + labels: + severity: warning + annotations: + summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe" + description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless." + + # This change trades single-request latency for fairness: capping a long + # prompt at 2048 tokens/step means it takes more steps to finish, so a + # lone large prompt can now be slower even while preemptions stay 0. + # Without this alert the change has no way to show a regression. + - alert: LLMPrefillStarvation + expr: | + (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5) + or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60 + for: 10m + labels: + severity: warning + annotations: + summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity" + description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)." diff --git a/k8s/infra/monitoring/dashboards/generate-dashboards.py b/k8s/infra/monitoring/dashboards/generate-dashboards.py index 2123342..d21daf0 100755 --- a/k8s/infra/monitoring/dashboards/generate-dashboards.py +++ b/k8s/infra/monitoring/dashboards/generate-dashboards.py @@ -26,20 +26,23 @@ def stat_panel(id, title, expr, x, y, w=4, h=4, unit="short", mappings=None, thr return p -def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short"): +def ts_panel(id, title, exprs, x, y, w=8, h=8, unit="short", description=None): targets = [] for e in exprs: if isinstance(e, tuple): targets.append({"expr": e[0], "legendFormat": e[1]}) else: targets.append({"expr": e, "legendFormat": "{{pod}}"}) - return { + p = { "id": id, "title": title, "type": "timeseries", "gridPos": {"h": h, "w": w, "x": x, "y": y}, "datasource": DS_PROM, "fieldConfig": {"defaults": {"unit": unit}}, "targets": targets, } + if description: + p["description"] = description + return p def table_panel(id, title, expr, x, y, w=12, h=8): @@ -274,23 +277,29 @@ def build_api_gateway(): # vLLM + TEI (llm-serving) — engine-truth, but only for the three engines # that expose /metrics. Ollama (ornith:35b, # qwen2.5:3b-instruct) exposes nothing at all. -# api-gateway (/metrics) — intended to cover EVERY model including the -# Ollama ones, but see the dead-metrics note -# below: its latency series are not recorded. +# api-gateway (/metrics) — covers EVERY model including the two Ollama +# ones, derived from the SSE byte stream: TTFT is +# request-start to first byte, ITL is the gap +# between SSE writes carrying "data: ", so it +# counts SSE chunks rather than true tokens. +# vLLM's own histograms remain authoritative for +# `reasoning`. # -# NOTE — llm_ttft_seconds, llm_itl_seconds and llm_tokens_total are DEAD as of -# 2026-09-29. The only recorder is LLMMetricsCapture -# (internal/proxy/llm_metrics.go), and NewLLMMetricsCapture() has ZERO call -# sites in the whole repo, so the exporter emits HELP-only stubs and any panel -# querying them stays empty forever. Verified by grep, not assumed. +# These three series ARE live as of 2026-09-29 — 3 series each, one per replica, +# verified against the running gateway and Prometheus: +# llm_ttft_seconds, llm_itl_seconds, llm_tokens_total # -# The gateway latency panels are kept deliberately so the gap is visible — they -# light up unchanged once the capture wrapper is wired into the streaming proxy -# path. Until then vLLM's own histograms are the latency source for `reasoning`, -# and the two Ollama models have no latency visibility from anywhere. +# History, recorded so nobody re-derives it the hard way: on 2026-09-29 these +# recorded nothing at all. internal/proxy/llm_metrics.go declared a capture +# wrapper that was never constructed, so the exporter emitted HELP-only stubs. +# homelab-frontend#63 fixed exactly that — the wrapper is now +# ResponseWriterWrapper, constructed by NewResponseWriterWrapper +# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy path. +# The "dead code" note that used to live in this header is superseded. # # Panels backed by queue_full_requests_total, warmup_requests_delayed_total and -# vllm_prefix_cache_hit_tokens_total DO work today (3 series, one per replica). +# vllm_prefix_cache_hit_tokens_total also work (3 series, one per replica). +# # # All metric names below were verified against a live scrape, not assumed: # vLLM 0.11.0 exposes prefix_cache_hits_total / prefix_cache_queries_total, @@ -436,6 +445,48 @@ def build_llm_observability(): 16, 15, w=8, h=8, unit="s"), ])) + # ── Admission control ─────────────────────────────────────────────────── + # These four panels read the gateway's admission gate, which is the + # companion change in homelab-frontend#64. That emitter is NOT deployed + # yet (verified against the live /metrics endpoint: 0 series matching + # ^(admission_|observer_|engine_state_)), so they render "No data" until + # #64 ships. That is expected, and each panel carries a description + # saying so — an unexplained blank panel would read as a broken + # dashboard, which is exactly the failure mode this row exists to catch. + panels.append(row(160, "Admission Control", 16, [ + ts_panel(161, "Admission parked (5m)", + [("increase(admission_parked_total[5m])", "parked")], + 0, 17, w=8, h=8, unit="short", + description="Requests the gateway held back rather than admitting. " + "Emitted by the admission gate (homelab-frontend#64); " + "empty until that change is deployed."), + ts_panel(162, "Admission rejected by reason", + [("sum by (reason) (increase(admission_rejected_total[5m]))", "{{reason}}")], + 8, 17, w=8, h=8, unit="short", + description="Admission rejections split by the gate's own reason label. " + "Emitted by homelab-frontend#64; empty until deployed."), + ts_panel(163, "Engine state stale", + [("increase(engine_state_stale_total[5m])", "stale")], + 16, 17, w=8, h=8, unit="short", + description="Times the gateway's view of the engine gauges went stale. " + "The gate fails OPEN when this happens, so any non-zero rate " + "means admission is silently degraded to admit-everything. " + "Emitted by homelab-frontend#64; empty until deployed."), + ts_panel(164, "Gateway view vs engine truth", + [("max(vllm_requests_running)", "gateway: running"), + ("sum(vllm:num_requests_running)", "engine: running"), + ("max(vllm_kv_usage_perc)", "gateway: kv (fraction) [same as engine if healthy]"), + ("max(vllm:kv_cache_usage_perc)", "engine: kv (fraction)")], + 0, 18, w=24, h=8, unit="short", + description="The gateway's own reading of the engine against the engine's " + "self-reported value. A disagreement between a pair is how a " + "stale observer or a mistuned admission target shows itself, " + "and nothing else in this dashboard reveals it. The two " + "gateway series (vllm_requests_running, vllm_kv_usage_perc) " + "ship with homelab-frontend#64 and are absent until it is " + "deployed — until then this panel shows the engine's pair only."), + ])) + return { "title": "LLM Observability", "uid": "llm-observability", diff --git a/k8s/infra/monitoring/dashboards/llm-observability.yaml b/k8s/infra/monitoring/dashboards/llm-observability.yaml index 8b6d3cf..9332c67 100644 --- a/k8s/infra/monitoring/dashboards/llm-observability.yaml +++ b/k8s/infra/monitoring/dashboards/llm-observability.yaml @@ -64,7 +64,27 @@ data: / rerank latency p95","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":15},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"s"}},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(te_embed_duration_bucket[5m])) by (le, app))","legendFormat":"{{app}} embed"},{"expr":"histogram_quantile(0.95, sum(rate(te_embed_queue_duration_bucket[5m])) - by (le, app))","legendFormat":"{{app}} queue"}]}]}]}' + by (le, app))","legendFormat":"{{app}} queue"}]}]},{"id":160,"title":"Admission + Control","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":16},"panels":[{"id":161,"title":"Admission + parked (5m)","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(admission_parked_total[5m])","legendFormat":"parked"}],"description":"Requests + the gateway held back rather than admitting. Emitted by the admission gate (homelab-frontend#64); + empty until that change is deployed."},{"id":162,"title":"Admission rejected by + reason","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"sum + by (reason) (increase(admission_rejected_total[5m]))","legendFormat":"{{reason}}"}],"description":"Admission + rejections split by the gate''s own reason label. Emitted by homelab-frontend#64; + empty until deployed."},{"id":163,"title":"Engine state stale","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":17},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"increase(engine_state_stale_total[5m])","legendFormat":"stale"}],"description":"Times + the gateway''s view of the engine gauges went stale. The gate fails OPEN when + this happens, so any non-zero rate means admission is silently degraded to admit-everything. + Emitted by homelab-frontend#64; empty until deployed."},{"id":164,"title":"Gateway + view vs engine truth","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":18},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"unit":"short"}},"targets":[{"expr":"max(vllm_requests_running)","legendFormat":"gateway: + running"},{"expr":"sum(vllm:num_requests_running)","legendFormat":"engine: running"},{"expr":"max(vllm_kv_usage_perc)","legendFormat":"gateway: + kv (fraction) [same as engine if healthy]"},{"expr":"max(vllm:kv_cache_usage_perc)","legendFormat":"engine: + kv (fraction)"}],"description":"The gateway''s own reading of the engine against + the engine''s self-reported value. A disagreement between a pair is how a stale + observer or a mistuned admission target shows itself, and nothing else in this + dashboard reveals it. The two gateway series (vllm_requests_running, vllm_kv_usage_perc) + ship with homelab-frontend#64 and are absent until it is deployed \u2014 until + then this panel shows the engine''s pair only."}]}]}' kind: ConfigMap metadata: annotations: diff --git a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml index 216c5fd..815c7fb 100644 --- a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml +++ b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml @@ -1,25 +1,36 @@ -# WARNING — three of the series documented below are DEAD CODE as of -# 2026-09-29 and will never produce data: +# History, recorded so nobody re-derives it the hard way: on 2026-09-29 the +# three latency series below recorded nothing. internal/proxy/llm_metrics.go +# declared a capture wrapper (LLMMetricsCapture) that was never constructed +# anywhere, so the exporter emitted HELP-only stubs and the dashboard panels +# stayed empty. # -# llm_ttft_seconds, llm_itl_seconds, llm_tokens_total +# homelab-frontend#63 fixed exactly that. The wrapper is now +# ResponseWriterWrapper, constructed by NewResponseWriterWrapper +# (internal/proxy/llm_metrics.go:38) and installed on the streaming proxy +# request path. Verified live on 2026-09-29: +# llm_ttft_seconds count=7 sum=598.212s +# llm_itl_seconds count=16116 +# llm_tokens_total 3 series # -# They are declared and exported in internal/observability/prometheus.go, but -# the only thing that would ever record samples is LLMMetricsCapture -# (internal/proxy/llm_metrics.go) — and NewLLMMetricsCapture() has ZERO call -# sites. Verified: `grep -rn NewLLMMetricsCapture` returns only the definition, -# and RecordTTFT/RecordITL/RecordTokens are called from nowhere else. The -# exporter skips keys with no samples, so these render as HELP-only stubs. +# These derived-from-SSE series cover EVERY model including the two Ollama ones +# (ornith:35b, qwen2.5:3b-instruct), which expose no metrics endpoint at all. +# They are proxy-side: TTFT is request-start to first byte and ITL is the gap +# between SSE writes, so both are granular to SSE write boundaries rather than +# to the engine's own token cadence. llm_tokens_total counts `data:` events, +# and the documented assumption in internal/proxy/llm_metrics.go is that these +# upstreams emit one such event per token — so it is a token count only as far +# as that assumption holds. vLLM's own histograms remain authoritative. # -# The gateway latency panels on the llm-observability dashboard depend on that -# wiring. Until it lands, use vLLM's own histograms as the latency source for -# `reasoning`; the two Ollama models (ornith:35b, qwen2.5:3b-instruct) have no -# latency visibility at all, because Ollama exposes no metrics endpoint either. -# -# These three ARE recorded and do work (3 series, one per replica): +# Also recorded and working (3 series, one per replica): # queue_full_requests_total counter backpressure rejects # warmup_requests_delayed_total counter vLLM slot contention # vllm_prefix_cache_hit_tokens_total counter prefix-cache hits -# request_duration_seconds histogram full request +# +# NOT working, do not build on it: request_duration_seconds is declared and +# exported but has 0 samples, and its only writer Metrics.RecordRequest has +# zero call sites outside _test.go — verified by ast-grep, not assumed. It is +# the same dead-code shape this branch already corrected for the latency +# series, so it is listed here rather than left to be rediscovered. # # Also note the Deployment already carried prometheus.io/scrape annotations, # but kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs,