diff --git a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml index bf56bd5..1a7bbfa 100644 --- a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml +++ b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml @@ -1,3 +1,25 @@ +# The gateway is the only source of per-request LLM latency covering EVERY +# model, including the two Ollama ones (ornith:35b, qwen2.5:3b-instruct) that +# expose no metrics endpoint at all. Series it emits, per +# internal/observability/prometheus.go: +# llm_ttft_seconds histogram time-to-first-token +# llm_itl_seconds histogram inter-token latency +# llm_tokens_total counter SSE chunks generated +# queue_full_requests_total counter backpressure rejects +# warmup_requests_delayed_total counter vLLM slot contention +# vllm_prefix_cache_hit_tokens_total counter prefix-cache hits +# request_duration_seconds histogram full request +# +# Caveat: these are derived from the SSE byte stream, not from the engine. +# TTFT is request-start to first byte, ITL is the gap between SSE writes +# carrying "data: ", and llm_tokens_total counts SSE chunks rather than true +# tokens. They are also streaming-only — non-streaming requests record +# nothing. Treat vLLM's own histograms as authoritative for `reasoning`. +# +# The Deployment already carried prometheus.io/scrape annotations, but +# kube-prometheus-stack ignores pod annotations (no additionalScrapeConfigs, +# serviceMonitorSelectorNilUsesHelmValues: false), so none of the above was +# ever collected. This ServiceMonitor is what actually wires it up. apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -22,27 +44,3 @@ spec: path: /metrics scheme: http - # The gateway is the only source of per-request LLM latency that covers - # EVERY model, including the two Ollama ones that expose no metrics at - # all. Series it produces (internal/observability/prometheus.go): - # llm_ttft_seconds histogram time-to-first-token - # llm_itl_seconds histogram inter-token latency - # llm_tokens_total counter tokens generated - # queue_full_requests_total counter backpressure rejects - # warmup_requests_delayed_total counter vLLM slot contention - # vllm_prefix_cache_hit_tokens_total counter prefix-cache hits - # request_duration_seconds histogram full request - # - # The Deployment already carried prometheus.io/scrape annotations, but - # kube-prometheus-stack ignores pod annotations (no additionalScrape- - # Configs, serviceMonitorSelectorNilUsesHelmValues: false) — so none of - # the above was being collected. This ServiceMonitor is what actually - # wires it up. - metricRelabelings: - # 3 gateway replicas each build their own in-process histogram. Without - # this the label set is otherwise identical, so per-instance series - # would collide conceptually in sum()-by aggregations; instance is - # retained deliberately so per-replica skew stays visible. - - sourceLabels: [__address__] - targetLabel: instance - replacement: $1 diff --git a/k8s/infra/monitoring/servicemonitors/vllm.yaml b/k8s/infra/monitoring/servicemonitors/vllm.yaml index dfbd574..e2b0eb9 100644 --- a/k8s/infra/monitoring/servicemonitors/vllm.yaml +++ b/k8s/infra/monitoring/servicemonitors/vllm.yaml @@ -36,11 +36,14 @@ spec: path: /metrics scheme: http relabelings: - - sourceLabels: [__meta_kubernetes_namespace] - targetLabel: namespace - - sourceLabels: [__meta_kubernetes_pod_name] - targetLabel: pod - - sourceLabels: [__meta_kubernetes_service_name] - targetLabel: service - - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + # Prometheus's Kubernetes service discovery already supplies the + # namespace, pod and service labels; the only one that needs deriving + # is `app`, since the TEI panels aggregate by it (by (app)) and SD does + # not promote arbitrary pod labels. + # + # `action: replace` is written explicitly. It is the API server's + # default for relabeling rules, so omitting it leaves the live object + # diverged from this manifest and ArgoCD reports permanent OutOfSync. + - action: replace + sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] targetLabel: app