From 6ae36981bbac82030667bb96a01cceb89d1d9ef8 Mon Sep 17 00:00:00 2001 From: rock Date: Tue, 29 Sep 2026 22:05:00 +0800 Subject: [PATCH] fix(monitoring): wire llm-serving + api-gateway metric scraping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both metric sources existed but neither was scraped: - servicemonitors/vllm.yaml was orphaned (absent from kustomization resources:) and would not have matched anyway — it used 'port: http' but KServe-generated Services in llm-serving have UNNAMED ports. Switched to targetPort: 8080 (prometheus-operator v0.94.1 supports it), which avoids editing the InferenceServices and therefore avoids a predictor restart / weights reload. - api-gw exposes llm_ttft_seconds, llm_itl_seconds, llm_tokens_total, queue_full_requests_total, warmup_requests_delayed_total and vllm_prefix_cache_hit_tokens_total, but the Deployment's prometheus.io/scrape annotation is ignored by kube-prometheus-stack (no additionalScrapeConfigs). Added a ServiceMonitor for it. This is the only per-request latency source covering the two Ollama models, which expose no metrics endpoint at all. Ollama services are excluded from the vllm ServiceMonitor selector to avoid permanently-DOWN targets. Note: alerts/cag-kv-cache-rules.yaml references vllm:prefix_cache_hit_rate derived from vllm:cache_query_hit_total/miss, which were never scraped. Those alerts were dead; verify the metric names after the first scrape. --- k8s/infra/monitoring/kustomization.yaml | 2 + .../servicemonitors/api-gateway.yaml | 48 +++++++++++++++++++ .../monitoring/servicemonitors/vllm.yaml | 22 +++++++-- 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 k8s/infra/monitoring/servicemonitors/api-gateway.yaml diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 6d7e629..8b24910 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -20,5 +20,7 @@ resources: - servicemonitors/forgejo.yaml - servicemonitors/minio.yaml - servicemonitors/ingress-nginx.yaml + - servicemonitors/vllm.yaml + - servicemonitors/api-gateway.yaml - dashboards/cluster-infrastructure.yaml - dashboards/api-gateway.yaml diff --git a/k8s/infra/monitoring/servicemonitors/api-gateway.yaml b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml new file mode 100644 index 0000000..bf56bd5 --- /dev/null +++ b/k8s/infra/monitoring/servicemonitors/api-gateway.yaml @@ -0,0 +1,48 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: api-gateway + namespace: monitoring + labels: + app.kubernetes.io/name: api-gateway + app.kubernetes.io/part-of: api +spec: + namespaceSelector: + matchNames: + - api + selector: + matchLabels: + app: api-gateway + endpoints: + # api-gateway Service port IS named `http` (unlike the KServe ones), so + # the name form works here. + - port: http + interval: 30s + scrapeTimeout: 10s + path: /metrics + scheme: http + + # The gateway is the only source of per-request LLM latency that covers + # EVERY model, including the two Ollama ones that expose no metrics at + # all. Series it produces (internal/observability/prometheus.go): + # llm_ttft_seconds histogram time-to-first-token + # llm_itl_seconds histogram inter-token latency + # llm_tokens_total counter tokens generated + # queue_full_requests_total counter backpressure rejects + # warmup_requests_delayed_total counter vLLM slot contention + # vllm_prefix_cache_hit_tokens_total counter prefix-cache hits + # request_duration_seconds histogram full request + # + # The Deployment already carried prometheus.io/scrape annotations, but + # kube-prometheus-stack ignores pod annotations (no additionalScrape- + # Configs, serviceMonitorSelectorNilUsesHelmValues: false) — so none of + # the above was being collected. This ServiceMonitor is what actually + # wires it up. + metricRelabelings: + # 3 gateway replicas each build their own in-process histogram. Without + # this the label set is otherwise identical, so per-instance series + # would collide conceptually in sum()-by aggregations; instance is + # retained deliberately so per-replica skew stays visible. + - sourceLabels: [__address__] + targetLabel: instance + replacement: $1 diff --git a/k8s/infra/monitoring/servicemonitors/vllm.yaml b/k8s/infra/monitoring/servicemonitors/vllm.yaml index 9218dd1..dfbd574 100644 --- a/k8s/infra/monitoring/servicemonitors/vllm.yaml +++ b/k8s/infra/monitoring/servicemonitors/vllm.yaml @@ -11,10 +11,26 @@ spec: matchNames: - llm-serving selector: - matchLabels: - app.kubernetes.io/part-of: llm-serving + # Only the three engines that actually serve /metrics. + # ornith-predictor and qwen-cpu are Ollama, which has no Prometheus + # endpoint (/metrics returns 404) — selecting them would add permanently + # DOWN targets. Ollama-model visibility comes from the api-gateway + # ServiceMonitor instead (llm_ttft_seconds / llm_itl_seconds are + # measured at the proxy, so they cover every model including Ollama). + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - llm-reasoning # vLLM 0.11.0 — full engine metrics + - llm-embeddings # TEI 1.8.2 — CPU + - llm-reranker # TEI 1.8.2 — CPU endpoints: - - port: http + # targetPort, not port: the KServe-generated Services on llm-serving have + # UNNAMED ports, so `port: ` can never match. Naming them would mean + # editing the InferenceServices (predictor restart = full weights reload, + # tens of seconds on the vLLM slot), so scrape the pod port directly. + # prometheus-operator v0.94.1 supports targetPort on ServiceMonitor. + - targetPort: 8080 interval: 30s scrapeTimeout: 10s path: /metrics