From 28753d11ed9867c1f78db0881e0de367c25171b7 Mon Sep 17 00:00:00 2001 From: Rock Date: Tue, 29 Sep 2026 14:40:31 +0000 Subject: [PATCH] fix(netpol): allow Prometheus scrape of gateway /metrics (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The new `api-gateway` ServiceMonitor discovers all 3 replicas, but every scrape fails with `context deadline exceeded`: ``` down api-gateway api-gateway-6986b7c889-pmt4b err=Get "http://10.244.0.180:8080/metrics": context deadline down api-gateway api-gateway-6986b7c889-c28dt err=Get "http://10.244.1.1:8080/metrics": context deadline down api-gateway api-gateway-6986b7c889-552gl err=Get "http://10.244.2.61:8080/metrics": context deadline ``` ## Cause `network-policy.yaml` allowed Prometheus ingress with: ```yaml namespaceSelector: matchLabels: name: monitoring ``` Only `ingress-nginx` carries a plain `name:` label in this cluster. Every other namespace has just `kubernetes.io/metadata.name`, so this rule matched **nothing** and silently denied the scrape. (The rule directly above it — `name: ingress-nginx` — works only because that namespace is the documented exception.) ## Fix Select on `kubernetes.io/metadata.name: monitoring`. ## Impact Unblocks gateway metrics: `llm_ttft_seconds`, `llm_itl_seconds`, `llm_tokens_total`, `queue_full_requests_total`, `warmup_requests_delayed_total`, `vllm_prefix_cache_hit_tokens_total`. These are the only per-request LLM latency series that cover the two Ollama models (`ornith:35b`, `qwen2.5:3b-instruct`), which expose no metrics endpoint of their own. --------- Co-authored-by: Admin Bot Reviewed-on: https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend/pulls/62 --- k8s/network-policy.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/k8s/network-policy.yaml b/k8s/network-policy.yaml index c82ec15..619dace 100644 --- a/k8s/network-policy.yaml +++ b/k8s/network-policy.yaml @@ -21,11 +21,17 @@ spec: ports: - protocol: TCP port: 8080 - # Allow from Prometheus scraping (if in monitoring namespace) + # Allow from Prometheus scraping (monitoring namespace) + # MUST select on kubernetes.io/metadata.name, not a plain `name:` label. + # Only ingress-nginx carries a plain `name` label in this cluster; every + # other namespace has just kubernetes.io/metadata.name, so `name: monitoring` + # matched nothing and silently denied every Prometheus scrape of :8080 + # (surfaced as context-deadline-exceeded on all 3 gateway replicas while the + # ServiceMonitor itself was discovered and healthy). - from: - namespaceSelector: matchLabels: - name: monitoring + kubernetes.io/metadata.name: monitoring ports: - protocol: TCP port: 8080