From 0c52dec155c9296ad92d182854512fcaa50a8195 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Sun, 16 Aug 2026 07:01:15 -0700 Subject: [PATCH] feat(api): add Kong RED metrics for LLM routes Cluster-wide prometheus KongClusterPlugin (kong-metrics.yaml) + chart-native ServiceMonitor (kong-values.yaml) expose kong_http_requests_total/kong_latency_bucket/kong_bandwidth_bytes for every route, LLM and otherwise. Dashboard filters to route=~"llm-.*" for request rate, error rate, p95 upstream latency, and bandwidth. Token-count metrics still need ai-proxy-advanced (Enterprise-only); not attempted. --- k8s/apps/api/kong-metrics.yaml | 19 +++++++++++++++++++ k8s/apps/api/kong-values.yaml | 10 ++++++++++ k8s/apps/api/kustomization.yaml | 1 + .../monitoring/dashboards/llm-frontend.yaml | 17 ++++++++++------- 4 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 k8s/apps/api/kong-metrics.yaml diff --git a/k8s/apps/api/kong-metrics.yaml b/k8s/apps/api/kong-metrics.yaml new file mode 100644 index 0000000..d490e06 --- /dev/null +++ b/k8s/apps/api/kong-metrics.yaml @@ -0,0 +1,19 @@ +# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the +# ingress controller apply it to every route on this Kong instance, so all +# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics +# without touching llm-routes.yaml. Scraped via kong-values.yaml's +# serviceMonitor (status listener, already on by chart default at :8100). +apiVersion: configuration.konghq.com/v1 +kind: KongClusterPlugin +metadata: + name: prometheus + annotations: + kubernetes.io/ingress.class: kong + labels: + global: "true" +plugin: prometheus +config: + status_code_metrics: true + latency_metrics: true + bandwidth_metrics: true + upstream_health_metrics: true diff --git a/k8s/apps/api/kong-values.yaml b/k8s/apps/api/kong-values.yaml index f737bf5..8f284e5 100644 --- a/k8s/apps/api/kong-values.yaml +++ b/k8s/apps/api/kong-values.yaml @@ -118,6 +118,16 @@ podDisruptionBudget: enabled: true minAvailable: 1 +# Status listener (metrics/health) is on by default at :8100 (chart default, +# verified via `helm show values`). This just wires the ServiceMonitor the +# chart already knows how to generate for it, so kong_http_requests_total / +# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the +# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml. +serviceMonitor: + enabled: true + labels: + release: kube-prometheus-stack + # Spread the two replicas across nodes; `ScheduleAnyway` so a single-node # situation degrades to co-location instead of leaving a pod Pending. topologySpreadConstraints: diff --git a/k8s/apps/api/kustomization.yaml b/k8s/apps/api/kustomization.yaml index 35023fd..5a9ea06 100644 --- a/k8s/apps/api/kustomization.yaml +++ b/k8s/apps/api/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization # or it is silently dropped with no error and no drift shown. resources: - ingress.yaml + - kong-metrics.yaml - llm-routes.yaml - model-auth.yaml # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own diff --git a/k8s/infra/monitoring/dashboards/llm-frontend.yaml b/k8s/infra/monitoring/dashboards/llm-frontend.yaml index 41f9a28..8537562 100644 --- a/k8s/infra/monitoring/dashboards/llm-frontend.yaml +++ b/k8s/infra/monitoring/dashboards/llm-frontend.yaml @@ -7,12 +7,15 @@ metadata: grafana_dashboard: "1" annotations: grafana_folder: "LLM" - # No ServiceMonitor/PodMonitor exists yet for Kong or the KServe - # predictors (ornith/reasoning/qwen/embeddings/reranker/verifier), so - # there is no request-rate/latency/token metric to show. This dashboard - # is resources + logs only, sourced from cAdvisor/kube-state-metrics - # (cluster-wide, no extra scrape config needed) and Loki. Add RED-metric - # panels once a metrics exporter exists for those services. + # Request rate/error/latency/bandwidth now come from Kong's prometheus + # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the + # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) -- + # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/ + # embeddings/rerank uniformly without per-backend instrumentation. + # Token-count metrics are still not available: that needs response-body + # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only). + # Predictor-level metrics (native Ollama/vLLM stats) also still need a + # dedicated exporter -- not added here. data: llm-frontend.json: | - {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":3},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":13},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":23},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]} + {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}