apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: api-gateway
  namespace: monitoring
  labels:
    app.kubernetes.io/name: api-gateway
    app.kubernetes.io/part-of: api
spec:
  namespaceSelector:
    matchNames:
      - api
  selector:
    matchLabels:
      app: api-gateway
  endpoints:
    # api-gateway Service port IS named `http` (unlike the KServe ones), so
    # the name form works here.
    - port: http
      interval: 30s
      scrapeTimeout: 10s
      path: /metrics
      scheme: http

      # The gateway is the only source of per-request LLM latency that covers
      # EVERY model, including the two Ollama ones that expose no metrics at
      # all. Series it produces (internal/observability/prometheus.go):
      #   llm_ttft_seconds              histogram  time-to-first-token
      #   llm_itl_seconds               histogram  inter-token latency
      #   llm_tokens_total              counter    tokens generated
      #   queue_full_requests_total     counter    backpressure rejects
      #   warmup_requests_delayed_total counter    vLLM slot contention
      #   vllm_prefix_cache_hit_tokens_total counter prefix-cache hits
      #   request_duration_seconds      histogram  full request
      #
      # The Deployment already carried prometheus.io/scrape annotations, but
      # kube-prometheus-stack ignores pod annotations (no additionalScrape-
      # Configs, serviceMonitorSelectorNilUsesHelmValues: false) — so none of
      # the above was being collected. This ServiceMonitor is what actually
      # wires it up.
      metricRelabelings:
        # 3 gateway replicas each build their own in-process histogram. Without
        # this the label set is otherwise identical, so per-instance series
        # would collide conceptually in sum()-by aggregations; instance is
        # retained deliberately so per-replica skew stays visible.
        - sourceLabels: [__address__]
          targetLabel: instance
          replacement: $1