apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: api-gateway-alerts
  namespace: monitoring
  labels:
    release: prometheus
spec:
  groups:
    # ================================================================
    # SLA Targets (based on canary traffic baselines):
    #
    # Availability:  99.9% (43.8 min downtime/month)
    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
    # Embeddings:    p95 < 500ms
    # Rerank:        p95 < 500ms
    # Models list:   p95 < 300ms
    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
    #
    # Baselines from 200-request canary run:
    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
    #   rerank p99=218ms, models p99=277ms
    # SLA set at ~2x p99 for headroom.
    # ================================================================

    - name: api-gateway.availability
      rules:
        # Gateway pods not ready
        - alert: APIGatewayDown
          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway has zero ready pods"

        # Gateway pod count below desired
        - alert: APIGatewayDegraded
          expr: |
            sum(kube_pod_status_ready{namespace="api",condition="true"})
            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway {{ $value }} ready pods below desired replica count"

        # Blackbox probe down
        - alert: APIGatewayProbeDown
          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway probe failed: {{ $labels.instance }}"

        # LLM serving pods not ready
        - alert: LLMServingDown
          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "All LLM serving pods down"

        # Individual predictor down
        - alert: LLMPredictorDown
          expr: |
            kube_deployment_status_replicas_ready{namespace="llm-serving"}
            < kube_deployment_spec_replicas{namespace="llm-serving"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"

    - name: api-gateway.latency
      # SLA: latency thresholds at ~2x measured p99
      rules:
        # Ingress-level latency (all requests through nginx)
        - alert: APIGatewayLatencyHigh
          expr: |
            histogram_quantile(0.95,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"

        # Extreme latency (p99 > 5s)
        - alert: APIGatewayLatencyCritical
          expr: |
            histogram_quantile(0.99,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 5
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"

    - name: api-gateway.errors
      rules:
        # 5xx error rate > 1%
        - alert: APIGateway5xxErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.01
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"

        # Total error rate > 10% (including 4xx)
        - alert: APIGatewayHighErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.10
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"

    - name: api-gateway.resources
      rules:
        # Gateway pod restart
        - alert: APIGatewayRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway pod {{ $labels.pod }} restarted"

        # LLM predictor restart
        - alert: LLMPredictorRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "LLM predictor {{ $labels.pod }} restarted"

        # Gateway high memory (>80% of limit)
        - alert: APIGatewayHighMemory
          expr: |
            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
            > 0.8
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"

    - name: api-gateway.admission
      # The admission gate fails OPEN when its view of the engine gauges goes
      # stale, so a broken observer degrades silently to the pre-#64 behaviour
      # of admitting everything. These two series are the only detection —
      # the gate has no automated tests.
      rules:
        # The gate fails OPEN when it cannot read the engine, so "no data" must
        # alert: otherwise a missing emitter, a renamed metric, or a broken
        # scrape is indistinguishable from a healthy observer. Both expressions
        # therefore OR in an absent() guard on the metric itself.
        - alert: APIGatewayObserverDegraded
          expr: observer_degraded == 1 or absent(observer_degraded)
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: "Gateway admission observer degraded or not reporting — gate is failing open"
            description: "observer_degraded is either 1 or absent. Absent means the emitter is missing, renamed, or unscraped, which is the fail-open case."

        # Engine gauges went stale, or are not being reported at all
        - alert: GatewayEngineStateStale
          expr: increase(engine_state_stale_total[15m]) > 0 or absent(engine_state_stale_total)
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: "Gateway engine state stale or not reporting — admission may be based on stale gauges"
            description: "Non-zero means the observer lost the engine; absent means the metric is not being emitted at all."

        # The acceptance metric for reasoning.yaml's prefill change. Preemption
        # of a 65k prompt costs a full re-prefill (~250s at the measured
        # 262 tok/s), so any sustained rate is a regression, not noise.
        - alert: LLMPreemptions
          # Guarded absent() is deliberately NOT used bare here: the engine
          # restarts (strategy Recreate) and the series legitimately vanishes
          # for the length of a boot, so a bare absent() would false-fire on
          # every rollout. It only fires when the metric is missing WHILE the
          # predictor is up, which is the real "renamed or unscraped" case.
          expr: |
            increase(vllm:num_preemptions_total[15m]) > 0
            or (absent(vllm:num_preemptions_total) and on() up{job="reasoning-predictor"} == 1)
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "vLLM preempting requests ({{ $value }} in 15m) — KV cache over-committed, re-prefill cost is severe"
            description: "Any sustained preemption rate is a regression for the reasoning prefill change: evicting a 65k-token prompt costs a full re-prefill (~250s at the measured 262 tok/s). Orchestrator verified `up{job=\"reasoning-predictor\"}` is the live job label — if that label is wrong the guard is inert but harmless."

        # This change trades single-request latency for fairness: capping a long
        # prompt at 2048 tokens/step means it takes more steps to finish, so a
        # lone large prompt can now be slower even while preemptions stay 0.
        # Without this alert the change has no way to show a regression.
        - alert: LLMPrefillStarvation
          expr: |
            (sum(vllm:num_requests_waiting) > 0 and max(vllm:kv_cache_usage_perc) < 0.5)
            or histogram_quantile(0.95, sum(rate(vllm:time_to_first_token_seconds_bucket[10m])) by (le)) > 60
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "reasoning queueing or TTFT p95 above 60s while KV is under 50% — scheduling, not capacity"
            description: "Queue depth with spare KV means requests are being starved rather than blocked on memory; a high TTFT p95 with headroom points at the same. Both series verified live (vllm:num_requests_waiting and vllm:time_to_first_token_seconds exist)."