homelab/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: api-gateway-alerts
  namespace: monitoring
  labels:
    release: prometheus
spec:
  groups:
    # ================================================================
    # SLA Targets (based on canary traffic baselines):
    #
    # Availability:  99.9% (43.8 min downtime/month)
    # LLM Chat:      p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
    # Embeddings:    p95 < 500ms
    # Rerank:        p95 < 500ms
    # Models list:   p95 < 300ms
    # Error rate:    < 1% (5xx), < 5% (4xx excluding auth)
    #
    # Baselines from 200-request canary run:
    #   qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
    #   rerank p99=218ms, models p99=277ms
    # SLA set at ~2x p99 for headroom.
    # ================================================================

    - name: api-gateway.availability
      rules:
        # Gateway pods not ready
        - alert: APIGatewayDown
          expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway has zero ready pods"

        # Gateway pod count below desired
        - alert: APIGatewayDegraded
          expr: |
            sum(kube_pod_status_ready{namespace="api",condition="true"})
            < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway {{ $value }} ready pods below desired replica count"

        # Blackbox probe down
        - alert: APIGatewayProbeDown
          expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway probe failed: {{ $labels.instance }}"

        # LLM serving pods not ready
        - alert: LLMServingDown
          expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "All LLM serving pods down"

        # Individual predictor down
        - alert: LLMPredictorDown
          expr: |
            kube_deployment_status_replicas_ready{namespace="llm-serving"}
            < kube_deployment_spec_replicas{namespace="llm-serving"}
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"

    - name: api-gateway.latency
      # SLA: latency thresholds at ~2x measured p99
      rules:
        # Ingress-level latency (all requests through nginx)
        - alert: APIGatewayLatencyHigh
          expr: |
            histogram_quantile(0.95,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"

        # Extreme latency (p99 > 5s)
        - alert: APIGatewayLatencyCritical
          expr: |
            histogram_quantile(0.99,
              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
            ) > 5
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"

    - name: api-gateway.errors
      rules:
        # 5xx error rate > 1%
        - alert: APIGateway5xxErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.01
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"

        # Total error rate > 10% (including 4xx)
        - alert: APIGatewayHighErrorRate
          expr: |
            sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
            / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
            > 0.10
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"

    - name: api-gateway.resources
      rules:
        # Gateway pod restart
        - alert: APIGatewayRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "API Gateway pod {{ $labels.pod }} restarted"

        # LLM predictor restart
        - alert: LLMPredictorRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "LLM predictor {{ $labels.pod }} restarted"

        # Gateway high memory (>80% of limit)
        - alert: APIGatewayHighMemory
          expr: |
            sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
            / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
            > 0.8
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"