feat: add cluster and api-gateway alert rules with SLA targets
This commit is contained in:
@@ -0,0 +1,160 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: api-gateway-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
groups:
|
||||
# ================================================================
|
||||
# SLA Targets (based on canary traffic baselines):
|
||||
#
|
||||
# Availability: 99.9% (43.8 min downtime/month)
|
||||
# LLM Chat: p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith)
|
||||
# Embeddings: p95 < 500ms
|
||||
# Rerank: p95 < 500ms
|
||||
# Models list: p95 < 300ms
|
||||
# Error rate: < 1% (5xx), < 5% (4xx excluding auth)
|
||||
#
|
||||
# Baselines from 200-request canary run:
|
||||
# qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms,
|
||||
# rerank p99=218ms, models p99=277ms
|
||||
# SLA set at ~2x p99 for headroom.
|
||||
# ================================================================
|
||||
|
||||
- name: api-gateway.availability
|
||||
rules:
|
||||
# Gateway pods not ready
|
||||
- alert: APIGatewayDown
|
||||
expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "API Gateway has zero ready pods"
|
||||
|
||||
# Gateway pod count below desired
|
||||
- alert: APIGatewayDegraded
|
||||
expr: |
|
||||
sum(kube_pod_status_ready{namespace="api",condition="true"})
|
||||
< kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "API Gateway {{ $value }} ready pods below desired replica count"
|
||||
|
||||
# Blackbox probe down
|
||||
- alert: APIGatewayProbeDown
|
||||
expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "API Gateway probe failed: {{ $labels.instance }}"
|
||||
|
||||
# LLM serving pods not ready
|
||||
- alert: LLMServingDown
|
||||
expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "All LLM serving pods down"
|
||||
|
||||
# Individual predictor down
|
||||
- alert: LLMPredictorDown
|
||||
expr: |
|
||||
kube_deployment_status_replicas_ready{namespace="llm-serving"}
|
||||
< kube_deployment_spec_replicas{namespace="llm-serving"}
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)"
|
||||
|
||||
- name: api-gateway.latency
|
||||
# SLA: latency thresholds at ~2x measured p99
|
||||
rules:
|
||||
# Ingress-level latency (all requests through nginx)
|
||||
- alert: APIGatewayLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
|
||||
) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
|
||||
|
||||
# Extreme latency (p99 > 5s)
|
||||
- alert: APIGatewayLatencyCritical
|
||||
expr: |
|
||||
histogram_quantile(0.99,
|
||||
sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le)
|
||||
) > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)"
|
||||
|
||||
- name: api-gateway.errors
|
||||
rules:
|
||||
# 5xx error rate > 1%
|
||||
- alert: APIGateway5xxErrorRate
|
||||
expr: |
|
||||
sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m]))
|
||||
/ sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
|
||||
> 0.01
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)"
|
||||
|
||||
# Total error rate > 10% (including 4xx)
|
||||
- alert: APIGatewayHighErrorRate
|
||||
expr: |
|
||||
sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m]))
|
||||
/ sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m]))
|
||||
> 0.10
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)"
|
||||
|
||||
- name: api-gateway.resources
|
||||
rules:
|
||||
# Gateway pod restart
|
||||
- alert: APIGatewayRestarted
|
||||
expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "API Gateway pod {{ $labels.pod }} restarted"
|
||||
|
||||
# LLM predictor restart
|
||||
- alert: LLMPredictorRestarted
|
||||
expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "LLM predictor {{ $labels.pod }} restarted"
|
||||
|
||||
# Gateway high memory (>80% of limit)
|
||||
- alert: APIGatewayHighMemory
|
||||
expr: |
|
||||
sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod)
|
||||
/ sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod)
|
||||
> 0.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit"
|
||||
@@ -0,0 +1,177 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: cluster-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
groups:
|
||||
- name: cluster.availability
|
||||
rules:
|
||||
# Node down
|
||||
- alert: NodeNotReady
|
||||
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.node }} is NotReady"
|
||||
|
||||
# Pod stuck pending (scheduling failure)
|
||||
- alert: PodStuckPending
|
||||
expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
|
||||
|
||||
# CrashLoopBackOff
|
||||
- alert: PodCrashLooping
|
||||
expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"
|
||||
|
||||
# OOMKilled spike
|
||||
- alert: OOMKilledSpike
|
||||
expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $value }} OOMKilled events in last hour"
|
||||
|
||||
# Deployment replicas unavailable
|
||||
- alert: DeploymentReplicasUnavailable
|
||||
expr: kube_deployment_status_replicas_unavailable > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"
|
||||
|
||||
- name: cluster.jobs
|
||||
rules:
|
||||
# Job failed
|
||||
- alert: JobFailed
|
||||
expr: kube_job_status_failed > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"
|
||||
|
||||
# Job stuck running >2h
|
||||
- alert: JobStuckRunning
|
||||
expr: |
|
||||
kube_job_status_active == 1
|
||||
and on(job_name,namespace)
|
||||
(time() - kube_job_status_start_time) > 7200
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"
|
||||
|
||||
# CronJob missed schedule
|
||||
- alert: CronJobMissedSchedule
|
||||
expr: |
|
||||
(time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"
|
||||
|
||||
- name: cluster.resources
|
||||
rules:
|
||||
# Node CPU >90% sustained
|
||||
- alert: NodeHighCPU
|
||||
expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"
|
||||
|
||||
# Node memory >90% sustained
|
||||
- alert: NodeHighMemory
|
||||
expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"
|
||||
|
||||
# Node disk >85%
|
||||
- alert: NodeDiskFull
|
||||
expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"
|
||||
|
||||
# Container restart storm (>5 restarts in 15m)
|
||||
- alert: ContainerRestartStorm
|
||||
expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"
|
||||
|
||||
- name: cluster.storage
|
||||
rules:
|
||||
# Longhorn drive offline
|
||||
- alert: LonghornDriveOffline
|
||||
expr: longhorn_disk_health != 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Longhorn disk {{ $labels.node }} unhealthy"
|
||||
|
||||
- name: cluster.dns
|
||||
rules:
|
||||
# CoreDNS errors spike
|
||||
- alert: CoreDNSErrorSpike
|
||||
expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"
|
||||
|
||||
- name: cluster.probes
|
||||
rules:
|
||||
# Any blackbox probe down
|
||||
- alert: ServiceProbeDown
|
||||
expr: probe_success == 0
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Probe failed: {{ $labels.instance }}"
|
||||
|
||||
# Probe latency >2s
|
||||
- alert: ServiceProbeSlow
|
||||
expr: probe_duration_seconds > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"
|
||||
|
||||
# Certificate expiry <14 days
|
||||
- alert: CertificateExpiringSoon
|
||||
expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"
|
||||
Reference in New Issue
Block a user