diff --git a/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml b/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml new file mode 100644 index 0000000..49e904b --- /dev/null +++ b/k8s/infra/monitoring/alerts/api-gateway-alerts.yaml @@ -0,0 +1,160 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: api-gateway-alerts + namespace: monitoring + labels: + release: prometheus +spec: + groups: + # ================================================================ + # SLA Targets (based on canary traffic baselines): + # + # Availability: 99.9% (43.8 min downtime/month) + # LLM Chat: p95 < 1s (qwen), p95 < 2s (reasoning), p95 < 5s (ornith) + # Embeddings: p95 < 500ms + # Rerank: p95 < 500ms + # Models list: p95 < 300ms + # Error rate: < 1% (5xx), < 5% (4xx excluding auth) + # + # Baselines from 200-request canary run: + # qwen p99=609ms, reasoning p99=328ms, embeddings p99=287ms, + # rerank p99=218ms, models p99=277ms + # SLA set at ~2x p99 for headroom. + # ================================================================ + + - name: api-gateway.availability + rules: + # Gateway pods not ready + - alert: APIGatewayDown + expr: sum(kube_pod_status_ready{namespace="api",condition="true"}) == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "API Gateway has zero ready pods" + + # Gateway pod count below desired + - alert: APIGatewayDegraded + expr: | + sum(kube_pod_status_ready{namespace="api",condition="true"}) + < kube_deployment_spec_replicas{namespace="api",deployment="api-gateway"} + for: 5m + labels: + severity: warning + annotations: + summary: "API Gateway {{ $value }} ready pods below desired replica count" + + # Blackbox probe down + - alert: APIGatewayProbeDown + expr: probe_success{instance=~".*api.riotpiao.com.*"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "API Gateway probe failed: {{ $labels.instance }}" + + # LLM serving pods not ready + - alert: LLMServingDown + expr: sum(kube_pod_status_ready{namespace="llm-serving",condition="true"}) == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "All LLM serving pods down" + + # Individual predictor down + - alert: LLMPredictorDown + expr: | + kube_deployment_status_replicas_ready{namespace="llm-serving"} + < kube_deployment_spec_replicas{namespace="llm-serving"} + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $labels.deployment }} has {{ $value }} ready (below desired)" + + - name: api-gateway.latency + # SLA: latency thresholds at ~2x measured p99 + rules: + # Ingress-level latency (all requests through nginx) + - alert: APIGatewayLatencyHigh + expr: | + histogram_quantile(0.95, + sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le) + ) > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "API Gateway p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)" + + # Extreme latency (p99 > 5s) + - alert: APIGatewayLatencyCritical + expr: | + histogram_quantile(0.99, + sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress="api"}[5m])) by (le) + ) > 5 + for: 5m + labels: + severity: critical + annotations: + summary: "API Gateway p99 latency {{ $value | printf \"%.1f\" }}s (SLA: <5s)" + + - name: api-gateway.errors + rules: + # 5xx error rate > 1% + - alert: APIGateway5xxErrorRate + expr: | + sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"5.."}[5m])) + / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) + > 0.01 + for: 5m + labels: + severity: critical + annotations: + summary: "API Gateway 5xx rate {{ $value | humanizePercentage }} (SLA: <1%)" + + # Total error rate > 10% (including 4xx) + - alert: APIGatewayHighErrorRate + expr: | + sum(rate(nginx_ingress_controller_requests{ingress="api",status=~"[45].."}[5m])) + / sum(rate(nginx_ingress_controller_requests{ingress="api"}[5m])) + > 0.10 + for: 10m + labels: + severity: warning + annotations: + summary: "API Gateway total error rate {{ $value | humanizePercentage }} (SLA: <10%)" + + - name: api-gateway.resources + rules: + # Gateway pod restart + - alert: APIGatewayRestarted + expr: increase(kube_pod_container_status_restarts_total{namespace="api"}[15m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: "API Gateway pod {{ $labels.pod }} restarted" + + # LLM predictor restart + - alert: LLMPredictorRestarted + expr: increase(kube_pod_container_status_restarts_total{namespace="llm-serving"}[15m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: "LLM predictor {{ $labels.pod }} restarted" + + # Gateway high memory (>80% of limit) + - alert: APIGatewayHighMemory + expr: | + sum(container_memory_working_set_bytes{namespace="api",container="gateway"}) by (pod) + / sum(kube_pod_container_resource_limits{namespace="api",container="gateway",resource="memory"}) by (pod) + > 0.8 + for: 10m + labels: + severity: warning + annotations: + summary: "Gateway pod {{ $labels.pod }} memory at {{ $value | humanizePercentage }} of limit" diff --git a/k8s/infra/monitoring/alerts/cluster-alerts.yaml b/k8s/infra/monitoring/alerts/cluster-alerts.yaml new file mode 100644 index 0000000..040055b --- /dev/null +++ b/k8s/infra/monitoring/alerts/cluster-alerts.yaml @@ -0,0 +1,177 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: cluster-alerts + namespace: monitoring + labels: + release: prometheus +spec: + groups: + - name: cluster.availability + rules: + # Node down + - alert: NodeNotReady + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Node {{ $labels.node }} is NotReady" + + # Pod stuck pending (scheduling failure) + - alert: PodStuckPending + expr: sum(kube_pod_status_phase{phase="Pending"}) > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $value }} pod(s) stuck in Pending state for >10m" + + # CrashLoopBackOff + - alert: PodCrashLooping + expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff" + + # OOMKilled spike + - alert: OOMKilledSpike + expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3 + for: 0m + labels: + severity: warning + annotations: + summary: "{{ $value }} OOMKilled events in last hour" + + # Deployment replicas unavailable + - alert: DeploymentReplicasUnavailable + expr: kube_deployment_status_replicas_unavailable > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas" + + - name: cluster.jobs + rules: + # Job failed + - alert: JobFailed + expr: kube_job_status_failed > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed" + + # Job stuck running >2h + - alert: JobStuckRunning + expr: | + kube_job_status_active == 1 + and on(job_name,namespace) + (time() - kube_job_status_start_time) > 7200 + for: 0m + labels: + severity: warning + annotations: + summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h" + + # CronJob missed schedule + - alert: CronJobMissedSchedule + expr: | + (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time) + for: 10m + labels: + severity: warning + annotations: + summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule" + + - name: cluster.resources + rules: + # Node CPU >90% sustained + - alert: NodeHighCPU + expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%" + + # Node memory >90% sustained + - alert: NodeHighMemory + expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%" + + # Node disk >85% + - alert: NodeDiskFull + expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85 + for: 5m + labels: + severity: critical + annotations: + summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%" + + # Container restart storm (>5 restarts in 15m) + - alert: ContainerRestartStorm + expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5 + for: 0m + labels: + severity: warning + annotations: + summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m" + + - name: cluster.storage + rules: + # Longhorn drive offline + - alert: LonghornDriveOffline + expr: longhorn_disk_health != 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Longhorn disk {{ $labels.node }} unhealthy" + + - name: cluster.dns + rules: + # CoreDNS errors spike + - alert: CoreDNSErrorSpike + expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5 + for: 5m + labels: + severity: warning + annotations: + summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s" + + - name: cluster.probes + rules: + # Any blackbox probe down + - alert: ServiceProbeDown + expr: probe_success == 0 + for: 3m + labels: + severity: critical + annotations: + summary: "Probe failed: {{ $labels.instance }}" + + # Probe latency >2s + - alert: ServiceProbeSlow + expr: probe_duration_seconds > 2 + for: 5m + labels: + severity: warning + annotations: + summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}" + + # Certificate expiry <14 days + - alert: CertificateExpiringSoon + expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14 + for: 0m + labels: + severity: warning + annotations: + summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days" diff --git a/k8s/infra/monitoring/kustomization.yaml b/k8s/infra/monitoring/kustomization.yaml index 0dffaad..549ee07 100644 --- a/k8s/infra/monitoring/kustomization.yaml +++ b/k8s/infra/monitoring/kustomization.yaml @@ -12,6 +12,8 @@ resources: - alerts/svc-grafana-rules.yaml - alerts/svc-minio-rules.yaml - alerts/svc-vault-rules.yaml + - alerts/cluster-alerts.yaml + - alerts/api-gateway-alerts.yaml - servicemonitors/argocd.yaml - servicemonitors/authentik.yaml - servicemonitors/forgejo.yaml