homelab/k8s/infra/monitoring/alerts/cluster-alerts.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cluster-alerts
  namespace: monitoring
  labels:
    release: prometheus
spec:
  groups:
    - name: cluster.availability
      rules:
        # Node down
        - alert: NodeNotReady
          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Node {{ $labels.node }} is NotReady"

        # Pod stuck pending (scheduling failure)
        - alert: PodStuckPending
          expr: sum(kube_pod_status_phase{phase="Pending"}) > 0
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "{{ $value }} pod(s) stuck in Pending state for >10m"

        # CrashLoopBackOff
        - alert: PodCrashLooping
          expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} in CrashLoopBackOff"

        # OOMKilled spike
        - alert: OOMKilledSpike
          expr: sum(increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[1h])) > 3
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "{{ $value }} OOMKilled events in last hour"

        # Deployment replicas unavailable
        - alert: DeploymentReplicasUnavailable
          expr: kube_deployment_status_replicas_unavailable > 0
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.deployment }} has {{ $value }} unavailable replicas"

    - name: cluster.jobs
      rules:
        # Job failed
        - alert: JobFailed
          expr: kube_job_status_failed > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed"

        # Job stuck running >2h
        - alert: JobStuckRunning
          expr: |
            kube_job_status_active == 1
            and on(job_name,namespace)
            (time() - kube_job_status_start_time) > 7200
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Job {{ $labels.namespace }}/{{ $labels.job_name }} running >2h"

        # CronJob missed schedule
        - alert: CronJobMissedSchedule
          expr: |
            (time() - kube_cronjob_status_last_schedule_time) > 2 * (kube_cronjob_spec_next_schedule_time - kube_cronjob_status_last_schedule_time)
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} missed schedule"

    - name: cluster.resources
      rules:
        # Node CPU >90% sustained
        - alert: NodeHighCPU
          expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Node {{ $labels.instance }} CPU at {{ $value | printf \"%.0f\" }}%"

        # Node memory >90% sustained
        - alert: NodeHighMemory
          expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Node {{ $labels.instance }} memory at {{ $value | printf \"%.0f\" }}%"

        # Node disk >85%
        - alert: NodeDiskFull
          expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Node {{ $labels.instance }} disk at {{ $value | printf \"%.0f\" }}%"

        # Container restart storm (>5 restarts in 15m)
        - alert: ContainerRestartStorm
          expr: sum(increase(kube_pod_container_status_restarts_total[15m])) by (namespace, pod) > 5
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted {{ $value | printf \"%.0f\" }} times in 15m"

    - name: cluster.storage
      rules:
        # Longhorn drive offline
        - alert: LonghornDriveOffline
          expr: longhorn_disk_health != 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Longhorn disk {{ $labels.node }} unhealthy"

    - name: cluster.dns
      rules:
        # CoreDNS errors spike
        - alert: CoreDNSErrorSpike
          expr: sum(rate(coredns_dns_responses_total{rcode=~"SERVFAIL"}[5m])) > 0.5
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "CoreDNS SERVFAIL rate {{ $value | printf \"%.2f\" }}/s"

    - name: cluster.probes
      rules:
        # Any blackbox probe down
        - alert: ServiceProbeDown
          expr: probe_success == 0
          for: 3m
          labels:
            severity: critical
          annotations:
            summary: "Probe failed: {{ $labels.instance }}"

        # Probe latency >2s
        - alert: ServiceProbeSlow
          expr: probe_duration_seconds > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Probe slow ({{ $value | printf \"%.1f\" }}s): {{ $labels.instance }}"

        # Certificate expiry <14 days
        - alert: CertificateExpiringSoon
          expr: (certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Certificate {{ $labels.name }} expires in {{ $value | printf \"%.0f\" }} days"