poimen-memory/k8s/monitoring/alerts.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: poimen-memory-alerts
  namespace: monitoring
  labels:
    release: prometheus
spec:
  groups:
    # ================================================================
    # SLA Targets:
    #   Availability:  99.9%
    #   Ingest p95:    < 10s (includes LLM extraction)
    #   Query p95:     < 2s
    #   Error rate:    < 5%
    #   DB connection: always available
    # ================================================================

    - name: poimen-memory.availability
      rules:
        - alert: PoimenMemoryDown
          expr: sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"}) == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Poimen Memory has zero ready pods"

        - alert: PoimenMemoryRestarted
          expr: increase(kube_pod_container_status_restarts_total{namespace="poimen",pod=~"poimen-memory.*"}[15m]) > 0
          for: 0m
          labels:
            severity: warning
          annotations:
            summary: "Poimen Memory pod {{ $labels.pod }} restarted"

        - alert: PoimenMemoryHealthCheckFailing
          expr: rate(memory_health_check_failures_total[5m]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen Memory health checks failing"

    - name: poimen-memory.ingest
      rules:
        - alert: PoimenIngestErrorRate
          expr: |
            rate(memory_ingest_errors_total[5m])
            / (rate(memory_ingest_requests_total[5m]) > 0)
            > 0.05
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen ingest error rate {{ $value | humanizePercentage }}"

        - alert: PoimenIngestLatencyHigh
          expr: |
            histogram_quantile(0.95,
              rate(memory_ingest_duration_seconds_bucket[5m])
            ) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen ingest p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <10s)"

        - alert: PoimenLLMExtractionSlow
          expr: |
            histogram_quantile(0.95,
              rate(memory_ingest_llm_extract_duration_seconds_bucket[5m])
            ) > 30
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "LLM entity extraction p95 {{ $value | printf \"%.1f\" }}s"

        - alert: PoimenIngestQueueBacklog
          expr: memory_ingest_queue_size > 100
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Poimen ingest queue size {{ $value }} (backlog)"

    - name: poimen-memory.query
      rules:
        - alert: PoimenQueryErrorRate
          expr: |
            rate(memory_query_errors_total[5m])
            / (rate(memory_query_requests_total[5m]) > 0)
            > 0.05
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen query error rate {{ $value | humanizePercentage }}"

        - alert: PoimenQueryLatencyHigh
          expr: |
            histogram_quantile(0.95,
              rate(memory_query_duration_seconds_bucket[5m])
            ) > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen query p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"

        - alert: PoimenEmbeddingFailures
          expr: rate(memory_query_embedding_failures_total[5m]) > 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Poimen embedding failures — semantic search degraded"

        - alert: PoimenQueryEmptyResultRate
          expr: |
            rate(memory_query_empty_results_total[5m])
            / (rate(memory_query_requests_total[5m]) > 0)
            > 0.5
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "{{ $value | humanizePercentage }} of queries returning empty results"

    - name: poimen-memory.dependencies
      rules:
        - alert: PoimenDatabaseDown
          expr: memory_dependency_db_up == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Poimen Memory cannot reach database"

        - alert: PoimenDatabaseLatencyHigh
          expr: memory_dependency_db_latency_seconds > 1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen DB latency {{ $value | printf \"%.1f\" }}s"

        - alert: PoimenEmbeddingServiceDown
          expr: memory_dependency_embedding_up == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Poimen embedding service unreachable — ingest + semantic query broken"

        - alert: PoimenLLMServiceDown
          expr: memory_dependency_llm_up == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen LLM service unreachable — entity extraction degraded"

    - name: poimen-memory.resources
      rules:
        - alert: PoimenMemoryHighMemory
          expr: |
            sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)
            / sum(kube_pod_container_resource_limits{namespace="poimen",container="poimen-memory",resource="memory"}) by (pod)
            > 0.8
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Poimen Memory pod {{ $labels.pod }} at {{ $value | humanizePercentage }} memory limit"

        - alert: PoimenDBPoolExhausted
          expr: memory_db_pool_active >= memory_db_pool_size
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Poimen DB connection pool exhausted: {{ $value }} active = pool size"

    - name: poimen-memory.errors
      rules:
        - alert: PoimenAuthFailureSpike
          expr: |
            sum(rate(memory_ingest_auth_failures_total[5m]))
            + sum(rate(memory_query_auth_failures_total[5m]))
            > 1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen auth failures: {{ $value | printf \"%.1f\" }}/sec"

        - alert: PoimenUnexpectedErrors
          expr: rate(memory_error_unexpected_total[5m]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Poimen unexpected errors at {{ $value | printf \"%.2f\" }}/sec"