poimen-memory/k8s/infra/prometheus-alerts.yaml

# Prometheus alerting rules for Poimen Memory (O12)
# Deploy: kubectl apply -f k8s/infra/prometheus-alerts.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: poimen-memory-alerts
  namespace: poimen
  labels:
    app: poimen-memory
    prometheus: k8s
    role: alert-rules
spec:
  groups:
    - name: poimen-memory.availability
      rules:
        - alert: MemoryServiceDown
          expr: up{job="poimen-memory"} == 0
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Poimen memory service is down"
            description: "Memory service has been unreachable for > 2 minutes"

        - alert: MemoryDBDown
          expr: memory_dependency_db_up == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Memory service cannot reach database"
            description: "DB dependency health check failing for > 1 minute"

        - alert: MemoryEmbeddingDown
          expr: memory_dependency_embedding_up == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Embedding service unreachable"
            description: "Embedding dependency health check failing for > 5 minutes"

    - name: poimen-memory.latency
      rules:
        - alert: MemoryIngestLatencyHigh
          expr: histogram_quantile(0.95, rate(memory_ingest_duration_seconds_bucket[5m])) > 5
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Ingest p95 latency > 5s"
            description: "95th percentile ingest latency is {{ $value }}s"

        - alert: MemoryQueryLatencyHigh
          expr: histogram_quantile(0.95, rate(memory_query_duration_seconds_bucket[5m])) > 2
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Query p95 latency > 2s"
            description: "95th percentile query latency is {{ $value }}s"

        - alert: MemoryEmbeddingLatencyHigh
          expr: histogram_quantile(0.95, rate(memory_query_embedding_duration_seconds_bucket[5m])) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Embedding p95 latency > 10s"
            description: "95th percentile embedding call latency is {{ $value }}s"

    - name: poimen-memory.errors
      rules:
        - alert: MemoryIngestErrorRateHigh
          expr: rate(memory_ingest_errors_total[5m]) / rate(memory_ingest_requests_total[5m]) > 0.1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Ingest error rate > 10%"
            description: "{{ $value | humanizePercentage }} of ingest requests are failing"

        - alert: MemoryQueryErrorRateHigh
          expr: rate(memory_query_errors_total[5m]) / rate(memory_query_requests_total[5m]) > 0.1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Query error rate > 10%"
            description: "{{ $value | humanizePercentage }} of query requests are failing"

        - alert: MemoryEmbeddingFailureRate
          expr: rate(memory_query_embedding_failures_total[5m]) > 0.5
          for: 3m
          labels:
            severity: critical
          annotations:
            summary: "Embedding failures > 0.5/s"
            description: "Embedding service failing at {{ $value }}/s — queries cannot embed"

    - name: poimen-memory.storage
      rules:
        - alert: MemoryDBPoolExhausted
          expr: memory_db_pool_idle == 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "DB connection pool exhausted"
            description: "No idle DB connections for > 5 minutes"

        - alert: MemoryWriteErrorsHigh
          expr: rate(memory_write_errors_total[5m]) > 1
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Write errors > 1/s"
            description: "Database write errors at {{ $value }}/s"

    - name: poimen-memory.quality
      rules:
        - alert: MemoryRelevanceLow
          expr: memory_relevance_precision < 0.3
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Retrieval relevance precision < 30%"
            description: "Relevance precision is {{ $value | humanizePercentage }}"