feat: add monitoring resources (alerts, dashboard, service-monitor)
- PrometheusRule with 16 alerts across 6 groups: availability, ingest, query, dependencies, resources, errors - Grafana dashboard (ConfigMap, sidecar auto-discovery): 8 rows covering overview, ingest pipeline, query perf, context retrieval, database, relevance, error breakdown, app resources, logs - ServiceMonitor moved from k8s/app/ to k8s/monitoring/ - Dashboard generator script (Python, matches homelab pattern) - Separate k8s/monitoring/ path for ArgoCD Application - sqlx::migrate!() added to server startup for seamless migrations
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: poimen-memory-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: prometheus
|
||||
spec:
|
||||
groups:
|
||||
# ================================================================
|
||||
# SLA Targets:
|
||||
# Availability: 99.9%
|
||||
# Ingest p95: < 10s (includes LLM extraction)
|
||||
# Query p95: < 2s
|
||||
# Error rate: < 5%
|
||||
# DB connection: always available
|
||||
# ================================================================
|
||||
|
||||
- name: poimen-memory.availability
|
||||
rules:
|
||||
- alert: PoimenMemoryDown
|
||||
expr: sum(kube_pod_status_ready{namespace="poimen",pod=~"poimen-memory.*",condition="true"}) == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen Memory has zero ready pods"
|
||||
|
||||
- alert: PoimenMemoryRestarted
|
||||
expr: increase(kube_pod_container_status_restarts_total{namespace="poimen",pod=~"poimen-memory.*"}[15m]) > 0
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory pod {{ $labels.pod }} restarted"
|
||||
|
||||
- alert: PoimenMemoryHealthCheckFailing
|
||||
expr: rate(memory_health_check_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory health checks failing"
|
||||
|
||||
- name: poimen-memory.ingest
|
||||
rules:
|
||||
- alert: PoimenIngestErrorRate
|
||||
expr: |
|
||||
rate(memory_ingest_errors_total[5m])
|
||||
/ (rate(memory_ingest_requests_total[5m]) > 0)
|
||||
> 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest error rate {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: PoimenIngestLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_ingest_duration_seconds_bucket[5m])
|
||||
) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <10s)"
|
||||
|
||||
- alert: PoimenLLMExtractionSlow
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_ingest_llm_extract_duration_seconds_bucket[5m])
|
||||
) > 30
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "LLM entity extraction p95 {{ $value | printf \"%.1f\" }}s"
|
||||
|
||||
- alert: PoimenIngestQueueBacklog
|
||||
expr: memory_ingest_queue_size > 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen ingest queue size {{ $value }} (backlog)"
|
||||
|
||||
- name: poimen-memory.query
|
||||
rules:
|
||||
- alert: PoimenQueryErrorRate
|
||||
expr: |
|
||||
rate(memory_query_errors_total[5m])
|
||||
/ (rate(memory_query_requests_total[5m]) > 0)
|
||||
> 0.05
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen query error rate {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: PoimenQueryLatencyHigh
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
rate(memory_query_duration_seconds_bucket[5m])
|
||||
) > 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen query p95 latency {{ $value | printf \"%.1f\" }}s (SLA: <2s)"
|
||||
|
||||
- alert: PoimenEmbeddingFailures
|
||||
expr: rate(memory_query_embedding_failures_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen embedding failures — semantic search degraded"
|
||||
|
||||
- alert: PoimenQueryEmptyResultRate
|
||||
expr: |
|
||||
rate(memory_query_empty_results_total[5m])
|
||||
/ (rate(memory_query_requests_total[5m]) > 0)
|
||||
> 0.5
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "{{ $value | humanizePercentage }} of queries returning empty results"
|
||||
|
||||
- name: poimen-memory.dependencies
|
||||
rules:
|
||||
- alert: PoimenDatabaseDown
|
||||
expr: memory_dependency_db_up == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen Memory cannot reach database"
|
||||
|
||||
- alert: PoimenDatabaseLatencyHigh
|
||||
expr: memory_dependency_db_latency_seconds > 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen DB latency {{ $value | printf \"%.1f\" }}s"
|
||||
|
||||
- alert: PoimenEmbeddingServiceDown
|
||||
expr: memory_dependency_embedding_up == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen embedding service unreachable — ingest + semantic query broken"
|
||||
|
||||
- alert: PoimenLLMServiceDown
|
||||
expr: memory_dependency_llm_up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen LLM service unreachable — entity extraction degraded"
|
||||
|
||||
- name: poimen-memory.resources
|
||||
rules:
|
||||
- alert: PoimenMemoryHighMemory
|
||||
expr: |
|
||||
sum(container_memory_working_set_bytes{namespace="poimen",container="poimen-memory"}) by (pod)
|
||||
/ sum(kube_pod_container_resource_limits{namespace="poimen",container="poimen-memory",resource="memory"}) by (pod)
|
||||
> 0.8
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen Memory pod {{ $labels.pod }} at {{ $value | humanizePercentage }} memory limit"
|
||||
|
||||
- alert: PoimenDBPoolExhausted
|
||||
expr: memory_db_pool_active >= memory_db_pool_size
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Poimen DB connection pool exhausted: {{ $value }} active = pool size"
|
||||
|
||||
- name: poimen-memory.errors
|
||||
rules:
|
||||
- alert: PoimenAuthFailureSpike
|
||||
expr: |
|
||||
sum(rate(memory_ingest_auth_failures_total[5m]))
|
||||
+ sum(rate(memory_query_auth_failures_total[5m]))
|
||||
> 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen auth failures: {{ $value | printf \"%.1f\" }}/sec"
|
||||
|
||||
- alert: PoimenUnexpectedErrors
|
||||
expr: rate(memory_error_unexpected_total[5m]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Poimen unexpected errors at {{ $value | printf \"%.2f\" }}/sec"
|
||||
Reference in New Issue
Block a user