k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention) - Promtail daemonset (pod + talos journal logs) - Prometheus + kube-state-metrics - Grafana dashboards (6-row template per service)
2026-07-11 19:17:28 -07:00
parent 674c8f0d66
commit 63d7256b9e
44 changed files with 3306 additions and 0 deletions
@@ -0,0 +1,34 @@
+# k8s/monitoring/ingress-alerts.yaml
+# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
+# a standalone CRD instance. Applied via the prometheus release's postsync hook
+# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ingress-nginx-rules
+  namespace: ingress-nginx
+spec:
+  groups:
+    - name: ingress-nginx.rules
+      rules:
+        - alert: IngressHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
+            / sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High 5xx rate on {{ $labels.ingress }}"
+            description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
+        - alert: IngressHighLatencyP95
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
+            ) > 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "p95 latency > 1s on {{ $labels.ingress }}"
+            description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: argocd-rules
+  namespace: argocd
+spec:
+  groups:
+    - name: argocd.rules
+      interval: 15s
+      rules:
+        - alert: ArgoCDHighErrorRate
+          expr: |
+            (
+              sum(rate(argocd_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(argocd_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Argo CD"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: ArgoCDApplicationSyncFailure
+          expr: |
+            argocd_app_health_degraded_total > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Argo CD applications in degraded state"
+            description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes"
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: authentik-rules
+  namespace: iam
+spec:
+  groups:
+    - name: authentik.rules
+      interval: 15s
+      rules:
+        - alert: AuthentikHighErrorRate
+          expr: |
+            (
+              sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
+              /
+              sum(rate(authentik_http_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Authentik"
+            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: AuthentikOutpostDown
+          expr: |
+            authentik_outpost_total_up == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Authentik outpost is down"
+            description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"
@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: forgejo-rules
+  namespace: forgejo
+spec:
+  groups:
+    - name: forgejo.rules
+      interval: 15s
+      rules:
+        - alert: ForgejoHighErrorRate
+          expr: |
+            (
+              sum(rate(forgejo_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(forgejo_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Forgejo"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: grafana-rules
+  namespace: logging
+spec:
+  groups:
+    - name: grafana.rules
+      interval: 15s
+      rules:
+        - alert: GrafanaHighErrorRate
+          expr: |
+            (
+              sum(rate(grafana_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(grafana_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Grafana"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
@@ -0,0 +1,57 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: minio-rules
+  namespace: storage
+spec:
+  groups:
+    - name: minio.rules
+      interval: 15s
+      rules:
+        - alert: MinIOHighErrorRate
+          expr: |
+            (
+              sum(rate(minio_s3_requests_total{error="true"}[5m]))
+              /
+              sum(rate(minio_s3_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on MinIO"
+            description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: MinIODiskSpaceLow
+          expr: |
+            (
+              minio_cluster_capacity_usable_bytes
+              /
+              minio_cluster_capacity_raw_total_bytes
+            ) < 0.1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO disk space critically low"
+            description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
+
+        - alert: MinIOReplicationLag
+          expr: |
+            minio_replication_metrics_replicating_byte_count > 1073741824
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "MinIO replication lag detected"
+            description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
+
+        - alert: MinIODriveOffline
+          expr: |
+            minio_cluster_health_drives_offline > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO drive offline"
+            description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: story-crater-backend-rules
+  namespace: story-crater-backend
+spec:
+  groups:
+    - name: story-crater-backend.rules
+      interval: 15s
+      rules:
+        - alert: StoryCraterBackendHighErrorRate
+          expr: |
+            (
+              sum(rate(story_crater_app_metric_total{severity="error"}[5m]))
+              /
+              sum(rate(story_crater_messages_handled_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High application error rate on story-crater-backend"
+            description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: StoryCraterBackendCheckLatencySLOBreach
+          expr: |
+            histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)"
+            description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms"
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: story-crater-frontend-rules
+  namespace: story-crater-frontend
+spec:
+  groups:
+    - name: story-crater-frontend.rules
+      interval: 15s
+      rules:
+        - alert: StoryCraterFrontendHighErrorRate
+          expr: |
+            (
+              sum(rate(story_crater_frontend_http_requests_total{status=~"5.."}[5m]))
+              /
+              sum(rate(story_crater_frontend_http_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on story-crater-frontend"
+            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: StoryCraterFrontendWebVitalsLCPRegression
+          expr: |
+            histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms) > 2500
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LCP (Largest Contentful Paint) regression on story-crater-frontend"
+            description: "LCP p75 exceeded 2500ms for 10 minutes. Value: {{ $value }}ms"
@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: vault-rules
+  namespace: storage
+spec:
+  groups:
+    - name: vault.rules
+      interval: 15s
+      rules:
+        - alert: VaultHighErrorRate
+          expr: |
+            (
+              sum(rate(vault_core_handle_request_total{error="true"}[5m]))
+              /
+              sum(rate(vault_core_handle_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Vault"
+            description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: VaultSealed
+          expr: |
+            vault_core_unsealed == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Vault is sealed"
+            description: "Vault has been sealed for 1 minute. Immediate attention required."