refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,34 @@
|
||||
# k8s/monitoring/ingress-alerts.yaml
|
||||
# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
|
||||
# a standalone CRD instance. Applied via the prometheus release's postsync hook
|
||||
# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: ingress-nginx-rules
|
||||
namespace: ingress-nginx
|
||||
spec:
|
||||
groups:
|
||||
- name: ingress-nginx.rules
|
||||
rules:
|
||||
- alert: IngressHighErrorRate
|
||||
expr: |
|
||||
sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
|
||||
/ sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High 5xx rate on {{ $labels.ingress }}"
|
||||
description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
|
||||
- alert: IngressHighLatencyP95
|
||||
expr: |
|
||||
histogram_quantile(0.95,
|
||||
sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
|
||||
) > 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "p95 latency > 1s on {{ $labels.ingress }}"
|
||||
description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: argocd-rules
|
||||
namespace: argocd
|
||||
spec:
|
||||
groups:
|
||||
- name: argocd.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: ArgoCDHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(argocd_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(argocd_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Argo CD"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: ArgoCDApplicationSyncFailure
|
||||
expr: |
|
||||
argocd_app_health_degraded_total > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Argo CD applications in degraded state"
|
||||
description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes"
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: authentik-rules
|
||||
namespace: iam
|
||||
spec:
|
||||
groups:
|
||||
- name: authentik.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: AuthentikHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(authentik_http_requests_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Authentik"
|
||||
description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: AuthentikOutpostDown
|
||||
expr: |
|
||||
authentik_outpost_total_up == 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Authentik outpost is down"
|
||||
description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: forgejo-rules
|
||||
namespace: forgejo
|
||||
spec:
|
||||
groups:
|
||||
- name: forgejo.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: ForgejoHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(forgejo_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(forgejo_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Forgejo"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
@@ -0,0 +1,23 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: grafana-rules
|
||||
namespace: logging
|
||||
spec:
|
||||
groups:
|
||||
- name: grafana.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: GrafanaHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(grafana_http_request_total{status=~"5.."}[5m]))
|
||||
/
|
||||
sum(rate(grafana_http_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Grafana"
|
||||
description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
@@ -0,0 +1,57 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: minio-rules
|
||||
namespace: storage
|
||||
spec:
|
||||
groups:
|
||||
- name: minio.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: MinIOHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(minio_s3_requests_total{error="true"}[5m]))
|
||||
/
|
||||
sum(rate(minio_s3_requests_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on MinIO"
|
||||
description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: MinIODiskSpaceLow
|
||||
expr: |
|
||||
(
|
||||
minio_cluster_capacity_usable_bytes
|
||||
/
|
||||
minio_cluster_capacity_raw_total_bytes
|
||||
) < 0.1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MinIO disk space critically low"
|
||||
description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: MinIOReplicationLag
|
||||
expr: |
|
||||
minio_replication_metrics_replicating_byte_count > 1073741824
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "MinIO replication lag detected"
|
||||
description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
|
||||
|
||||
- alert: MinIODriveOffline
|
||||
expr: |
|
||||
minio_cluster_health_drives_offline > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "MinIO drive offline"
|
||||
description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: vault-rules
|
||||
namespace: storage
|
||||
spec:
|
||||
groups:
|
||||
- name: vault.rules
|
||||
interval: 15s
|
||||
rules:
|
||||
- alert: VaultHighErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(rate(vault_core_handle_request_total{error="true"}[5m]))
|
||||
/
|
||||
sum(rate(vault_core_handle_request_total[5m]))
|
||||
) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "High error rate on Vault"
|
||||
description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
|
||||
|
||||
- alert: VaultSealed
|
||||
expr: |
|
||||
vault_core_unsealed == 0
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Vault is sealed"
|
||||
description: "Vault has been sealed for 1 minute. Immediate attention required."
|
||||
Reference in New Issue
Block a user