diff --git a/k8s/logging/.env.example b/k8s/logging/.env.example new file mode 100644 index 0000000..a1ad07e --- /dev/null +++ b/k8s/logging/.env.example @@ -0,0 +1,13 @@ +# logging/.env.example +# Copy to logging/.env and fill in real values. +# The real .env is gitignored — never commit it. +# Used by: k8s/logging/bootstrap.sh and k8s/monitoring/bootstrap.sh (Grafana upgrade step) + +# Grafana admin UI password +# openssl rand -base64 24 +GRAFANA_ADMIN_PASSWORD= + +# MinIO credentials — must match the values used in k8s/storage/.env +# Loki uses these to authenticate to the MinIO S3 backend +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= diff --git a/k8s/logging/bootstrap.sh b/k8s/logging/bootstrap.sh new file mode 100755 index 0000000..d82d397 --- /dev/null +++ b/k8s/logging/bootstrap.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# logging/bootstrap.sh +# Deploys MinIO + Loki + Promtail + Grafana into the logging namespace. +# +# Prerequisites: +# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) +# - helm >= 3.x installed +# - logging/.env file containing: +# GRAFANA_ADMIN_PASSWORD=... +# MINIO_ROOT_USER=... +# MINIO_ROOT_PASSWORD=... +# OR those variables already exported in the calling shell. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}" +export KUBECONFIG + +# ── Load credentials ────────────────────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + # shellcheck source=/dev/null + source "${SCRIPT_DIR}/.env" +fi + +for var in GRAFANA_ADMIN_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do + if [[ -z "${!var:-}" ]]; then + echo "ERROR: ${var} is not set. Export it or place it in logging/.env" + exit 1 + fi +done + +# ── Namespace ───────────────────────────────────────────────────────────────── +echo "==> Creating logging namespace with privileged pod security..." +kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace logging \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/enforce-version=latest \ + --overwrite + +# ── Helm repos ──────────────────────────────────────────────────────────────── +echo "==> Adding Helm repositories..." +helm repo add minio https://charts.min.io +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update minio grafana + +# ── MinIO ───────────────────────────────────────────────────────────────────── +echo "==> Installing MinIO..." +helm upgrade --install minio minio/minio \ + --namespace logging \ + --values "${SCRIPT_DIR}/minio-values.yaml" \ + --set auth.rootUser="${MINIO_ROOT_USER}" \ + --set auth.rootPassword="${MINIO_ROOT_PASSWORD}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for MinIO Deployment to be ready..." +kubectl rollout status deployment/minio -n logging --timeout=120s + +# ── Create Loki buckets in MinIO ────────────────────────────────────────────── +echo "==> Creating Loki buckets in MinIO..." +MINIO_POD=$(kubectl get pod -n logging -l app=minio,release=minio -o jsonpath='{.items[0].metadata.name}') + +# Read the actual credentials MinIO is running with from the cluster secret +ACTUAL_MINIO_USER=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootUser}' | base64 --decode) +ACTUAL_MINIO_PASS=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootPassword}' | base64 --decode) + +kubectl exec -n logging "${MINIO_POD}" -- \ + mc alias set local http://localhost:9000 "${ACTUAL_MINIO_USER}" "${ACTUAL_MINIO_PASS}" +for bucket in loki-chunks loki-ruler loki-admin; do + kubectl exec -n logging "${MINIO_POD}" -- \ + mc mb --ignore-existing "local/${bucket}" + echo " bucket: ${bucket} ready" +done + +# ── Loki ────────────────────────────────────────────────────────────────────── +echo "==> Installing Loki (SingleBinary + MinIO backend)..." +helm upgrade --install loki grafana/loki \ + --namespace logging \ + --values "${SCRIPT_DIR}/loki-values.yaml" \ + --set loki.storage.s3.access_key_id="${ACTUAL_MINIO_USER}" \ + --set loki.storage.s3.secret_access_key="${ACTUAL_MINIO_PASS}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for Loki StatefulSet to be ready..." +kubectl rollout status statefulset/loki -n logging --timeout=120s + +# ── Promtail ────────────────────────────────────────────────────────────────── +echo "==> Installing Promtail..." +helm upgrade --install promtail grafana/promtail \ + --namespace logging \ + --values "${SCRIPT_DIR}/promtail-values.yaml" \ + --wait \ + --timeout 3m + +echo "==> Waiting for Promtail DaemonSet to be ready..." +kubectl rollout status daemonset/promtail -n logging --timeout=60s + +# ── Grafana ─────────────────────────────────────────────────────────────────── +echo "==> Installing Grafana..." +helm upgrade --install grafana grafana/grafana \ + --namespace logging \ + --values "${SCRIPT_DIR}/grafana-values.yaml" \ + --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for Grafana Deployment to be ready..." +kubectl rollout status deployment/grafana -n logging --timeout=120s + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo "==> Logging stack is up." +echo "" +echo "Grafana (log explorer UI):" +echo " kubectl port-forward -n logging svc/grafana 3000:80" +echo " http://localhost:3000 (admin / )" +echo "" +echo "MinIO console (S3 object browser):" +echo " kubectl port-forward -n logging svc/minio 9001:9001" +echo " http://localhost:9001 (${MINIO_ROOT_USER} / )" +echo "" +echo "MinIO S3 endpoint for other apps:" +echo " http://minio.logging.svc.cluster.local:9000" diff --git a/k8s/logging/grafana-values.yaml b/k8s/logging/grafana-values.yaml new file mode 100644 index 0000000..919d61c --- /dev/null +++ b/k8s/logging/grafana-values.yaml @@ -0,0 +1,185 @@ +# logging/grafana-values.yaml +# Grafana — dashboarding and log/metrics exploration UI. +# Deployed in the logging namespace alongside Loki and Promtail. +# +# Secrets never set here: +# adminPassword — injected via helmfile --set (GRAFANA_ADMIN_PASSWORD) +# OAuth client secret — mounted from the grafana-oidc K8s Secret (envFromSecret below) + +replicas: 1 + +# RWO PVC (Longhorn) — old pod must fully terminate before the new one can +# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure. +deploymentStrategy: + type: Recreate + +podAnnotations: + secret.reloader.stakater.com/reload: "grafana-oidc" + +adminUser: admin + +resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + +# ── Persistence ─────────────────────────────────────────────────────────────── +# Stores dashboards saved in the UI, datasource edits, and user sessions. +# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON. +persistence: + enabled: true + storageClassName: longhorn + accessModes: + - ReadWriteOnce + size: 5Gi + +# ── Grafana config ──────────────────────────────────────────────────────────── +grafana.ini: + server: + root_url: https://grafana.riotpiao.homelab.com + + # No anonymous read access — every user must log in via Authentik SSO. + auth.anonymous: + enabled: false + + # Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus. + explore: + enabled: true + + # WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown. + database: + wal: true + + # ── Authentik OIDC (generic OAuth2) ─────────────────────────────────────── + # Grafana v10+ supports OIDC auto-discovery; we wire it manually here because + # Authentik's discovery endpoint is internal-only (no external DNS for iam.svc). + # All URLs use the external hostname so CoreDNS rewrites them to + # authentik-server.iam.svc — this keeps the Host header correct so Authentik + # doesn't return localhost redirects in its token responses. + # + # role_attribute_path: JMESPath expression evaluated against the userinfo + # response. Members of the 'grafana-admins' Authentik group get Admin role; + # everyone else gets Viewer. The group name must match exactly what Authentik + # sends in the 'groups' claim. + auth.generic_oauth: + enabled: true + name: Authentik + allow_sign_up: true + client_id: grafana + scopes: openid email profile + auth_url: https://authentik.riotpiao.homelab.com/application/o/authorize/ + token_url: https://authentik.riotpiao.homelab.com/application/o/token/ + api_url: https://authentik.riotpiao.homelab.com/application/o/userinfo/ + role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'" + use_pkce: false + use_refresh_token: false + skip_org_role_sync: false + tls_skip_verify_insecure: true # Authentik uses self-signed cert; verify in prod + +# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s +# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts +# every key in that secret as an environment variable — keeps secrets out of +# values files and out of git. +envFromSecret: grafana-oidc + +# ── Datasources ─────────────────────────────────────────────────────────────── +# Provisioned at install — no manual "Add datasource" step in the UI. +# Loki is the default (log exploration); Prometheus is secondary (metrics). +# Both use in-cluster service DNS — Grafana never leaves the cluster for data. +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Loki + type: loki + uid: loki + access: proxy + url: http://loki.logging.svc.cluster.local:3100 + isDefault: true + version: 1 + editable: true + jsonData: + maxLines: 1000 + timeout: 60 + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090 + isDefault: false + version: 1 + editable: true + jsonData: + timeInterval: 30s + timeout: 60 + +# ── Dashboard providers ─────────────────────────────────────────────────────── +# Tells Grafana to watch a directory for dashboard JSON files. +# The `dashboards` block below populates that directory via an init container +# that downloads from grafana.com at install time. +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + +# ── Pre-loaded dashboards ────────────────────────────────────────────────────── +# Fetched from grafana.com by an init container at helm-install time and baked +# into ConfigMaps. gnetId + revision pin the exact dashboard version so an +# upstream change on grafana.com doesn't silently alter what's deployed. +dashboards: + default: + node-exporter-full: + gnetId: 1860 + revision: 37 + datasource: Prometheus + kubernetes-cluster: + gnetId: 7249 + revision: 1 + datasource: Prometheus + +# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint). +tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# ClusterIP only — access via ingress (grafana.riotpiao.homelab.com) or port-forward. +service: + type: ClusterIP + port: 80 + +# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress. +ingress: + enabled: false + +serviceMonitor: + enabled: false + +# ── Dashboard-as-code (sidecar discovery) ────────────────────────────────────── +# Additive to the gnetId-download mechanism above. The sidecar watches for +# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live — +# no Grafana restart needed when a new dashboard ConfigMap is applied. +sidecar: + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + folder: /var/lib/grafana/dashboards/custom + folderAnnotation: grafana_folder + provider: + name: custom + folder: "Homelab" + disableDelete: false + foldersFromFilesStructure: true + searchNamespace: ALL diff --git a/k8s/logging/loki-values.yaml b/k8s/logging/loki-values.yaml new file mode 100644 index 0000000..43c6b15 --- /dev/null +++ b/k8s/logging/loki-values.yaml @@ -0,0 +1,163 @@ +# logging/loki-values.yaml +# Grafana Loki — log aggregation backend for the homelab. +# Deployed in SingleBinary mode: one pod handles ingest, query, and compaction. +# Chunks are stored in MinIO (S3-compatible) — no local PVC needed for log data. +# +# MinIO credentials are injected at deploy time via helmfile --set: +# loki.storage.s3.accessKeyId ← MINIO_ROOT_USER +# loki.storage.s3.secretAccessKey ← MINIO_ROOT_PASSWORD +# The placeholder values below are overridden and never used. + +# ── Deployment mode ─────────────────────────────────────────────────────────── +# SingleBinary collapses all Loki components (ingester, querier, compactor, ruler) +# into one Deployment. Simpler ops for a homelab — no inter-component networking +# or separate scaling to worry about. The tradeoff is no horizontal scaling. +deploymentMode: SingleBinary + +loki: + # auth_enabled: false skips tenant header (X-Scope-OrgID) enforcement. + # All Promtail → Loki traffic is internal; multi-tenancy adds no value here. + auth_enabled: false + + # ── Replication ───────────────────────────────────────────────────────────── + # replication_factor: 1 — single replica, no write quorum needed. + # Higher values require multiple ingesters (only valid outside SingleBinary). + commonConfig: + replication_factor: 1 + + # ── Storage backend ────────────────────────────────────────────────────────── + # s3 type with s3ForcePathStyle: MinIO exposes buckets as paths + # (http://host:9000/bucket) not subdomains (http://bucket.host:9000). + # insecure: true — MinIO in this cluster has no TLS; traffic stays in-cluster. + # Three buckets: chunks (log data), ruler (recording/alerting rules), admin (index). + storage: + type: s3 + s3: + endpoint: minio.storage.svc.cluster.local:9000 + region: us-east-1 # MinIO ignores region but Loki's S3 client requires it + s3ForcePathStyle: true + insecure: true + access_key_id: "" # overridden by helmfile --set (MINIO_ROOT_USER) + secret_access_key: "" # overridden by helmfile --set (MINIO_ROOT_PASSWORD) + bucketNames: + chunks: loki-chunks + ruler: loki-ruler + admin: loki-admin + + # ── Schema ─────────────────────────────────────────────────────────────────── + # v13 + TSDB is the current recommended schema (Loki 3.x). + # from: sets the date after which this schema applies — logs before this date + # would use a previous schema config (none exists here, so all logs use v13). + # period: 24h means one index table per day in the object store. + schemaConfig: + configs: + - from: "2024-01-01" + store: boltdb-shipper + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + + # ── Ingester ───────────────────────────────────────────────────────────────── + # Controls how log chunks are buffered before being flushed to MinIO. + # chunk_idle_period: flush a chunk if no new logs arrive for 3m (reduces + # open chunk count). chunk_retain_period: keep flushed chunks in memory + # briefly so late-arriving out-of-order logs can still be appended. + # WAL persists in-memory chunks to disk — required for boltdb-shipper. + ingester: + chunk_idle_period: 3m + chunk_block_size: 262144 + chunk_retain_period: 1m + wal: + dir: /var/loki/wal + + # ── Compactor ──────────────────────────────────────────────────────────────── + # Merges small index files written by ingesters into larger ones, and + # enforces retention by deleting chunks older than retention_period. + # retention_delete_delay: waits 2h after marking chunks for deletion before + # actually removing them — safety window if a query is still reading them. + compactor: + working_directory: /var/loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: s3 + + # ── Limits ─────────────────────────────────────────────────────────────────── + # retention_period: 10 days. Homelab — no long-term log storage needed. + # ingestion_rate_mb / burst: rate limits per tenant (single tenant here). + # 4 MB/s steady, 6 MB/s burst — plenty for a 3-node cluster. + # max_query_series: caps how many unique label combinations a single query + # can return — prevents runaway cardinality queries from OOMing the pod. + # max_query_lookback: hard cap matching retention_period (no point querying + # further back than what's stored). + # allow_structured_metadata: false — required for boltdb-shipper index store. + limits_config: + retention_period: 240h + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_query_series: 5000 + max_query_lookback: 240h + max_label_names_per_series: 30 + allow_structured_metadata: false + # Query timeout: increased to 120s to tolerate 5+ second network latency spikes + # Default: 30s — too aggressive when pod-to-pod latency hits 5-10s + query_timeout: 120s + +# ── Single binary pod ───────────────────────────────────────────────────────── +singleBinary: + replicas: 1 + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 1000m + memory: 512Mi + + # Persistent storage for indices + WAL. Durable log chunks live in MinIO. + persistence: + enabled: true + storageClassName: longhorn + size: 5Gi + +# ── Disable micro-service replicas ─────────────────────────────────────────── +# In SingleBinary mode the chart still templates read/write/backend Deployments +# unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster. +read: + replicas: 0 +write: + replicas: 0 +backend: + replicas: 0 + +# Nginx gateway is only needed for multi-replica deployments that split read +# and write paths. Not used in SingleBinary. +gateway: + enabled: false + +# Disable the bundled MinIO subchart — we run our own minio-az-a/az-b releases +# in the storage namespace with site replication. +minio: + enabled: false + +# ── Monitoring ──────────────────────────────────────────────────────────────── +# Self-monitoring ships a Grafana Agent operator to scrape Loki's own metrics. +# We use kube-prometheus-stack for that instead — avoid running two agents. +# lokiCanary sends synthetic log lines to verify the write→read pipeline; +# useful in production, too noisy for a homelab. +monitoring: + selfMonitoring: + enabled: false + grafanaAgent: + installOperator: false + lokiCanary: + enabled: false + serviceMonitor: + enabled: false + +test: + enabled: false diff --git a/k8s/logging/minio-values.yaml b/k8s/logging/minio-values.yaml new file mode 100644 index 0000000..529737b --- /dev/null +++ b/k8s/logging/minio-values.yaml @@ -0,0 +1,42 @@ +# logging/minio-values.yaml +# Official MinIO chart (minio/minio from https://charts.min.io). +# Single pod with console built-in on port 9001. +# rootUser and rootPassword are injected via --set at install time from .env. + +mode: standalone + +rootUser: "" # injected via --set +rootPassword: "" # injected via --set + +persistence: + enabled: true + storageClass: longhorn + accessMode: ReadWriteOnce + size: 100Gi + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +service: + type: ClusterIP + port: 9000 + +consoleService: + type: ClusterIP + port: 9001 + +# Disable ingress — access via kubectl port-forward +ingress: + enabled: false + +consoleIngress: + enabled: false + +metrics: + serviceMonitor: + enabled: false diff --git a/k8s/logging/promtail-values.yaml b/k8s/logging/promtail-values.yaml new file mode 100644 index 0000000..df8758d --- /dev/null +++ b/k8s/logging/promtail-values.yaml @@ -0,0 +1,152 @@ +# logging/promtail-values.yaml +# Promtail — log shipper DaemonSet. One pod per node; reads container logs +# from /var/log/pods and Talos kernel/service logs from /var/log/journal, +# then pushes them to Loki. + +daemonset: + enabled: true + +config: + logLevel: warn # info is noisy at homelab scale; warn keeps Promtail quiet + serverPort: 3101 + + # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false). + clients: + - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push + + snippets: + scrapeConfigs: | + # ── Pod logs ────────────────────────────────────────────────────────── + # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod). + # cri pipeline stage parses the CRI-O/containerd log format so timestamps + # and stream (stdout/stderr) are extracted properly before the line is + # forwarded to Loki. + # relabel_configs build useful labels: namespace, pod, container, node, + # and a job label of the form "namespace/pod-name" for easy filtering. + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + pipeline_stages: + - drop: + expression: '(health|heartbeat|ping|keepalive|level="debug"|"timeout".*"retrying")' + - sampling: + rate: 0.1 + enabled: true + relabel_configs: + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: __host__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_name + target_label: job + - action: replace + source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - action: replace + source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - action: replace + source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - action: replace + source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # ── Talos systemd journal ────────────────────────────────────────────── + # Talos runs containerd, kubelet, and kernel messages through systemd- + # journald — they never appear in /var/log/pods. This job reads the + # binary journal directly and emits unit (systemd unit name) and node + # labels so you can filter by service (e.g. unit="kubelet.service"). + # max_age: 12h — only tail recent journal entries on startup; prevents + # Promtail from replaying hours of history after a pod restart. + - job_name: systemd-journal + journal: + path: /var/log/journal + max_age: 12h + labels: + job: systemd-journal + relabel_configs: + - source_labels: [__journal__systemd_unit] + target_label: unit + - source_labels: [__journal__hostname] + target_label: node + +# ── Volume mounts ───────────────────────────────────────────────────────────── +# hostPath mounts give Promtail access to the node's log directories. +# /var/log/pods — container stdout/stderr (written by containerd's CRI layer) +# /var/log/journal — Talos systemd journal (binary format, read via journald API) +# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes +# before journald has written anything. +defaultVolumes: + - name: pods-logs + hostPath: + path: /var/log/pods + - name: journal + hostPath: + path: /var/log/journal + type: DirectoryOrCreate + +defaultVolumeMounts: + - name: pods-logs + mountPath: /var/log/pods + readOnly: true + - name: journal + mountPath: /var/log/journal + readOnly: true + +resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + +# ── Security context ────────────────────────────────────────────────────────── +# privileged: true — required to open the binary journal on Talos (journald +# uses file locking that non-privileged processes can't bypass). +# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods +# even with a read-only root filesystem. Without this, pod logs from containers +# running as non-root UIDs would be unreadable. +# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to +# write to its own container filesystem. +# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+ +# rejects privileged containers that explicitly set allowPrivilegeEscalation: false. +containerSecurityContext: + privileged: true + allowPrivilegeEscalation: true + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - DAC_READ_SEARCH + +# runAsUser/Group: 0 (root) — needed to access journal files and pod log +# directories that are owned by root on Talos nodes. +podSecurityContext: + runAsUser: 0 + runAsGroup: 0 + +# Tolerate every taint so Promtail runs on ALL nodes including the CP. +# Without this, the control-plane node's logs (etcd, kube-apiserver) would +# be missing from Loki entirely. +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +serviceMonitor: + enabled: false diff --git a/k8s/monitoring/alerts/ingress-alerts.yaml b/k8s/monitoring/alerts/ingress-alerts.yaml new file mode 100644 index 0000000..a845426 --- /dev/null +++ b/k8s/monitoring/alerts/ingress-alerts.yaml @@ -0,0 +1,34 @@ +# k8s/monitoring/ingress-alerts.yaml +# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are +# a standalone CRD instance. Applied via the prometheus release's postsync hook +# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ingress-nginx-rules + namespace: ingress-nginx +spec: + groups: + - name: ingress-nginx.rules + rules: + - alert: IngressHighErrorRate + expr: | + sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress) + / sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High 5xx rate on {{ $labels.ingress }}" + description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes." + - alert: IngressHighLatencyP95 + expr: | + histogram_quantile(0.95, + sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le) + ) > 1 + for: 10m + labels: + severity: warning + annotations: + summary: "p95 latency > 1s on {{ $labels.ingress }}" + description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes." diff --git a/k8s/monitoring/alerts/svc-argocd-rules.yaml b/k8s/monitoring/alerts/svc-argocd-rules.yaml new file mode 100644 index 0000000..7f98542 --- /dev/null +++ b/k8s/monitoring/alerts/svc-argocd-rules.yaml @@ -0,0 +1,33 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: argocd-rules + namespace: argocd +spec: + groups: + - name: argocd.rules + interval: 15s + rules: + - alert: ArgoCDHighErrorRate + expr: | + ( + sum(rate(argocd_http_request_total{status=~"5.."}[5m])) + / + sum(rate(argocd_http_request_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on Argo CD" + description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: ArgoCDApplicationSyncFailure + expr: | + argocd_app_health_degraded_total > 0 + for: 10m + labels: + severity: warning + annotations: + summary: "Argo CD applications in degraded state" + description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes" diff --git a/k8s/monitoring/alerts/svc-authentik-rules.yaml b/k8s/monitoring/alerts/svc-authentik-rules.yaml new file mode 100644 index 0000000..3833e0d --- /dev/null +++ b/k8s/monitoring/alerts/svc-authentik-rules.yaml @@ -0,0 +1,33 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: authentik-rules + namespace: iam +spec: + groups: + - name: authentik.rules + interval: 15s + rules: + - alert: AuthentikHighErrorRate + expr: | + ( + sum(rate(authentik_http_requests_total{status=~"5.."}[5m])) + / + sum(rate(authentik_http_requests_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on Authentik" + description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: AuthentikOutpostDown + expr: | + authentik_outpost_total_up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Authentik outpost is down" + description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes" diff --git a/k8s/monitoring/alerts/svc-forgejo-rules.yaml b/k8s/monitoring/alerts/svc-forgejo-rules.yaml new file mode 100644 index 0000000..218aa32 --- /dev/null +++ b/k8s/monitoring/alerts/svc-forgejo-rules.yaml @@ -0,0 +1,23 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: forgejo-rules + namespace: forgejo +spec: + groups: + - name: forgejo.rules + interval: 15s + rules: + - alert: ForgejoHighErrorRate + expr: | + ( + sum(rate(forgejo_http_request_total{status=~"5.."}[5m])) + / + sum(rate(forgejo_http_request_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on Forgejo" + description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}" diff --git a/k8s/monitoring/alerts/svc-grafana-rules.yaml b/k8s/monitoring/alerts/svc-grafana-rules.yaml new file mode 100644 index 0000000..e3ad9d6 --- /dev/null +++ b/k8s/monitoring/alerts/svc-grafana-rules.yaml @@ -0,0 +1,23 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: grafana-rules + namespace: logging +spec: + groups: + - name: grafana.rules + interval: 15s + rules: + - alert: GrafanaHighErrorRate + expr: | + ( + sum(rate(grafana_http_request_total{status=~"5.."}[5m])) + / + sum(rate(grafana_http_request_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on Grafana" + description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}" diff --git a/k8s/monitoring/alerts/svc-minio-rules.yaml b/k8s/monitoring/alerts/svc-minio-rules.yaml new file mode 100644 index 0000000..7974ab1 --- /dev/null +++ b/k8s/monitoring/alerts/svc-minio-rules.yaml @@ -0,0 +1,57 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: minio-rules + namespace: storage +spec: + groups: + - name: minio.rules + interval: 15s + rules: + - alert: MinIOHighErrorRate + expr: | + ( + sum(rate(minio_s3_requests_total{error="true"}[5m])) + / + sum(rate(minio_s3_requests_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on MinIO" + description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: MinIODiskSpaceLow + expr: | + ( + minio_cluster_capacity_usable_bytes + / + minio_cluster_capacity_raw_total_bytes + ) < 0.1 + for: 5m + labels: + severity: critical + annotations: + summary: "MinIO disk space critically low" + description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}" + + - alert: MinIOReplicationLag + expr: | + minio_replication_metrics_replicating_byte_count > 1073741824 + for: 15m + labels: + severity: warning + annotations: + summary: "MinIO replication lag detected" + description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B" + + - alert: MinIODriveOffline + expr: | + minio_cluster_health_drives_offline > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "MinIO drive offline" + description: "{{ $value | humanize }} drive(s) offline in MinIO cluster" diff --git a/k8s/monitoring/alerts/svc-story-crater-backend-rules.yaml b/k8s/monitoring/alerts/svc-story-crater-backend-rules.yaml new file mode 100644 index 0000000..ed90c69 --- /dev/null +++ b/k8s/monitoring/alerts/svc-story-crater-backend-rules.yaml @@ -0,0 +1,33 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: story-crater-backend-rules + namespace: story-crater-backend +spec: + groups: + - name: story-crater-backend.rules + interval: 15s + rules: + - alert: StoryCraterBackendHighErrorRate + expr: | + ( + sum(rate(story_crater_app_metric_total{severity="error"}[5m])) + / + sum(rate(story_crater_messages_handled_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High application error rate on story-crater-backend" + description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: StoryCraterBackendCheckLatencySLOBreach + expr: | + histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200 + for: 10m + labels: + severity: critical + annotations: + summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)" + description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms" diff --git a/k8s/monitoring/alerts/svc-story-crater-frontend-rules.yaml b/k8s/monitoring/alerts/svc-story-crater-frontend-rules.yaml new file mode 100644 index 0000000..b2dc55c --- /dev/null +++ b/k8s/monitoring/alerts/svc-story-crater-frontend-rules.yaml @@ -0,0 +1,33 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: story-crater-frontend-rules + namespace: story-crater-frontend +spec: + groups: + - name: story-crater-frontend.rules + interval: 15s + rules: + - alert: StoryCraterFrontendHighErrorRate + expr: | + ( + sum(rate(story_crater_frontend_http_requests_total{status=~"5.."}[5m])) + / + sum(rate(story_crater_frontend_http_requests_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on story-crater-frontend" + description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: StoryCraterFrontendWebVitalsLCPRegression + expr: | + histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms) > 2500 + for: 10m + labels: + severity: warning + annotations: + summary: "LCP (Largest Contentful Paint) regression on story-crater-frontend" + description: "LCP p75 exceeded 2500ms for 10 minutes. Value: {{ $value }}ms" diff --git a/k8s/monitoring/alerts/svc-vault-rules.yaml b/k8s/monitoring/alerts/svc-vault-rules.yaml new file mode 100644 index 0000000..6bf49a1 --- /dev/null +++ b/k8s/monitoring/alerts/svc-vault-rules.yaml @@ -0,0 +1,33 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: vault-rules + namespace: storage +spec: + groups: + - name: vault.rules + interval: 15s + rules: + - alert: VaultHighErrorRate + expr: | + ( + sum(rate(vault_core_handle_request_total{error="true"}[5m])) + / + sum(rate(vault_core_handle_request_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "High error rate on Vault" + description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}" + + - alert: VaultSealed + expr: | + vault_core_unsealed == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Vault is sealed" + description: "Vault has been sealed for 1 minute. Immediate attention required." diff --git a/k8s/monitoring/blackbox-exporter-values.yaml b/k8s/monitoring/blackbox-exporter-values.yaml new file mode 100644 index 0000000..0f7ae29 --- /dev/null +++ b/k8s/monitoring/blackbox-exporter-values.yaml @@ -0,0 +1,75 @@ +# k8s/monitoring/blackbox-exporter-values.yaml +# Active black-box HTTP probing of every ingress-exposed service — gives an +# uptime/availability signal independent of real traffic. Homelab services +# like Vault/MinIO/Longhorn UI get almost no organic requests, so ingress RED +# metrics alone can't tell "idle" from "down"; this closes that gap. + +config: + modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [] # any 2xx + follow_redirects: true + preferred_ip_protocol: "ip4" + tls_config: + insecure_skip_verify: true # homelab-ca is a private CA; skip verify for simplicity + +resources: + requests: + cpu: 20m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + +tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +serviceMonitor: + enabled: true + selfMonitor: + enabled: true + defaults: + interval: 30s + scrapeTimeout: 10s + module: http_2xx + targets: + - name: grafana + url: https://grafana.riotpiao.homelab.com/ + - name: loki + url: https://loki.riotpiao.homelab.com/ready + - name: authentik + url: https://authentik.riotpiao.homelab.com/-/health/ready/ + - name: vault + url: https://vault.riotpiao.homelab.com/v1/sys/health + - name: minio-console + url: https://minio.riotpiao.homelab.com/ + - name: minio-api + url: https://minio-api.riotpiao.homelab.com/minio/health/live + - name: prometheus + url: https://prometheus.riotpiao.homelab.com/-/healthy + - name: portainer + url: https://portainer.riotpiao.homelab.com/ + - name: forgejo + url: https://forgejo.riotpiao.homelab.com/api/healthz + - name: argocd + url: https://argocd.riotpiao.homelab.com/healthz + - name: longhorn + url: https://longhorn.riotpiao.homelab.com/ + +prometheusRule: + enabled: true + rules: + - alert: ServiceProbeDown + expr: probe_success == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Probe failing for {{ $labels.instance }}" + description: "Blackbox probe for {{ $labels.instance }} has failed for more than 5 minutes." diff --git a/k8s/monitoring/bootstrap.sh b/k8s/monitoring/bootstrap.sh new file mode 100755 index 0000000..cffe4d7 --- /dev/null +++ b/k8s/monitoring/bootstrap.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# monitoring/bootstrap.sh +# Deploys kube-prometheus-stack into the monitoring namespace, then upgrades +# Grafana (in the logging namespace) to add the Prometheus datasource and +# pre-built dashboards. +# +# Prerequisites: +# - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig) +# - helm >= 3.x +# - GRAFANA_ADMIN_PASSWORD set, or present in k8s/logging/.env +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}" +export KUBECONFIG + +# ── Namespace ───────────────────────────────────────────────────────────────── +echo "==> Creating monitoring namespace..." +kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - +kubectl label namespace monitoring \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/enforce-version=latest \ + --overwrite + +# ── Helm repo ───────────────────────────────────────────────────────────────── +echo "==> Adding prometheus-community Helm repo..." +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update prometheus-community + +# ── kube-prometheus-stack ───────────────────────────────────────────────────── +echo "==> Installing kube-prometheus-stack..." +helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \ + --namespace monitoring \ + --values "${SCRIPT_DIR}/prometheus-values.yaml" \ + --wait \ + --timeout 10m + +echo "==> Waiting for Prometheus StatefulSet..." +kubectl rollout status \ + statefulset/prometheus-prometheus-kube-prometheus-prometheus \ + -n monitoring --timeout=180s + +echo "==> Waiting for node-exporter DaemonSet..." +kubectl rollout status \ + daemonset/prometheus-prometheus-node-exporter \ + -n monitoring --timeout=60s + +echo "==> Waiting for kube-state-metrics..." +kubectl rollout status \ + deployment/prometheus-kube-state-metrics \ + -n monitoring --timeout=60s + +# ── Upgrade Grafana with Prometheus datasource + dashboards ─────────────────── +echo "" +echo "==> Upgrading Grafana to wire in Prometheus datasource + dashboards..." +LOGGING_DIR="${REPO_ROOT}/k8s/logging" + +if [[ -f "${LOGGING_DIR}/.env" ]]; then + # shellcheck source=/dev/null + source "${LOGGING_DIR}/.env" +fi + +if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then + echo "ERROR: GRAFANA_ADMIN_PASSWORD is not set. Export it or place it in k8s/logging/.env" + exit 1 +fi + +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update grafana + +helm upgrade grafana grafana/grafana \ + --namespace logging \ + --values "${LOGGING_DIR}/grafana-values.yaml" \ + --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \ + --wait \ + --timeout 5m + +echo "==> Waiting for Grafana rollout..." +kubectl rollout status deployment/grafana -n logging --timeout=120s + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo "==> Monitoring stack is up." +echo "" +echo "Prometheus UI:" +echo " kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090" +echo " http://localhost:9090 (Targets page shows node-exporter + kube-state-metrics)" +echo "" +echo "Grafana:" +echo " kubectl port-forward -n logging svc/grafana 3000:80" +echo " http://localhost:3000" +echo " Datasources: Loki (logs, default) + Prometheus (metrics)" +echo " Dashboards → General:" +echo " - Node Exporter Full (per-node CPU, RAM, disk, network)" +echo " - Kubernetes Cluster (pod resource usage across namespaces)" diff --git a/k8s/monitoring/dashboards/control-plane-logs.yaml b/k8s/monitoring/dashboards/control-plane-logs.yaml new file mode 100644 index 0000000..bd5458b --- /dev/null +++ b/k8s/monitoring/dashboards/control-plane-logs.yaml @@ -0,0 +1,55 @@ +# k8s/monitoring/dashboards/control-plane-logs.yaml +# Surfaces controller/control-plane logs that are already in Loki today +# (Promtail scrapes every namespace with no filter) — this dashboard is the +# "make it visible" piece, not new log collection. +apiVersion: v1 +kind: ConfigMap +metadata: + name: control-plane-logs-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + control-plane-logs.json: | + { + "title": "Cluster Control Plane & Controllers (Logs)", + "uid": "control-plane-logs", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-1h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, + "title": "Error rate by namespace", + "type": "timeseries", + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))" + } + ] + }, + { + "id": 2, + "title": "Control plane (kube-apiserver, controller-manager, scheduler)", + "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { "expr": "{namespace=\"kube-system\"}" } + ] + }, + { + "id": 3, + "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)", + "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/hardware-overview.yaml b/k8s/monitoring/dashboards/hardware-overview.yaml new file mode 100644 index 0000000..93307d9 --- /dev/null +++ b/k8s/monitoring/dashboards/hardware-overview.yaml @@ -0,0 +1,121 @@ +# k8s/monitoring/dashboards/hardware-overview.yaml +# Trimmed operator at-a-glance view across all nodes — node-exporter already +# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml), +# this is the quick health-check version, not a replacement for it. +apiVersion: v1 +kind: ConfigMap +metadata: + name: hardware-overview-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + hardware-overview.json: | + { + "title": "Hardware Statistics (Operator Overview)", + "uid": "hardware-overview", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, + "title": "Nodes up / down", + "type": "stat", + "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, + { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } + ] + } + }, + "targets": [ + { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" } + ] + }, + { + "id": 2, + "title": "CPU usage % by node", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, + "targets": [ + { + "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100", + "legendFormat": "{{instance}}" + } + ] + }, + { + "id": 3, + "title": "Memory usage % by node", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, + "targets": [ + { + "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100", + "legendFormat": "{{instance}}" + } + ] + }, + { + "id": 4, + "title": "Root filesystem usage % by node", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, + "targets": [ + { + "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100", + "legendFormat": "{{instance}}" + } + ] + }, + { + "id": 5, + "title": "Root filesystem space remaining", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "targets": [ + { + "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}", + "legendFormat": "{{instance}}" + } + ] + }, + { + "id": 6, + "title": "Network errors/drops by node", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" }, + { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" }, + { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" }, + { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" } + ] + }, + { + "id": 7, + "title": "Load average (1m / 5m) by node", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "node_load1", "legendFormat": "{{instance}} load1" }, + { "expr": "node_load5", "legendFormat": "{{instance}} load5" } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/kube-controller-health.yaml b/k8s/monitoring/dashboards/kube-controller-health.yaml new file mode 100644 index 0000000..4cf8cbe --- /dev/null +++ b/k8s/monitoring/dashboards/kube-controller-health.yaml @@ -0,0 +1,185 @@ +# k8s/monitoring/dashboards/kube-controller-health.yaml +# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus +# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is +# the one control-plane component that's still reachable (its ServiceMonitor +# targets the in-cluster `kubernetes` service, not localhost) — paired with +# kube-state-metrics signals as a proxy for controller/scheduler health. +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-controller-health-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + kube-controller-health.json: | + { + "title": "Kube-Controller Health", + "uid": "kube-controller-health", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, + "title": "API server — up", + "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, + { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } + ] + } + }, + "targets": [ + { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" } + ] + }, + { + "id": 2, + "title": "API server — request rate by verb/code", + "type": "timeseries", + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)", + "legendFormat": "{{verb}} {{code}}" + } + ] + }, + { + "id": 3, + "title": "API server — error rate % (5xx)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100", + "legendFormat": "5xx %" + } + ] + }, + { + "id": 4, + "title": "API server — latency p95 / p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 5, + "title": "Pods stuck Pending", + "type": "stat", + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "yellow" }, + { "value": 5, "color": "red" } + ] + } + } + }, + "targets": [ + { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" } + ] + }, + { + "id": 6, + "title": "CrashLoopBackOff containers", + "type": "stat", + "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "red" } + ] + } + } + }, + "targets": [ + { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" } + ] + }, + { + "id": 7, + "title": "Nodes NotReady", + "type": "stat", + "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "red" } + ] + } + } + }, + "targets": [ + { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" } + ] + }, + { + "id": 8, + "title": "Failed Jobs", + "type": "table", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true } + ] + }, + { + "id": 9, + "title": "Deployments with unavailable replicas", + "type": "table", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true } + ] + }, + { + "id": 10, + "title": "Container restart rate by pod", + "type": "timeseries", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)", + "legendFormat": "{{namespace}}/{{pod}}" + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/service-availability.yaml b/k8s/monitoring/dashboards/service-availability.yaml new file mode 100644 index 0000000..172e552 --- /dev/null +++ b/k8s/monitoring/dashboards/service-availability.yaml @@ -0,0 +1,172 @@ +# k8s/monitoring/dashboards/service-availability.yaml +# Active uptime/availability from blackbox-exporter probes — the signal that +# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics +# alone can't distinguish "idle" from "down". +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-availability-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + service-availability.json: | + { + "title": "Service Availability & Certificate Expiration", + "uid": "svc-availability", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-24h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, + "title": "Up / Down — all probed services", + "type": "stat", + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } }, + { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "red" }, + { "value": 1, "color": "green" } + ] + } + } + }, + "targets": [ + { "expr": "probe_success", "legendFormat": "{{instance}}" } + ] + }, + { + "id": 2, + "title": "Uptime % trend", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } }, + "targets": [ + { + "expr": "avg_over_time(probe_success[$__rate_interval]) * 100", + "legendFormat": "{{instance}}" + } + ] + }, + { + "id": 3, + "title": "Probe latency", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" } + ] + }, + { + "id": 4, + "title": "7-day SLO (% successful probes)", + "type": "table", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "red" }, + { "value": 99, "color": "yellow" }, + { "value": 99.9, "color": "green" } + ] + } + } + }, + "targets": [ + { + "expr": "avg_over_time(probe_success[7d]) * 100", + "format": "table", + "instant": true + } + ] + }, + { + "id": 5, + "title": "Services DOWN right now", + "type": "stat", + "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "red" } + ] + } + } + }, + "targets": [ + { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" } + ] + }, + { + "id": 6, + "title": "Certs expiring in < 14 days", + "type": "stat", + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "red" } + ] + } + } + }, + "targets": [ + { + "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)", + "legendFormat": "expiring" + } + ] + }, + { + "id": 7, + "title": "Certificate expiry — days remaining", + "type": "table", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "red" }, + { "value": 14, "color": "yellow" }, + { "value": 30, "color": "green" } + ] + } + } + }, + "targets": [ + { + "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", + "legendFormat": "{{name}}", + "format": "table", + "instant": true + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/service-golden-signals.yaml b/k8s/monitoring/dashboards/service-golden-signals.yaml new file mode 100644 index 0000000..9a0093a --- /dev/null +++ b/k8s/monitoring/dashboards/service-golden-signals.yaml @@ -0,0 +1,141 @@ +# k8s/monitoring/dashboards/service-golden-signals.yaml +# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx. +# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see +# sidecar.dashboards in k8s/logging/grafana-values.yaml. +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-golden-signals-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + service-golden-signals.json: | + { + "title": "Latency & Golden Signals (Ingress RED)", + "uid": "svc-golden-signals", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "ingress", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(nginx_ingress_controller_requests, ingress)", + "refresh": 2, + "includeAll": false + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Request rate by status — $ingress", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)", + "legendFormat": "{{status}}" + } + ] + }, + { + "id": 2, + "title": "Error rate % (4xx / 5xx) — $ingress", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100", + "legendFormat": "5xx" + }, + { + "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100", + "legendFormat": "4xx" + } + ] + }, + { + "id": 3, + "title": "Latency p50 / p95 / p99 — $ingress", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 4, + "title": "All services — traffic overview", + "type": "table", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))", + "format": "table", + "instant": true + } + ] + }, + { + "id": 5, + "title": "Customer-facing failures (5xx count, window total)", + "type": "stat", + "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "green" }, + { "value": 1, "color": "yellow" }, + { "value": 50, "color": "red" } + ] + } + } + }, + "targets": [ + { + "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)", + "legendFormat": "5xx total" + } + ] + }, + { + "id": 6, + "title": "Top 5 error-contributing services", + "type": "table", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))", + "format": "table", + "instant": true + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/service-internals.yaml b/k8s/monitoring/dashboards/service-internals.yaml new file mode 100644 index 0000000..17d115f --- /dev/null +++ b/k8s/monitoring/dashboards/service-internals.yaml @@ -0,0 +1,109 @@ +# k8s/monitoring/dashboards/service-internals.yaml +# Native per-service metrics — the "why" layer behind the ingress RED/uptime +# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline). +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-internals-dashboard + namespace: logging + labels: + grafana_dashboard: "1" +data: + service-internals.json: | + { + "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)", + "uid": "svc-internals", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" }, + { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" } + ] + }, + { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" } + ] + }, + { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "targets": [ + { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" } + ] + }, + { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "gitea_repositories", "legendFormat": "repos" }, + { "expr": "gitea_organizations", "legendFormat": "orgs" } + ] + }, + { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" }, + { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" } + ] + }, + { "id": 6, "title": "Argo CD — app sync/health status", "type": "table", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "argocd_app_info", "format": "table", "instant": true } + ] + }, + { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": 0, "color": "red" }, + { "value": 14, "color": "yellow" }, + { "value": 30, "color": "green" } + ] + } + } + }, + "targets": [ + { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" } + ] + }, + { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } }, + { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } } + ] + } + }, + "targets": [ + { "expr": "vault_core_unsealed", "legendFormat": "vault" } + ] + }, + { "id": 9, "title": "Longhorn — volume robustness", "type": "table", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "longhorn_volume_robustness", "format": "table", "instant": true } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/services/argocd.descriptor.yaml b/k8s/monitoring/dashboards/services/argocd.descriptor.yaml new file mode 100644 index 0000000..b31dcee --- /dev/null +++ b/k8s/monitoring/dashboards/services/argocd.descriptor.yaml @@ -0,0 +1,48 @@ +service: argocd +display_name: "Argo CD" +namespace: argocd +service_type: web-app +folder: "Argo CD" +jobs: [argocd-server, argocd-repo-server] +metrics: + prefix: argocd + up_selector: 'job=~"argocd-.*"' + rate: + metric: argocd_http_request_total + labels: [method, path, status] + duration: + metric: argocd_http_request_duration_seconds + type: histogram + unit: s + error: + metric: argocd_http_request_total + filter: 'status=~"5.."' + domain: + - metric: argocd_app_total + kind: gauge + - metric: argocd_app_sync_total + kind: counter + labels: [sync_status] + - metric: argocd_app_health_degraded_total + kind: counter + - metric: argocd_git_sync_total + kind: counter + labels: [git_operation, git_status] + - metric: argocd_reconcile_total + kind: counter + labels: [app_name] +correlation_ids: [] +logs: + loki_namespace_selector: 'namespace="argocd"' +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: AppSyncFailure + expr_template: gauge_above_threshold + metric: argocd_app_health_degraded_total + threshold: 1 + for: 10m + severity: warning diff --git a/k8s/monitoring/dashboards/services/authentik.descriptor.yaml b/k8s/monitoring/dashboards/services/authentik.descriptor.yaml new file mode 100644 index 0000000..6095e89 --- /dev/null +++ b/k8s/monitoring/dashboards/services/authentik.descriptor.yaml @@ -0,0 +1,64 @@ +# authentik.descriptor.yaml +# Build input for generating svc-authentik.yaml dashboard ConfigMap +# and svc-authentik-rules.yaml PrometheusRule. +# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. + +service: authentik +display_name: "Authentik" +namespace: iam +service_type: identity-provider +folder: "Authentik" + +jobs: + - authentik-server + - authentik-worker + +metrics: + prefix: authentik + up_selector: 'job="authentik-server"' + + rate: + metric: authentik_flows_execution_stage_time_count + labels: [flow_slug, stage_name] + + duration: + metric: authentik_main_request_duration_seconds + type: histogram + unit: s + + error: + metric: authentik_flows_cached + filter: null + + domain: + - metric: authentik_outpost_connection + kind: gauge + group_by: [outpost_name, outpost_type] + - metric: authentik_outposts_connected + kind: gauge + - metric: authentik_flows_cached + kind: gauge + - metric: authentik_policies_cached + kind: gauge + - metric: authentik_tasks_queued + kind: gauge + - metric: authentik_admin_workers + kind: gauge + +correlation_ids: [] + +logs: + loki_namespace_selector: 'namespace="iam"' + +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: OutpostDown + expr_template: gauge_below_threshold + metric: authentik_outpost_total_up + threshold: 1 + for: 5m + severity: warning diff --git a/k8s/monitoring/dashboards/services/forgejo.descriptor.yaml b/k8s/monitoring/dashboards/services/forgejo.descriptor.yaml new file mode 100644 index 0000000..b8242d9 --- /dev/null +++ b/k8s/monitoring/dashboards/services/forgejo.descriptor.yaml @@ -0,0 +1,39 @@ +service: forgejo +display_name: "Forgejo" +namespace: forgejo +service_type: web-app +folder: "Forgejo" +jobs: [forgejo] +metrics: + prefix: forgejo + up_selector: 'job="forgejo"' + rate: + metric: forgejo_http_request_total + labels: [method, status] + duration: + metric: forgejo_http_request_duration_seconds + type: histogram + unit: s + error: + metric: forgejo_http_request_total + filter: 'status=~"5.."' + domain: + - metric: forgejo_repositories_total + kind: gauge + - metric: forgejo_users_total + kind: gauge + - metric: forgejo_git_operations_total + kind: counter + labels: [operation_type] + - metric: forgejo_runner_tasks_total + kind: counter + labels: [status] +correlation_ids: [] +logs: + loki_namespace_selector: 'namespace="forgejo"' +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning diff --git a/k8s/monitoring/dashboards/services/grafana.descriptor.yaml b/k8s/monitoring/dashboards/services/grafana.descriptor.yaml new file mode 100644 index 0000000..d04ee39 --- /dev/null +++ b/k8s/monitoring/dashboards/services/grafana.descriptor.yaml @@ -0,0 +1,35 @@ +service: grafana +display_name: "Grafana" +namespace: logging +service_type: web-app +folder: "Grafana" +jobs: [grafana] +metrics: + prefix: grafana + up_selector: 'job="grafana"' + rate: + metric: grafana_http_request_total + labels: [handler, status] + duration: + metric: grafana_http_request_duration_seconds + type: histogram + unit: s + error: + metric: grafana_http_request_total + filter: 'status=~"5.."' + domain: + - metric: grafana_dashboard_total + kind: gauge + - metric: grafana_user_total + kind: gauge + - metric: grafana_alerts_total + kind: gauge +correlation_ids: [] +logs: + loki_namespace_selector: 'namespace="logging"' +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning diff --git a/k8s/monitoring/dashboards/services/minio.descriptor.yaml b/k8s/monitoring/dashboards/services/minio.descriptor.yaml new file mode 100644 index 0000000..e948d4d --- /dev/null +++ b/k8s/monitoring/dashboards/services/minio.descriptor.yaml @@ -0,0 +1,68 @@ +# minio.descriptor.yaml +# Build input for generating svc-minio.yaml dashboard ConfigMap +# and svc-minio-rules.yaml PrometheusRule. + +service: minio +display_name: "MinIO" +namespace: storage +service_type: stateful-store +folder: "MinIO" + +jobs: + - minio + +metrics: + prefix: minio + up_selector: 'job="minio"' + + rate: + metric: minio_s3_requests_total + labels: [method, bucket] + + duration: + metric: minio_s3_requests_duration_seconds + type: histogram + unit: s + + error: + metric: minio_s3_requests_total + filter: 'error="true"' + + domain: + - metric: minio_cluster_capacity_usable_bytes + kind: gauge + - metric: minio_cluster_capacity_raw_total_bytes + kind: gauge + - metric: minio_replication_metrics_failed_byte_count + kind: gauge + - metric: minio_replication_metrics_replicating_byte_count + kind: gauge + - metric: minio_cluster_health_drives_online + kind: gauge + - metric: minio_cluster_health_drives_offline + kind: gauge + +correlation_ids: [] + +logs: + loki_namespace_selector: 'namespace="storage"' + +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: DiskSpaceLow + expr_template: gauge_below_percentage + metric: minio_cluster_capacity_usable_bytes + threshold: 0.1 + base_metric: minio_cluster_capacity_raw_total_bytes + for: 5m + severity: critical + - name: ReplicationLag + expr_template: gauge_above_threshold + metric: minio_replication_metrics_replicating_byte_count + threshold: 1073741824 + for: 15m + severity: warning diff --git a/k8s/monitoring/dashboards/services/story-crater-backend.descriptor.yaml b/k8s/monitoring/dashboards/services/story-crater-backend.descriptor.yaml new file mode 100644 index 0000000..b64b998 --- /dev/null +++ b/k8s/monitoring/dashboards/services/story-crater-backend.descriptor.yaml @@ -0,0 +1,74 @@ +# story-crater-backend.descriptor.yaml +# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap +# and svc-story-crater-backend-rules.yaml PrometheusRule. +# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. + +service: story-crater-backend +display_name: "Story Crater Backend" +namespace: story-crater-backend +service_type: message-worker +folder: "Story Crater Backend" + +jobs: + - agent-worker + - check + - edit-collab + - canary + +metrics: + prefix: story_crater + up_selector: 'job=~"agent-worker|check|edit-collab|canary"' + + rate: + metric: story_crater_messages_handled_total + labels: [agent, queue, status] # status: succeeded | failed + + duration: + metric: story_crater_message_handle_duration_ms + type: histogram + unit: ms + + error: + metric: story_crater_app_metric_total + filter: 'severity="error"' + + domain: + - metric: story_crater_queue_depth + kind: gauge + group_by: [queue] + - metric: story_crater_dedup_redeliveries_total + kind: counter + group_by: [agent, queue] + - metric: story_crater_outbox_publish_lag_ms + kind: histogram + - metric: story_crater_llm_tokens_used_total + kind: counter + group_by: [agent, kind] + - metric: story_crater_llm_call_duration_seconds + kind: histogram + group_by: [agent, model_provider] + - metric: story_crater_check_latency_ms + kind: histogram + slo_ms: 1200 + - metric: story_crater_degrade_state + kind: gauge + group_by: [service] + +correlation_ids: [session_id, trace_id, tenant] + +logs: + loki_namespace_selector: 'namespace="story-crater-backend"' + +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: CheckLatencySLOBreach + expr_template: histogram_quantile_over_threshold + metric: story_crater_check_latency_ms + quantile: 0.95 + threshold: 1200 + for: 10m + severity: critical diff --git a/k8s/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml b/k8s/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml new file mode 100644 index 0000000..9196d7f --- /dev/null +++ b/k8s/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml @@ -0,0 +1,63 @@ +# story-crater-frontend.descriptor.yaml +# Build input for generating svc-story-crater-frontend.yaml dashboard ConfigMap +# and svc-story-crater-frontend-rules.yaml PrometheusRule. +# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation. + +service: story-crater-frontend +display_name: "Story Crater Frontend" +namespace: story-crater-frontend +service_type: web-app +folder: "Story Crater Frontend" + +jobs: + - story-crater-frontend + +metrics: + prefix: story_crater_frontend + up_selector: 'job="story-crater-frontend"' + + rate: + metric: story_crater_frontend_http_requests_total + labels: [method, route, status] + + duration: + metric: story_crater_frontend_http_request_duration_ms + type: histogram + unit: ms + + error: + metric: story_crater_frontend_http_requests_total + filter: 'status=~"5.."' + + domain: + - metric: story_crater_frontend_web_vitals_lcp_ms + kind: gauge + group_by: [page] + - metric: story_crater_frontend_web_vitals_fid_ms + kind: gauge + group_by: [page] + - metric: story_crater_frontend_web_vitals_cls + kind: gauge + group_by: [page] + - metric: story_crater_frontend_web_vitals_inp_ms + kind: gauge + group_by: [page] + +correlation_ids: [] # Frontend doesn't have session_id/trace_id in metrics + +logs: + loki_namespace_selector: 'namespace="story-crater-frontend"' + +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: WebVitalsLCPRegression + expr_template: histogram_quantile_over_threshold + metric: story_crater_frontend_web_vitals_lcp_ms + quantile: 0.75 + threshold: 2500 + for: 10m + severity: warning diff --git a/k8s/monitoring/dashboards/services/vault.descriptor.yaml b/k8s/monitoring/dashboards/services/vault.descriptor.yaml new file mode 100644 index 0000000..cc76479 --- /dev/null +++ b/k8s/monitoring/dashboards/services/vault.descriptor.yaml @@ -0,0 +1,45 @@ +service: vault +display_name: "Vault" +namespace: storage +service_type: stateful-store +folder: "Vault" +jobs: [vault] +metrics: + prefix: vault + up_selector: 'job="vault"' + rate: + metric: vault_core_handle_request_total + labels: [method, path] + duration: + metric: vault_core_handle_request_duration_seconds + type: histogram + unit: s + error: + metric: vault_core_handle_request_total + filter: 'error="true"' + domain: + - metric: vault_core_unsealed + kind: gauge + - metric: vault_core_active + kind: gauge + - metric: vault_core_replication_primary + kind: gauge + - metric: vault_token_total + kind: gauge + - metric: vault_database_connection_close_total + kind: counter +correlation_ids: [] +logs: + loki_namespace_selector: 'namespace="iam"' +alerts: + - name: HighErrorRate + expr_template: rate_error_ratio + threshold: 0.05 + for: 10m + severity: warning + - name: VaultSealed + expr_template: gauge_below_threshold + metric: vault_core_unsealed + threshold: 1 + for: 1m + severity: critical diff --git a/k8s/monitoring/dashboards/svc-argocd.yaml b/k8s/monitoring/dashboards/svc-argocd.yaml new file mode 100644 index 0000000..8d62d1b --- /dev/null +++ b/k8s/monitoring/dashboards/svc-argocd.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-argocd-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Argo CD" +data: + svc-argocd.json: | + {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]} diff --git a/k8s/monitoring/dashboards/svc-authentik.yaml b/k8s/monitoring/dashboards/svc-authentik.yaml new file mode 100644 index 0000000..e9e6371 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-authentik.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-authentik-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Authentik" +data: + svc-authentik.json: | + { + "title": "Authentik — Service Overview", + "uid": "svc-authentik", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [ + { + "id": 2, "title": "Up", "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } + } + }, + "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }] + }, + { + "id": 3, "title": "HTTP request rate by status", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }] + }, + { + "id": 4, "title": "Error rate % (5xx)", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }] + }, + { + "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" }, + { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" }, + { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" } + ] + } + ] + }, + { + "id": 10, "title": "Row: Resource Usage", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, + "panels": [ + { + "id": 11, "title": "CPU by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 12, "title": "Memory by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 13, "title": "Restart rate by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] + } + ] + }, + { + "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, + "panels": [ + { + "id": 21, "title": "Outpost connections", "type": "stat", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "authentik_outposts_connected" }] + }, + { + "id": 22, "title": "Flows cached", "type": "stat", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "authentik_flows_cached" }] + }, + { + "id": 23, "title": "Policies cached", "type": "stat", + "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "authentik_policies_cached" }] + }, + { + "id": 24, "title": "Queued tasks", "type": "stat", + "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "authentik_tasks_queued" }] + }, + { + "id": 25, "title": "Admin workers", "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "authentik_admin_workers" }] + } + ] + }, + { + "id": 30, "title": "Row: Logs", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "panels": [ + { + "id": 31, "title": "Recent logs", "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [{ "expr": "{namespace=\"iam\"}" }] + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/svc-forgejo.yaml b/k8s/monitoring/dashboards/svc-forgejo.yaml new file mode 100644 index 0000000..2f17c86 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-forgejo.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-forgejo-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Forgejo" +data: + svc-forgejo.json: | + {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]} diff --git a/k8s/monitoring/dashboards/svc-grafana.yaml b/k8s/monitoring/dashboards/svc-grafana.yaml new file mode 100644 index 0000000..ed28281 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-grafana.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-grafana-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Grafana" +data: + svc-grafana.json: | + {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]} diff --git a/k8s/monitoring/dashboards/svc-minio.yaml b/k8s/monitoring/dashboards/svc-minio.yaml new file mode 100644 index 0000000..4c6ec94 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-minio.yaml @@ -0,0 +1,143 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-minio-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "MinIO" +data: + svc-minio.json: | + { + "title": "MinIO — Service Overview", + "uid": "svc-minio", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [ + { + "id": 2, "title": "Up", "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } + } + }, + "targets": [{ "expr": "min(up{job=\"minio\"})" }] + }, + { + "id": 3, "title": "S3 request rate by method", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }] + }, + { + "id": 4, "title": "Error rate %", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }] + }, + { + "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" }, + { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" }, + { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" } + ] + } + ] + }, + { + "id": 10, "title": "Row: Resource Usage", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, + "panels": [ + { + "id": 11, "title": "CPU by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 12, "title": "Memory by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 13, "title": "Restart rate by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] + } + ] + }, + { + "id": 20, "title": "Row: Storage & Replication", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, + "panels": [ + { + "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" }, + { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" } + ] + }, + { + "id": 22, "title": "Drive health (online/offline)", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "short" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" }, + { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" } + ] + }, + { + "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }] + }, + { + "id": 24, "title": "Replication failures (bytes)", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }] + } + ] + }, + { + "id": 30, "title": "Row: Logs", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "panels": [ + { + "id": 31, "title": "Recent logs", "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [{ "expr": "{namespace=\"storage\"}" }] + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/svc-story-crater-backend.yaml b/k8s/monitoring/dashboards/svc-story-crater-backend.yaml new file mode 100644 index 0000000..88d3c72 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-story-crater-backend.yaml @@ -0,0 +1,191 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-story-crater-backend-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Story Crater Backend" +data: + svc-story-crater-backend.json: | + { + "title": "Story Crater Backend — Service Overview", + "uid": "svc-story-crater-backend", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [ + { + "id": 2, "title": "Up", "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } + } + }, + "targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }] + }, + { + "id": 3, "title": "Message rate by status", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }] + }, + { + "id": 4, "title": "Error rate %", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }] + }, + { + "id": 5, "title": "Duration p50/p95/p99", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "ms" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" }, + { "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" }, + { "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" } + ] + } + ] + }, + { + "id": 10, "title": "Row: Resource Usage", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, + "panels": [ + { + "id": 11, "title": "CPU by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 12, "title": "Memory by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 13, "title": "Restart rate by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] + } + ] + }, + { + "id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, + "panels": [ + { + "id": 21, "title": "Queue depth", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }] + }, + { + "id": 22, "title": "Outbox publish lag p95", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 }, + "fieldConfig": { "defaults": { "unit": "ms" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }] + }, + { + "id": 23, "title": "Dedup redeliveries rate", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }] + } + ] + }, + { + "id": 30, "title": "Row: LLM / Inference", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "panels": [ + { + "id": 31, "title": "Token usage rate", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }] + }, + { + "id": 32, "title": "LLM call duration p95", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, + "fieldConfig": { "defaults": { "unit": "s" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }] + }, + { + "id": 33, "title": "LLM call errors rate", "type": "timeseries", + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }] + } + ] + }, + { + "id": 40, "title": "Row: SLOs / Degradation", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, + "panels": [ + { + "id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 1200, "color": "red" } + ] + } + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }] + }, + { + "id": 42, "title": "Degrade state by service", "type": "stat", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } }, + { "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } } + ] + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }] + } + ] + }, + { + "id": 50, "title": "Row: Logs", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "panels": [ + { + "id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }] + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/svc-story-crater-frontend.yaml b/k8s/monitoring/dashboards/svc-story-crater-frontend.yaml new file mode 100644 index 0000000..ebf672c --- /dev/null +++ b/k8s/monitoring/dashboards/svc-story-crater-frontend.yaml @@ -0,0 +1,184 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-story-crater-frontend-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Story Crater Frontend" +data: + svc-story-crater-frontend.json: | + { + "title": "Story Crater Frontend — Service Overview", + "uid": "svc-story-crater-frontend", + "schemaVersion": 39, + "timezone": "browser", + "time": { "from": "now-6h", "to": "now" }, + "refresh": "30s", + "panels": [ + { + "id": 1, "title": "Row: Availability & Golden Signals", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [ + { + "id": 2, "title": "Up", "type": "stat", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] } + } + }, + "targets": [{ "expr": "min(up{job=\"story-crater-frontend\"})" }] + }, + { + "id": 3, "title": "HTTP request rate by status", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total[5m])) by (status)", "legendFormat": "{{status}}" }] + }, + { + "id": 4, "title": "Error rate % (5xx)", "type": "timeseries", + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 }, + "fieldConfig": { "defaults": { "unit": "percent" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(story_crater_frontend_http_requests_total[5m])) * 100" }] + }, + { + "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { "defaults": { "unit": "ms" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { "expr": "histogram_quantile(0.50, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" }, + { "expr": "histogram_quantile(0.95, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" }, + { "expr": "histogram_quantile(0.99, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" } + ] + } + ] + }, + { + "id": 10, "title": "Row: Resource Usage", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, + "panels": [ + { + "id": 11, "title": "CPU by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-frontend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 12, "title": "Memory by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, + "fieldConfig": { "defaults": { "unit": "bytes" } }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-frontend\"}) by (pod)", "legendFormat": "{{pod}}" }] + }, + { + "id": 13, "title": "Restart rate by pod", "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-frontend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }] + } + ] + }, + { + "id": 20, "title": "Row: Web Vitals", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, + "panels": [ + { + "id": 21, "title": "LCP p75 (Largest Contentful Paint)", "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 2500, "color": "orange" }, + { "value": 4000, "color": "red" } + ] + } + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms)", "legendFormat": "LCP p75" }] + }, + { + "id": 22, "title": "FID p75 (First Input Delay)", "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 100, "color": "orange" }, + { "value": 300, "color": "red" } + ] + } + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_fid_ms)", "legendFormat": "FID p75" }] + }, + { + "id": 23, "title": "CLS (Cumulative Layout Shift)", "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 0.1, "color": "orange" }, + { "value": 0.25, "color": "red" } + ] + } + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "max(story_crater_frontend_web_vitals_cls)", "legendFormat": "CLS" }] + }, + { + "id": 24, "title": "INP p75 (Interaction to Next Paint)", "type": "timeseries", + "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { "value": null, "color": "green" }, + { "value": 200, "color": "orange" }, + { "value": 500, "color": "red" } + ] + } + } + }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_inp_ms)", "legendFormat": "INP p75" }] + } + ] + }, + { + "id": 30, "title": "Row: Logs", "type": "row", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "panels": [ + { + "id": 31, "title": "Recent logs", "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [{ "expr": "{namespace=\"story-crater-frontend\"}" }] + } + ] + } + ] + } diff --git a/k8s/monitoring/dashboards/svc-vault.yaml b/k8s/monitoring/dashboards/svc-vault.yaml new file mode 100644 index 0000000..2ac8663 --- /dev/null +++ b/k8s/monitoring/dashboards/svc-vault.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: svc-vault-dashboard + namespace: logging + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: "Vault" +data: + svc-vault.json: | + {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]} diff --git a/k8s/monitoring/prometheus-values.yaml b/k8s/monitoring/prometheus-values.yaml new file mode 100644 index 0000000..6e2923e --- /dev/null +++ b/k8s/monitoring/prometheus-values.yaml @@ -0,0 +1,133 @@ +# monitoring/prometheus-values.yaml +# Chart: prometheus-community/kube-prometheus-stack +# Release name: prometheus (affects all generated resource names) +# +# What this installs: +# - Prometheus Operator (manages the Prometheus CRD) +# - Prometheus StatefulSet (scrapes metrics, stores on Longhorn PVC) +# - node-exporter DaemonSet (kernel CPU/RAM/disk/network per node) +# - kube-state-metrics Deployment (K8s object state — pod resource requests, phases) +# +# What this deliberately omits: +# - Grafana: already deployed in the logging namespace +# - Alertmanager: enable later when you want Slack/PagerDuty routing +# - kubeControllerManager / kubeScheduler / kubeEtcd: Talos only binds these +# on 127.0.0.1 — the default ServiceMonitors can't reach them +# - kubeProxy: removed cluster-wide; Cilium handles routing instead + +# ── Grafana ─────────────────────────────────────────────────────────────────── +grafana: + enabled: false + +# ── Alertmanager ────────────────────────────────────────────────────────────── +alertmanager: + enabled: false + +# ── Prometheus ──────────────────────────────────────────────────────────────── +prometheus: + prometheusSpec: + retention: 15d + retentionSize: "18GB" + + # Scrape timeout: increased to 60s to tolerate 5+ second network latency spikes + # Default: 10s — too aggressive for latency-prone clusters + scrapeInterval: 30s + scrapeTimeout: 60s + evaluationInterval: 30s + + # Persistent storage — metrics survive node reboots and pod restarts. + # Longhorn reattaches the PVC automatically when the pod reschedules. + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: longhorn + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi + + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + + # Match ServiceMonitors/PodMonitors from all namespaces, not just the ones + # the chart itself creates. Required to scrape workloads in other namespaces. + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + +# ── Prometheus Operator ──────────────────────────────────────────────────────── +prometheusOperator: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + +# ── node-exporter ───────────────────────────────────────────────────────────── +# DaemonSet: one pod per node, reads /proc and /sys directly via hostPID. +# These filesystem excludes prevent scrape errors on Talos's read-only overlayfs +# mounts and containerd's ephemeral snapshot filesystems. +nodeExporter: + enabled: true + +prometheus-node-exporter: + extraArgs: + - --collector.filesystem.mount-points-exclude=^/(dev|proc|run/credentials/.+|sys|var/lib/containerd/.+|var/lib/kubelet/.+|run/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + tolerations: + - operator: Exists # schedule on every node regardless of taints + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + +# ── kube-state-metrics ──────────────────────────────────────────────────────── +# Watches the K8s API; surfaces pod CPU/memory requests, deployment replica +# counts, pod phase, etc. — the "are my workloads healthy?" layer. +kubeStateMetrics: + enabled: true + +kube-state-metrics: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + +# ── Disable unreachable control-plane scrape targets ────────────────────────── +kubeControllerManager: + enabled: false + +kubeScheduler: + enabled: false + +kubeEtcd: + enabled: false + +kubeProxy: + enabled: false diff --git a/k8s/monitoring/servicemonitors/argocd.yaml b/k8s/monitoring/servicemonitors/argocd.yaml new file mode 100644 index 0000000..31480b5 --- /dev/null +++ b/k8s/monitoring/servicemonitors/argocd.yaml @@ -0,0 +1,37 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: argocd + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - cicd + selector: + matchLabels: + app.kubernetes.io/name: argocd-metrics + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: argocd-server + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - cicd + selector: + matchLabels: + app.kubernetes.io/name: argocd-server-metrics + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s diff --git a/k8s/monitoring/servicemonitors/authentik.yaml b/k8s/monitoring/servicemonitors/authentik.yaml new file mode 100644 index 0000000..d993276 --- /dev/null +++ b/k8s/monitoring/servicemonitors/authentik.yaml @@ -0,0 +1,18 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: authentik + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - iam + selector: + matchLabels: + app.kubernetes.io/name: authentik + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 10s diff --git a/k8s/monitoring/servicemonitors/forgejo.yaml b/k8s/monitoring/servicemonitors/forgejo.yaml new file mode 100644 index 0000000..0685650 --- /dev/null +++ b/k8s/monitoring/servicemonitors/forgejo.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: forgejo + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - cicd + selector: + matchLabels: + app.kubernetes.io/name: gitea + endpoints: + - port: http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/k8s/monitoring/servicemonitors/minio.yaml b/k8s/monitoring/servicemonitors/minio.yaml new file mode 100644 index 0000000..5563323 --- /dev/null +++ b/k8s/monitoring/servicemonitors/minio.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: minio + namespace: monitoring + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - storage + selector: + matchLabels: + app: minio + endpoints: + - port: minio-api + path: /minio/v2/metrics/cluster + interval: 30s + scrapeTimeout: 10s