k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)

k8s/logging/.env.example

@@ -0,0 +1,13 @@
+# logging/.env.example
+# Copy to logging/.env and fill in real values.
+# The real .env is gitignored — never commit it.
+# Used by: k8s/logging/bootstrap.sh and k8s/monitoring/bootstrap.sh (Grafana upgrade step)
+
+# Grafana admin UI password
+#   openssl rand -base64 24
+GRAFANA_ADMIN_PASSWORD=
+
+# MinIO credentials — must match the values used in k8s/storage/.env
+# Loki uses these to authenticate to the MinIO S3 backend
+MINIO_ROOT_USER=
+MINIO_ROOT_PASSWORD=

k8s/logging/bootstrap.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# logging/bootstrap.sh
+# Deploys MinIO + Loki + Promtail + Grafana into the logging namespace.
+#
+# Prerequisites:
+#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
+#   - helm >= 3.x installed
+#   - logging/.env file containing:
+#       GRAFANA_ADMIN_PASSWORD=...
+#       MINIO_ROOT_USER=...
+#       MINIO_ROOT_PASSWORD=...
+#     OR those variables already exported in the calling shell.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+KUBECONFIG="${KUBECONFIG:-${SCRIPT_DIR}/../cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Load credentials ──────────────────────────────────────────────────────────
+if [[ -f "${SCRIPT_DIR}/.env" ]]; then
+  # shellcheck source=/dev/null
+  source "${SCRIPT_DIR}/.env"
+fi
+
+for var in GRAFANA_ADMIN_PASSWORD MINIO_ROOT_USER MINIO_ROOT_PASSWORD; do
+  if [[ -z "${!var:-}" ]]; then
+    echo "ERROR: ${var} is not set. Export it or place it in logging/.env"
+    exit 1
+  fi
+done
+
+# ── Namespace ─────────────────────────────────────────────────────────────────
+echo "==> Creating logging namespace with privileged pod security..."
+kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace logging \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# ── Helm repos ────────────────────────────────────────────────────────────────
+echo "==> Adding Helm repositories..."
+helm repo add minio https://charts.min.io
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update minio grafana
+
+# ── MinIO ─────────────────────────────────────────────────────────────────────
+echo "==> Installing MinIO..."
+helm upgrade --install minio minio/minio \
+  --namespace logging \
+  --values "${SCRIPT_DIR}/minio-values.yaml" \
+  --set auth.rootUser="${MINIO_ROOT_USER}" \
+  --set auth.rootPassword="${MINIO_ROOT_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for MinIO Deployment to be ready..."
+kubectl rollout status deployment/minio -n logging --timeout=120s
+
+# ── Create Loki buckets in MinIO ──────────────────────────────────────────────
+echo "==> Creating Loki buckets in MinIO..."
+MINIO_POD=$(kubectl get pod -n logging -l app=minio,release=minio -o jsonpath='{.items[0].metadata.name}')
+
+# Read the actual credentials MinIO is running with from the cluster secret
+ACTUAL_MINIO_USER=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootUser}' | base64 --decode)
+ACTUAL_MINIO_PASS=$(kubectl get secret minio -n logging -o jsonpath='{.data.rootPassword}' | base64 --decode)
+
+kubectl exec -n logging "${MINIO_POD}" -- \
+  mc alias set local http://localhost:9000 "${ACTUAL_MINIO_USER}" "${ACTUAL_MINIO_PASS}"
+for bucket in loki-chunks loki-ruler loki-admin; do
+  kubectl exec -n logging "${MINIO_POD}" -- \
+    mc mb --ignore-existing "local/${bucket}"
+  echo "  bucket: ${bucket} ready"
+done
+
+# ── Loki ──────────────────────────────────────────────────────────────────────
+echo "==> Installing Loki (SingleBinary + MinIO backend)..."
+helm upgrade --install loki grafana/loki \
+  --namespace logging \
+  --values "${SCRIPT_DIR}/loki-values.yaml" \
+  --set loki.storage.s3.access_key_id="${ACTUAL_MINIO_USER}" \
+  --set loki.storage.s3.secret_access_key="${ACTUAL_MINIO_PASS}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for Loki StatefulSet to be ready..."
+kubectl rollout status statefulset/loki -n logging --timeout=120s
+
+# ── Promtail ──────────────────────────────────────────────────────────────────
+echo "==> Installing Promtail..."
+helm upgrade --install promtail grafana/promtail \
+  --namespace logging \
+  --values "${SCRIPT_DIR}/promtail-values.yaml" \
+  --wait \
+  --timeout 3m
+
+echo "==> Waiting for Promtail DaemonSet to be ready..."
+kubectl rollout status daemonset/promtail -n logging --timeout=60s
+
+# ── Grafana ───────────────────────────────────────────────────────────────────
+echo "==> Installing Grafana..."
+helm upgrade --install grafana grafana/grafana \
+  --namespace logging \
+  --values "${SCRIPT_DIR}/grafana-values.yaml" \
+  --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for Grafana Deployment to be ready..."
+kubectl rollout status deployment/grafana -n logging --timeout=120s
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> Logging stack is up."
+echo ""
+echo "Grafana (log explorer UI):"
+echo "  kubectl port-forward -n logging svc/grafana 3000:80"
+echo "  http://localhost:3000  (admin / <your GRAFANA_ADMIN_PASSWORD>)"
+echo ""
+echo "MinIO console (S3 object browser):"
+echo "  kubectl port-forward -n logging svc/minio 9001:9001"
+echo "  http://localhost:9001  (${MINIO_ROOT_USER} / <your MINIO_ROOT_PASSWORD>)"
+echo ""
+echo "MinIO S3 endpoint for other apps:"
+echo "  http://minio.logging.svc.cluster.local:9000"

k8s/logging/grafana-values.yaml

@@ -0,0 +1,185 @@
+# logging/grafana-values.yaml
+# Grafana — dashboarding and log/metrics exploration UI.
+# Deployed in the logging namespace alongside Loki and Promtail.
+#
+# Secrets never set here:
+#   adminPassword        — injected via helmfile --set (GRAFANA_ADMIN_PASSWORD)
+#   OAuth client secret  — mounted from the grafana-oidc K8s Secret (envFromSecret below)
+
+replicas: 1
+
+# RWO PVC (Longhorn) — old pod must fully terminate before the new one can
+# mount the volume. Recreate avoids the "two pods fighting over one PVC" failure.
+deploymentStrategy:
+  type: Recreate
+
+podAnnotations:
+  secret.reloader.stakater.com/reload: "grafana-oidc"
+
+adminUser: admin
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 128Mi
+  limits:
+    cpu: 500m
+    memory: 256Mi
+
+# ── Persistence ───────────────────────────────────────────────────────────────
+# Stores dashboards saved in the UI, datasource edits, and user sessions.
+# Longhorn provides the RWO block volume; 5Gi is ample for dashboard JSON.
+persistence:
+  enabled: true
+  storageClassName: longhorn
+  accessModes:
+    - ReadWriteOnce
+  size: 5Gi
+
+# ── Grafana config ────────────────────────────────────────────────────────────
+grafana.ini:
+  server:
+    root_url: https://grafana.riotpiao.homelab.com
+
+  # No anonymous read access — every user must log in via Authentik SSO.
+  auth.anonymous:
+    enabled: false
+
+  # Explore tab: required for ad-hoc LogQL/PromQL queries against Loki/Prometheus.
+  explore:
+    enabled: true
+
+  # WAL for the embedded SQLite DB — prevents corruption on ungraceful shutdown.
+  database:
+    wal: true
+
+  # ── Authentik OIDC (generic OAuth2) ───────────────────────────────────────
+  # Grafana v10+ supports OIDC auto-discovery; we wire it manually here because
+  # Authentik's discovery endpoint is internal-only (no external DNS for iam.svc).
+  # All URLs use the external hostname so CoreDNS rewrites them to
+  # authentik-server.iam.svc — this keeps the Host header correct so Authentik
+  # doesn't return localhost redirects in its token responses.
+  #
+  # role_attribute_path: JMESPath expression evaluated against the userinfo
+  # response. Members of the 'grafana-admins' Authentik group get Admin role;
+  # everyone else gets Viewer. The group name must match exactly what Authentik
+  # sends in the 'groups' claim.
+  auth.generic_oauth:
+    enabled: true
+    name: Authentik
+    allow_sign_up: true
+    client_id: grafana
+    scopes: openid email profile
+    auth_url: https://authentik.riotpiao.homelab.com/application/o/authorize/
+    token_url: https://authentik.riotpiao.homelab.com/application/o/token/
+    api_url: https://authentik.riotpiao.homelab.com/application/o/userinfo/
+    role_attribute_path: "contains(groups[*], 'grafana-admins') && 'Admin' || 'Viewer'"
+    use_pkce: false
+    use_refresh_token: false
+    skip_org_role_sync: false
+    tls_skip_verify_insecure: true   # Authentik uses self-signed cert; verify in prod
+
+# GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET is injected from the grafana-oidc K8s
+# Secret (created by k8s/talos-iam/setup_talos_iam.sh). envFromSecret mounts
+# every key in that secret as an environment variable — keeps secrets out of
+# values files and out of git.
+envFromSecret: grafana-oidc
+
+# ── Datasources ───────────────────────────────────────────────────────────────
+# Provisioned at install — no manual "Add datasource" step in the UI.
+# Loki is the default (log exploration); Prometheus is secondary (metrics).
+# Both use in-cluster service DNS — Grafana never leaves the cluster for data.
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: Loki
+        type: loki
+        uid: loki
+        access: proxy
+        url: http://loki.logging.svc.cluster.local:3100
+        isDefault: true
+        version: 1
+        editable: true
+        jsonData:
+          maxLines: 1000
+          timeout: 60
+      - name: Prometheus
+        type: prometheus
+        uid: prometheus
+        access: proxy
+        url: http://prometheus-kube-prometheus-prometheus.monitoring.svc.cluster.local:9090
+        isDefault: false
+        version: 1
+        editable: true
+        jsonData:
+          timeInterval: 30s
+          timeout: 60
+
+# ── Dashboard providers ───────────────────────────────────────────────────────
+# Tells Grafana to watch a directory for dashboard JSON files.
+# The `dashboards` block below populates that directory via an init container
+# that downloads from grafana.com at install time.
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+      - name: default
+        orgId: 1
+        folder: ""
+        type: file
+        disableDeletion: false
+        editable: true
+        options:
+          path: /var/lib/grafana/dashboards/default
+
+# ── Pre-loaded dashboards ──────────────────────────────────────────────────────
+# Fetched from grafana.com by an init container at helm-install time and baked
+# into ConfigMaps. gnetId + revision pin the exact dashboard version so an
+# upstream change on grafana.com doesn't silently alter what's deployed.
+dashboards:
+  default:
+    node-exporter-full:
+      gnetId: 1860
+      revision: 37
+      datasource: Prometheus
+    kubernetes-cluster:
+      gnetId: 7249
+      revision: 1
+      datasource: Prometheus
+
+# Allow scheduling on the control-plane node (talos-cp-1 carries NoSchedule taint).
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+# ClusterIP only — access via ingress (grafana.riotpiao.homelab.com) or port-forward.
+service:
+  type: ClusterIP
+  port: 80
+
+# Ingress managed by k8s/ingress/ingress.yaml — not the chart's built-in ingress.
+ingress:
+  enabled: false
+
+serviceMonitor:
+  enabled: false
+
+# ── Dashboard-as-code (sidecar discovery) ──────────────────────────────────────
+# Additive to the gnetId-download mechanism above. The sidecar watches for
+# ConfigMaps labeled grafana_dashboard=1 in ANY namespace and loads them live —
+# no Grafana restart needed when a new dashboard ConfigMap is applied.
+sidecar:
+  dashboards:
+    enabled: true
+    label: grafana_dashboard
+    labelValue: "1"
+    folder: /var/lib/grafana/dashboards/custom
+    folderAnnotation: grafana_folder
+    provider:
+      name: custom
+      folder: "Homelab"
+      disableDelete: false
+      foldersFromFilesStructure: true
+    searchNamespace: ALL

k8s/logging/loki-values.yaml

@@ -0,0 +1,163 @@
+# logging/loki-values.yaml
+# Grafana Loki — log aggregation backend for the homelab.
+# Deployed in SingleBinary mode: one pod handles ingest, query, and compaction.
+# Chunks are stored in MinIO (S3-compatible) — no local PVC needed for log data.
+#
+# MinIO credentials are injected at deploy time via helmfile --set:
+#   loki.storage.s3.accessKeyId    ← MINIO_ROOT_USER
+#   loki.storage.s3.secretAccessKey ← MINIO_ROOT_PASSWORD
+# The placeholder values below are overridden and never used.
+
+# ── Deployment mode ───────────────────────────────────────────────────────────
+# SingleBinary collapses all Loki components (ingester, querier, compactor, ruler)
+# into one Deployment. Simpler ops for a homelab — no inter-component networking
+# or separate scaling to worry about. The tradeoff is no horizontal scaling.
+deploymentMode: SingleBinary
+
+loki:
+  # auth_enabled: false skips tenant header (X-Scope-OrgID) enforcement.
+  # All Promtail → Loki traffic is internal; multi-tenancy adds no value here.
+  auth_enabled: false
+
+  # ── Replication ─────────────────────────────────────────────────────────────
+  # replication_factor: 1 — single replica, no write quorum needed.
+  # Higher values require multiple ingesters (only valid outside SingleBinary).
+  commonConfig:
+    replication_factor: 1
+
+  # ── Storage backend ──────────────────────────────────────────────────────────
+  # s3 type with s3ForcePathStyle: MinIO exposes buckets as paths
+  # (http://host:9000/bucket) not subdomains (http://bucket.host:9000).
+  # insecure: true — MinIO in this cluster has no TLS; traffic stays in-cluster.
+  # Three buckets: chunks (log data), ruler (recording/alerting rules), admin (index).
+  storage:
+    type: s3
+    s3:
+      endpoint: minio.storage.svc.cluster.local:9000
+      region: us-east-1          # MinIO ignores region but Loki's S3 client requires it
+      s3ForcePathStyle: true
+      insecure: true
+      access_key_id: ""          # overridden by helmfile --set (MINIO_ROOT_USER)
+      secret_access_key: ""      # overridden by helmfile --set (MINIO_ROOT_PASSWORD)
+    bucketNames:
+      chunks: loki-chunks
+      ruler: loki-ruler
+      admin: loki-admin
+
+  # ── Schema ───────────────────────────────────────────────────────────────────
+  # v13 + TSDB is the current recommended schema (Loki 3.x).
+  # from: sets the date after which this schema applies — logs before this date
+  # would use a previous schema config (none exists here, so all logs use v13).
+  # period: 24h means one index table per day in the object store.
+  schemaConfig:
+    configs:
+      - from: "2024-01-01"
+        store: boltdb-shipper
+        object_store: s3
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+
+  # ── Ingester ─────────────────────────────────────────────────────────────────
+  # Controls how log chunks are buffered before being flushed to MinIO.
+  # chunk_idle_period: flush a chunk if no new logs arrive for 3m (reduces
+  # open chunk count). chunk_retain_period: keep flushed chunks in memory
+  # briefly so late-arriving out-of-order logs can still be appended.
+  # WAL persists in-memory chunks to disk — required for boltdb-shipper.
+  ingester:
+    chunk_idle_period: 3m
+    chunk_block_size: 262144
+    chunk_retain_period: 1m
+    wal:
+      dir: /var/loki/wal
+
+  # ── Compactor ────────────────────────────────────────────────────────────────
+  # Merges small index files written by ingesters into larger ones, and
+  # enforces retention by deleting chunks older than retention_period.
+  # retention_delete_delay: waits 2h after marking chunks for deletion before
+  # actually removing them — safety window if a query is still reading them.
+  compactor:
+    working_directory: /var/loki/compactor
+    compaction_interval: 10m
+    retention_enabled: true
+    retention_delete_delay: 2h
+    retention_delete_worker_count: 150
+    delete_request_store: s3
+
+  # ── Limits ───────────────────────────────────────────────────────────────────
+  # retention_period: 10 days. Homelab — no long-term log storage needed.
+  # ingestion_rate_mb / burst: rate limits per tenant (single tenant here).
+  #   4 MB/s steady, 6 MB/s burst — plenty for a 3-node cluster.
+  # max_query_series: caps how many unique label combinations a single query
+  #   can return — prevents runaway cardinality queries from OOMing the pod.
+  # max_query_lookback: hard cap matching retention_period (no point querying
+  #   further back than what's stored).
+  # allow_structured_metadata: false — required for boltdb-shipper index store.
+  limits_config:
+    retention_period: 240h
+    ingestion_rate_mb: 4
+    ingestion_burst_size_mb: 6
+    max_query_series: 5000
+    max_query_lookback: 240h
+    max_label_names_per_series: 30
+    allow_structured_metadata: false
+    # Query timeout: increased to 120s to tolerate 5+ second network latency spikes
+    # Default: 30s — too aggressive when pod-to-pod latency hits 5-10s
+    query_timeout: 120s
+
+# ── Single binary pod ─────────────────────────────────────────────────────────
+singleBinary:
+  replicas: 1
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 1000m
+      memory: 512Mi
+
+  # Persistent storage for indices + WAL. Durable log chunks live in MinIO.
+  persistence:
+    enabled: true
+    storageClassName: longhorn
+    size: 5Gi
+
+# ── Disable micro-service replicas ───────────────────────────────────────────
+# In SingleBinary mode the chart still templates read/write/backend Deployments
+# unless explicitly set to 0. Setting replicas: 0 keeps them out of the cluster.
+read:
+  replicas: 0
+write:
+  replicas: 0
+backend:
+  replicas: 0
+
+# Nginx gateway is only needed for multi-replica deployments that split read
+# and write paths. Not used in SingleBinary.
+gateway:
+  enabled: false
+
+# Disable the bundled MinIO subchart — we run our own minio-az-a/az-b releases
+# in the storage namespace with site replication.
+minio:
+  enabled: false
+
+# ── Monitoring ────────────────────────────────────────────────────────────────
+# Self-monitoring ships a Grafana Agent operator to scrape Loki's own metrics.
+# We use kube-prometheus-stack for that instead — avoid running two agents.
+# lokiCanary sends synthetic log lines to verify the write→read pipeline;
+# useful in production, too noisy for a homelab.
+monitoring:
+  selfMonitoring:
+    enabled: false
+    grafanaAgent:
+      installOperator: false
+  lokiCanary:
+    enabled: false
+  serviceMonitor:
+    enabled: false
+
+test:
+  enabled: false

k8s/logging/minio-values.yaml

@@ -0,0 +1,42 @@
+# logging/minio-values.yaml
+# Official MinIO chart (minio/minio from https://charts.min.io).
+# Single pod with console built-in on port 9001.
+# rootUser and rootPassword are injected via --set at install time from .env.
+
+mode: standalone
+
+rootUser: ""      # injected via --set
+rootPassword: ""  # injected via --set
+
+persistence:
+  enabled: true
+  storageClass: longhorn
+  accessMode: ReadWriteOnce
+  size: 100Gi
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+service:
+  type: ClusterIP
+  port: 9000
+
+consoleService:
+  type: ClusterIP
+  port: 9001
+
+# Disable ingress — access via kubectl port-forward
+ingress:
+  enabled: false
+
+consoleIngress:
+  enabled: false
+
+metrics:
+  serviceMonitor:
+    enabled: false

k8s/logging/promtail-values.yaml

@@ -0,0 +1,152 @@
+# logging/promtail-values.yaml
+# Promtail — log shipper DaemonSet. One pod per node; reads container logs
+# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
+# then pushes them to Loki.
+
+daemonset:
+  enabled: true
+
+config:
+  logLevel: warn   # info is noisy at homelab scale; warn keeps Promtail quiet
+  serverPort: 3101
+
+  # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
+  clients:
+    - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
+
+  snippets:
+    scrapeConfigs: |
+      # ── Pod logs ──────────────────────────────────────────────────────────
+      # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
+      # cri pipeline stage parses the CRI-O/containerd log format so timestamps
+      # and stream (stdout/stderr) are extracted properly before the line is
+      # forwarded to Loki.
+      # relabel_configs build useful labels: namespace, pod, container, node,
+      # and a job label of the form "namespace/pod-name" for easy filtering.
+      - job_name: kubernetes-pods
+        kubernetes_sd_configs:
+          - role: pod
+        pipeline_stages:
+          - drop:
+              expression: '(health|heartbeat|ping|keepalive|level="debug"|"timeout".*"retrying")'
+          - sampling:
+              rate: 0.1
+              enabled: true
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: __host__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - action: replace
+            replacement: $1
+            separator: /
+            source_labels:
+              - __meta_kubernetes_namespace
+              - __meta_kubernetes_pod_name
+            target_label: job
+          - action: replace
+            source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_container_name]
+            target_label: container
+          - replacement: /var/log/pods/*$1/*.log
+            separator: /
+            source_labels:
+              - __meta_kubernetes_pod_uid
+              - __meta_kubernetes_pod_container_name
+            target_label: __path__
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+
+      # ── Talos systemd journal ──────────────────────────────────────────────
+      # Talos runs containerd, kubelet, and kernel messages through systemd-
+      # journald — they never appear in /var/log/pods. This job reads the
+      # binary journal directly and emits unit (systemd unit name) and node
+      # labels so you can filter by service (e.g. unit="kubelet.service").
+      # max_age: 12h — only tail recent journal entries on startup; prevents
+      # Promtail from replaying hours of history after a pod restart.
+      - job_name: systemd-journal
+        journal:
+          path: /var/log/journal
+          max_age: 12h
+          labels:
+            job: systemd-journal
+        relabel_configs:
+          - source_labels: [__journal__systemd_unit]
+            target_label: unit
+          - source_labels: [__journal__hostname]
+            target_label: node
+
+# ── Volume mounts ─────────────────────────────────────────────────────────────
+# hostPath mounts give Promtail access to the node's log directories.
+# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
+# /var/log/journal — Talos systemd journal (binary format, read via journald API)
+# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
+# before journald has written anything.
+defaultVolumes:
+  - name: pods-logs
+    hostPath:
+      path: /var/log/pods
+  - name: journal
+    hostPath:
+      path: /var/log/journal
+      type: DirectoryOrCreate
+
+defaultVolumeMounts:
+  - name: pods-logs
+    mountPath: /var/log/pods
+    readOnly: true
+  - name: journal
+    mountPath: /var/log/journal
+    readOnly: true
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
+
+# ── Security context ──────────────────────────────────────────────────────────
+# privileged: true — required to open the binary journal on Talos (journald
+# uses file locking that non-privileged processes can't bypass).
+# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
+# even with a read-only root filesystem. Without this, pod logs from containers
+# running as non-root UIDs would be unreadable.
+# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
+# write to its own container filesystem.
+# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
+# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
+containerSecurityContext:
+  privileged: true
+  allowPrivilegeEscalation: true
+  readOnlyRootFilesystem: true
+  capabilities:
+    drop:
+      - ALL
+    add:
+      - DAC_READ_SEARCH
+
+# runAsUser/Group: 0 (root) — needed to access journal files and pod log
+# directories that are owned by root on Talos nodes.
+podSecurityContext:
+  runAsUser: 0
+  runAsGroup: 0
+
+# Tolerate every taint so Promtail runs on ALL nodes including the CP.
+# Without this, the control-plane node's logs (etcd, kube-apiserver) would
+# be missing from Loki entirely.
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
+  - effect: NoExecute
+    operator: Exists
+
+serviceMonitor:
+  enabled: false

k8s/monitoring/alerts/ingress-alerts.yaml

@@ -0,0 +1,34 @@
+# k8s/monitoring/ingress-alerts.yaml
+# ingress-nginx's chart has no built-in PrometheusRule block, so these rules are
+# a standalone CRD instance. Applied via the prometheus release's postsync hook
+# (after the operator/CRDs are confirmed up) — see helmfile.yaml.gotmpl.
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ingress-nginx-rules
+  namespace: ingress-nginx
+spec:
+  groups:
+    - name: ingress-nginx.rules
+      rules:
+        - alert: IngressHighErrorRate
+          expr: |
+            sum(rate(nginx_ingress_controller_requests{status=~"5.."}[5m])) by (ingress)
+            / sum(rate(nginx_ingress_controller_requests[5m])) by (ingress) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High 5xx rate on {{ $labels.ingress }}"
+            description: "More than 5% of requests to {{ $labels.ingress }} have returned 5xx for 10 minutes."
+        - alert: IngressHighLatencyP95
+          expr: |
+            histogram_quantile(0.95,
+              sum(rate(nginx_ingress_controller_request_duration_seconds_bucket[5m])) by (ingress, le)
+            ) > 1
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "p95 latency > 1s on {{ $labels.ingress }}"
+            description: "95th percentile request latency for {{ $labels.ingress }} has exceeded 1s for 10 minutes."

k8s/monitoring/alerts/svc-argocd-rules.yaml

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: argocd-rules
+  namespace: argocd
+spec:
+  groups:
+    - name: argocd.rules
+      interval: 15s
+      rules:
+        - alert: ArgoCDHighErrorRate
+          expr: |
+            (
+              sum(rate(argocd_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(argocd_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Argo CD"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: ArgoCDApplicationSyncFailure
+          expr: |
+            argocd_app_health_degraded_total > 0
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Argo CD applications in degraded state"
+            description: "{{ $value | humanize }} application(s) have been degraded for 10 minutes"

k8s/monitoring/alerts/svc-authentik-rules.yaml

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: authentik-rules
+  namespace: iam
+spec:
+  groups:
+    - name: authentik.rules
+      interval: 15s
+      rules:
+        - alert: AuthentikHighErrorRate
+          expr: |
+            (
+              sum(rate(authentik_http_requests_total{status=~"5.."}[5m]))
+              /
+              sum(rate(authentik_http_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Authentik"
+            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: AuthentikOutpostDown
+          expr: |
+            authentik_outpost_total_up == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Authentik outpost is down"
+            description: "Outpost {{ $labels.outpost_name }} (type: {{ $labels.outpost_type }}) has been offline for 5 minutes"

k8s/monitoring/alerts/svc-forgejo-rules.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: forgejo-rules
+  namespace: forgejo
+spec:
+  groups:
+    - name: forgejo.rules
+      interval: 15s
+      rules:
+        - alert: ForgejoHighErrorRate
+          expr: |
+            (
+              sum(rate(forgejo_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(forgejo_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Forgejo"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"

k8s/monitoring/alerts/svc-grafana-rules.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: grafana-rules
+  namespace: logging
+spec:
+  groups:
+    - name: grafana.rules
+      interval: 15s
+      rules:
+        - alert: GrafanaHighErrorRate
+          expr: |
+            (
+              sum(rate(grafana_http_request_total{status=~"5.."}[5m]))
+              /
+              sum(rate(grafana_http_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Grafana"
+            description: "5xx error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"

k8s/monitoring/alerts/svc-minio-rules.yaml

@@ -0,0 +1,57 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: minio-rules
+  namespace: storage
+spec:
+  groups:
+    - name: minio.rules
+      interval: 15s
+      rules:
+        - alert: MinIOHighErrorRate
+          expr: |
+            (
+              sum(rate(minio_s3_requests_total{error="true"}[5m]))
+              /
+              sum(rate(minio_s3_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on MinIO"
+            description: "S3 request error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: MinIODiskSpaceLow
+          expr: |
+            (
+              minio_cluster_capacity_usable_bytes
+              /
+              minio_cluster_capacity_raw_total_bytes
+            ) < 0.1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO disk space critically low"
+            description: "Usable capacity < 10% of raw capacity. Free space: {{ $value | humanizePercentage }}"
+
+        - alert: MinIOReplicationLag
+          expr: |
+            minio_replication_metrics_replicating_byte_count > 1073741824
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "MinIO replication lag detected"
+            description: "Bytes pending replication > 1GB for 15 minutes. Value: {{ $value | humanize1024 }}B"
+
+        - alert: MinIODriveOffline
+          expr: |
+            minio_cluster_health_drives_offline > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "MinIO drive offline"
+            description: "{{ $value | humanize }} drive(s) offline in MinIO cluster"

k8s/monitoring/alerts/svc-story-crater-backend-rules.yaml

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: story-crater-backend-rules
+  namespace: story-crater-backend
+spec:
+  groups:
+    - name: story-crater-backend.rules
+      interval: 15s
+      rules:
+        - alert: StoryCraterBackendHighErrorRate
+          expr: |
+            (
+              sum(rate(story_crater_app_metric_total{severity="error"}[5m]))
+              /
+              sum(rate(story_crater_messages_handled_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High application error rate on story-crater-backend"
+            description: "Error events exceeded 5% of message volume for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: StoryCraterBackendCheckLatencySLOBreach
+          expr: |
+            histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le)) > 1200
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "CheckScene p95 latency breaching NFR-01 (1200ms SLO)"
+            description: "p95 CheckScene latency has exceeded 1200ms for 10 minutes. Value: {{ $value }}ms"

k8s/monitoring/alerts/svc-story-crater-frontend-rules.yaml

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: story-crater-frontend-rules
+  namespace: story-crater-frontend
+spec:
+  groups:
+    - name: story-crater-frontend.rules
+      interval: 15s
+      rules:
+        - alert: StoryCraterFrontendHighErrorRate
+          expr: |
+            (
+              sum(rate(story_crater_frontend_http_requests_total{status=~"5.."}[5m]))
+              /
+              sum(rate(story_crater_frontend_http_requests_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on story-crater-frontend"
+            description: "5xx error rate exceeded 5% of total requests for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: StoryCraterFrontendWebVitalsLCPRegression
+          expr: |
+            histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms) > 2500
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "LCP (Largest Contentful Paint) regression on story-crater-frontend"
+            description: "LCP p75 exceeded 2500ms for 10 minutes. Value: {{ $value }}ms"

k8s/monitoring/alerts/svc-vault-rules.yaml

@@ -0,0 +1,33 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: vault-rules
+  namespace: storage
+spec:
+  groups:
+    - name: vault.rules
+      interval: 15s
+      rules:
+        - alert: VaultHighErrorRate
+          expr: |
+            (
+              sum(rate(vault_core_handle_request_total{error="true"}[5m]))
+              /
+              sum(rate(vault_core_handle_request_total[5m]))
+            ) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "High error rate on Vault"
+            description: "Error rate exceeded 5% for 10 minutes. Value: {{ $value | humanizePercentage }}"
+
+        - alert: VaultSealed
+          expr: |
+            vault_core_unsealed == 0
+          for: 1m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Vault is sealed"
+            description: "Vault has been sealed for 1 minute. Immediate attention required."

k8s/monitoring/blackbox-exporter-values.yaml

@@ -0,0 +1,75 @@
+# k8s/monitoring/blackbox-exporter-values.yaml
+# Active black-box HTTP probing of every ingress-exposed service — gives an
+# uptime/availability signal independent of real traffic. Homelab services
+# like Vault/MinIO/Longhorn UI get almost no organic requests, so ingress RED
+# metrics alone can't tell "idle" from "down"; this closes that gap.
+
+config:
+  modules:
+    http_2xx:
+      prober: http
+      timeout: 5s
+      http:
+        valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+        valid_status_codes: []   # any 2xx
+        follow_redirects: true
+        preferred_ip_protocol: "ip4"
+        tls_config:
+          insecure_skip_verify: true   # homelab-ca is a private CA; skip verify for simplicity
+
+resources:
+  requests:
+    cpu: 20m
+    memory: 32Mi
+  limits:
+    cpu: 100m
+    memory: 64Mi
+
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+
+serviceMonitor:
+  enabled: true
+  selfMonitor:
+    enabled: true
+  defaults:
+    interval: 30s
+    scrapeTimeout: 10s
+    module: http_2xx
+  targets:
+    - name: grafana
+      url: https://grafana.riotpiao.homelab.com/
+    - name: loki
+      url: https://loki.riotpiao.homelab.com/ready
+    - name: authentik
+      url: https://authentik.riotpiao.homelab.com/-/health/ready/
+    - name: vault
+      url: https://vault.riotpiao.homelab.com/v1/sys/health
+    - name: minio-console
+      url: https://minio.riotpiao.homelab.com/
+    - name: minio-api
+      url: https://minio-api.riotpiao.homelab.com/minio/health/live
+    - name: prometheus
+      url: https://prometheus.riotpiao.homelab.com/-/healthy
+    - name: portainer
+      url: https://portainer.riotpiao.homelab.com/
+    - name: forgejo
+      url: https://forgejo.riotpiao.homelab.com/api/healthz
+    - name: argocd
+      url: https://argocd.riotpiao.homelab.com/healthz
+    - name: longhorn
+      url: https://longhorn.riotpiao.homelab.com/
+
+prometheusRule:
+  enabled: true
+  rules:
+    - alert: ServiceProbeDown
+      expr: probe_success == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: "Probe failing for {{ $labels.instance }}"
+        description: "Blackbox probe for {{ $labels.instance }} has failed for more than 5 minutes."

k8s/monitoring/bootstrap.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# monitoring/bootstrap.sh
+# Deploys kube-prometheus-stack into the monitoring namespace, then upgrades
+# Grafana (in the logging namespace) to add the Prometheus datasource and
+# pre-built dashboards.
+#
+# Prerequisites:
+#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
+#   - helm >= 3.x
+#   - GRAFANA_ADMIN_PASSWORD set, or present in k8s/logging/.env
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Namespace ─────────────────────────────────────────────────────────────────
+echo "==> Creating monitoring namespace..."
+kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace monitoring \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# ── Helm repo ─────────────────────────────────────────────────────────────────
+echo "==> Adding prometheus-community Helm repo..."
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update prometheus-community
+
+# ── kube-prometheus-stack ─────────────────────────────────────────────────────
+echo "==> Installing kube-prometheus-stack..."
+helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
+  --namespace monitoring \
+  --values "${SCRIPT_DIR}/prometheus-values.yaml" \
+  --wait \
+  --timeout 10m
+
+echo "==> Waiting for Prometheus StatefulSet..."
+kubectl rollout status \
+  statefulset/prometheus-prometheus-kube-prometheus-prometheus \
+  -n monitoring --timeout=180s
+
+echo "==> Waiting for node-exporter DaemonSet..."
+kubectl rollout status \
+  daemonset/prometheus-prometheus-node-exporter \
+  -n monitoring --timeout=60s
+
+echo "==> Waiting for kube-state-metrics..."
+kubectl rollout status \
+  deployment/prometheus-kube-state-metrics \
+  -n monitoring --timeout=60s
+
+# ── Upgrade Grafana with Prometheus datasource + dashboards ───────────────────
+echo ""
+echo "==> Upgrading Grafana to wire in Prometheus datasource + dashboards..."
+LOGGING_DIR="${REPO_ROOT}/k8s/logging"
+
+if [[ -f "${LOGGING_DIR}/.env" ]]; then
+  # shellcheck source=/dev/null
+  source "${LOGGING_DIR}/.env"
+fi
+
+if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
+  echo "ERROR: GRAFANA_ADMIN_PASSWORD is not set. Export it or place it in k8s/logging/.env"
+  exit 1
+fi
+
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update grafana
+
+helm upgrade grafana grafana/grafana \
+  --namespace logging \
+  --values "${LOGGING_DIR}/grafana-values.yaml" \
+  --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for Grafana rollout..."
+kubectl rollout status deployment/grafana -n logging --timeout=120s
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> Monitoring stack is up."
+echo ""
+echo "Prometheus UI:"
+echo "  kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090"
+echo "  http://localhost:9090  (Targets page shows node-exporter + kube-state-metrics)"
+echo ""
+echo "Grafana:"
+echo "  kubectl port-forward -n logging svc/grafana 3000:80"
+echo "  http://localhost:3000"
+echo "  Datasources: Loki (logs, default) + Prometheus (metrics)"
+echo "  Dashboards → General:"
+echo "    - Node Exporter Full       (per-node CPU, RAM, disk, network)"
+echo "    - Kubernetes Cluster       (pod resource usage across namespaces)"

k8s/monitoring/dashboards/control-plane-logs.yaml

@@ -0,0 +1,55 @@
+# k8s/monitoring/dashboards/control-plane-logs.yaml
+# Surfaces controller/control-plane logs that are already in Loki today
+# (Promtail scrapes every namespace with no filter) — this dashboard is the
+# "make it visible" piece, not new log collection.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: control-plane-logs-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  control-plane-logs.json: |
+    {
+      "title": "Cluster Control Plane & Controllers (Logs)",
+      "uid": "control-plane-logs",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-1h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1,
+          "title": "Error rate by namespace",
+          "type": "timeseries",
+          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
+          "datasource": { "type": "loki", "uid": "loki" },
+          "targets": [
+            {
+              "expr": "sum by (namespace) (count_over_time({namespace=~\"kube-system|cert-manager|ingress-nginx|longhorn-system\"} |= \"error\" [5m]))"
+            }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "Control plane (kube-apiserver, controller-manager, scheduler)",
+          "type": "logs",
+          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
+          "datasource": { "type": "loki", "uid": "loki" },
+          "targets": [
+            { "expr": "{namespace=\"kube-system\"}" }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "Cluster add-ons (cert-manager, ingress-nginx, longhorn)",
+          "type": "logs",
+          "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 },
+          "datasource": { "type": "loki", "uid": "loki" },
+          "targets": [
+            { "expr": "{namespace=~\"cert-manager|ingress-nginx|longhorn-system\"}" }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/hardware-overview.yaml

@@ -0,0 +1,121 @@
+# k8s/monitoring/dashboards/hardware-overview.yaml
+# Trimmed operator at-a-glance view across all nodes — node-exporter already
+# powers the deep-dive "Node Exporter Full" (#1860, see grafana-values.yaml),
+# this is the quick health-check version, not a replacement for it.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hardware-overview-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  hardware-overview.json: |
+    {
+      "title": "Hardware Statistics (Operator Overview)",
+      "uid": "hardware-overview",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1,
+          "title": "Nodes up / down",
+          "type": "stat",
+          "gridPos": { "h": 5, "w": 24, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "mappings": [
+                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
+                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
+              ]
+            }
+          },
+          "targets": [
+            { "expr": "up{job=~\".*node-exporter.*\"}", "legendFormat": "{{instance}}" }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "CPU usage % by node",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
+          "targets": [
+            {
+              "expr": "(1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (instance)) * 100",
+              "legendFormat": "{{instance}}"
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "Memory usage % by node",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
+          "targets": [
+            {
+              "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100",
+              "legendFormat": "{{instance}}"
+            }
+          ]
+        },
+        {
+          "id": 4,
+          "title": "Root filesystem usage % by node",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
+          "targets": [
+            {
+              "expr": "(1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}) * 100",
+              "legendFormat": "{{instance}}"
+            }
+          ]
+        },
+        {
+          "id": 5,
+          "title": "Root filesystem space remaining",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "bytes" } },
+          "targets": [
+            {
+              "expr": "node_filesystem_avail_bytes{mountpoint=\"/\"}",
+              "legendFormat": "{{instance}}"
+            }
+          ]
+        },
+        {
+          "id": 6,
+          "title": "Network errors/drops by node",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "rate(node_network_receive_errs_total[5m])", "legendFormat": "{{instance}} rx errs" },
+            { "expr": "rate(node_network_transmit_errs_total[5m])", "legendFormat": "{{instance}} tx errs" },
+            { "expr": "rate(node_network_receive_drop_total[5m])", "legendFormat": "{{instance}} rx drops" },
+            { "expr": "rate(node_network_transmit_drop_total[5m])", "legendFormat": "{{instance}} tx drops" }
+          ]
+        },
+        {
+          "id": 7,
+          "title": "Load average (1m / 5m) by node",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "node_load1", "legendFormat": "{{instance}} load1" },
+            { "expr": "node_load5", "legendFormat": "{{instance}} load5" }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/kube-controller-health.yaml

@@ -0,0 +1,185 @@
+# k8s/monitoring/dashboards/kube-controller-health.yaml
+# Talos binds controller-manager/scheduler/etcd to 127.0.0.1, so Prometheus
+# can't scrape them directly (see prometheus-values.yaml). kube-apiserver is
+# the one control-plane component that's still reachable (its ServiceMonitor
+# targets the in-cluster `kubernetes` service, not localhost) — paired with
+# kube-state-metrics signals as a proxy for controller/scheduler health.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-controller-health-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  kube-controller-health.json: |
+    {
+      "title": "Kube-Controller Health",
+      "uid": "kube-controller-health",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1,
+          "title": "API server — up",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "mappings": [
+                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
+                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
+              ]
+            }
+          },
+          "targets": [
+            { "expr": "min(up{job=\"apiserver\"})", "legendFormat": "apiserver" }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "API server — request rate by verb/code",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 18, "x": 6, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "sum(rate(apiserver_request_total[5m])) by (verb, code)",
+              "legendFormat": "{{verb}} {{code}}"
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "API server — error rate % (5xx)",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent" } },
+          "targets": [
+            {
+              "expr": "sum(rate(apiserver_request_total{code=~\"5..\"}[5m])) / sum(rate(apiserver_request_total[5m])) * 100",
+              "legendFormat": "5xx %"
+            }
+          ]
+        },
+        {
+          "id": 4,
+          "title": "API server — latency p95 / p99",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "s" } },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.95, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
+              "legendFormat": "p95"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket[5m])) by (le))",
+              "legendFormat": "p99"
+            }
+          ]
+        },
+        {
+          "id": 5,
+          "title": "Pods stuck Pending",
+          "type": "stat",
+          "gridPos": { "h": 5, "w": 8, "x": 0, "y": 16 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "yellow" },
+                  { "value": 5, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "sum(kube_pod_status_phase{phase=\"Pending\"}) OR on() vector(0)", "legendFormat": "pending" }
+          ]
+        },
+        {
+          "id": 6,
+          "title": "CrashLoopBackOff containers",
+          "type": "stat",
+          "gridPos": { "h": 5, "w": 8, "x": 8, "y": 16 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "sum(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}) OR on() vector(0)", "legendFormat": "crashlooping" }
+          ]
+        },
+        {
+          "id": 7,
+          "title": "Nodes NotReady",
+          "type": "stat",
+          "gridPos": { "h": 5, "w": 8, "x": 16, "y": 16 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "count(kube_node_status_condition{condition=\"Ready\", status=\"true\"} == 0) OR on() vector(0)", "legendFormat": "not ready" }
+          ]
+        },
+        {
+          "id": 8,
+          "title": "Failed Jobs",
+          "type": "table",
+          "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "kube_job_status_failed > 0", "format": "table", "instant": true }
+          ]
+        },
+        {
+          "id": 9,
+          "title": "Deployments with unavailable replicas",
+          "type": "table",
+          "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "kube_deployment_status_replicas_unavailable > 0", "format": "table", "instant": true }
+          ]
+        },
+        {
+          "id": 10,
+          "title": "Container restart rate by pod",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "sum(rate(kube_pod_container_status_restarts_total[15m])) by (namespace, pod)",
+              "legendFormat": "{{namespace}}/{{pod}}"
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/service-availability.yaml

@@ -0,0 +1,172 @@
+# k8s/monitoring/dashboards/service-availability.yaml
+# Active uptime/availability from blackbox-exporter probes — the signal that
+# covers low-traffic services (Vault, MinIO, Longhorn UI) where RED metrics
+# alone can't distinguish "idle" from "down".
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: service-availability-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  service-availability.json: |
+    {
+      "title": "Service Availability & Certificate Expiration",
+      "uid": "svc-availability",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-24h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1,
+          "title": "Up / Down — all probed services",
+          "type": "stat",
+          "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "mappings": [
+                { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } },
+                { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }
+              ],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "red" },
+                  { "value": 1, "color": "green" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "probe_success", "legendFormat": "{{instance}}" }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "Uptime % trend",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent", "max": 100, "min": 0 } },
+          "targets": [
+            {
+              "expr": "avg_over_time(probe_success[$__rate_interval]) * 100",
+              "legendFormat": "{{instance}}"
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "Probe latency",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "s" } },
+          "targets": [
+            { "expr": "probe_duration_seconds", "legendFormat": "{{instance}}" }
+          ]
+        },
+        {
+          "id": 4,
+          "title": "7-day SLO (% successful probes)",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percent",
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "red" },
+                  { "value": 99, "color": "yellow" },
+                  { "value": 99.9, "color": "green" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "avg_over_time(probe_success[7d]) * 100",
+              "format": "table",
+              "instant": true
+            }
+          ]
+        },
+        {
+          "id": 5,
+          "title": "Services DOWN right now",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 12, "x": 0, "y": 22 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "count(probe_success == 0) OR on() vector(0)", "legendFormat": "down" }
+          ]
+        },
+        {
+          "id": 6,
+          "title": "Certs expiring in < 14 days",
+          "type": "stat",
+          "gridPos": { "h": 4, "w": 12, "x": 12, "y": 22 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "count((certmanager_certificate_expiration_timestamp_seconds - time()) / 86400 < 14) OR on() vector(0)",
+              "legendFormat": "expiring"
+            }
+          ]
+        },
+        {
+          "id": 7,
+          "title": "Certificate expiry — days remaining",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "red" },
+                  { "value": 14, "color": "yellow" },
+                  { "value": 30, "color": "green" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400",
+              "legendFormat": "{{name}}",
+              "format": "table",
+              "instant": true
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/service-golden-signals.yaml

@@ -0,0 +1,141 @@
+# k8s/monitoring/dashboards/service-golden-signals.yaml
+# RED metrics (rate/errors/duration) for every service fronted by ingress-nginx.
+# Picked up automatically by Grafana's sidecar (grafana_dashboard=1 label) — see
+# sidecar.dashboards in k8s/logging/grafana-values.yaml.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: service-golden-signals-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  service-golden-signals.json: |
+    {
+      "title": "Latency & Golden Signals (Ingress RED)",
+      "uid": "svc-golden-signals",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "templating": {
+        "list": [
+          {
+            "name": "ingress",
+            "type": "query",
+            "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "query": "label_values(nginx_ingress_controller_requests, ingress)",
+            "refresh": 2,
+            "includeAll": false
+          }
+        ]
+      },
+      "panels": [
+        {
+          "id": 1,
+          "title": "Request rate by status — $ingress",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) by (status)",
+              "legendFormat": "{{status}}"
+            }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "Error rate % (4xx / 5xx) — $ingress",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "percent" } },
+          "targets": [
+            {
+              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"5..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
+              "legendFormat": "5xx"
+            },
+            {
+              "expr": "sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\", status=~\"4..\"}[5m])) / sum(rate(nginx_ingress_controller_requests{ingress=\"$ingress\"}[5m])) * 100",
+              "legendFormat": "4xx"
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "Latency p50 / p95 / p99 — $ingress",
+          "type": "timeseries",
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "s" } },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
+              "legendFormat": "p50"
+            },
+            {
+              "expr": "histogram_quantile(0.95, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
+              "legendFormat": "p95"
+            },
+            {
+              "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress=\"$ingress\"}[5m])) by (le))",
+              "legendFormat": "p99"
+            }
+          ]
+        },
+        {
+          "id": 4,
+          "title": "All services — traffic overview",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "topk(11, sum(rate(nginx_ingress_controller_requests[5m])) by (ingress))",
+              "format": "table",
+              "instant": true
+            }
+          ]
+        },
+        {
+          "id": 5,
+          "title": "Customer-facing failures (5xx count, window total)",
+          "type": "stat",
+          "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "green" },
+                  { "value": 1, "color": "yellow" },
+                  { "value": 50, "color": "red" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "sum(increase(nginx_ingress_controller_requests{status=~\"5..\"}[$__range])) OR on() vector(0)",
+              "legendFormat": "5xx total"
+            }
+          ]
+        },
+        {
+          "id": 6,
+          "title": "Top 5 error-contributing services",
+          "type": "table",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            {
+              "expr": "topk(5, sum(rate(nginx_ingress_controller_requests{status=~\"5..\"}[5m])) by (ingress))",
+              "format": "table",
+              "instant": true
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/service-internals.yaml

@@ -0,0 +1,109 @@
+# k8s/monitoring/dashboards/service-internals.yaml
+# Native per-service metrics — the "why" layer behind the ingress RED/uptime
+# dashboards (e.g. ingress shows MinIO is slow; this shows disk offline).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: service-internals-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+data:
+  service-internals.json: |
+    {
+      "title": "Service Internals (MinIO / Forgejo / Argo CD / cert-manager / Vault / Longhorn)",
+      "uid": "svc-internals",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        { "id": 1, "title": "MinIO — disk/node offline", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "minio_cluster_disk_offline_total", "legendFormat": "disks offline" },
+            { "expr": "minio_cluster_nodes_offline_total", "legendFormat": "nodes offline" }
+          ]
+        },
+        { "id": 2, "title": "MinIO — S3 request errors", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "sum(rate(minio_s3_requests_errors_total[5m])) by (api)", "legendFormat": "{{api}}" }
+          ]
+        },
+        { "id": 3, "title": "MinIO — S3 TTFB latency", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": { "defaults": { "unit": "s" } },
+          "targets": [
+            { "expr": "minio_s3_time_ttfb_seconds_distribution", "legendFormat": "{{api}}" }
+          ]
+        },
+        { "id": 4, "title": "Forgejo — repos / orgs", "type": "stat",
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 6 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "gitea_repositories", "legendFormat": "repos" },
+            { "expr": "gitea_organizations", "legendFormat": "orgs" }
+          ]
+        },
+        { "id": 5, "title": "Forgejo — process health (CPU/mem)", "type": "timeseries",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 12 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "rate(process_cpu_seconds_total{job=~\".*forgejo.*|.*gitea.*\"}[5m])", "legendFormat": "cpu" },
+            { "expr": "process_resident_memory_bytes{job=~\".*forgejo.*|.*gitea.*\"}", "legendFormat": "mem" }
+          ]
+        },
+        { "id": 6, "title": "Argo CD — app sync/health status", "type": "table",
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "argocd_app_info", "format": "table", "instant": true }
+          ]
+        },
+        { "id": 7, "title": "cert-manager — days to cert expiry", "type": "stat",
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "value": 0, "color": "red" },
+                  { "value": 14, "color": "yellow" },
+                  { "value": 30, "color": "green" }
+                ]
+              }
+            }
+          },
+          "targets": [
+            { "expr": "(certmanager_certificate_expiration_timestamp_seconds - time()) / 86400", "legendFormat": "{{name}}" }
+          ]
+        },
+        { "id": 8, "title": "Vault — sealed/unsealed", "type": "stat",
+          "gridPos": { "h": 6, "w": 6, "x": 12, "y": 20 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "mappings": [
+                { "type": "value", "options": { "0": { "text": "SEALED", "color": "red" } } },
+                { "type": "value", "options": { "1": { "text": "UNSEALED", "color": "green" } } }
+              ]
+            }
+          },
+          "targets": [
+            { "expr": "vault_core_unsealed", "legendFormat": "vault" }
+          ]
+        },
+        { "id": 9, "title": "Longhorn — volume robustness", "type": "table",
+          "gridPos": { "h": 6, "w": 6, "x": 18, "y": 20 },
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "expr": "longhorn_volume_robustness", "format": "table", "instant": true }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/services/argocd.descriptor.yaml

@@ -0,0 +1,48 @@
+service: argocd
+display_name: "Argo CD"
+namespace: argocd
+service_type: web-app
+folder: "Argo CD"
+jobs: [argocd-server, argocd-repo-server]
+metrics:
+  prefix: argocd
+  up_selector: 'job=~"argocd-.*"'
+  rate:
+    metric: argocd_http_request_total
+    labels: [method, path, status]
+  duration:
+    metric: argocd_http_request_duration_seconds
+    type: histogram
+    unit: s
+  error:
+    metric: argocd_http_request_total
+    filter: 'status=~"5.."'
+  domain:
+    - metric: argocd_app_total
+      kind: gauge
+    - metric: argocd_app_sync_total
+      kind: counter
+      labels: [sync_status]
+    - metric: argocd_app_health_degraded_total
+      kind: counter
+    - metric: argocd_git_sync_total
+      kind: counter
+      labels: [git_operation, git_status]
+    - metric: argocd_reconcile_total
+      kind: counter
+      labels: [app_name]
+correlation_ids: []
+logs:
+  loki_namespace_selector: 'namespace="argocd"'
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: AppSyncFailure
+    expr_template: gauge_above_threshold
+    metric: argocd_app_health_degraded_total
+    threshold: 1
+    for: 10m
+    severity: warning

k8s/monitoring/dashboards/services/authentik.descriptor.yaml

@@ -0,0 +1,64 @@
+# authentik.descriptor.yaml
+# Build input for generating svc-authentik.yaml dashboard ConfigMap
+# and svc-authentik-rules.yaml PrometheusRule.
+# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
+
+service: authentik
+display_name: "Authentik"
+namespace: iam
+service_type: identity-provider
+folder: "Authentik"
+
+jobs:
+  - authentik-server
+  - authentik-worker
+
+metrics:
+  prefix: authentik
+  up_selector: 'job="authentik-server"'
+
+  rate:
+    metric: authentik_flows_execution_stage_time_count
+    labels: [flow_slug, stage_name]
+
+  duration:
+    metric: authentik_main_request_duration_seconds
+    type: histogram
+    unit: s
+
+  error:
+    metric: authentik_flows_cached
+    filter: null
+
+  domain:
+    - metric: authentik_outpost_connection
+      kind: gauge
+      group_by: [outpost_name, outpost_type]
+    - metric: authentik_outposts_connected
+      kind: gauge
+    - metric: authentik_flows_cached
+      kind: gauge
+    - metric: authentik_policies_cached
+      kind: gauge
+    - metric: authentik_tasks_queued
+      kind: gauge
+    - metric: authentik_admin_workers
+      kind: gauge
+
+correlation_ids: []
+
+logs:
+  loki_namespace_selector: 'namespace="iam"'
+
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: OutpostDown
+    expr_template: gauge_below_threshold
+    metric: authentik_outpost_total_up
+    threshold: 1
+    for: 5m
+    severity: warning

k8s/monitoring/dashboards/services/forgejo.descriptor.yaml

@@ -0,0 +1,39 @@
+service: forgejo
+display_name: "Forgejo"
+namespace: forgejo
+service_type: web-app
+folder: "Forgejo"
+jobs: [forgejo]
+metrics:
+  prefix: forgejo
+  up_selector: 'job="forgejo"'
+  rate:
+    metric: forgejo_http_request_total
+    labels: [method, status]
+  duration:
+    metric: forgejo_http_request_duration_seconds
+    type: histogram
+    unit: s
+  error:
+    metric: forgejo_http_request_total
+    filter: 'status=~"5.."'
+  domain:
+    - metric: forgejo_repositories_total
+      kind: gauge
+    - metric: forgejo_users_total
+      kind: gauge
+    - metric: forgejo_git_operations_total
+      kind: counter
+      labels: [operation_type]
+    - metric: forgejo_runner_tasks_total
+      kind: counter
+      labels: [status]
+correlation_ids: []
+logs:
+  loki_namespace_selector: 'namespace="forgejo"'
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning

k8s/monitoring/dashboards/services/grafana.descriptor.yaml

@@ -0,0 +1,35 @@
+service: grafana
+display_name: "Grafana"
+namespace: logging
+service_type: web-app
+folder: "Grafana"
+jobs: [grafana]
+metrics:
+  prefix: grafana
+  up_selector: 'job="grafana"'
+  rate:
+    metric: grafana_http_request_total
+    labels: [handler, status]
+  duration:
+    metric: grafana_http_request_duration_seconds
+    type: histogram
+    unit: s
+  error:
+    metric: grafana_http_request_total
+    filter: 'status=~"5.."'
+  domain:
+    - metric: grafana_dashboard_total
+      kind: gauge
+    - metric: grafana_user_total
+      kind: gauge
+    - metric: grafana_alerts_total
+      kind: gauge
+correlation_ids: []
+logs:
+  loki_namespace_selector: 'namespace="logging"'
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning

k8s/monitoring/dashboards/services/minio.descriptor.yaml

@@ -0,0 +1,68 @@
+# minio.descriptor.yaml
+# Build input for generating svc-minio.yaml dashboard ConfigMap
+# and svc-minio-rules.yaml PrometheusRule.
+
+service: minio
+display_name: "MinIO"
+namespace: storage
+service_type: stateful-store
+folder: "MinIO"
+
+jobs:
+  - minio
+
+metrics:
+  prefix: minio
+  up_selector: 'job="minio"'
+
+  rate:
+    metric: minio_s3_requests_total
+    labels: [method, bucket]
+
+  duration:
+    metric: minio_s3_requests_duration_seconds
+    type: histogram
+    unit: s
+
+  error:
+    metric: minio_s3_requests_total
+    filter: 'error="true"'
+
+  domain:
+    - metric: minio_cluster_capacity_usable_bytes
+      kind: gauge
+    - metric: minio_cluster_capacity_raw_total_bytes
+      kind: gauge
+    - metric: minio_replication_metrics_failed_byte_count
+      kind: gauge
+    - metric: minio_replication_metrics_replicating_byte_count
+      kind: gauge
+    - metric: minio_cluster_health_drives_online
+      kind: gauge
+    - metric: minio_cluster_health_drives_offline
+      kind: gauge
+
+correlation_ids: []
+
+logs:
+  loki_namespace_selector: 'namespace="storage"'
+
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: DiskSpaceLow
+    expr_template: gauge_below_percentage
+    metric: minio_cluster_capacity_usable_bytes
+    threshold: 0.1
+    base_metric: minio_cluster_capacity_raw_total_bytes
+    for: 5m
+    severity: critical
+  - name: ReplicationLag
+    expr_template: gauge_above_threshold
+    metric: minio_replication_metrics_replicating_byte_count
+    threshold: 1073741824
+    for: 15m
+    severity: warning

k8s/monitoring/dashboards/services/story-crater-backend.descriptor.yaml

@@ -0,0 +1,74 @@
+# story-crater-backend.descriptor.yaml
+# Build input for generating svc-story-crater-backend.yaml dashboard ConfigMap
+# and svc-story-crater-backend-rules.yaml PrometheusRule.
+# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
+
+service: story-crater-backend
+display_name: "Story Crater Backend"
+namespace: story-crater-backend
+service_type: message-worker
+folder: "Story Crater Backend"
+
+jobs:
+  - agent-worker
+  - check
+  - edit-collab
+  - canary
+
+metrics:
+  prefix: story_crater
+  up_selector: 'job=~"agent-worker|check|edit-collab|canary"'
+
+  rate:
+    metric: story_crater_messages_handled_total
+    labels: [agent, queue, status]  # status: succeeded | failed
+
+  duration:
+    metric: story_crater_message_handle_duration_ms
+    type: histogram
+    unit: ms
+
+  error:
+    metric: story_crater_app_metric_total
+    filter: 'severity="error"'
+
+  domain:
+    - metric: story_crater_queue_depth
+      kind: gauge
+      group_by: [queue]
+    - metric: story_crater_dedup_redeliveries_total
+      kind: counter
+      group_by: [agent, queue]
+    - metric: story_crater_outbox_publish_lag_ms
+      kind: histogram
+    - metric: story_crater_llm_tokens_used_total
+      kind: counter
+      group_by: [agent, kind]
+    - metric: story_crater_llm_call_duration_seconds
+      kind: histogram
+      group_by: [agent, model_provider]
+    - metric: story_crater_check_latency_ms
+      kind: histogram
+      slo_ms: 1200
+    - metric: story_crater_degrade_state
+      kind: gauge
+      group_by: [service]
+
+correlation_ids: [session_id, trace_id, tenant]
+
+logs:
+  loki_namespace_selector: 'namespace="story-crater-backend"'
+
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: CheckLatencySLOBreach
+    expr_template: histogram_quantile_over_threshold
+    metric: story_crater_check_latency_ms
+    quantile: 0.95
+    threshold: 1200
+    for: 10m
+    severity: critical

k8s/monitoring/dashboards/services/story-crater-frontend.descriptor.yaml

@@ -0,0 +1,63 @@
+# story-crater-frontend.descriptor.yaml
+# Build input for generating svc-story-crater-frontend.yaml dashboard ConfigMap
+# and svc-story-crater-frontend-rules.yaml PrometheusRule.
+# NOT applied to cluster — this is a reference for Claude's mechanical dashboard generation.
+
+service: story-crater-frontend
+display_name: "Story Crater Frontend"
+namespace: story-crater-frontend
+service_type: web-app
+folder: "Story Crater Frontend"
+
+jobs:
+  - story-crater-frontend
+
+metrics:
+  prefix: story_crater_frontend
+  up_selector: 'job="story-crater-frontend"'
+
+  rate:
+    metric: story_crater_frontend_http_requests_total
+    labels: [method, route, status]
+
+  duration:
+    metric: story_crater_frontend_http_request_duration_ms
+    type: histogram
+    unit: ms
+
+  error:
+    metric: story_crater_frontend_http_requests_total
+    filter: 'status=~"5.."'
+
+  domain:
+    - metric: story_crater_frontend_web_vitals_lcp_ms
+      kind: gauge
+      group_by: [page]
+    - metric: story_crater_frontend_web_vitals_fid_ms
+      kind: gauge
+      group_by: [page]
+    - metric: story_crater_frontend_web_vitals_cls
+      kind: gauge
+      group_by: [page]
+    - metric: story_crater_frontend_web_vitals_inp_ms
+      kind: gauge
+      group_by: [page]
+
+correlation_ids: []  # Frontend doesn't have session_id/trace_id in metrics
+
+logs:
+  loki_namespace_selector: 'namespace="story-crater-frontend"'
+
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: WebVitalsLCPRegression
+    expr_template: histogram_quantile_over_threshold
+    metric: story_crater_frontend_web_vitals_lcp_ms
+    quantile: 0.75
+    threshold: 2500
+    for: 10m
+    severity: warning

k8s/monitoring/dashboards/services/vault.descriptor.yaml

@@ -0,0 +1,45 @@
+service: vault
+display_name: "Vault"
+namespace: storage
+service_type: stateful-store
+folder: "Vault"
+jobs: [vault]
+metrics:
+  prefix: vault
+  up_selector: 'job="vault"'
+  rate:
+    metric: vault_core_handle_request_total
+    labels: [method, path]
+  duration:
+    metric: vault_core_handle_request_duration_seconds
+    type: histogram
+    unit: s
+  error:
+    metric: vault_core_handle_request_total
+    filter: 'error="true"'
+  domain:
+    - metric: vault_core_unsealed
+      kind: gauge
+    - metric: vault_core_active
+      kind: gauge
+    - metric: vault_core_replication_primary
+      kind: gauge
+    - metric: vault_token_total
+      kind: gauge
+    - metric: vault_database_connection_close_total
+      kind: counter
+correlation_ids: []
+logs:
+  loki_namespace_selector: 'namespace="iam"'
+alerts:
+  - name: HighErrorRate
+    expr_template: rate_error_ratio
+    threshold: 0.05
+    for: 10m
+    severity: warning
+  - name: VaultSealed
+    expr_template: gauge_below_threshold
+    metric: vault_core_unsealed
+    threshold: 1
+    for: 1m
+    severity: critical

k8s/monitoring/dashboards/svc-argocd.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-argocd-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Argo CD"
+data:
+  svc-argocd.json: |
+    {"title":"Argo CD — Service Overview","uid":"svc-argocd","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=~\"argocd-.*\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_http_request_total{status=~\"5..\"}[5m])) / sum(rate(argocd_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(argocd_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"argocd\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"argocd\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"argocd\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Applications & Sync","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Applications","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_total"}]},{"id":22,"title":"Sync by status","type":"timeseries","gridPos":{"h":6,"w":9,"x":6,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_app_sync_total[5m])) by (sync_status)","legendFormat":"{{sync_status}}"}]},{"id":23,"title":"Degraded apps","type":"stat","gridPos":{"h":6,"w":6,"x":15,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"argocd_app_health_degraded_total"}]},{"id":24,"title":"Git sync ops","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(argocd_git_sync_total[5m])) by (git_operation,git_status)","legendFormat":"{{git_operation}}/{{git_status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"argocd\"}"}]}]}]}

k8s/monitoring/dashboards/svc-authentik.yaml

@@ -0,0 +1,143 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-authentik-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Authentik"
+data:
+  svc-authentik.json: |
+    {
+      "title": "Authentik — Service Overview",
+      "uid": "svc-authentik",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "panels": [
+            {
+              "id": 2, "title": "Up", "type": "stat",
+              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "fieldConfig": {
+                "defaults": {
+                  "color": { "mode": "thresholds" },
+                  "mappings": [
+                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+                  ],
+                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
+                }
+              },
+              "targets": [{ "expr": "min(up{job=\"authentik-server\"})" }]
+            },
+            {
+              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(authentik_flows_execution_stage_time_count[5m])) by (flow_slug)", "legendFormat": "{{flow_slug}}" }]
+            },
+            {
+              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
+              "fieldConfig": { "defaults": { "unit": "percent" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "(1 - (authentik_flows_cached / authentik_flows_execution_stage_time_count)) * 100" }]
+            },
+            {
+              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+              "fieldConfig": { "defaults": { "unit": "s" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "histogram_quantile(0.50, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
+                { "expr": "histogram_quantile(0.95, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
+                { "expr": "histogram_quantile(0.99, sum(rate(authentik_main_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
+              ]
+            }
+          ]
+        },
+        {
+          "id": 10, "title": "Row: Resource Usage", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
+          "panels": [
+            {
+              "id": 11, "title": "CPU by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"authentik.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 12, "title": "Memory by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"authentik.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"authentik.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
+            }
+          ]
+        },
+        {
+          "id": 20, "title": "Row: Identity Provider (OIDC / OAuth2)", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
+          "panels": [
+            {
+              "id": 21, "title": "Outpost connections", "type": "stat",
+              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "short" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "authentik_outposts_connected" }]
+            },
+            {
+              "id": 22, "title": "Flows cached", "type": "stat",
+              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "short" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "authentik_flows_cached" }]
+            },
+            {
+              "id": 23, "title": "Policies cached", "type": "stat",
+              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "short" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "authentik_policies_cached" }]
+            },
+            {
+              "id": 24, "title": "Queued tasks", "type": "stat",
+              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
+              "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "unit": "short" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "authentik_tasks_queued" }]
+            },
+            {
+              "id": 25, "title": "Admin workers", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "authentik_admin_workers" }]
+            }
+          ]
+        },
+        {
+          "id": 30, "title": "Row: Logs", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
+          "panels": [
+            {
+              "id": 31, "title": "Recent logs", "type": "logs",
+              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
+              "datasource": { "type": "loki", "uid": "loki" },
+              "targets": [{ "expr": "{namespace=\"iam\"}" }]
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/svc-forgejo.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-forgejo-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Forgejo"
+data:
+  svc-forgejo.json: |
+    {"title":"Forgejo — Service Overview","uid":"svc-forgejo","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"}}},"targets":[{"expr":"min(up{job=\"forgejo\"})"}]},{"id":3,"title":"HTTP requests by method","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_http_request_total{status=~\"5..\"}[5m])) / sum(rate(forgejo_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(forgejo_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"forgejo\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"forgejo\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"forgejo\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Git Operations","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Repositories","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_repositories_total"}]},{"id":22,"title":"Users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"forgejo_users_total"}]},{"id":23,"title":"Git ops rate","type":"timeseries","gridPos":{"h":6,"w":12,"x":12,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_git_operations_total[5m])) by (operation_type)","legendFormat":"{{operation_type}}"}]},{"id":24,"title":"Runner tasks","type":"timeseries","gridPos":{"h":6,"w":12,"x":0,"y":9},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(forgejo_runner_tasks_total[5m])) by (status)","legendFormat":"{{status}}"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"forgejo\"}"}]}]}]}

k8s/monitoring/dashboards/svc-grafana.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-grafana-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Grafana"
+data:
+  svc-grafana.json: |
+    {"title":"Grafana — Service Overview","uid":"svc-grafana","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}]}},"targets":[{"expr":"min(up{job=\"grafana\"})"}]},{"id":3,"title":"HTTP requests","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total[5m])) by (status)","legendFormat":"{{status}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(grafana_http_request_total{status=~\"5..\"}[5m])) / sum(rate(grafana_http_request_total[5m])) * 100"}]},{"id":5,"title":"Request latency","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(grafana_http_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"logging\",pod=~\"grafana.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"logging\",pod=~\"grafana.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restarts","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"logging\",pod=~\"grafana.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Dashboards & Users","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Total dashboards","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_dashboard_total"}]},{"id":22,"title":"Total users","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_user_total"}]},{"id":23,"title":"Total alerts","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"grafana_alerts_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"logging\",container=\"grafana\"}"}]}]}]}

k8s/monitoring/dashboards/svc-minio.yaml

@@ -0,0 +1,143 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-minio-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "MinIO"
+data:
+  svc-minio.json: |
+    {
+      "title": "MinIO — Service Overview",
+      "uid": "svc-minio",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "panels": [
+            {
+              "id": 2, "title": "Up", "type": "stat",
+              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "fieldConfig": {
+                "defaults": {
+                  "color": { "mode": "thresholds" },
+                  "mappings": [
+                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+                  ],
+                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
+                }
+              },
+              "targets": [{ "expr": "min(up{job=\"minio\"})" }]
+            },
+            {
+              "id": 3, "title": "S3 request rate by method", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(minio_s3_requests_total[5m])) by (method)", "legendFormat": "{{method}}" }]
+            },
+            {
+              "id": 4, "title": "Error rate %", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
+              "fieldConfig": { "defaults": { "unit": "percent" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(minio_s3_requests_total{error=\"true\"}[5m])) / sum(rate(minio_s3_requests_total[5m])) * 100" }]
+            },
+            {
+              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+              "fieldConfig": { "defaults": { "unit": "s" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "histogram_quantile(0.50, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p50" },
+                { "expr": "histogram_quantile(0.95, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95" },
+                { "expr": "histogram_quantile(0.99, sum(rate(minio_s3_requests_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p99" }
+              ]
+            }
+          ]
+        },
+        {
+          "id": 10, "title": "Row: Resource Usage", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
+          "panels": [
+            {
+              "id": 11, "title": "CPU by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"storage\",pod=~\"minio.*\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 12, "title": "Memory by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"storage\",pod=~\"minio.*\"}) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"storage\",pod=~\"minio.*\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
+            }
+          ]
+        },
+        {
+          "id": 20, "title": "Row: Storage & Replication", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
+          "panels": [
+            {
+              "id": 21, "title": "Usable vs Raw capacity", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "lineWidth": 2 } } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "minio_cluster_capacity_usable_bytes", "legendFormat": "Usable" },
+                { "expr": "minio_cluster_capacity_raw_total_bytes", "legendFormat": "Raw Total" }
+              ]
+            },
+            {
+              "id": 22, "title": "Drive health (online/offline)", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "short" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "minio_cluster_health_drives_online", "legendFormat": "Online" },
+                { "expr": "minio_cluster_health_drives_offline", "legendFormat": "Offline" }
+              ]
+            },
+            {
+              "id": 23, "title": "Replication lag (bytes pending)", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "minio_replication_metrics_replicating_byte_count", "legendFormat": "Pending replication" }]
+            },
+            {
+              "id": 24, "title": "Replication failures (bytes)", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "minio_replication_metrics_failed_byte_count", "legendFormat": "Failed replication" }]
+            }
+          ]
+        },
+        {
+          "id": 30, "title": "Row: Logs", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
+          "panels": [
+            {
+              "id": 31, "title": "Recent logs", "type": "logs",
+              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
+              "datasource": { "type": "loki", "uid": "loki" },
+              "targets": [{ "expr": "{namespace=\"storage\"}" }]
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/svc-story-crater-backend.yaml

@@ -0,0 +1,191 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-story-crater-backend-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Story Crater Backend"
+data:
+  svc-story-crater-backend.json: |
+    {
+      "title": "Story Crater Backend — Service Overview",
+      "uid": "svc-story-crater-backend",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "panels": [
+            {
+              "id": 2, "title": "Up", "type": "stat",
+              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "fieldConfig": {
+                "defaults": {
+                  "color": { "mode": "thresholds" },
+                  "mappings": [
+                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+                  ],
+                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
+                }
+              },
+              "targets": [{ "expr": "min(up{job=~\"agent-worker|check|edit-collab|canary\"})" }]
+            },
+            {
+              "id": 3, "title": "Message rate by status", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_messages_handled_total[5m])) by (status)", "legendFormat": "{{status}}" }]
+            },
+            {
+              "id": 4, "title": "Error rate %", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
+              "fieldConfig": { "defaults": { "unit": "percent" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_app_metric_total{severity=\"error\"}[5m])) / sum(rate(story_crater_messages_handled_total[5m])) * 100" }]
+            },
+            {
+              "id": 5, "title": "Duration p50/p95/p99", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+              "fieldConfig": { "defaults": { "unit": "ms" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "histogram_quantile(0.50, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" },
+                { "expr": "histogram_quantile(0.95, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" },
+                { "expr": "histogram_quantile(0.99, sum(rate(story_crater_message_handle_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" }
+              ]
+            }
+          ]
+        },
+        {
+          "id": 10, "title": "Row: Resource Usage", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
+          "panels": [
+            {
+              "id": 11, "title": "CPU by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-backend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 12, "title": "Memory by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-backend\"}) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-backend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
+            }
+          ]
+        },
+        {
+          "id": 20, "title": "Row: Broker / Outbox / Dedup", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
+          "panels": [
+            {
+              "id": 21, "title": "Queue depth", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 0, "y": 3 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "story_crater_queue_depth", "legendFormat": "{{queue}}" }]
+            },
+            {
+              "id": 22, "title": "Outbox publish lag p95", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 8, "y": 3 },
+              "fieldConfig": { "defaults": { "unit": "ms" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_outbox_publish_lag_ms_bucket[5m])) by (le))" }]
+            },
+            {
+              "id": 23, "title": "Dedup redeliveries rate", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 16, "y": 3 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_dedup_redeliveries_total[5m])) by (agent, queue)", "legendFormat": "{{agent}}-{{queue}}" }]
+            }
+          ]
+        },
+        {
+          "id": 30, "title": "Row: LLM / Inference", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
+          "panels": [
+            {
+              "id": 31, "title": "Token usage rate", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_llm_tokens_used_total[5m])) by (agent, kind)", "legendFormat": "{{agent}}-{{kind}}" }]
+            },
+            {
+              "id": 32, "title": "LLM call duration p95", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 },
+              "fieldConfig": { "defaults": { "unit": "s" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_llm_call_duration_seconds_bucket[5m])) by (le, agent))", "legendFormat": "{{agent}}" }]
+            },
+            {
+              "id": 33, "title": "LLM call errors rate", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_llm_call_errors_total[5m])) by (agent, error_class)", "legendFormat": "{{agent}}-{{error_class}}" }]
+            }
+          ]
+        },
+        {
+          "id": 40, "title": "Row: SLOs / Degradation", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 },
+          "panels": [
+            {
+              "id": 41, "title": "CheckScene p95 latency (NFR-01: <1200ms)", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 },
+              "fieldConfig": {
+                "defaults": {
+                  "unit": "ms",
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      { "value": null, "color": "green" },
+                      { "value": 1200, "color": "red" }
+                    ]
+                  }
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.95, sum(rate(story_crater_check_latency_ms_bucket[5m])) by (le))" }]
+            },
+            {
+              "id": 42, "title": "Degrade state by service", "type": "stat",
+              "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 },
+              "fieldConfig": {
+                "defaults": {
+                  "mappings": [
+                    { "type": "value", "options": { "0": { "text": "NORMAL", "color": "green" } } },
+                    { "type": "value", "options": { "1": { "text": "SHEDDING", "color": "red" } } }
+                  ]
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "story_crater_degrade_state", "legendFormat": "{{service}}" }]
+            }
+          ]
+        },
+        {
+          "id": 50, "title": "Row: Logs", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
+          "panels": [
+            {
+              "id": 51, "title": "Recent logs (session_id / trace_id searchable)", "type": "logs",
+              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
+              "datasource": { "type": "loki", "uid": "loki" },
+              "targets": [{ "expr": "{namespace=\"story-crater-backend\"} | json" }]
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/svc-story-crater-frontend.yaml

@@ -0,0 +1,184 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-story-crater-frontend-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Story Crater Frontend"
+data:
+  svc-story-crater-frontend.json: |
+    {
+      "title": "Story Crater Frontend — Service Overview",
+      "uid": "svc-story-crater-frontend",
+      "schemaVersion": 39,
+      "timezone": "browser",
+      "time": { "from": "now-6h", "to": "now" },
+      "refresh": "30s",
+      "panels": [
+        {
+          "id": 1, "title": "Row: Availability & Golden Signals", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
+          "panels": [
+            {
+              "id": 2, "title": "Up", "type": "stat",
+              "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "fieldConfig": {
+                "defaults": {
+                  "color": { "mode": "thresholds" },
+                  "mappings": [
+                    { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
+                  ],
+                  "thresholds": { "mode": "absolute", "steps": [ { "value": null, "color": "red" }, { "value": 1, "color": "green" } ] }
+                }
+              },
+              "targets": [{ "expr": "min(up{job=\"story-crater-frontend\"})" }]
+            },
+            {
+              "id": 3, "title": "HTTP request rate by status", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 6, "y": 1 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total[5m])) by (status)", "legendFormat": "{{status}}" }]
+            },
+            {
+              "id": 4, "title": "Error rate % (5xx)", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 9, "x": 15, "y": 1 },
+              "fieldConfig": { "defaults": { "unit": "percent" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(story_crater_frontend_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(story_crater_frontend_http_requests_total[5m])) * 100" }]
+            },
+            {
+              "id": 5, "title": "Request duration p50/p95/p99", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
+              "fieldConfig": { "defaults": { "unit": "ms" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [
+                { "expr": "histogram_quantile(0.50, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p50" },
+                { "expr": "histogram_quantile(0.95, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p95" },
+                { "expr": "histogram_quantile(0.99, sum(rate(story_crater_frontend_http_request_duration_ms_bucket[5m])) by (le))", "legendFormat": "p99" }
+              ]
+            }
+          ]
+        },
+        {
+          "id": 10, "title": "Row: Resource Usage", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 },
+          "panels": [
+            {
+              "id": 11, "title": "CPU by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"story-crater-frontend\"}[5m])) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 12, "title": "Memory by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 },
+              "fieldConfig": { "defaults": { "unit": "bytes" } },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(container_memory_working_set_bytes{namespace=\"story-crater-frontend\"}) by (pod)", "legendFormat": "{{pod}}" }]
+            },
+            {
+              "id": 13, "title": "Restart rate by pod", "type": "timeseries",
+              "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "sum(rate(kube_pod_container_status_restarts_total{namespace=\"story-crater-frontend\"}[15m])) by (pod)", "legendFormat": "{{pod}}" }]
+            }
+          ]
+        },
+        {
+          "id": 20, "title": "Row: Web Vitals", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 },
+          "panels": [
+            {
+              "id": 21, "title": "LCP p75 (Largest Contentful Paint)", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 6, "x": 0, "y": 3 },
+              "fieldConfig": {
+                "defaults": {
+                  "unit": "ms",
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      { "value": null, "color": "green" },
+                      { "value": 2500, "color": "orange" },
+                      { "value": 4000, "color": "red" }
+                    ]
+                  }
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_lcp_ms)", "legendFormat": "LCP p75" }]
+            },
+            {
+              "id": 22, "title": "FID p75 (First Input Delay)", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 6, "x": 6, "y": 3 },
+              "fieldConfig": {
+                "defaults": {
+                  "unit": "ms",
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      { "value": null, "color": "green" },
+                      { "value": 100, "color": "orange" },
+                      { "value": 300, "color": "red" }
+                    ]
+                  }
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_fid_ms)", "legendFormat": "FID p75" }]
+            },
+            {
+              "id": 23, "title": "CLS (Cumulative Layout Shift)", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 6, "x": 12, "y": 3 },
+              "fieldConfig": {
+                "defaults": {
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      { "value": null, "color": "green" },
+                      { "value": 0.1, "color": "orange" },
+                      { "value": 0.25, "color": "red" }
+                    ]
+                  }
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "max(story_crater_frontend_web_vitals_cls)", "legendFormat": "CLS" }]
+            },
+            {
+              "id": 24, "title": "INP p75 (Interaction to Next Paint)", "type": "timeseries",
+              "gridPos": { "h": 7, "w": 6, "x": 18, "y": 3 },
+              "fieldConfig": {
+                "defaults": {
+                  "unit": "ms",
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      { "value": null, "color": "green" },
+                      { "value": 200, "color": "orange" },
+                      { "value": 500, "color": "red" }
+                    ]
+                  }
+                }
+              },
+              "datasource": { "type": "prometheus", "uid": "prometheus" },
+              "targets": [{ "expr": "histogram_quantile(0.75, story_crater_frontend_web_vitals_inp_ms)", "legendFormat": "INP p75" }]
+            }
+          ]
+        },
+        {
+          "id": 30, "title": "Row: Logs", "type": "row",
+          "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 },
+          "panels": [
+            {
+              "id": 31, "title": "Recent logs", "type": "logs",
+              "gridPos": { "h": 10, "w": 24, "x": 0, "y": 4 },
+              "datasource": { "type": "loki", "uid": "loki" },
+              "targets": [{ "expr": "{namespace=\"story-crater-frontend\"}" }]
+            }
+          ]
+        }
+      ]
+    }

k8s/monitoring/dashboards/svc-vault.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: svc-vault-dashboard
+  namespace: logging
+  labels:
+    grafana_dashboard: "1"
+  annotations:
+    grafana_folder: "Vault"
+data:
+  svc-vault.json: |
+    {"title":"Vault — Service Overview","uid":"svc-vault","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability & Golden Signals","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"Up","type":"stat","gridPos":{"h":4,"w":6,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"DOWN","color":"red"},"1":{"text":"UP","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"targets":[{"expr":"min(up{job=\"vault\"})"}]},{"id":3,"title":"Request rate by status","type":"timeseries","gridPos":{"h":8,"w":9,"x":6,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total[5m])) by (method)","legendFormat":"{{method}}"}]},{"id":4,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":9,"x":15,"y":1},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(vault_core_handle_request_total{error=\"true\"}[5m])) / sum(rate(vault_core_handle_request_total[5m])) * 100"}]},{"id":5,"title":"Request duration p50/p95/p99","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":9},"fieldConfig":{"defaults":{"unit":"s"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.50, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p50"},{"expr":"histogram_quantile(0.95, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p95"},{"expr":"histogram_quantile(0.99, sum(rate(vault_core_handle_request_duration_seconds_bucket[5m])) by (le))","legendFormat":"p99"}]}]},{"id":10,"title":"Row: Resource Usage","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"iam\",pod=~\"vault.*\"}[5m])) by (pod)","legendFormat":"{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=\"iam\",pod=~\"vault.*\"}) by (pod)","legendFormat":"{{pod}}"}]},{"id":13,"title":"Restart rate","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"iam\",pod=~\"vault.*\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Vault Seal State","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"Sealed","type":"stat","gridPos":{"h":6,"w":6,"x":0,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"UNSEALED","color":"green"},"1":{"text":"SEALED","color":"red"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"green"},{"value":1,"color":"red"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_unsealed"}]},{"id":22,"title":"Active","type":"stat","gridPos":{"h":6,"w":6,"x":6,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"INACTIVE","color":"red"},"1":{"text":"ACTIVE","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"red"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_active"}]},{"id":23,"title":"Replication (Primary)","type":"stat","gridPos":{"h":6,"w":6,"x":12,"y":3},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"type":"value","options":{"0":{"text":"SECONDARY","color":"orange"},"1":{"text":"PRIMARY","color":"green"}}}],"thresholds":{"mode":"absolute","steps":[{"value":null,"color":"orange"},{"value":1,"color":"green"}]}}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_core_replication_primary"}]},{"id":24,"title":"Active tokens","type":"stat","gridPos":{"h":6,"w":6,"x":18,"y":3},"fieldConfig":{"defaults":{"unit":"short"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"vault_token_total"}]}]},{"id":30,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":31,"title":"Recent logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"iam\",container=\"vault\"}"}]}]}]}

k8s/monitoring/prometheus-values.yaml

@@ -0,0 +1,133 @@
+# monitoring/prometheus-values.yaml
+# Chart: prometheus-community/kube-prometheus-stack
+# Release name: prometheus  (affects all generated resource names)
+#
+# What this installs:
+#   - Prometheus Operator (manages the Prometheus CRD)
+#   - Prometheus StatefulSet (scrapes metrics, stores on Longhorn PVC)
+#   - node-exporter DaemonSet (kernel CPU/RAM/disk/network per node)
+#   - kube-state-metrics Deployment (K8s object state — pod resource requests, phases)
+#
+# What this deliberately omits:
+#   - Grafana: already deployed in the logging namespace
+#   - Alertmanager: enable later when you want Slack/PagerDuty routing
+#   - kubeControllerManager / kubeScheduler / kubeEtcd: Talos only binds these
+#     on 127.0.0.1 — the default ServiceMonitors can't reach them
+#   - kubeProxy: removed cluster-wide; Cilium handles routing instead
+
+# ── Grafana ───────────────────────────────────────────────────────────────────
+grafana:
+  enabled: false
+
+# ── Alertmanager ──────────────────────────────────────────────────────────────
+alertmanager:
+  enabled: false
+
+# ── Prometheus ────────────────────────────────────────────────────────────────
+prometheus:
+  prometheusSpec:
+    retention: 15d
+    retentionSize: "18GB"
+
+    # Scrape timeout: increased to 60s to tolerate 5+ second network latency spikes
+    # Default: 10s — too aggressive for latency-prone clusters
+    scrapeInterval: 30s
+    scrapeTimeout: 60s
+    evaluationInterval: 30s
+
+    # Persistent storage — metrics survive node reboots and pod restarts.
+    # Longhorn reattaches the PVC automatically when the pod reschedules.
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: longhorn
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 20Gi
+
+    resources:
+      requests:
+        cpu: 200m
+        memory: 512Mi
+      limits:
+        cpu: 1000m
+        memory: 1Gi
+
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+
+    # Match ServiceMonitors/PodMonitors from all namespaces, not just the ones
+    # the chart itself creates. Required to scrape workloads in other namespaces.
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+
+# ── Prometheus Operator ────────────────────────────────────────────────────────
+prometheusOperator:
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 200m
+      memory: 256Mi
+
+# ── node-exporter ─────────────────────────────────────────────────────────────
+# DaemonSet: one pod per node, reads /proc and /sys directly via hostPID.
+# These filesystem excludes prevent scrape errors on Talos's read-only overlayfs
+# mounts and containerd's ephemeral snapshot filesystems.
+nodeExporter:
+  enabled: true
+
+prometheus-node-exporter:
+  extraArgs:
+    - --collector.filesystem.mount-points-exclude=^/(dev|proc|run/credentials/.+|sys|var/lib/containerd/.+|var/lib/kubelet/.+|run/.+)($|/)
+    - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
+  tolerations:
+    - operator: Exists  # schedule on every node regardless of taints
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# ── kube-state-metrics ────────────────────────────────────────────────────────
+# Watches the K8s API; surfaces pod CPU/memory requests, deployment replica
+# counts, pod phase, etc. — the "are my workloads healthy?" layer.
+kubeStateMetrics:
+  enabled: true
+
+kube-state-metrics:
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# ── Disable unreachable control-plane scrape targets ──────────────────────────
+kubeControllerManager:
+  enabled: false
+
+kubeScheduler:
+  enabled: false
+
+kubeEtcd:
+  enabled: false
+
+kubeProxy:
+  enabled: false

k8s/monitoring/servicemonitors/argocd.yaml

@@ -0,0 +1,37 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: argocd
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - cicd
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: argocd-metrics
+  endpoints:
+    - port: metrics
+      interval: 30s
+      scrapeTimeout: 10s
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: argocd-server
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - cicd
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: argocd-server-metrics
+  endpoints:
+    - port: metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s/monitoring/servicemonitors/authentik.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: authentik
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - iam
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: authentik
+  endpoints:
+    - port: metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s/monitoring/servicemonitors/forgejo.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: forgejo
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - cicd
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: gitea
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: 30s
+      scrapeTimeout: 10s

k8s/monitoring/servicemonitors/minio.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: minio
+  namespace: monitoring
+  labels:
+    release: kube-prometheus-stack
+spec:
+  namespaceSelector:
+    matchNames:
+      - storage
+  selector:
+    matchLabels:
+      app: minio
+  endpoints:
+    - port: minio-api
+      path: /minio/v2/metrics/cluster
+      interval: 30s
+      scrapeTimeout: 10s