homelab/k8s/infra/monitoring/prometheus-values.yaml

# monitoring/prometheus-values.yaml
# Chart: prometheus-community/kube-prometheus-stack
# Release name: prometheus  (affects all generated resource names)
#
# What this installs:
#   - Prometheus Operator (manages the Prometheus CRD)
#   - Prometheus StatefulSet (scrapes metrics, stores on Longhorn PVC)
#   - node-exporter DaemonSet (kernel CPU/RAM/disk/network per node)
#   - kube-state-metrics Deployment (K8s object state — pod resource requests, phases)
#
# What this deliberately omits:
#   - Grafana: already deployed in the logging namespace
#   - Alertmanager: enable later when you want Slack/PagerDuty routing
#   - kubeControllerManager / kubeScheduler / kubeEtcd: Talos only binds these
#     on 127.0.0.1 — the default ServiceMonitors can't reach them
#   - kubeProxy: removed cluster-wide; Cilium handles routing instead

# ── Grafana ───────────────────────────────────────────────────────────────────
grafana:
  enabled: false

# ── Alertmanager ──────────────────────────────────────────────────────────────
# Enabled with a default (null) receiver — every firing PrometheusRule lands in
# the Alertmanager UI and Grafana's Alerting view; no external Slack/email/
# PagerDuty notifier is wired yet (add a receiver + route later). Storage pinned
# to az-a (sole Longhorn node) like Prometheus so the RWO PVC can attach.
alertmanager:
  enabled: true
  alertmanagerSpec:
    nodeSelector:
      topology.kubernetes.io/zone: az-a
    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: longhorn
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 2Gi
  config:
    route:
      group_by: ["alertname", "namespace"]
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      receiver: "null"
    receivers:
      - name: "null"

# ── Prometheus ────────────────────────────────────────────────────────────────
prometheus:
  prometheusSpec:
    retention: 30d
    # Must stay under the PVC size below (25Gi). At 95GB the limit never
    # triggered, the volume filled, and Prometheus crashlooped with
    # "no space left on device" writing /prometheus/queries.active.
    retentionSize: "18GB"

    # scrapeTimeout MUST be <= scrapeInterval or the operator refuses to generate
    # the Prometheus config ("scrapeTimeout greater than scrapeInterval") and no
    # STS is created. Use a larger interval to keep the 60s timeout headroom.
    scrapeInterval: 60s
    scrapeTimeout: 60s
    evaluationInterval: 30s

    # Pin to worker-1 for more storage capacity
    nodeSelector:
      kubernetes.io/hostname: worker-1

    # Persistent storage — metrics survive node reboots and pod restarts.
    # Uses the default `longhorn` StorageClass; nodeSelector above already pins
    # the pod to az-a (the sole Longhorn node), so Immediate binding is fine.
    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: longhorn
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 25Gi

    resources:
      requests:
        cpu: 200m
        memory: 512Mi
      limits:
        cpu: 1000m
        memory: 1Gi

    tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
        effect: NoSchedule

    # Match ServiceMonitors/PodMonitors from all namespaces, not just the ones
    # the chart itself creates. Required to scrape workloads in other namespaces.
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false

# ── Prometheus Operator ────────────────────────────────────────────────────────
prometheusOperator:
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  resources:
    requests:
      cpu: 100m
      memory: 128Mi
    limits:
      cpu: 200m
      memory: 256Mi

# ── node-exporter ─────────────────────────────────────────────────────────────
# DaemonSet: one pod per node, reads /proc and /sys directly via hostPID.
# These filesystem excludes prevent scrape errors on Talos's read-only overlayfs
# mounts and containerd's ephemeral snapshot filesystems.
nodeExporter:
  enabled: true

prometheus-node-exporter:
  extraArgs:
    - --collector.filesystem.mount-points-exclude=^/(dev|proc|run/credentials/.+|sys|var/lib/containerd/.+|var/lib/kubelet/.+|run/.+)($|/)
    - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
  tolerations:
    - operator: Exists  # schedule on every node regardless of taints
  resources:
    requests:
      cpu: 50m
      memory: 64Mi
    limits:
      cpu: 100m
      memory: 128Mi

# ── kube-state-metrics ────────────────────────────────────────────────────────
# Watches the K8s API; surfaces pod CPU/memory requests, deployment replica
# counts, pod phase, etc. — the "are my workloads healthy?" layer.
kubeStateMetrics:
  enabled: true

kube-state-metrics:
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule
  resources:
    requests:
      cpu: 50m
      memory: 64Mi
    limits:
      cpu: 100m
      memory: 128Mi

# ── Disable unreachable control-plane scrape targets ──────────────────────────
kubeControllerManager:
  enabled: false

kubeScheduler:
  enabled: false

kubeEtcd:
  enabled: false

kubeProxy:
  enabled: false