homelab/k8s/logging/promtail-values.yaml

# logging/promtail-values.yaml
# Promtail — log shipper DaemonSet. One pod per node; reads container logs
# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
# then pushes them to Loki.

daemonset:
  enabled: true

config:
  logLevel: warn   # info is noisy at homelab scale; warn keeps Promtail quiet
  serverPort: 3101

  # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
  clients:
    - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push

  snippets:
    scrapeConfigs: |
      # ═══════════════════════════════════════════════════════════════════════
      # LOG AGGREGATION STRATEGY
      # ═══════════════════════════════════════════════════════════════════════
      # Three scrape jobs balance log volume, visibility, and cost:
      #
      # 1. kubernetes-pods-critical (no sampling)
      #    Namespaces: iam, monitoring, temporal, cicd, cert-manager
      #    Strategy: Keep ALL logs (full fidelity for security/observability)
      #    Rationale: Critical services justify full log retention
      #
      # 2. kubernetes-pods (50% sampling + selective drops)
      #    Namespaces: all others (sqs, storage, ingress-nginx, llm, etc.)
      #    Drops: health/heartbeat/ping/keepalive + DEBUG from verbose services
      #    Sampling: 50% of INFO/DEBUG (ERROR/WARN always kept)
      #    Rationale: Reduce noise while preserving error visibility
      #
      # 3. systemd-journal (no filtering)
      #    Source: Talos node kernel/kubelet logs
      #    Strategy: Keep all for system-level debugging
      #
      # Result: ~60-70% total log reduction vs. 90% before
      #         All errors/warnings captured; info logs sampled intelligently
      # ═══════════════════════════════════════════════════════════════════════

      # ── Pod logs ──────────────────────────────────────────────────────────
      # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
      # cri pipeline stage parses the CRI-O/containerd log format so timestamps
      # and stream (stdout/stderr) are extracted properly before the line is
      # forwarded to Loki.
      # relabel_configs build useful labels: namespace, pod, container, node,
      # and a job label of the form "namespace/pod-name" for easy filtering.
      - job_name: kubernetes-pods
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          # Exclude critical namespaces (handled separately by kubernetes-pods-critical)
          - source_labels: [__meta_kubernetes_namespace]
            regex: '(iam|monitoring|temporal|cicd|cert-manager)'
            action: drop
          # Apply standard pod labels
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: __host__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - action: replace
            replacement: $1
            separator: /
            source_labels:
              - __meta_kubernetes_namespace
              - __meta_kubernetes_pod_name
            target_label: job
          - action: replace
            source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - action: replace
            source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - action: replace
            source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
              - __meta_kubernetes_pod_uid
              - __meta_kubernetes_pod_container_name
            target_label: __path__
          - action: replace
            source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node
        pipeline_stages:
          # ── Drop Rule 1: Noise — all services ──────────────────────────
          # Discard health checks, keep-alives, pings, trace logs
          - drop:
              expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE|timeout.*retrying|level="trace"|level="TRACE")'

          # ── Drop Rule 2: DEBUG from verbose services ────────────────────
          # storage, ingress-nginx, kube-system have chatty DEBUG logs
          # Example: {namespace="storage"...level="debug"}
          - drop:
              expression: 'namespace="(storage|ingress-nginx|kube-system|sqs)".*level="(debug|DEBUG)"'

          # ── Sampling: 50% keep rate on INFO + DEBUG after drops ────────
          # Keep 100% of ERROR/WARN; keep 50% of INFO (if not dropped above)
          # This balances log volume with visibility
          - sampling:
              rate: 0.5
              enabled: true

      # ── Pod logs from critical services (no sampling) ────────────────────
      # iam, monitoring, temporal, cicd = always keep all logs (no sampling)
      # These services are critical for debugging and deserve full log retention
      - job_name: kubernetes-pods-critical
        kubernetes_sd_configs:
          - role: pod
        pipeline_stages:
          # Only drop obvious noise; keep everything else
          - drop:
              expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE)'
        relabel_configs:
          # Only select pods from critical namespaces
          - source_labels: [__meta_kubernetes_namespace]
            regex: '(iam|monitoring|temporal|cicd|cert-manager)'
            action: keep
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: __host__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - action: replace
            replacement: $1
            separator: /
            source_labels:
              - __meta_kubernetes_namespace
              - __meta_kubernetes_pod_name
            target_label: job
          - action: replace
            source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - action: replace
            source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - action: replace
            source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
              - __meta_kubernetes_pod_uid
              - __meta_kubernetes_pod_container_name
            target_label: __path__
          - action: replace
            source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # ── Talos systemd journal ──────────────────────────────────────────────
      # Talos runs containerd, kubelet, and kernel messages through systemd-
      # journald — they never appear in /var/log/pods. This job reads the
      # binary journal directly and emits unit (systemd unit name) and node
      # labels so you can filter by service (e.g. unit="kubelet.service").
      # max_age: 12h — only tail recent journal entries on startup; prevents
      # Promtail from replaying hours of history after a pod restart.
      - job_name: systemd-journal
        journal:
          path: /var/log/journal
          max_age: 12h
          labels:
            job: systemd-journal
        relabel_configs:
          - source_labels: [__journal__systemd_unit]
            target_label: unit
          - source_labels: [__journal__hostname]
            target_label: node

# ── Volume mounts ─────────────────────────────────────────────────────────────
# hostPath mounts give Promtail access to the node's log directories.
# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
# /var/log/journal — Talos systemd journal (binary format, read via journald API)
# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
# before journald has written anything.
defaultVolumes:
  - name: pods-logs
    hostPath:
      path: /var/log/pods
  - name: journal
    hostPath:
      path: /var/log/journal
      type: DirectoryOrCreate

defaultVolumeMounts:
  - name: pods-logs
    mountPath: /var/log/pods
    readOnly: true
  - name: journal
    mountPath: /var/log/journal
    readOnly: true

resources:
  requests:
    cpu: 50m
    memory: 64Mi
  limits:
    cpu: 200m
    memory: 128Mi

# ── Security context ──────────────────────────────────────────────────────────
# privileged: true — required to open the binary journal on Talos (journald
# uses file locking that non-privileged processes can't bypass).
# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
# even with a read-only root filesystem. Without this, pod logs from containers
# running as non-root UIDs would be unreadable.
# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
# write to its own container filesystem.
# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
containerSecurityContext:
  privileged: true
  allowPrivilegeEscalation: true
  readOnlyRootFilesystem: true
  capabilities:
    drop:
      - ALL
    add:
      - DAC_READ_SEARCH

# runAsUser/Group: 0 (root) — needed to access journal files and pod log
# directories that are owned by root on Talos nodes.
podSecurityContext:
  runAsUser: 0
  runAsGroup: 0

# Tolerate every taint so Promtail runs on ALL nodes including the CP.
# Without this, the control-plane node's logs (etcd, kube-apiserver) would
# be missing from Loki entirely.
tolerations:
  - effect: NoSchedule
    operator: Exists
  - effect: NoExecute
    operator: Exists

serviceMonitor:
  enabled: false