homelab/k8s/logging/promtail-values.yaml

# logging/promtail-values.yaml
# Promtail — log shipper DaemonSet. One pod per node; reads container logs
# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
# then pushes them to Loki.

daemonset:
  enabled: true

config:
  logLevel: warn   # info is noisy at homelab scale; warn keeps Promtail quiet
  serverPort: 3101

  # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
  clients:
    - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push

  snippets:
    scrapeConfigs: |
      # ── Pod logs ──────────────────────────────────────────────────────────
      # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
      # cri pipeline stage parses the CRI-O/containerd log format so timestamps
      # and stream (stdout/stderr) are extracted properly before the line is
      # forwarded to Loki.
      # relabel_configs build useful labels: namespace, pod, container, node,
      # and a job label of the form "namespace/pod-name" for easy filtering.
      - job_name: kubernetes-pods
        kubernetes_sd_configs:
          - role: pod
        pipeline_stages:
          - drop:
              expression: '(health|heartbeat|ping|keepalive|level="debug"|"timeout".*"retrying")'
          - sampling:
              rate: 0.1
              enabled: true
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_node_name]
            target_label: __host__
          - action: labelmap
            regex: __meta_kubernetes_pod_label_(.+)
          - action: replace
            replacement: $1
            separator: /
            source_labels:
              - __meta_kubernetes_namespace
              - __meta_kubernetes_pod_name
            target_label: job
          - action: replace
            source_labels: [__meta_kubernetes_namespace]
            target_label: namespace
          - action: replace
            source_labels: [__meta_kubernetes_pod_name]
            target_label: pod
          - action: replace
            source_labels: [__meta_kubernetes_pod_container_name]
            target_label: container
          - replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
              - __meta_kubernetes_pod_uid
              - __meta_kubernetes_pod_container_name
            target_label: __path__
          - action: replace
            source_labels: [__meta_kubernetes_pod_node_name]
            target_label: node

      # ── Talos systemd journal ──────────────────────────────────────────────
      # Talos runs containerd, kubelet, and kernel messages through systemd-
      # journald — they never appear in /var/log/pods. This job reads the
      # binary journal directly and emits unit (systemd unit name) and node
      # labels so you can filter by service (e.g. unit="kubelet.service").
      # max_age: 12h — only tail recent journal entries on startup; prevents
      # Promtail from replaying hours of history after a pod restart.
      - job_name: systemd-journal
        journal:
          path: /var/log/journal
          max_age: 12h
          labels:
            job: systemd-journal
        relabel_configs:
          - source_labels: [__journal__systemd_unit]
            target_label: unit
          - source_labels: [__journal__hostname]
            target_label: node

# ── Volume mounts ─────────────────────────────────────────────────────────────
# hostPath mounts give Promtail access to the node's log directories.
# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
# /var/log/journal — Talos systemd journal (binary format, read via journald API)
# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
# before journald has written anything.
defaultVolumes:
  - name: pods-logs
    hostPath:
      path: /var/log/pods
  - name: journal
    hostPath:
      path: /var/log/journal
      type: DirectoryOrCreate

defaultVolumeMounts:
  - name: pods-logs
    mountPath: /var/log/pods
    readOnly: true
  - name: journal
    mountPath: /var/log/journal
    readOnly: true

resources:
  requests:
    cpu: 50m
    memory: 64Mi
  limits:
    cpu: 200m
    memory: 128Mi

# ── Security context ──────────────────────────────────────────────────────────
# privileged: true — required to open the binary journal on Talos (journald
# uses file locking that non-privileged processes can't bypass).
# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
# even with a read-only root filesystem. Without this, pod logs from containers
# running as non-root UIDs would be unreadable.
# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
# write to its own container filesystem.
# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
containerSecurityContext:
  privileged: true
  allowPrivilegeEscalation: true
  readOnlyRootFilesystem: true
  capabilities:
    drop:
      - ALL
    add:
      - DAC_READ_SEARCH

# runAsUser/Group: 0 (root) — needed to access journal files and pod log
# directories that are owned by root on Talos nodes.
podSecurityContext:
  runAsUser: 0
  runAsGroup: 0

# Tolerate every taint so Promtail runs on ALL nodes including the CP.
# Without this, the control-plane node's logs (etcd, kube-apiserver) would
# be missing from Loki entirely.
tolerations:
  - effect: NoSchedule
    operator: Exists
  - effect: NoExecute
    operator: Exists

serviceMonitor:
  enabled: false