# logging/promtail-values.yaml # Promtail — log shipper DaemonSet. One pod per node; reads container logs # from /var/log/pods and Talos kernel/service logs from /var/log/journal, # then pushes them to Loki. daemonset: enabled: true config: logLevel: warn # info is noisy at homelab scale; warn keeps Promtail quiet serverPort: 3101 # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false). clients: - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push snippets: scrapeConfigs: | # ═══════════════════════════════════════════════════════════════════════ # LOG AGGREGATION STRATEGY # ═══════════════════════════════════════════════════════════════════════ # Three scrape jobs balance log volume, visibility, and cost: # # 1. kubernetes-pods-critical (no sampling) # Namespaces: iam, monitoring, temporal, cicd, cert-manager # Strategy: Keep ALL logs (full fidelity for security/observability) # Rationale: Critical services justify full log retention # # 2. kubernetes-pods (50% sampling + selective drops) # Namespaces: all others (sqs, storage, ingress-nginx, llm, etc.) # Drops: health/heartbeat/ping/keepalive + DEBUG from verbose services # Sampling: 50% of INFO/DEBUG (ERROR/WARN always kept) # Rationale: Reduce noise while preserving error visibility # # 3. systemd-journal (no filtering) # Source: Talos node kernel/kubelet logs # Strategy: Keep all for system-level debugging # # Result: ~60-70% total log reduction vs. 90% before # All errors/warnings captured; info logs sampled intelligently # ═══════════════════════════════════════════════════════════════════════ # ── Pod logs ────────────────────────────────────────────────────────── # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod). # cri pipeline stage parses the CRI-O/containerd log format so timestamps # and stream (stdout/stderr) are extracted properly before the line is # forwarded to Loki. # relabel_configs build useful labels: namespace, pod, container, node, # and a job label of the form "namespace/pod-name" for easy filtering. - job_name: kubernetes-pods kubernetes_sd_configs: - role: pod relabel_configs: # Exclude critical namespaces (handled separately by kubernetes-pods-critical) - source_labels: [__meta_kubernetes_namespace] regex: '(iam|monitoring|temporal|cicd|cert-manager)' action: drop # Apply standard pod labels - source_labels: [__meta_kubernetes_pod_node_name] target_label: __host__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __meta_kubernetes_pod_name target_label: job - action: replace source_labels: [__meta_kubernetes_namespace] target_label: namespace - action: replace source_labels: [__meta_kubernetes_pod_name] target_label: pod - action: replace source_labels: [__meta_kubernetes_pod_container_name] target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - action: replace source_labels: [__meta_kubernetes_pod_node_name] target_label: node pipeline_stages: # ── Drop Rule 1: Noise — all services ────────────────────────── # Discard health checks, keep-alives, pings, trace logs - drop: expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE|timeout.*retrying|level="trace"|level="TRACE")' # ── Drop Rule 2: DEBUG from verbose services ──────────────────── # storage, ingress-nginx, kube-system have chatty DEBUG logs # Example: {namespace="storage"...level="debug"} - drop: expression: 'namespace="(storage|ingress-nginx|kube-system|sqs)".*level="(debug|DEBUG)"' # ── Sampling: 50% keep rate on INFO + DEBUG after drops ──────── # Keep 100% of ERROR/WARN; keep 50% of INFO (if not dropped above) # This balances log volume with visibility - sampling: rate: 0.5 enabled: true # ── Pod logs from critical services (no sampling) ──────────────────── # iam, monitoring, temporal, cicd = always keep all logs (no sampling) # These services are critical for debugging and deserve full log retention - job_name: kubernetes-pods-critical kubernetes_sd_configs: - role: pod pipeline_stages: # Only drop obvious noise; keep everything else - drop: expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE)' relabel_configs: # Only select pods from critical namespaces - source_labels: [__meta_kubernetes_namespace] regex: '(iam|monitoring|temporal|cicd|cert-manager)' action: keep - source_labels: [__meta_kubernetes_pod_node_name] target_label: __host__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __meta_kubernetes_pod_name target_label: job - action: replace source_labels: [__meta_kubernetes_namespace] target_label: namespace - action: replace source_labels: [__meta_kubernetes_pod_name] target_label: pod - action: replace source_labels: [__meta_kubernetes_pod_container_name] target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - action: replace source_labels: [__meta_kubernetes_pod_node_name] target_label: node # ── Talos systemd journal ────────────────────────────────────────────── # Talos runs containerd, kubelet, and kernel messages through systemd- # journald — they never appear in /var/log/pods. This job reads the # binary journal directly and emits unit (systemd unit name) and node # labels so you can filter by service (e.g. unit="kubelet.service"). # max_age: 12h — only tail recent journal entries on startup; prevents # Promtail from replaying hours of history after a pod restart. - job_name: systemd-journal journal: path: /var/log/journal max_age: 12h labels: job: systemd-journal relabel_configs: - source_labels: [__journal__systemd_unit] target_label: unit - source_labels: [__journal__hostname] target_label: node # ── Volume mounts ───────────────────────────────────────────────────────────── # hostPath mounts give Promtail access to the node's log directories. # /var/log/pods — container stdout/stderr (written by containerd's CRI layer) # /var/log/journal — Talos systemd journal (binary format, read via journald API) # DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes # before journald has written anything. defaultVolumes: - name: pods-logs hostPath: path: /var/log/pods - name: journal hostPath: path: /var/log/journal type: DirectoryOrCreate defaultVolumeMounts: - name: pods-logs mountPath: /var/log/pods readOnly: true - name: journal mountPath: /var/log/journal readOnly: true resources: requests: cpu: 50m memory: 64Mi limits: cpu: 200m memory: 128Mi # ── Security context ────────────────────────────────────────────────────────── # privileged: true — required to open the binary journal on Talos (journald # uses file locking that non-privileged processes can't bypass). # DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods # even with a read-only root filesystem. Without this, pod logs from containers # running as non-root UIDs would be unreadable. # readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to # write to its own container filesystem. # allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+ # rejects privileged containers that explicitly set allowPrivilegeEscalation: false. containerSecurityContext: privileged: true allowPrivilegeEscalation: true readOnlyRootFilesystem: true capabilities: drop: - ALL add: - DAC_READ_SEARCH # runAsUser/Group: 0 (root) — needed to access journal files and pod log # directories that are owned by root on Talos nodes. podSecurityContext: runAsUser: 0 runAsGroup: 0 # Tolerate every taint so Promtail runs on ALL nodes including the CP. # Without this, the control-plane node's logs (etcd, kube-apiserver) would # be missing from Loki entirely. tolerations: - effect: NoSchedule operator: Exists - effect: NoExecute operator: Exists serviceMonitor: enabled: false