refactor(k8s): consolidate to infra/+apps/ single-source tree, dedicated per-app CNPG (authentik-db/temporal-db), wire monitoring-config, forgejo→cicd ns, drop orphan/stale (data-schemas, ollama, story-crater, sqs/argocd, key-rotation)
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,238 @@
|
||||
# logging/promtail-values.yaml
|
||||
# Promtail — log shipper DaemonSet. One pod per node; reads container logs
|
||||
# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
|
||||
# then pushes them to Loki.
|
||||
|
||||
daemonset:
|
||||
enabled: true
|
||||
|
||||
config:
|
||||
logLevel: warn # info is noisy at homelab scale; warn keeps Promtail quiet
|
||||
serverPort: 3101
|
||||
|
||||
# Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
|
||||
clients:
|
||||
- url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
|
||||
|
||||
snippets:
|
||||
scrapeConfigs: |
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# LOG AGGREGATION STRATEGY
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Three scrape jobs balance log volume, visibility, and cost:
|
||||
#
|
||||
# 1. kubernetes-pods-critical (no sampling)
|
||||
# Namespaces: iam, monitoring, temporal, cicd, cert-manager
|
||||
# Strategy: Keep ALL logs (full fidelity for security/observability)
|
||||
# Rationale: Critical services justify full log retention
|
||||
#
|
||||
# 2. kubernetes-pods (50% sampling + selective drops)
|
||||
# Namespaces: all others (sqs, storage, ingress-nginx, llm, etc.)
|
||||
# Drops: health/heartbeat/ping/keepalive + DEBUG from verbose services
|
||||
# Sampling: 50% of INFO/DEBUG (ERROR/WARN always kept)
|
||||
# Rationale: Reduce noise while preserving error visibility
|
||||
#
|
||||
# 3. systemd-journal (no filtering)
|
||||
# Source: Talos node kernel/kubelet logs
|
||||
# Strategy: Keep all for system-level debugging
|
||||
#
|
||||
# Result: ~60-70% total log reduction vs. 90% before
|
||||
# All errors/warnings captured; info logs sampled intelligently
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ── Pod logs ──────────────────────────────────────────────────────────
|
||||
# Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
|
||||
# cri pipeline stage parses the CRI-O/containerd log format so timestamps
|
||||
# and stream (stdout/stderr) are extracted properly before the line is
|
||||
# forwarded to Loki.
|
||||
# relabel_configs build useful labels: namespace, pod, container, node,
|
||||
# and a job label of the form "namespace/pod-name" for easy filtering.
|
||||
- job_name: kubernetes-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
# Exclude critical namespaces (handled separately by kubernetes-pods-critical)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
regex: '(iam|monitoring|temporal|cicd|cert-manager)'
|
||||
action: drop
|
||||
# Apply standard pod labels
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: __host__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- action: replace
|
||||
replacement: $1
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
- __meta_kubernetes_pod_name
|
||||
target_label: job
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: /var/log/pods/*$1/*.log
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_pod_uid
|
||||
- __meta_kubernetes_pod_container_name
|
||||
target_label: __path__
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
pipeline_stages:
|
||||
# ── Drop Rule 1: Noise — all services ──────────────────────────
|
||||
# Discard health checks, keep-alives, pings, trace logs
|
||||
- drop:
|
||||
expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE|timeout.*retrying|level="trace"|level="TRACE")'
|
||||
|
||||
# ── Drop Rule 2: DEBUG from verbose services ────────────────────
|
||||
# storage, ingress-nginx, kube-system have chatty DEBUG logs
|
||||
# Example: {namespace="storage"...level="debug"}
|
||||
- drop:
|
||||
expression: 'namespace="(storage|ingress-nginx|kube-system|sqs)".*level="(debug|DEBUG)"'
|
||||
|
||||
# ── Sampling: 50% keep rate on INFO + DEBUG after drops ────────
|
||||
# Keep 100% of ERROR/WARN; keep 50% of INFO (if not dropped above)
|
||||
# This balances log volume with visibility
|
||||
- sampling:
|
||||
rate: 0.5
|
||||
enabled: true
|
||||
|
||||
# ── Pod logs from critical services (no sampling) ────────────────────
|
||||
# iam, monitoring, temporal, cicd = always keep all logs (no sampling)
|
||||
# These services are critical for debugging and deserve full log retention
|
||||
- job_name: kubernetes-pods-critical
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
pipeline_stages:
|
||||
# Only drop obvious noise; keep everything else
|
||||
- drop:
|
||||
expression: '(health|heartbeat|ping|pong|keepalive|KEEPALIVE)'
|
||||
relabel_configs:
|
||||
# Only select pods from critical namespaces
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
regex: '(iam|monitoring|temporal|cicd|cert-manager)'
|
||||
action: keep
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: __host__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- action: replace
|
||||
replacement: $1
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_namespace
|
||||
- __meta_kubernetes_pod_name
|
||||
target_label: job
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: /var/log/pods/*$1/*.log
|
||||
separator: /
|
||||
source_labels:
|
||||
- __meta_kubernetes_pod_uid
|
||||
- __meta_kubernetes_pod_container_name
|
||||
target_label: __path__
|
||||
- action: replace
|
||||
source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# ── Talos systemd journal ──────────────────────────────────────────────
|
||||
# Talos runs containerd, kubelet, and kernel messages through systemd-
|
||||
# journald — they never appear in /var/log/pods. This job reads the
|
||||
# binary journal directly and emits unit (systemd unit name) and node
|
||||
# labels so you can filter by service (e.g. unit="kubelet.service").
|
||||
# max_age: 12h — only tail recent journal entries on startup; prevents
|
||||
# Promtail from replaying hours of history after a pod restart.
|
||||
- job_name: systemd-journal
|
||||
journal:
|
||||
path: /var/log/journal
|
||||
max_age: 12h
|
||||
labels:
|
||||
job: systemd-journal
|
||||
relabel_configs:
|
||||
- source_labels: [__journal__systemd_unit]
|
||||
target_label: unit
|
||||
- source_labels: [__journal__hostname]
|
||||
target_label: node
|
||||
|
||||
# ── Volume mounts ─────────────────────────────────────────────────────────────
|
||||
# hostPath mounts give Promtail access to the node's log directories.
|
||||
# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
|
||||
# /var/log/journal — Talos systemd journal (binary format, read via journald API)
|
||||
# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
|
||||
# before journald has written anything.
|
||||
defaultVolumes:
|
||||
- name: pods-logs
|
||||
hostPath:
|
||||
path: /var/log/pods
|
||||
- name: journal
|
||||
hostPath:
|
||||
path: /var/log/journal
|
||||
type: DirectoryOrCreate
|
||||
|
||||
defaultVolumeMounts:
|
||||
- name: pods-logs
|
||||
mountPath: /var/log/pods
|
||||
readOnly: true
|
||||
- name: journal
|
||||
mountPath: /var/log/journal
|
||||
readOnly: true
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
|
||||
# ── Security context ──────────────────────────────────────────────────────────
|
||||
# privileged: true — required to open the binary journal on Talos (journald
|
||||
# uses file locking that non-privileged processes can't bypass).
|
||||
# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
|
||||
# even with a read-only root filesystem. Without this, pod logs from containers
|
||||
# running as non-root UIDs would be unreadable.
|
||||
# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
|
||||
# write to its own container filesystem.
|
||||
# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
|
||||
# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
add:
|
||||
- DAC_READ_SEARCH
|
||||
|
||||
# runAsUser/Group: 0 (root) — needed to access journal files and pod log
|
||||
# directories that are owned by root on Talos nodes.
|
||||
podSecurityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
|
||||
# Tolerate every taint so Promtail runs on ALL nodes including the CP.
|
||||
# Without this, the control-plane node's logs (etcd, kube-apiserver) would
|
||||
# be missing from Loki entirely.
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
operator: Exists
|
||||
- effect: NoExecute
|
||||
operator: Exists
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
Reference in New Issue
Block a user