k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)

k8s/logging/promtail-values.yaml

@@ -0,0 +1,152 @@
+# logging/promtail-values.yaml
+# Promtail — log shipper DaemonSet. One pod per node; reads container logs
+# from /var/log/pods and Talos kernel/service logs from /var/log/journal,
+# then pushes them to Loki.
+
+daemonset:
+  enabled: true
+
+config:
+  logLevel: warn   # info is noisy at homelab scale; warn keeps Promtail quiet
+  serverPort: 3101
+
+  # Push endpoint — in-cluster DNS, no auth (Loki has auth_enabled: false).
+  clients:
+    - url: http://loki.logging.svc.cluster.local:3100/loki/api/v1/push
+
+  snippets:
+    scrapeConfigs: |
+      # ── Pod logs ──────────────────────────────────────────────────────────
+      # Discovers pods via the Kubernetes API (kubernetes_sd_configs role: pod).
+      # cri pipeline stage parses the CRI-O/containerd log format so timestamps
+      # and stream (stdout/stderr) are extracted properly before the line is
+      # forwarded to Loki.
+      # relabel_configs build useful labels: namespace, pod, container, node,
+      # and a job label of the form "namespace/pod-name" for easy filtering.
+      - job_name: kubernetes-pods
+        kubernetes_sd_configs:
+          - role: pod
+        pipeline_stages:
+          - drop:
+              expression: '(health|heartbeat|ping|keepalive|level="debug"|"timeout".*"retrying")'
+          - sampling:
+              rate: 0.1
+              enabled: true
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: __host__
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          - action: replace
+            replacement: $1
+            separator: /
+            source_labels:
+              - __meta_kubernetes_namespace
+              - __meta_kubernetes_pod_name
+            target_label: job
+          - action: replace
+            source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_name]
+            target_label: pod
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_container_name]
+            target_label: container
+          - replacement: /var/log/pods/*$1/*.log
+            separator: /
+            source_labels:
+              - __meta_kubernetes_pod_uid
+              - __meta_kubernetes_pod_container_name
+            target_label: __path__
+          - action: replace
+            source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+
+      # ── Talos systemd journal ──────────────────────────────────────────────
+      # Talos runs containerd, kubelet, and kernel messages through systemd-
+      # journald — they never appear in /var/log/pods. This job reads the
+      # binary journal directly and emits unit (systemd unit name) and node
+      # labels so you can filter by service (e.g. unit="kubelet.service").
+      # max_age: 12h — only tail recent journal entries on startup; prevents
+      # Promtail from replaying hours of history after a pod restart.
+      - job_name: systemd-journal
+        journal:
+          path: /var/log/journal
+          max_age: 12h
+          labels:
+            job: systemd-journal
+        relabel_configs:
+          - source_labels: [__journal__systemd_unit]
+            target_label: unit
+          - source_labels: [__journal__hostname]
+            target_label: node
+
+# ── Volume mounts ─────────────────────────────────────────────────────────────
+# hostPath mounts give Promtail access to the node's log directories.
+# /var/log/pods — container stdout/stderr (written by containerd's CRI layer)
+# /var/log/journal — Talos systemd journal (binary format, read via journald API)
+# DirectoryOrCreate on journal ensures the mount doesn't fail on fresh nodes
+# before journald has written anything.
+defaultVolumes:
+  - name: pods-logs
+    hostPath:
+      path: /var/log/pods
+  - name: journal
+    hostPath:
+      path: /var/log/journal
+      type: DirectoryOrCreate
+
+defaultVolumeMounts:
+  - name: pods-logs
+    mountPath: /var/log/pods
+    readOnly: true
+  - name: journal
+    mountPath: /var/log/journal
+    readOnly: true
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 64Mi
+  limits:
+    cpu: 200m
+    memory: 128Mi
+
+# ── Security context ──────────────────────────────────────────────────────────
+# privileged: true — required to open the binary journal on Talos (journald
+# uses file locking that non-privileged processes can't bypass).
+# DAC_READ_SEARCH — lets Promtail read files owned by other UIDs in /var/log/pods
+# even with a read-only root filesystem. Without this, pod logs from containers
+# running as non-root UIDs would be unreadable.
+# readOnlyRootFilesystem: true — defence in depth; Promtail doesn't need to
+# write to its own container filesystem.
+# allowPrivilegeEscalation must be true when privileged: true — Kubernetes 1.26+
+# rejects privileged containers that explicitly set allowPrivilegeEscalation: false.
+containerSecurityContext:
+  privileged: true
+  allowPrivilegeEscalation: true
+  readOnlyRootFilesystem: true
+  capabilities:
+    drop:
+      - ALL
+    add:
+      - DAC_READ_SEARCH
+
+# runAsUser/Group: 0 (root) — needed to access journal files and pod log
+# directories that are owned by root on Talos nodes.
+podSecurityContext:
+  runAsUser: 0
+  runAsGroup: 0
+
+# Tolerate every taint so Promtail runs on ALL nodes including the CP.
+# Without this, the control-plane node's logs (etcd, kube-apiserver) would
+# be missing from Loki entirely.
+tolerations:
+  - effect: NoSchedule
+    operator: Exists
+  - effect: NoExecute
+    operator: Exists
+
+serviceMonitor:
+  enabled: false