k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)

k8s/monitoring/prometheus-values.yaml

@@ -0,0 +1,133 @@
+# monitoring/prometheus-values.yaml
+# Chart: prometheus-community/kube-prometheus-stack
+# Release name: prometheus  (affects all generated resource names)
+#
+# What this installs:
+#   - Prometheus Operator (manages the Prometheus CRD)
+#   - Prometheus StatefulSet (scrapes metrics, stores on Longhorn PVC)
+#   - node-exporter DaemonSet (kernel CPU/RAM/disk/network per node)
+#   - kube-state-metrics Deployment (K8s object state — pod resource requests, phases)
+#
+# What this deliberately omits:
+#   - Grafana: already deployed in the logging namespace
+#   - Alertmanager: enable later when you want Slack/PagerDuty routing
+#   - kubeControllerManager / kubeScheduler / kubeEtcd: Talos only binds these
+#     on 127.0.0.1 — the default ServiceMonitors can't reach them
+#   - kubeProxy: removed cluster-wide; Cilium handles routing instead
+
+# ── Grafana ───────────────────────────────────────────────────────────────────
+grafana:
+  enabled: false
+
+# ── Alertmanager ──────────────────────────────────────────────────────────────
+alertmanager:
+  enabled: false
+
+# ── Prometheus ────────────────────────────────────────────────────────────────
+prometheus:
+  prometheusSpec:
+    retention: 15d
+    retentionSize: "18GB"
+
+    # Scrape timeout: increased to 60s to tolerate 5+ second network latency spikes
+    # Default: 10s — too aggressive for latency-prone clusters
+    scrapeInterval: 30s
+    scrapeTimeout: 60s
+    evaluationInterval: 30s
+
+    # Persistent storage — metrics survive node reboots and pod restarts.
+    # Longhorn reattaches the PVC automatically when the pod reschedules.
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: longhorn
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 20Gi
+
+    resources:
+      requests:
+        cpu: 200m
+        memory: 512Mi
+      limits:
+        cpu: 1000m
+        memory: 1Gi
+
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+
+    # Match ServiceMonitors/PodMonitors from all namespaces, not just the ones
+    # the chart itself creates. Required to scrape workloads in other namespaces.
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+
+# ── Prometheus Operator ────────────────────────────────────────────────────────
+prometheusOperator:
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  resources:
+    requests:
+      cpu: 100m
+      memory: 128Mi
+    limits:
+      cpu: 200m
+      memory: 256Mi
+
+# ── node-exporter ─────────────────────────────────────────────────────────────
+# DaemonSet: one pod per node, reads /proc and /sys directly via hostPID.
+# These filesystem excludes prevent scrape errors on Talos's read-only overlayfs
+# mounts and containerd's ephemeral snapshot filesystems.
+nodeExporter:
+  enabled: true
+
+prometheus-node-exporter:
+  extraArgs:
+    - --collector.filesystem.mount-points-exclude=^/(dev|proc|run/credentials/.+|sys|var/lib/containerd/.+|var/lib/kubelet/.+|run/.+)($|/)
+    - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
+  tolerations:
+    - operator: Exists  # schedule on every node regardless of taints
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# ── kube-state-metrics ────────────────────────────────────────────────────────
+# Watches the K8s API; surfaces pod CPU/memory requests, deployment replica
+# counts, pod phase, etc. — the "are my workloads healthy?" layer.
+kubeStateMetrics:
+  enabled: true
+
+kube-state-metrics:
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+  resources:
+    requests:
+      cpu: 50m
+      memory: 64Mi
+    limits:
+      cpu: 100m
+      memory: 128Mi
+
+# ── Disable unreachable control-plane scrape targets ──────────────────────────
+kubeControllerManager:
+  enabled: false
+
+kubeScheduler:
+  enabled: false
+
+kubeEtcd:
+  enabled: false
+
+kubeProxy:
+  enabled: false