k8s/monitoring: add prometheus grafana loki observability

- Loki log aggregation (MinIO backed, 10-day retention)
- Promtail daemonset (pod + talos journal logs)
- Prometheus + kube-state-metrics
- Grafana dashboards (6-row template per service)

k8s/monitoring/bootstrap.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# monitoring/bootstrap.sh
+# Deploys kube-prometheus-stack into the monitoring namespace, then upgrades
+# Grafana (in the logging namespace) to add the Prometheus datasource and
+# pre-built dashboards.
+#
+# Prerequisites:
+#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
+#   - helm >= 3.x
+#   - GRAFANA_ADMIN_PASSWORD set, or present in k8s/logging/.env
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
+export KUBECONFIG
+
+# ── Namespace ─────────────────────────────────────────────────────────────────
+echo "==> Creating monitoring namespace..."
+kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace monitoring \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# ── Helm repo ─────────────────────────────────────────────────────────────────
+echo "==> Adding prometheus-community Helm repo..."
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm repo update prometheus-community
+
+# ── kube-prometheus-stack ─────────────────────────────────────────────────────
+echo "==> Installing kube-prometheus-stack..."
+helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
+  --namespace monitoring \
+  --values "${SCRIPT_DIR}/prometheus-values.yaml" \
+  --wait \
+  --timeout 10m
+
+echo "==> Waiting for Prometheus StatefulSet..."
+kubectl rollout status \
+  statefulset/prometheus-prometheus-kube-prometheus-prometheus \
+  -n monitoring --timeout=180s
+
+echo "==> Waiting for node-exporter DaemonSet..."
+kubectl rollout status \
+  daemonset/prometheus-prometheus-node-exporter \
+  -n monitoring --timeout=60s
+
+echo "==> Waiting for kube-state-metrics..."
+kubectl rollout status \
+  deployment/prometheus-kube-state-metrics \
+  -n monitoring --timeout=60s
+
+# ── Upgrade Grafana with Prometheus datasource + dashboards ───────────────────
+echo ""
+echo "==> Upgrading Grafana to wire in Prometheus datasource + dashboards..."
+LOGGING_DIR="${REPO_ROOT}/k8s/logging"
+
+if [[ -f "${LOGGING_DIR}/.env" ]]; then
+  # shellcheck source=/dev/null
+  source "${LOGGING_DIR}/.env"
+fi
+
+if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
+  echo "ERROR: GRAFANA_ADMIN_PASSWORD is not set. Export it or place it in k8s/logging/.env"
+  exit 1
+fi
+
+helm repo add grafana https://grafana.github.io/helm-charts
+helm repo update grafana
+
+helm upgrade grafana grafana/grafana \
+  --namespace logging \
+  --values "${LOGGING_DIR}/grafana-values.yaml" \
+  --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
+  --wait \
+  --timeout 5m
+
+echo "==> Waiting for Grafana rollout..."
+kubectl rollout status deployment/grafana -n logging --timeout=120s
+
+# ── Done ──────────────────────────────────────────────────────────────────────
+echo ""
+echo "==> Monitoring stack is up."
+echo ""
+echo "Prometheus UI:"
+echo "  kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090"
+echo "  http://localhost:9090  (Targets page shows node-exporter + kube-state-metrics)"
+echo ""
+echo "Grafana:"
+echo "  kubectl port-forward -n logging svc/grafana 3000:80"
+echo "  http://localhost:3000"
+echo "  Datasources: Loki (logs, default) + Prometheus (metrics)"
+echo "  Dashboards → General:"
+echo "    - Node Exporter Full       (per-node CPU, RAM, disk, network)"
+echo "    - Kubernetes Cluster       (pod resource usage across namespaces)"