homelab/k8s/monitoring/bootstrap.sh

#!/usr/bin/env bash
# monitoring/bootstrap.sh
# Deploys kube-prometheus-stack into the monitoring namespace, then upgrades
# Grafana (in the logging namespace) to add the Prometheus datasource and
# pre-built dashboards.
#
# Prerequisites:
#   - kubectl configured (KUBECONFIG pointing to cluster-config/kubeconfig)
#   - helm >= 3.x
#   - GRAFANA_ADMIN_PASSWORD set, or present in k8s/logging/.env
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
KUBECONFIG="${KUBECONFIG:-${REPO_ROOT}/cluster-config/kubeconfig}"
export KUBECONFIG

# ── Namespace ─────────────────────────────────────────────────────────────────
echo "==> Creating monitoring namespace..."
kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
kubectl label namespace monitoring \
  pod-security.kubernetes.io/enforce=privileged \
  pod-security.kubernetes.io/enforce-version=latest \
  --overwrite

# ── Helm repo ─────────────────────────────────────────────────────────────────
echo "==> Adding prometheus-community Helm repo..."
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update prometheus-community

# ── kube-prometheus-stack ─────────────────────────────────────────────────────
echo "==> Installing kube-prometheus-stack..."
helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --values "${SCRIPT_DIR}/prometheus-values.yaml" \
  --wait \
  --timeout 10m

echo "==> Waiting for Prometheus StatefulSet..."
kubectl rollout status \
  statefulset/prometheus-prometheus-kube-prometheus-prometheus \
  -n monitoring --timeout=180s

echo "==> Waiting for node-exporter DaemonSet..."
kubectl rollout status \
  daemonset/prometheus-prometheus-node-exporter \
  -n monitoring --timeout=60s

echo "==> Waiting for kube-state-metrics..."
kubectl rollout status \
  deployment/prometheus-kube-state-metrics \
  -n monitoring --timeout=60s

# ── Upgrade Grafana with Prometheus datasource + dashboards ───────────────────
echo ""
echo "==> Upgrading Grafana to wire in Prometheus datasource + dashboards..."
LOGGING_DIR="${REPO_ROOT}/k8s/logging"

if [[ -f "${LOGGING_DIR}/.env" ]]; then
  # shellcheck source=/dev/null
  source "${LOGGING_DIR}/.env"
fi

if [[ -z "${GRAFANA_ADMIN_PASSWORD:-}" ]]; then
  echo "ERROR: GRAFANA_ADMIN_PASSWORD is not set. Export it or place it in k8s/logging/.env"
  exit 1
fi

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update grafana

helm upgrade grafana grafana/grafana \
  --namespace logging \
  --values "${LOGGING_DIR}/grafana-values.yaml" \
  --set adminPassword="${GRAFANA_ADMIN_PASSWORD}" \
  --wait \
  --timeout 5m

echo "==> Waiting for Grafana rollout..."
kubectl rollout status deployment/grafana -n logging --timeout=120s

# ── Done ──────────────────────────────────────────────────────────────────────
echo ""
echo "==> Monitoring stack is up."
echo ""
echo "Prometheus UI:"
echo "  kubectl port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090"
echo "  http://localhost:9090  (Targets page shows node-exporter + kube-state-metrics)"
echo ""
echo "Grafana:"
echo "  kubectl port-forward -n logging svc/grafana 3000:80"
echo "  http://localhost:3000"
echo "  Datasources: Loki (logs, default) + Prometheus (metrics)"
echo "  Dashboards → General:"
echo "    - Node Exporter Full       (per-node CPU, RAM, disk, network)"
echo "    - Kubernetes Cluster       (pod resource usage across namespaces)"