homelab/terraform/scripts/phase-2-pilot-grafana.sh

#!/bin/bash
# Phase 2 Pilot — Grafana PVC import (full cycle: annotate → import → plan → values change → helmfile diff → helmfile apply)
# Lowest blast radius, validates entire workflow before rolling to other apps

set -e
cd "$(dirname "$0")/.."

echo "=== Phase 2 Pilot: Grafana PVC Import ==="
echo ""

# Step a) Protect the live PVC from Helm deletion
echo "--- Step a) Protect PVC from Helm deletion ---"
echo "Identifying grafana PVC in 'logging' namespace:"
kubectl get pvc -n logging -l app.kubernetes.io/instance=grafana -o wide || {
  echo "ERROR: Cannot find grafana PVC"
  exit 1
}
echo ""
echo "Annotating with helm.sh/resource-policy=keep (non-destructive, reversible):"
kubectl annotate pvc grafana -n logging helm.sh/resource-policy=keep --overwrite
echo "Annotation applied"
echo ""

# Step b) Capture the exact live PVC spec
echo "--- Step b) Capture live PVC spec ---"
kubectl get pvc grafana -n logging -o yaml > /tmp/grafana-pvc-live.yaml
echo "Saved to /tmp/grafana-pvc-live.yaml"
echo ""
echo "Extracting key fields:"
echo "Access modes:"
kubectl get pvc grafana -n logging -o jsonpath='{.spec.accessModes}' | tr ',' '\n'
echo "Storage class:"
kubectl get pvc grafana -n logging -o jsonpath='{.spec.storageClassName}'
echo ""
echo "Requested storage:"
kubectl get pvc grafana -n logging -o jsonpath='{.spec.resources.requests.storage}'
echo ""
echo "Bound PV name:"
PV_NAME=$(kubectl get pvc grafana -n logging -o jsonpath='{.spec.volumeName}')
echo "$PV_NAME"
echo ""
echo "MANUAL STEP: Update terraform/grafana.tf with the actual volumeName '$PV_NAME' (currently 'pvc-grafana' as placeholder)"
echo ""
read -p "Press Enter once grafana.tf is updated with the correct volumeName: " _ || true
echo ""

# Step d) Import and verify zero diff
echo "--- Step d) Import 'grafana' PVC ---"
terraform import kubernetes_persistent_volume_claim.grafana logging/grafana || {
  echo "ERROR: terraform import failed"
  exit 1
}
echo "Import successful"
echo ""

echo "--- Verifying zero diff (must show 0 to add/change/destroy) ---"
PLAN_OUTPUT=$(terraform plan 2>&1)
echo "$PLAN_OUTPUT"
if echo "$PLAN_OUTPUT" | grep -q "0 to add, 0 to change, 0 to destroy"; then
  echo "✓ Plan is clean"
else
  echo "✗ Plan shows changes — STOP, do not proceed"
  echo "  Options:"
  echo "    1. Fix terraform/grafana.tf and re-run terraform plan"
  echo "    2. Rollback with: terraform state rm kubernetes_persistent_volume_claim.grafana"
  exit 1
fi
echo ""

# Step f) Verify Longhorn replica health BEFORE values change
echo "--- Step f.1) Baseline Longhorn replica health ---"
LONGHORN_VOL=$(kubectl get pvc grafana -n logging -o jsonpath='{.spec.volumeName}' | sed 's/pvc-//' )
echo "Checking Longhorn volume health for: $LONGHORN_VOL"
kubectl get longhorn-volume -n longhorn-system "$LONGHORN_VOL" -o json | jq '.status.replicaStatus' 2>/dev/null || echo "  (could not get Longhorn status; continue)"
echo ""
read -p "Note the replica status above. Press Enter to continue: " _ || true
echo ""

# Step e) Update grafana values to use existingClaim (only if chart supports it)
echo "--- Step e) Update k8s/logging/grafana-values.yaml for existingClaim ---"
echo "Current grafana-values.yaml persistence section:"
grep -A 5 "^persistence:" k8s/logging/grafana-values.yaml || echo "  (no persistence section found)"
echo ""
echo "MANUAL STEP: Add/update to k8s/logging/grafana-values.yaml:"
echo "  persistence:"
echo "    existingClaim: grafana"
echo "    enabled: false"
echo ""
echo "If the chart does NOT support existingClaim (check Grafana chart docs), leave:"
echo "  persistence:"
echo "    enabled: true"
echo "    size: 5Gi"
echo "    storageClassName: longhorn"
echo "  (Helm will then see no diff and won't delete the PVC; the keep annotation is the backstop)"
echo ""
read -p "Press Enter once grafana-values.yaml is updated: " _ || true
echo ""

# Step e.2) helmfile diff to confirm no delete queued
echo "--- Step e.2) Helmfile diff to confirm no delete/replace ---"
echo "Running helmfile diff for grafana (in logging namespace, chart= from helmfile):"
cd "$(dirname "$0")/../.."  # go to repo root
helmfile -e logging -f helmfile.yaml.gotmpl -l name=grafana diff || {
  echo "WARNING: helmfile diff failed or returned nonzero exit; check output above"
  echo "  (helmfile may not be perfectly compatible with this session, but diff result should be visible)"
}
cd "$(dirname "$0")/../terraform"
echo ""
echo "Confirm no 'delete' or 'replace' operations on the grafana PVC are queued."
echo ""
read -p "Press Enter if helmfile diff shows no destructive ops on grafana PVC: " _ || true
echo ""

# Step e.3) helmfile apply
echo "--- Step e.3) Helmfile apply ---"
cd "$(dirname "$0")/../.."
echo "Applying logging/grafana via helmfile:"
helmfile -e logging -f helmfile.yaml.gotmpl -l name=grafana apply || {
  echo "WARNING: helmfile apply returned nonzero; check output above"
}
cd "$(dirname "$0")/../terraform"
echo ""
echo "Helmfile apply complete"
echo ""

# Step f.2) Verify Longhorn health AFTER helmfile apply
echo "--- Step f.2) Post-helmfile Longhorn replica health check ---"
echo "Checking Longhorn volume health for: $LONGHORN_VOL"
kubectl get longhorn-volume -n longhorn-system "$LONGHORN_VOL" -o json | jq '.status.replicaStatus' 2>/dev/null || echo "  (could not get Longhorn status)"
echo ""
echo "Confirm replica status is identical to baseline above."
read -p "Press Enter if replica health matches baseline: " _ || true
echo ""

# Final terraform plan
echo "--- Final terraform plan (must still be 0/0/0 after all changes) ---"
PLAN_OUTPUT=$(terraform plan 2>&1)
echo "$PLAN_OUTPUT"
if echo "$PLAN_OUTPUT" | grep -q "0 to add, 0 to change, 0 to destroy"; then
  echo "✓ Plan is still clean"
else
  echo "✗ Plan shows changes after helmfile apply — investigate"
  exit 1
fi
echo ""

echo "=== Phase 2 Pilot: Grafana Complete ==="
echo "Grafana PVC successfully imported. Ready for Phase 2 remaining apps."