#!/bin/bash # Phase 2 Pilot — Grafana PVC import (full cycle: annotate → import → plan → values change → helmfile diff → helmfile apply) # Lowest blast radius, validates entire workflow before rolling to other apps set -e cd "$(dirname "$0")/.." echo "=== Phase 2 Pilot: Grafana PVC Import ===" echo "" # Step a) Protect the live PVC from Helm deletion echo "--- Step a) Protect PVC from Helm deletion ---" echo "Identifying grafana PVC in 'logging' namespace:" kubectl get pvc -n logging -l app.kubernetes.io/instance=grafana -o wide || { echo "ERROR: Cannot find grafana PVC" exit 1 } echo "" echo "Annotating with helm.sh/resource-policy=keep (non-destructive, reversible):" kubectl annotate pvc grafana -n logging helm.sh/resource-policy=keep --overwrite echo "Annotation applied" echo "" # Step b) Capture the exact live PVC spec echo "--- Step b) Capture live PVC spec ---" kubectl get pvc grafana -n logging -o yaml > /tmp/grafana-pvc-live.yaml echo "Saved to /tmp/grafana-pvc-live.yaml" echo "" echo "Extracting key fields:" echo "Access modes:" kubectl get pvc grafana -n logging -o jsonpath='{.spec.accessModes}' | tr ',' '\n' echo "Storage class:" kubectl get pvc grafana -n logging -o jsonpath='{.spec.storageClassName}' echo "" echo "Requested storage:" kubectl get pvc grafana -n logging -o jsonpath='{.spec.resources.requests.storage}' echo "" echo "Bound PV name:" PV_NAME=$(kubectl get pvc grafana -n logging -o jsonpath='{.spec.volumeName}') echo "$PV_NAME" echo "" echo "MANUAL STEP: Update terraform/grafana.tf with the actual volumeName '$PV_NAME' (currently 'pvc-grafana' as placeholder)" echo "" read -p "Press Enter once grafana.tf is updated with the correct volumeName: " _ || true echo "" # Step d) Import and verify zero diff echo "--- Step d) Import 'grafana' PVC ---" terraform import kubernetes_persistent_volume_claim.grafana logging/grafana || { echo "ERROR: terraform import failed" exit 1 } echo "Import successful" echo "" echo "--- Verifying zero diff (must show 0 to add/change/destroy) ---" PLAN_OUTPUT=$(terraform plan 2>&1) echo "$PLAN_OUTPUT" if echo "$PLAN_OUTPUT" | grep -q "0 to add, 0 to change, 0 to destroy"; then echo "✓ Plan is clean" else echo "✗ Plan shows changes — STOP, do not proceed" echo " Options:" echo " 1. Fix terraform/grafana.tf and re-run terraform plan" echo " 2. Rollback with: terraform state rm kubernetes_persistent_volume_claim.grafana" exit 1 fi echo "" # Step f) Verify Longhorn replica health BEFORE values change echo "--- Step f.1) Baseline Longhorn replica health ---" LONGHORN_VOL=$(kubectl get pvc grafana -n logging -o jsonpath='{.spec.volumeName}' | sed 's/pvc-//' ) echo "Checking Longhorn volume health for: $LONGHORN_VOL" kubectl get longhorn-volume -n longhorn-system "$LONGHORN_VOL" -o json | jq '.status.replicaStatus' 2>/dev/null || echo " (could not get Longhorn status; continue)" echo "" read -p "Note the replica status above. Press Enter to continue: " _ || true echo "" # Step e) Update grafana values to use existingClaim (only if chart supports it) echo "--- Step e) Update k8s/logging/grafana-values.yaml for existingClaim ---" echo "Current grafana-values.yaml persistence section:" grep -A 5 "^persistence:" k8s/logging/grafana-values.yaml || echo " (no persistence section found)" echo "" echo "MANUAL STEP: Add/update to k8s/logging/grafana-values.yaml:" echo " persistence:" echo " existingClaim: grafana" echo " enabled: false" echo "" echo "If the chart does NOT support existingClaim (check Grafana chart docs), leave:" echo " persistence:" echo " enabled: true" echo " size: 5Gi" echo " storageClassName: longhorn" echo " (Helm will then see no diff and won't delete the PVC; the keep annotation is the backstop)" echo "" read -p "Press Enter once grafana-values.yaml is updated: " _ || true echo "" # Step e.2) helmfile diff to confirm no delete queued echo "--- Step e.2) Helmfile diff to confirm no delete/replace ---" echo "Running helmfile diff for grafana (in logging namespace, chart= from helmfile):" cd "$(dirname "$0")/../.." # go to repo root helmfile -e logging -f helmfile.yaml.gotmpl -l name=grafana diff || { echo "WARNING: helmfile diff failed or returned nonzero exit; check output above" echo " (helmfile may not be perfectly compatible with this session, but diff result should be visible)" } cd "$(dirname "$0")/../terraform" echo "" echo "Confirm no 'delete' or 'replace' operations on the grafana PVC are queued." echo "" read -p "Press Enter if helmfile diff shows no destructive ops on grafana PVC: " _ || true echo "" # Step e.3) helmfile apply echo "--- Step e.3) Helmfile apply ---" cd "$(dirname "$0")/../.." echo "Applying logging/grafana via helmfile:" helmfile -e logging -f helmfile.yaml.gotmpl -l name=grafana apply || { echo "WARNING: helmfile apply returned nonzero; check output above" } cd "$(dirname "$0")/../terraform" echo "" echo "Helmfile apply complete" echo "" # Step f.2) Verify Longhorn health AFTER helmfile apply echo "--- Step f.2) Post-helmfile Longhorn replica health check ---" echo "Checking Longhorn volume health for: $LONGHORN_VOL" kubectl get longhorn-volume -n longhorn-system "$LONGHORN_VOL" -o json | jq '.status.replicaStatus' 2>/dev/null || echo " (could not get Longhorn status)" echo "" echo "Confirm replica status is identical to baseline above." read -p "Press Enter if replica health matches baseline: " _ || true echo "" # Final terraform plan echo "--- Final terraform plan (must still be 0/0/0 after all changes) ---" PLAN_OUTPUT=$(terraform plan 2>&1) echo "$PLAN_OUTPUT" if echo "$PLAN_OUTPUT" | grep -q "0 to add, 0 to change, 0 to destroy"; then echo "✓ Plan is still clean" else echo "✗ Plan shows changes after helmfile apply — investigate" exit 1 fi echo "" echo "=== Phase 2 Pilot: Grafana Complete ===" echo "Grafana PVC successfully imported. Ready for Phase 2 remaining apps."