homelab/bootstrap.sh

#!/usr/bin/env bash
#
# Phase-0 bootstrap — bring a bare Talos cluster to a self-hosted GitOps control
# plane, breaking the ArgoCD <-> Forgejo circle via a GitHub seed + cutover.
# See docs/adr/0001-gitops-bootstrap-and-cd.md (Part A) and docs/plans/0001-EXECUTION.md.
#
# Order (all manual, once): Cilium -> Longhorn -> CNPG operator -> forgejo-db
# (wait Ready) -> Forgejo -> ArgoCD (seeded from GitHub) -> cutover to Forgejo.
# Everything ELSE is deployed by ArgoCD from the seed repo, in sync-wave order.
#
# Prereqs:
#   - Talos cluster up; kubectl context points at it
#   - helm 3, kubectl
#   - SOPS age key at $SOPS_KEY (for the ArgoCD SOPS CMP plugin)
#   - GitHub read-only deploy key private half at $DEPLOY_KEY (public half added
#     to the GitHub repo's Deploy keys)
#
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BOOT="$SCRIPT_DIR/k8s/bootstrap"
SOPS_KEY="${SOPS_KEY:-$HOME/.sops/key.txt}"
DEPLOY_KEY="${DEPLOY_KEY:-$HOME/.ssh/argocd_seed}"
GITHUB_SSH="[email protected]:Riotpiaole/riotpiao.homelab.com.git"

log()  { echo "[$(date +%H:%M:%S)] $*"; }
die()  { echo "ERROR: $*" >&2; exit 1; }
phase(){ echo; echo "━━━ $* ━━━"; echo; }

# Idempotent helm repo setup
ensure_helm_repo() {
  local name=$1 url=$2
  helm repo list 2>/dev/null | grep -q "^$name" || helm repo add "$name" "$url" >/dev/null
  helm repo update "$name" >/dev/null 2>&1 || true
}

preflight() {
  log "preflight…"
  kubectl cluster-info >/dev/null || die "kubectl not configured / cluster unreachable"
  command -v helm >/dev/null || die "helm 3 not found"
  [[ -f "$SOPS_KEY" ]]   || die "SOPS age key missing at $SOPS_KEY"
  [[ -f "$DEPLOY_KEY" ]] || die "GitHub deploy key missing at $DEPLOY_KEY (see phase4-argocd/seed-repo-secret.example.yaml)"
  log "✅ preflight ok"
}

p1_cilium() {
  phase "PHASE 1a: CNI (Cilium)"
  if kubectl -n kube-system get ds cilium >/dev/null 2>&1; then log "cilium present, skip"; return; fi
  ensure_helm_repo cilium https://helm.cilium.io
  helm install cilium cilium/cilium -n kube-system \
    --set ipam.mode=kubernetes --set kubeProxyReplacement=true --wait --timeout 10m
  log "✅ cilium installed"
}

p1_longhorn() {
  phase "PHASE 1b: STORAGE (Longhorn)"
  
  # Always ensure namespace + StorageClasses (idempotent, resumable)
  kubectl apply -f "$BOOT/phase1-storage/namespace.yaml"
  
  # Install Longhorn if not present
  if ! helm -n longhorn-system list 2>/dev/null | grep -q longhorn; then
    ensure_helm_repo longhorn https://charts.longhorn.io
    log "Installing Longhorn storage (this may take 5-10 minutes)..."
    if helm install longhorn longhorn/longhorn -n longhorn-system \
      --values "$BOOT/phase1-storage/longhorn-values.yaml" --wait --timeout 10m; then
      log "✅ Longhorn installed"
    else
      log "⚠️  Helm install failed, but continuing to ensure resources..."
    fi
  fi
  
  # Always apply StorageClasses (even if helm install partially failed)
  kubectl apply -f "$BOOT/phase1-storage/storageclasses.yaml"
  
  # Verify critical components (resumable check)
  if kubectl -n longhorn-system wait --for=condition=available --timeout=300s deploy/longhorn-manager 2>/dev/null; then
    log "✅ longhorn installed"
  else
    log "⚠️  longhorn-manager not ready yet, but StorageClasses applied. Re-run to verify."
  fi
}

p1_ingress() {
  phase "PHASE 1c: INGRESS (Nginx Ingress Controller)"
  
  # Install nginx-ingress if not present
  if kubectl get ingressclass nginx >/dev/null 2>&1; then
    log "nginx IngressClass present, skip install"
    return
  fi
  
  # Always ensure namespace with PodSecurity labels (idempotent)
  kubectl apply -f "$BOOT/ingress/namespace.yaml"
  
  ensure_helm_repo ingress-nginx https://kubernetes.github.io/ingress-nginx
  log "Installing nginx-ingress controller (this may take 2-3 minutes)..."
  if helm install ingress-nginx ingress-nginx/ingress-nginx -n ingress-nginx \
    --values "$BOOT/ingress/nginx-values.yaml" --timeout 5m; then
    log "✅ nginx-ingress installed"
  else
    log "❌ nginx-ingress install failed"
    return 1
  fi
  
  # Apply additional ingress resources (cert, ingress rules)
  log "Applying ingress manifests (ignoring cert-manager CRD errors)..."
  kubectl apply -k "$BOOT/ingress/" 2>&1 | grep -v "no matches for kind" || true
  log "✅ Ingress resources applied (cert-manager resources will be created by ArgoCD)"
}

p2_cnpg() {
  phase "PHASE 2: CNPG OPERATOR"
  if kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then log "cnpg CRD present, skip install"; return; fi
  ensure_helm_repo cnpg https://cloudnative-pg.github.io/charts
  log "Installing CloudNativePG operator (this may take 2-3 minutes)..."
  if helm install cnpg cnpg/cloudnative-pg -n cnpg-system --create-namespace \
    --values "$BOOT/phase2-cnpg/cnpg-values.yaml" --wait --timeout 5m; then
    log "✅ CNPG operator installed"
  else
    log "❌ CNPG operator install failed"
    return 1
  fi
  kubectl get crd clusters.postgresql.cnpg.io >/dev/null || die "CNPG CRD not registered"
  log "✅ cnpg operator installed"
}

p3_forgejo() {
  phase "PHASE 3: forgejo-db + Forgejo (ns cicd)"
  
  # Always ensure namespace + NetworkPolicy + Secrets (idempotent)
  kubectl apply -f "$BOOT/phase3-forgejo/namespace.yaml"
  
  # Clean up old Valkey NetworkPolicy if it exists (from bundled chart)
  kubectl delete networkpolicy forgejo-valkey-cluster -n cicd 2>/dev/null || true
  
  # Apply CNPG-specific NetworkPolicy
  kubectl apply -f "$BOOT/phase3-forgejo/cnpg-networkpolicy.yaml"
  
  # Create Forgejo admin secret (bootstrap-time only, before ArgoCD exists)
  # In GitOps mode, ArgoCD will sync the SOPS-encrypted version from git
  if ! kubectl get secret forgejo-admin -n cicd >/dev/null 2>&1; then
    log "Creating forgejo-admin secret from .env (bootstrap mode)"
    [ -f "$HOME/workplace/homelab/.env" ] && source "$HOME/workplace/homelab/.env"
    kubectl -n cicd create secret generic forgejo-admin \
      --from-literal=username=rock \
      --from-literal=password="${FORGEJO_ADMIN_PASSWORD}" \
      --from-literal=email=[email protected]
  else
    log "forgejo-admin secret exists, skip (managed by ArgoCD in GitOps mode)"
  fi
  
  # Check if forgejo-db cluster exists and is Ready
  if kubectl get cluster forgejo-db -n cicd >/dev/null 2>&1; then
    if kubectl get cluster forgejo-db -n cicd -o jsonpath='{.status.phase}' 2>/dev/null | grep -q "Cluster in healthy state"; then
      log "forgejo-db already Ready, skip wait"
    else
      log "forgejo-db exists but not Ready, waiting for all 3 instances (up to 30 min)…"
      if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
        log "✅ forgejo-db cluster is Ready"
      else
        log "❌ forgejo-db cluster failed to become Ready"
        return 1
      fi
    fi
  else
    log "creating forgejo-db cluster (3 instances)"
    kubectl apply -f "$BOOT/phase3-forgejo/forgejo-db.yaml"
    log "Waiting for all 3 CNPG instances to be Ready (up to 30 min)…"
    if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
      log "✅ forgejo-db cluster is Ready"
    else
      log "❌ forgejo-db cluster failed to become Ready"
      return 1
    fi
  fi
  
  kubectl -n cicd get secret forgejo-db-app >/dev/null || die "CNPG did not create forgejo-db-app secret"
  
  # Install Forgejo if not present
  if helm -n cicd list 2>/dev/null | grep -q forgejo; then log "forgejo helm release present, skip"; return; fi
  ensure_helm_repo forgejo https://dl.gitea.io/charts/
  log "Installing Forgejo (this may take 10-15 minutes on slow nodes)..."
  if helm install forgejo forgejo/gitea -n cicd \
    --values "$BOOT/phase3-forgejo/forgejo-values.yaml" --wait --timeout 10m; then
    log "✅ Forgejo installed"
  else
    log "❌ Forgejo install failed"
    return 1
  fi
  log "Forgejo is up — now push this repo to Forgejo and configure the GitHub pull-mirror"
}

p4_argocd() {
  phase "PHASE 4: ArgoCD (seeded from GitHub)"
  
  # Always ensure namespace + repository secret (idempotent)
  kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
  kubectl -n argocd create secret generic seed-github-repo \
    --from-literal=type=git --from-literal=url="$GITHUB_SSH" \
    --from-file=sshPrivateKey="$DEPLOY_KEY" --dry-run=client -o yaml | kubectl apply -f -
  kubectl -n argocd label secret seed-github-repo argocd.argoproj.io/secret-type=repository --overwrite 2>/dev/null || true
  
  # Decrypt and apply any encrypted secrets from bootstrap dir (local SOPS)
  if command -v sops &> /dev/null; then
    export SOPS_AGE_KEY_FILE="$SOPS_KEY"
    log "Decrypting encrypted secrets with local SOPS..."
    local decrypted_count=0
    for enc_file in "$BOOT"/phase*/**.enc.yaml; do
      [ -f "$enc_file" ] || continue
      log "  → Decrypting $(basename "$enc_file")..."
      if sops -d "$enc_file" | kubectl apply -f -; then
        decrypted_count=$((decrypted_count + 1))
        log "    ✅ Applied"
      else
        log "    ⚠️  Failed (may already exist)"
      fi
    done
    log "Decrypted and applied $decrypted_count secret(s)"
  else
    log "⚠️  SOPS not installed, skipping encrypted secret decryption"
  fi
  
  # Install ArgoCD if not present
  if ! helm -n argocd list 2>/dev/null | grep -q argocd; then
    ensure_helm_repo argo https://argoproj.github.io/argo-helm
    log "Installing ArgoCD via Helm (installing chart, pods will start afterward)..."
    if helm install argocd argo/argo-cd -n argocd \
      --values "$BOOT/phase4-argocd/argocd-values.yaml" --timeout 10m; then
      log "✅ ArgoCD Helm release installed (pods starting...)"
    else
      log "❌ ArgoCD Helm install failed"
      return 1
    fi
  else
    log "ArgoCD Helm release already exists, skipping install"
  fi
  
  # Wait for server ready (resumable - slow on talos-cp-2)
  log "Waiting for argocd-server deployment to be available (max 10 minutes)..."
  if kubectl -n argocd wait --for=condition=available --timeout=600s deploy/argocd-server; then
    log "✅ argocd-server is available"
  else
    log "❌ argocd-server failed to become available within 10 minutes"
    log "Check pods: kubectl get pods -n argocd"
    return 1
  fi
  
  # Always apply root app (idempotent)
  kubectl apply -f "$BOOT/phase4-argocd/root-app-github.yaml"
  log "✅ ArgoCD syncing from GitHub seed. Watch: kubectl get applications -n argocd"
  log "NOTE: SOPS CMP plugin not installed yet (bootstrap uses local SOPS decryption)."
  log "      To add SOPS plugin for GitOps, see k8s/bootstrap/phase4-argocd/argocd-cmp-cm.yaml"
}

p5_cutover() {
  phase "PHASE 5: CUTOVER GitHub -> Forgejo"
  read -rp "Forgejo healthy AND mirroring GitHub? (y/N) " r; [[ $r =~ ^[Yy]$ ]] || die "push+mirror to Forgejo first"
  kubectl apply -f "$BOOT/phase5-cutover/root-app-forgejo.yaml"
  log "✅ root app now sourced from Forgejo. GitHub mirror = DR seed. Circle dead."
}

case "${1:-all}" in
  all)      preflight; p1_cilium; p1_longhorn; p1_ingress; p2_cnpg; p3_forgejo; p4_argocd
            log "Phases 1-4 done. Push repo to Forgejo + set up pull-mirror, then: $0 cutover" ;;
  cilium)   preflight; p1_cilium ;;
  storage)  preflight; p1_longhorn ;;
  ingress)  preflight; p1_ingress ;;
  cnpg)     preflight; p2_cnpg ;;
  forgejo)  preflight; p3_forgejo ;;
  argocd)   preflight; p4_argocd ;;
  cutover)  preflight; p5_cutover ;;
  *) echo "usage: $0 {all|cilium|storage|ingress|cnpg|forgejo|argocd|cutover}"; exit 1 ;;
esac