fix(bootstrap): complete Phase 4 ArgoCD bootstrap with all permanent fixes
- Fix ArgoCD Application schema: move syncOptions under syncPolicy (00-secrets.yaml) - Remove helm install --wait flag (talos-cp-2 slow node timeout issue) - Add comprehensive progress logging with timestamps to bootstrap.sh - Fix SOPS key path (/Users/rockliang/.sops/key.txt, not homelab-age.key) - Add local SOPS decryption for bootstrap secrets - Add CNPG NetworkPolicy allowing app→database connectivity - Disable Forgejo bundled dependencies (saves 66Gi storage) - Inject database credentials via deployment.env (GITEA__DATABASE__*) - Remove invalid ext4 mount options from StorageClass - Add namespace manifests with PodSecurity labels - Add encrypted forgejo-admin secret (SOPS) - Reduce forgejo-db size 50Gi→25Gi per instance - Prepare ArgoCD SOPS CMP plugin (for post-bootstrap)
This commit is contained in:
Story Crater Bot
+155
-33
@@ -19,7 +19,7 @@ set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BOOT="$SCRIPT_DIR/k8s/bootstrap"
|
||||
SOPS_KEY="${SOPS_KEY:-$HOME/.sops/homelab-age.key}"
|
||||
SOPS_KEY="${SOPS_KEY:-$HOME/.sops/key.txt}"
|
||||
DEPLOY_KEY="${DEPLOY_KEY:-$HOME/.ssh/argocd_seed}"
|
||||
GITHUB_SSH="[email protected]:Riotpiaole/riotpiao.homelab.com.git"
|
||||
|
||||
@@ -27,6 +27,13 @@ log() { echo "[$(date +%H:%M:%S)] $*"; }
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
phase(){ echo; echo "━━━ $* ━━━"; echo; }
|
||||
|
||||
# Idempotent helm repo setup
|
||||
ensure_helm_repo() {
|
||||
local name=$1 url=$2
|
||||
helm repo list 2>/dev/null | grep -q "^$name" || helm repo add "$name" "$url" >/dev/null
|
||||
helm repo update "$name" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
preflight() {
|
||||
log "preflight…"
|
||||
kubectl cluster-info >/dev/null || die "kubectl not configured / cluster unreachable"
|
||||
@@ -39,7 +46,7 @@ preflight() {
|
||||
p1_cilium() {
|
||||
phase "PHASE 1a: CNI (Cilium)"
|
||||
if kubectl -n kube-system get ds cilium >/dev/null 2>&1; then log "cilium present, skip"; return; fi
|
||||
helm repo add cilium https://helm.cilium.io >/dev/null; helm repo update >/dev/null
|
||||
ensure_helm_repo cilium https://helm.cilium.io
|
||||
helm install cilium cilium/cilium -n kube-system \
|
||||
--set ipam.mode=kubernetes --set kubeProxyReplacement=true --wait --timeout 10m
|
||||
log "✅ cilium installed"
|
||||
@@ -47,60 +54,175 @@ p1_cilium() {
|
||||
|
||||
p1_longhorn() {
|
||||
phase "PHASE 1b: STORAGE (Longhorn)"
|
||||
if helm -n longhorn-system list 2>/dev/null | grep -q longhorn; then log "longhorn present, skip"; return; fi
|
||||
helm repo add longhorn https://charts.longhorn.io >/dev/null; helm repo update >/dev/null
|
||||
helm install longhorn longhorn/longhorn -n longhorn-system --create-namespace \
|
||||
--values "$BOOT/phase1-storage/longhorn-values.yaml" --wait --timeout 10m
|
||||
kubectl -n longhorn-system wait --for=condition=available --timeout=300s deploy/longhorn-manager
|
||||
|
||||
# Always ensure namespace + StorageClasses (idempotent, resumable)
|
||||
kubectl apply -f "$BOOT/phase1-storage/namespace.yaml"
|
||||
|
||||
# Install Longhorn if not present
|
||||
if ! helm -n longhorn-system list 2>/dev/null | grep -q longhorn; then
|
||||
ensure_helm_repo longhorn https://charts.longhorn.io
|
||||
log "Installing Longhorn storage (this may take 5-10 minutes)..."
|
||||
if helm install longhorn longhorn/longhorn -n longhorn-system \
|
||||
--values "$BOOT/phase1-storage/longhorn-values.yaml" --wait --timeout 10m; then
|
||||
log "✅ Longhorn installed"
|
||||
else
|
||||
log "⚠️ Helm install failed, but continuing to ensure resources..."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Always apply StorageClasses (even if helm install partially failed)
|
||||
kubectl apply -f "$BOOT/phase1-storage/storageclasses.yaml"
|
||||
log "✅ longhorn installed"
|
||||
|
||||
# Verify critical components (resumable check)
|
||||
if kubectl -n longhorn-system wait --for=condition=available --timeout=300s deploy/longhorn-manager 2>/dev/null; then
|
||||
log "✅ longhorn installed"
|
||||
else
|
||||
log "⚠️ longhorn-manager not ready yet, but StorageClasses applied. Re-run to verify."
|
||||
fi
|
||||
}
|
||||
|
||||
p2_cnpg() {
|
||||
phase "PHASE 2: CNPG OPERATOR"
|
||||
if helm -n cnpg-system list 2>/dev/null | grep -q cnpg; then log "cnpg present, skip"; return; fi
|
||||
helm repo add cnpg https://cloudnative-pg.github.io/charts >/dev/null; helm repo update >/dev/null
|
||||
helm install cnpg cnpg/cloudnative-pg -n cnpg-system --create-namespace \
|
||||
--values "$BOOT/phase2-cnpg/cnpg-values.yaml" --wait --timeout 5m
|
||||
if kubectl get crd clusters.postgresql.cnpg.io >/dev/null 2>&1; then log "cnpg CRD present, skip install"; return; fi
|
||||
ensure_helm_repo cnpg https://cloudnative-pg.github.io/charts
|
||||
log "Installing CloudNativePG operator (this may take 2-3 minutes)..."
|
||||
if helm install cnpg cnpg/cloudnative-pg -n cnpg-system --create-namespace \
|
||||
--values "$BOOT/phase2-cnpg/cnpg-values.yaml" --wait --timeout 5m; then
|
||||
log "✅ CNPG operator installed"
|
||||
else
|
||||
log "❌ CNPG operator install failed"
|
||||
return 1
|
||||
fi
|
||||
kubectl get crd clusters.postgresql.cnpg.io >/dev/null || die "CNPG CRD not registered"
|
||||
log "✅ cnpg operator installed"
|
||||
}
|
||||
|
||||
p3_forgejo() {
|
||||
phase "PHASE 3: forgejo-db + Forgejo (ns cicd)"
|
||||
kubectl create ns cicd --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl apply -f "$BOOT/phase3-forgejo/forgejo-db.yaml"
|
||||
log "waiting for forgejo-db Ready (3-5 min)…"
|
||||
kubectl wait --for=condition=Ready --timeout=600s cluster/forgejo-db -n cicd
|
||||
|
||||
# Always ensure namespace + NetworkPolicy + Secrets (idempotent)
|
||||
kubectl apply -f "$BOOT/phase3-forgejo/namespace.yaml"
|
||||
|
||||
# Clean up old Valkey NetworkPolicy if it exists (from bundled chart)
|
||||
kubectl delete networkpolicy forgejo-valkey-cluster -n cicd 2>/dev/null || true
|
||||
|
||||
# Apply CNPG-specific NetworkPolicy
|
||||
kubectl apply -f "$BOOT/phase3-forgejo/cnpg-networkpolicy.yaml"
|
||||
|
||||
# Create Forgejo admin secret (bootstrap-time only, before ArgoCD exists)
|
||||
# In GitOps mode, ArgoCD will sync the SOPS-encrypted version from git
|
||||
if ! kubectl get secret forgejo-admin -n cicd >/dev/null 2>&1; then
|
||||
log "Creating forgejo-admin secret from .env (bootstrap mode)"
|
||||
[ -f "$HOME/workplace/homelab/.env" ] && source "$HOME/workplace/homelab/.env"
|
||||
kubectl -n cicd create secret generic forgejo-admin \
|
||||
--from-literal=username=rock \
|
||||
--from-literal=password="${FORGEJO_ADMIN_PASSWORD}" \
|
||||
--from-literal=email=[email protected]
|
||||
else
|
||||
log "forgejo-admin secret exists, skip (managed by ArgoCD in GitOps mode)"
|
||||
fi
|
||||
|
||||
# Check if forgejo-db cluster exists and is Ready
|
||||
if kubectl get cluster forgejo-db -n cicd >/dev/null 2>&1; then
|
||||
if kubectl get cluster forgejo-db -n cicd -o jsonpath='{.status.phase}' 2>/dev/null | grep -q "Cluster in healthy state"; then
|
||||
log "forgejo-db already Ready, skip wait"
|
||||
else
|
||||
log "forgejo-db exists but not Ready, waiting for all 3 instances (up to 30 min)…"
|
||||
if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
|
||||
log "✅ forgejo-db cluster is Ready"
|
||||
else
|
||||
log "❌ forgejo-db cluster failed to become Ready"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "creating forgejo-db cluster (3 instances)"
|
||||
kubectl apply -f "$BOOT/phase3-forgejo/forgejo-db.yaml"
|
||||
log "Waiting for all 3 CNPG instances to be Ready (up to 30 min)…"
|
||||
if kubectl wait --for=condition=Ready --timeout=1800s cluster/forgejo-db -n cicd; then
|
||||
log "✅ forgejo-db cluster is Ready"
|
||||
else
|
||||
log "❌ forgejo-db cluster failed to become Ready"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
kubectl -n cicd get secret forgejo-db-app >/dev/null || die "CNPG did not create forgejo-db-app secret"
|
||||
if helm -n cicd list 2>/dev/null | grep -q forgejo; then log "forgejo present, skip"; return; fi
|
||||
helm repo add forgejo https://code.forgejo.org/forgejo-helm >/dev/null 2>&1 || \
|
||||
helm repo add forgejo https://dl.gitea.io/charts/ >/dev/null
|
||||
helm repo update >/dev/null
|
||||
helm install forgejo forgejo/forgejo -n cicd \
|
||||
--values "$BOOT/phase3-forgejo/forgejo-values.yaml" --wait --timeout 10m
|
||||
log "✅ forgejo up — now push this repo to Forgejo and configure the GitHub pull-mirror"
|
||||
|
||||
# Install Forgejo if not present
|
||||
if helm -n cicd list 2>/dev/null | grep -q forgejo; then log "forgejo helm release present, skip"; return; fi
|
||||
ensure_helm_repo forgejo https://dl.gitea.io/charts/
|
||||
log "Installing Forgejo (this may take 10-15 minutes on slow nodes)..."
|
||||
if helm install forgejo forgejo/gitea -n cicd \
|
||||
--values "$BOOT/phase3-forgejo/forgejo-values.yaml" --wait --timeout 10m; then
|
||||
log "✅ Forgejo installed"
|
||||
else
|
||||
log "❌ Forgejo install failed"
|
||||
return 1
|
||||
fi
|
||||
log "Forgejo is up — now push this repo to Forgejo and configure the GitHub pull-mirror"
|
||||
}
|
||||
|
||||
p4_argocd() {
|
||||
phase "PHASE 4: ArgoCD (seeded from GitHub)"
|
||||
|
||||
# Always ensure namespace + repository secret (idempotent)
|
||||
kubectl create ns argocd --dry-run=client -o yaml | kubectl apply -f -
|
||||
# SOPS age key for the repo-server CMP plugin
|
||||
kubectl -n argocd create secret generic sops-age \
|
||||
--from-file=keys.txt="$SOPS_KEY" --dry-run=client -o yaml | kubectl apply -f -
|
||||
# GitHub deploy-key repo credential (read-only)
|
||||
kubectl -n argocd create secret generic seed-github-repo \
|
||||
--from-literal=type=git --from-literal=url="$GITHUB_SSH" \
|
||||
--from-file=sshPrivateKey="$DEPLOY_KEY" --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl -n argocd label secret seed-github-repo argocd.argoproj.io/secret-type=repository --overwrite
|
||||
if ! helm -n argocd list 2>/dev/null | grep -q argocd; then
|
||||
helm repo add argo https://argoproj.github.io/argo-helm >/dev/null; helm repo update >/dev/null
|
||||
helm install argocd argo/argo-cd -n argocd \
|
||||
--values "$BOOT/phase4-argocd/argocd-values.yaml" --wait --timeout 10m
|
||||
kubectl -n argocd label secret seed-github-repo argocd.argoproj.io/secret-type=repository --overwrite 2>/dev/null || true
|
||||
|
||||
# Decrypt and apply any encrypted secrets from bootstrap dir (local SOPS)
|
||||
if command -v sops &> /dev/null; then
|
||||
export SOPS_AGE_KEY_FILE="$SOPS_KEY"
|
||||
log "Decrypting encrypted secrets with local SOPS..."
|
||||
local decrypted_count=0
|
||||
for enc_file in "$BOOT"/phase*/**.enc.yaml; do
|
||||
[ -f "$enc_file" ] || continue
|
||||
log " → Decrypting $(basename "$enc_file")..."
|
||||
if sops -d "$enc_file" | kubectl apply -f -; then
|
||||
decrypted_count=$((decrypted_count + 1))
|
||||
log " ✅ Applied"
|
||||
else
|
||||
log " ⚠️ Failed (may already exist)"
|
||||
fi
|
||||
done
|
||||
log "Decrypted and applied $decrypted_count secret(s)"
|
||||
else
|
||||
log "⚠️ SOPS not installed, skipping encrypted secret decryption"
|
||||
fi
|
||||
kubectl -n argocd wait --for=condition=available --timeout=300s deploy/argocd-server
|
||||
|
||||
# Install ArgoCD if not present
|
||||
if ! helm -n argocd list 2>/dev/null | grep -q argocd; then
|
||||
ensure_helm_repo argo https://argoproj.github.io/argo-helm
|
||||
log "Installing ArgoCD via Helm (installing chart, pods will start afterward)..."
|
||||
if helm install argocd argo/argo-cd -n argocd \
|
||||
--values "$BOOT/phase4-argocd/argocd-values.yaml" --timeout 10m; then
|
||||
log "✅ ArgoCD Helm release installed (pods starting...)"
|
||||
else
|
||||
log "❌ ArgoCD Helm install failed"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "ArgoCD Helm release already exists, skipping install"
|
||||
fi
|
||||
|
||||
# Wait for server ready (resumable - slow on talos-cp-2)
|
||||
log "Waiting for argocd-server deployment to be available (max 10 minutes)..."
|
||||
if kubectl -n argocd wait --for=condition=available --timeout=600s deploy/argocd-server; then
|
||||
log "✅ argocd-server is available"
|
||||
else
|
||||
log "❌ argocd-server failed to become available within 10 minutes"
|
||||
log "Check pods: kubectl get pods -n argocd"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Always apply root app (idempotent)
|
||||
kubectl apply -f "$BOOT/phase4-argocd/root-app-github.yaml"
|
||||
log "✅ ArgoCD syncing from GitHub seed. Watch: kubectl get applications -n argocd"
|
||||
log "NOTE: SOPS CMP plugin not installed yet (bootstrap uses local SOPS decryption)."
|
||||
log " To add SOPS plugin for GitOps, see k8s/bootstrap/phase4-argocd/argocd-cmp-cm.yaml"
|
||||
}
|
||||
|
||||
p5_cutover() {
|
||||
|
||||
Reference in New Issue
Block a user