diff --git a/k8s/infrastructure/longhorn/expand-replicas-job.yaml b/k8s/infrastructure/longhorn/expand-replicas-job.yaml new file mode 100644 index 0000000..b0609a5 --- /dev/null +++ b/k8s/infrastructure/longhorn/expand-replicas-job.yaml @@ -0,0 +1,96 @@ +# PostSync hook Job that expands all Longhorn volumes to 3 replicas. +# Runs after the longhorn-config Application syncs (which adds cp-2/cp-3 nodes +# + taint toleration), ensuring there are 3 nodes available before expanding. +apiVersion: batch/v1 +kind: Job +metadata: + name: longhorn-expand-replicas + namespace: longhorn-system + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 3 + template: + metadata: + name: longhorn-expand-replicas + spec: + restartPolicy: Never + serviceAccountName: longhorn-expand-replicas + containers: + - name: expand + image: bitnami/kubectl:latest + command: + - /bin/bash + - -c + - | + set -euo pipefail + + echo "Waiting for all 3 Longhorn nodes to be Ready..." + for i in {1..30}; do + READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io \ + -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="Ready")].status=="True")]}{.metadata.name}{"\n"}{end}' \ + | wc -l | tr -d ' ') + + if [ "$READY_COUNT" -ge 3 ]; then + echo "✓ All 3 nodes Ready" + break + fi + + echo " $READY_COUNT/3 nodes ready, waiting..." + sleep 10 + + if [ $i -eq 30 ]; then + echo "✗ Timeout waiting for 3 nodes" + exit 1 + fi + done + + echo + echo "Expanding volumes with < 3 replicas..." + + kubectl -n longhorn-system get volumes.longhorn.io \ + -o jsonpath='{range .items[?(@.spec.numberOfReplicas<3)]}{.metadata.name}{"\n"}{end}' | \ + while read -r vol; do + if [ -n "$vol" ]; then + CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}') + echo " $vol: $CURRENT → 3 replicas" + kubectl -n longhorn-system patch volume "$vol" --type merge \ + -p '{"spec":{"numberOfReplicas":3}}' + fi + done + + echo + echo "Done. Final replica counts:" + kubectl -n longhorn-system get volumes.longhorn.io \ + -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-expand-replicas + namespace: longhorn-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn-expand-replicas + namespace: longhorn-system +rules: +- apiGroups: ["longhorn.io"] + resources: ["volumes", "nodes"] + verbs: ["get", "list", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn-expand-replicas + namespace: longhorn-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn-expand-replicas +subjects: +- kind: ServiceAccount + name: longhorn-expand-replicas + namespace: longhorn-system diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infrastructure/longhorn/kustomization.yaml index e75ee28..8915071 100644 --- a/k8s/infrastructure/longhorn/kustomization.yaml +++ b/k8s/infrastructure/longhorn/kustomization.yaml @@ -6,7 +6,9 @@ resources: - longhorn-servicemonitor.yaml - longhorn-taint-toleration.yaml - longhorn-nodes.yaml + - expand-replicas-job.yaml # Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh). # These manifests configure it post-bootstrap: WFFC StorageClass (default), -# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and -# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints). +# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit +# Node CRDs for cp-2/cp-3, and a PostSync hook Job that expands all existing +# volumes from 1→3 replicas (runs after nodes are Ready).