feat(longhorn): auto-expand all volumes to 3 replicas via PostSync hook
Adds expand-replicas-job.yaml: PostSync hook Job that: - Waits for all 3 Longhorn nodes to be Ready - Patches every volume with numberOfReplicas < 3 to 3 - Runs idempotently on every longhorn-config sync (BeforeHookCreation deletes previous job, so re-runs are safe) This ensures existing 1-replica volumes (created before the HA setup) get expanded automatically via GitOps, not via manual kubectl patch. Why PostSync: needs to run AFTER the taint-toleration setting and Node CRDs are applied, otherwise there aren't 3 nodes available yet and the expansion would fail (Longhorn can't create replicas on nodes that don't exist).
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
|
||||
# Runs after the longhorn-config Application syncs (which adds cp-2/cp-3 nodes
|
||||
# + taint toleration), ensuring there are 3 nodes available before expanding.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: PostSync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
serviceAccountName: longhorn-expand-replicas
|
||||
containers:
|
||||
- name: expand
|
||||
image: bitnami/kubectl:latest
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Waiting for all 3 Longhorn nodes to be Ready..."
|
||||
for i in {1..30}; do
|
||||
READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io \
|
||||
-o jsonpath='{range .items[?(@.status.conditions[?(@.type=="Ready")].status=="True")]}{.metadata.name}{"\n"}{end}' \
|
||||
| wc -l | tr -d ' ')
|
||||
|
||||
if [ "$READY_COUNT" -ge 3 ]; then
|
||||
echo "✓ All 3 nodes Ready"
|
||||
break
|
||||
fi
|
||||
|
||||
echo " $READY_COUNT/3 nodes ready, waiting..."
|
||||
sleep 10
|
||||
|
||||
if [ $i -eq 30 ]; then
|
||||
echo "✗ Timeout waiting for 3 nodes"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Expanding volumes with < 3 replicas..."
|
||||
|
||||
kubectl -n longhorn-system get volumes.longhorn.io \
|
||||
-o jsonpath='{range .items[?(@.spec.numberOfReplicas<3)]}{.metadata.name}{"\n"}{end}' | \
|
||||
while read -r vol; do
|
||||
if [ -n "$vol" ]; then
|
||||
CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
|
||||
echo " $vol: $CURRENT → 3 replicas"
|
||||
kubectl -n longhorn-system patch volume "$vol" --type merge \
|
||||
-p '{"spec":{"numberOfReplicas":3}}'
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Done. Final replica counts:"
|
||||
kubectl -n longhorn-system get volumes.longhorn.io \
|
||||
-o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
rules:
|
||||
- apiGroups: ["longhorn.io"]
|
||||
resources: ["volumes", "nodes"]
|
||||
verbs: ["get", "list", "patch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: longhorn-expand-replicas
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: longhorn-expand-replicas
|
||||
namespace: longhorn-system
|
||||
@@ -6,7 +6,9 @@ resources:
|
||||
- longhorn-servicemonitor.yaml
|
||||
- longhorn-taint-toleration.yaml
|
||||
- longhorn-nodes.yaml
|
||||
- expand-replicas-job.yaml
|
||||
# Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh).
|
||||
# These manifests configure it post-bootstrap: WFFC StorageClass (default),
|
||||
# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and
|
||||
# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints).
|
||||
# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
|
||||
# Node CRDs for cp-2/cp-3, and a PostSync hook Job that expands all existing
|
||||
# volumes from 1→3 replicas (runs after nodes are Ready).
|
||||
|
||||
Reference in New Issue
Block a user