Story Crater Bot
e76ad914d2
Adds expand-replicas-job.yaml: PostSync hook Job that: - Waits for all 3 Longhorn nodes to be Ready - Patches every volume with numberOfReplicas < 3 to 3 - Runs idempotently on every longhorn-config sync (BeforeHookCreation deletes previous job, so re-runs are safe) This ensures existing 1-replica volumes (created before the HA setup) get expanded automatically via GitOps, not via manual kubectl patch. Why PostSync: needs to run AFTER the taint-toleration setting and Node CRDs are applied, otherwise there aren't 3 nodes available yet and the expansion would fail (Longhorn can't create replicas on nodes that don't exist).
97 lines
3.0 KiB
YAML
97 lines
3.0 KiB
YAML
# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
|
|
# Runs after the longhorn-config Application syncs (which adds cp-2/cp-3 nodes
|
|
# + taint toleration), ensuring there are 3 nodes available before expanding.
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: longhorn-expand-replicas
|
|
namespace: longhorn-system
|
|
annotations:
|
|
argocd.argoproj.io/hook: PostSync
|
|
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
|
|
spec:
|
|
backoffLimit: 3
|
|
template:
|
|
metadata:
|
|
name: longhorn-expand-replicas
|
|
spec:
|
|
restartPolicy: Never
|
|
serviceAccountName: longhorn-expand-replicas
|
|
containers:
|
|
- name: expand
|
|
image: bitnami/kubectl:latest
|
|
command:
|
|
- /bin/bash
|
|
- -c
|
|
- |
|
|
set -euo pipefail
|
|
|
|
echo "Waiting for all 3 Longhorn nodes to be Ready..."
|
|
for i in {1..30}; do
|
|
READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io \
|
|
-o jsonpath='{range .items[?(@.status.conditions[?(@.type=="Ready")].status=="True")]}{.metadata.name}{"\n"}{end}' \
|
|
| wc -l | tr -d ' ')
|
|
|
|
if [ "$READY_COUNT" -ge 3 ]; then
|
|
echo "✓ All 3 nodes Ready"
|
|
break
|
|
fi
|
|
|
|
echo " $READY_COUNT/3 nodes ready, waiting..."
|
|
sleep 10
|
|
|
|
if [ $i -eq 30 ]; then
|
|
echo "✗ Timeout waiting for 3 nodes"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "Expanding volumes with < 3 replicas..."
|
|
|
|
kubectl -n longhorn-system get volumes.longhorn.io \
|
|
-o jsonpath='{range .items[?(@.spec.numberOfReplicas<3)]}{.metadata.name}{"\n"}{end}' | \
|
|
while read -r vol; do
|
|
if [ -n "$vol" ]; then
|
|
CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
|
|
echo " $vol: $CURRENT → 3 replicas"
|
|
kubectl -n longhorn-system patch volume "$vol" --type merge \
|
|
-p '{"spec":{"numberOfReplicas":3}}'
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo "Done. Final replica counts:"
|
|
kubectl -n longhorn-system get volumes.longhorn.io \
|
|
-o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
|
|
---
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: longhorn-expand-replicas
|
|
namespace: longhorn-system
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: longhorn-expand-replicas
|
|
namespace: longhorn-system
|
|
rules:
|
|
- apiGroups: ["longhorn.io"]
|
|
resources: ["volumes", "nodes"]
|
|
verbs: ["get", "list", "patch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: longhorn-expand-replicas
|
|
namespace: longhorn-system
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: Role
|
|
name: longhorn-expand-replicas
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: longhorn-expand-replicas
|
|
namespace: longhorn-system
|