From e76ad914d25d1e624d7f24541140bbc423cc0577 Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Wed, 22 Jul 2026 08:50:22 -0700
Subject: [PATCH] feat(longhorn): auto-expand all volumes to 3 replicas via
 PostSync hook

Adds expand-replicas-job.yaml: PostSync hook Job that:
- Waits for all 3 Longhorn nodes to be Ready
- Patches every volume with numberOfReplicas < 3 to 3
- Runs idempotently on every longhorn-config sync (BeforeHookCreation
  deletes previous job, so re-runs are safe)

This ensures existing 1-replica volumes (created before the HA setup) get
expanded automatically via GitOps, not via manual kubectl patch.

Why PostSync: needs to run AFTER the taint-toleration setting and Node CRDs
are applied, otherwise there aren't 3 nodes available yet and the expansion
would fail (Longhorn can't create replicas on nodes that don't exist).
---
 .../longhorn/expand-replicas-job.yaml         | 96 +++++++++++++++++++
 .../longhorn/kustomization.yaml               |  6 +-
 2 files changed, 100 insertions(+), 2 deletions(-)
 create mode 100644 k8s/infrastructure/longhorn/expand-replicas-job.yaml

diff --git a/k8s/infrastructure/longhorn/expand-replicas-job.yaml b/k8s/infrastructure/longhorn/expand-replicas-job.yaml
new file mode 100644
index 0000000..b0609a5
--- /dev/null
+++ b/k8s/infrastructure/longhorn/expand-replicas-job.yaml
@@ -0,0 +1,96 @@
+# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
+# Runs after the longhorn-config Application syncs (which adds cp-2/cp-3 nodes
+# + taint toleration), ensuring there are 3 nodes available before expanding.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: longhorn-expand-replicas
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-expand-replicas
+      containers:
+      - name: expand
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+          
+          echo "Waiting for all 3 Longhorn nodes to be Ready..."
+          for i in {1..30}; do
+            READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io \
+              -o jsonpath='{range .items[?(@.status.conditions[?(@.type=="Ready")].status=="True")]}{.metadata.name}{"\n"}{end}' \
+              | wc -l | tr -d ' ')
+            
+            if [ "$READY_COUNT" -ge 3 ]; then
+              echo "✓ All 3 nodes Ready"
+              break
+            fi
+            
+            echo "  $READY_COUNT/3 nodes ready, waiting..."
+            sleep 10
+            
+            if [ $i -eq 30 ]; then
+              echo "✗ Timeout waiting for 3 nodes"
+              exit 1
+            fi
+          done
+          
+          echo
+          echo "Expanding volumes with < 3 replicas..."
+          
+          kubectl -n longhorn-system get volumes.longhorn.io \
+            -o jsonpath='{range .items[?(@.spec.numberOfReplicas<3)]}{.metadata.name}{"\n"}{end}' | \
+          while read -r vol; do
+            if [ -n "$vol" ]; then
+              CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
+              echo "  $vol: $CURRENT → 3 replicas"
+              kubectl -n longhorn-system patch volume "$vol" --type merge \
+                -p '{"spec":{"numberOfReplicas":3}}'
+            fi
+          done
+          
+          echo
+          echo "Done. Final replica counts:"
+          kubectl -n longhorn-system get volumes.longhorn.io \
+            -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+rules:
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "nodes"]
+  verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-expand-replicas
+subjects:
+- kind: ServiceAccount
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infrastructure/longhorn/kustomization.yaml
index e75ee28..8915071 100644
--- a/k8s/infrastructure/longhorn/kustomization.yaml
+++ b/k8s/infrastructure/longhorn/kustomization.yaml
@@ -6,7 +6,9 @@ resources:
   - longhorn-servicemonitor.yaml
   - longhorn-taint-toleration.yaml
   - longhorn-nodes.yaml
+  - expand-replicas-job.yaml
 # Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh).
 # These manifests configure it post-bootstrap: WFFC StorageClass (default),
-# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and
-# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints).
+# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
+# Node CRDs for cp-2/cp-3, and a PostSync hook Job that expands all existing
+# volumes from 1→3 replicas (runs after nodes are Ready).