feat(longhorn): auto-expand all volumes to 3 replicas via PostSync hook (jq query fix)

k8s/infrastructure/longhorn/expand-replicas-job.yaml

@@ -0,0 +1,96 @@
+# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: longhorn-expand-replicas
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-expand-replicas
+      containers:
+      - name: expand
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+          
+          echo "Waiting for all 3 Longhorn nodes to be Ready..."
+          for i in {1..30}; do
+            READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \
+              jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length')
+            
+            if [ "$READY_COUNT" -ge 3 ]; then
+              echo "✓ All 3 nodes Ready"
+              break
+            fi
+            
+            echo "  $READY_COUNT/3 nodes ready, waiting..."
+            sleep 10
+            
+            if [ $i -eq 30 ]; then
+              echo "✗ Timeout waiting for 3 nodes"
+              exit 1
+            fi
+          done
+          
+          echo
+          echo "Expanding volumes with < 3 replicas..."
+          
+          VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \
+            jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name')
+          
+          if [ -z "$VOLUMES" ]; then
+            echo "  No volumes need expansion"
+          else
+            echo "$VOLUMES" | while read -r vol; do
+              CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
+              echo "  $vol: $CURRENT → 3 replicas"
+              kubectl -n longhorn-system patch volume "$vol" --type merge \
+                -p '{"spec":{"numberOfReplicas":3}}'
+            done
+          fi
+          
+          echo
+          echo "Done. Final replica counts:"
+          kubectl -n longhorn-system get volumes.longhorn.io \
+            -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+rules:
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "nodes"]
+  verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-expand-replicas
+subjects:
+- kind: ServiceAccount
+  name: longhorn-expand-replicas
+  namespace: longhorn-system

k8s/infrastructure/longhorn/kustomization.yaml

@@ -6,7 +6,9 @@ resources:
   - longhorn-servicemonitor.yaml
   - longhorn-taint-toleration.yaml
   - longhorn-nodes.yaml
+  - expand-replicas-job.yaml
 # Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh).
 # These manifests configure it post-bootstrap: WFFC StorageClass (default),
-# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and
+# Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
-# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints).
+# Node CRDs for cp-2/cp-3, and a PostSync hook Job that expands all existing
+# volumes from 1→3 replicas (runs after nodes are Ready).