From e2fcfe1fa8c717fd8f198b4e41dc618bde1e008d Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:46:57 -0700 Subject: [PATCH] feat(storage): enable Longhorn on all 3 control-plane nodes for true HA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: - k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml: new Setting to tolerate node-role.kubernetes.io/control-plane:NoSchedule taint, allowing Longhorn DaemonSet to run on cp-2/cp-3 (not just cp-1) - k8s/infrastructure/longhorn/longhorn-nodes.yaml: explicit Node CRDs for talos-cp-2 and talos-cp-3 (auto-discovery doesn't work when nodes have taints; these define /var/lib/longhorn as the storage path) - k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml: bump numberOfReplicas from 1→3 (true HA: each volume gets 3 copies across 3 nodes; if one node fails, 2 others still have the data) - k8s/infrastructure/longhorn/kustomization.yaml: add new resources Root cause: Longhorn was only running on talos-cp-1 (.213) because cp-2/cp-3 have the control-plane taint and Longhorn DaemonSet had no matching toleration. Every workload with a PVC was forced to schedule on cp-1 (via nodeSelector or implicit co-location with the storage), defeating the entire purpose of a 3-node HA cluster. With this fix: - Longhorn manager runs on all 3 nodes - Storage is replicated 3x (erasure-coded across nodes) - Pods can schedule on any node without PVC attachment failures - True HA: lose 1 node, cluster still serves all volumes --- .../longhorn/kustomization.yaml | 10 ++++-- .../longhorn/longhorn-nodes.yaml | 33 +++++++++++++++++++ .../longhorn/longhorn-taint-toleration.yaml | 9 +++++ .../longhorn/longhorn-wffc-storageclass.yaml | 21 +++++------- 4 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 k8s/infrastructure/longhorn/longhorn-nodes.yaml create mode 100644 k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infrastructure/longhorn/kustomization.yaml index a587239..e75ee28 100644 --- a/k8s/infrastructure/longhorn/kustomization.yaml +++ b/k8s/infrastructure/longhorn/kustomization.yaml @@ -3,6 +3,10 @@ kind: Kustomization namespace: longhorn-system resources: - longhorn-wffc-storageclass.yaml -# Longhorn deployed via Helm chart (values in parent layer). The WFFC -# StorageClass above is the new default; the chart's Immediate `longhorn` SC is -# demoted (default-class annotation removed on the live object at bootstrap). + - longhorn-servicemonitor.yaml + - longhorn-taint-toleration.yaml + - longhorn-nodes.yaml +# Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh). +# These manifests configure it post-bootstrap: WFFC StorageClass (default), +# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and +# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints). diff --git a/k8s/infrastructure/longhorn/longhorn-nodes.yaml b/k8s/infrastructure/longhorn/longhorn-nodes.yaml new file mode 100644 index 0000000..187f730 --- /dev/null +++ b/k8s/infrastructure/longhorn/longhorn-nodes.yaml @@ -0,0 +1,33 @@ +# Longhorn Node CRDs for cp-2 and cp-3. +# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them. +# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes. +--- +apiVersion: longhorn.io/v1beta2 +kind: Node +metadata: + name: talos-cp-2 + namespace: longhorn-system +spec: + allowScheduling: true + disks: + default-disk: + allowScheduling: true + path: /var/lib/longhorn + storageReserved: 0 + tags: [] + tags: [] +--- +apiVersion: longhorn.io/v1beta2 +kind: Node +metadata: + name: talos-cp-3 + namespace: longhorn-system +spec: + allowScheduling: true + disks: + default-disk: + allowScheduling: true + path: /var/lib/longhorn + storageReserved: 0 + tags: [] + tags: [] diff --git a/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml b/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml new file mode 100644 index 0000000..be655e5 --- /dev/null +++ b/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml @@ -0,0 +1,9 @@ +# Longhorn taint-toleration setting - allows Longhorn to run on all control-plane nodes. +# Without this, Longhorn DaemonSet only runs on nodes without the control-plane taint, +# which in a 3-CP cluster with dedicated cp-2/cp-3 means only cp-1 gets storage. +apiVersion: longhorn.io/v1beta2 +kind: Setting +metadata: + name: taint-toleration + namespace: longhorn-system +value: "node-role.kubernetes.io/control-plane:NoSchedule" diff --git a/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml b/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml index 7cfd7e6..acbc034 100644 --- a/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml +++ b/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml @@ -1,17 +1,14 @@ # longhorn-wffc — Longhorn StorageClass with WaitForFirstConsumer binding. # -# The chart's default `longhorn` SC uses Immediate binding: the PV binds before -# the pod is scheduled, so on this single-storage-node topology (only talos-cp-1 -# runs Longhorn) the scheduler often places the pod on cp-2/cp-3 where the volume -# can't attach ("CSINode does not contain driver driver.longhorn.io"). +# WaitForFirstConsumer defers PV binding until the pod is scheduled, ensuring the +# volume is provisioned on a node where the pod can actually run. Critical for HA: +# with 3-replica volumes spread across 3 nodes, the scheduler needs to see which +# nodes already have replicas before placing the pod, avoiding situations where +# the pod lands on a node that can't reach any replica. # -# WaitForFirstConsumer defers binding until the pod is scheduled, so the volume -# is provisioned on the node the pod lands on — and with a single Longhorn node -# that co-locates pod + volume on cp-1 automatically. This is the new default; -# the chart's `longhorn` SC is demoted (see longhorn-values default-class=false). -# -# volumeBindingMode is immutable, so this is a distinct SC (not an edit of the -# chart's). Existing volumes stay on `longhorn`; new PVCs use this. +# numberOfReplicas=3 provides true HA: each volume has 3 copies across 3 nodes. +# If one node fails, the remaining 2 nodes still have the data and can serve it. +# volumeBindingMode is immutable, so this is a distinct SC from the chart's default. apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: @@ -23,7 +20,7 @@ allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer parameters: - numberOfReplicas: "1" + numberOfReplicas: "3" staleReplicaTimeout: "30" fromBackup: "" dataLocality: "best-effort"