diff --git a/k8s/infrastructure/longhorn/kustomization.yaml b/k8s/infrastructure/longhorn/kustomization.yaml index a587239..e75ee28 100644 --- a/k8s/infrastructure/longhorn/kustomization.yaml +++ b/k8s/infrastructure/longhorn/kustomization.yaml @@ -3,6 +3,10 @@ kind: Kustomization namespace: longhorn-system resources: - longhorn-wffc-storageclass.yaml -# Longhorn deployed via Helm chart (values in parent layer). The WFFC -# StorageClass above is the new default; the chart's Immediate `longhorn` SC is -# demoted (default-class annotation removed on the live object at bootstrap). + - longhorn-servicemonitor.yaml + - longhorn-taint-toleration.yaml + - longhorn-nodes.yaml +# Longhorn deployed via bootstrap script (cluster-config/longhorn_bootstrap.sh). +# These manifests configure it post-bootstrap: WFFC StorageClass (default), +# Prometheus ServiceMonitor, taint toleration for control-plane nodes, and +# explicit Node CRDs for cp-2/cp-3 (auto-discovery doesn't work with taints). diff --git a/k8s/infrastructure/longhorn/longhorn-nodes.yaml b/k8s/infrastructure/longhorn/longhorn-nodes.yaml new file mode 100644 index 0000000..187f730 --- /dev/null +++ b/k8s/infrastructure/longhorn/longhorn-nodes.yaml @@ -0,0 +1,33 @@ +# Longhorn Node CRDs for cp-2 and cp-3. +# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them. +# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes. +--- +apiVersion: longhorn.io/v1beta2 +kind: Node +metadata: + name: talos-cp-2 + namespace: longhorn-system +spec: + allowScheduling: true + disks: + default-disk: + allowScheduling: true + path: /var/lib/longhorn + storageReserved: 0 + tags: [] + tags: [] +--- +apiVersion: longhorn.io/v1beta2 +kind: Node +metadata: + name: talos-cp-3 + namespace: longhorn-system +spec: + allowScheduling: true + disks: + default-disk: + allowScheduling: true + path: /var/lib/longhorn + storageReserved: 0 + tags: [] + tags: [] diff --git a/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml b/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml new file mode 100644 index 0000000..be655e5 --- /dev/null +++ b/k8s/infrastructure/longhorn/longhorn-taint-toleration.yaml @@ -0,0 +1,9 @@ +# Longhorn taint-toleration setting - allows Longhorn to run on all control-plane nodes. +# Without this, Longhorn DaemonSet only runs on nodes without the control-plane taint, +# which in a 3-CP cluster with dedicated cp-2/cp-3 means only cp-1 gets storage. +apiVersion: longhorn.io/v1beta2 +kind: Setting +metadata: + name: taint-toleration + namespace: longhorn-system +value: "node-role.kubernetes.io/control-plane:NoSchedule" diff --git a/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml b/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml index 7cfd7e6..acbc034 100644 --- a/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml +++ b/k8s/infrastructure/longhorn/longhorn-wffc-storageclass.yaml @@ -1,17 +1,14 @@ # longhorn-wffc — Longhorn StorageClass with WaitForFirstConsumer binding. # -# The chart's default `longhorn` SC uses Immediate binding: the PV binds before -# the pod is scheduled, so on this single-storage-node topology (only talos-cp-1 -# runs Longhorn) the scheduler often places the pod on cp-2/cp-3 where the volume -# can't attach ("CSINode does not contain driver driver.longhorn.io"). +# WaitForFirstConsumer defers PV binding until the pod is scheduled, ensuring the +# volume is provisioned on a node where the pod can actually run. Critical for HA: +# with 3-replica volumes spread across 3 nodes, the scheduler needs to see which +# nodes already have replicas before placing the pod, avoiding situations where +# the pod lands on a node that can't reach any replica. # -# WaitForFirstConsumer defers binding until the pod is scheduled, so the volume -# is provisioned on the node the pod lands on — and with a single Longhorn node -# that co-locates pod + volume on cp-1 automatically. This is the new default; -# the chart's `longhorn` SC is demoted (see longhorn-values default-class=false). -# -# volumeBindingMode is immutable, so this is a distinct SC (not an edit of the -# chart's). Existing volumes stay on `longhorn`; new PVCs use this. +# numberOfReplicas=3 provides true HA: each volume has 3 copies across 3 nodes. +# If one node fails, the remaining 2 nodes still have the data and can serve it. +# volumeBindingMode is immutable, so this is a distinct SC from the chart's default. apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: @@ -23,7 +20,7 @@ allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer parameters: - numberOfReplicas: "1" + numberOfReplicas: "3" staleReplicaTimeout: "30" fromBackup: "" dataLocality: "best-effort"