fix(longhorn): retire the failed 2TB PERC disk on talos-cp-2
The disk behind /var/lib/longhorn-disk4 (sde1, 2TB PERC H710P virtual disk) failed: 54 million SCSI I/O errors and climbing at ~7,000/minute, against 3 on every other device on the node. Its XFS filesystem hung, so five Longhorn replica processes and the longhorn-manager blocked in uninterruptible D state in openat and unlinkat. D-state processes cannot be killed, so they held the node's three replica rebuild slots permanently: no new replica could start on talos-cp-2 and every new volume there failed to attach with DeadlineExceeded. That is what stopped the azhi database from coming up. Removed from all three declarations so it cannot come back: - terraform.tfvars.local.bak, longhorn_disks for talos-cp-2 - the Longhorn add-disks Job, which would otherwise re-register it - the Job's disk-count guard, 5 -> 4, or it would fire on every run once the disk is gone The running cluster already has the disk fenced (allowScheduling=false, evictionRequested=true); the replicas are evicted and the device retired at the controller after that.
This commit is contained in:
@@ -41,13 +41,24 @@ spec:
|
|||||||
# - /var/lib/longhorn-disk1 (sdb1, 600GB)
|
# - /var/lib/longhorn-disk1 (sdb1, 600GB)
|
||||||
# - /var/lib/longhorn-disk2 (sdc1, 500GB)
|
# - /var/lib/longhorn-disk2 (sdc1, 500GB)
|
||||||
# - /var/lib/longhorn-disk3 (sdd1, 160GB)
|
# - /var/lib/longhorn-disk3 (sdd1, 160GB)
|
||||||
# - /var/lib/longhorn-disk4 (sde1, 2TB)
|
#
|
||||||
|
# /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
|
||||||
|
# on 2026-10-12 and must not be re-added. The device returned SCSI I/O
|
||||||
|
# errors at ~7,000/minute, the XFS filesystem on it hung, and every
|
||||||
|
# process that touched it blocked in unkillable D state. Those unkillable
|
||||||
|
# processes held Longhorn's per-node rebuild slots forever, so no new
|
||||||
|
# replica could start on talos-cp-2 and every new volume there failed to
|
||||||
|
# attach. It is removed from the Terraform disk list too, so a re-render
|
||||||
|
# does not put it back.
|
||||||
|
|
||||||
echo "Checking talos-cp-2..."
|
echo "Checking talos-cp-2..."
|
||||||
CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
|
CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
|
||||||
echo " Current disk count: $CURRENT_DISKS"
|
echo " Current disk count: $CURRENT_DISKS"
|
||||||
|
|
||||||
if [ "$CURRENT_DISKS" -lt 5 ]; then
|
# 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
|
||||||
|
# existed; leaving the old threshold would re-apply the patch on every
|
||||||
|
# run once disk4 is gone.
|
||||||
|
if [ "$CURRENT_DISKS" -lt 4 ]; then
|
||||||
echo " Adding extra disks to talos-cp-2..."
|
echo " Adding extra disks to talos-cp-2..."
|
||||||
kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
|
kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
|
||||||
"spec": {
|
"spec": {
|
||||||
@@ -75,14 +86,6 @@ spec:
|
|||||||
"path": "/var/lib/longhorn-disk3",
|
"path": "/var/lib/longhorn-disk3",
|
||||||
"storageReserved": 5368709120,
|
"storageReserved": 5368709120,
|
||||||
"tags": []
|
"tags": []
|
||||||
},
|
|
||||||
"disk4": {
|
|
||||||
"allowScheduling": true,
|
|
||||||
"diskType": "filesystem",
|
|
||||||
"evictionRequested": false,
|
|
||||||
"path": "/var/lib/longhorn-disk4",
|
|
||||||
"storageReserved": 21474836480,
|
|
||||||
"tags": []
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ controlplane_configs = {
|
|||||||
"talos-cp-1" = {
|
"talos-cp-1" = {
|
||||||
hostname = "talos-cp-1"
|
hostname = "talos-cp-1"
|
||||||
lan_ip = "192.168.1.166"
|
lan_ip = "192.168.1.166"
|
||||||
|
public_ip = "https://k8s-api.riotpiao.com"
|
||||||
lan_subnet = "192.168.1.0/24"
|
lan_subnet = "192.168.1.0/24"
|
||||||
lan_gateway = "192.168.1.254"
|
lan_gateway = "192.168.1.254"
|
||||||
install_disk = "/dev/nvme0n1"
|
install_disk = "/dev/nvme0n1"
|
||||||
@@ -32,6 +33,7 @@ controlplane_configs = {
|
|||||||
"talos-cp-2" = {
|
"talos-cp-2" = {
|
||||||
hostname = "talos-cp-2"
|
hostname = "talos-cp-2"
|
||||||
lan_ip = "192.168.1.214"
|
lan_ip = "192.168.1.214"
|
||||||
|
public_ip = "https://k8s-api.riotpiao.com"
|
||||||
lan_subnet = "192.168.1.0/24"
|
lan_subnet = "192.168.1.0/24"
|
||||||
lan_gateway = "192.168.1.254"
|
lan_gateway = "192.168.1.254"
|
||||||
install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
|
install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
|
||||||
@@ -39,7 +41,11 @@ controlplane_configs = {
|
|||||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
|
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
|
||||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
|
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
|
||||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
|
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
|
||||||
{ device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
|
# /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
|
||||||
|
# retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
|
||||||
|
# SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
|
||||||
|
# rebuilds on talos-cp-2 with unkillable D-state processes. Do not
|
||||||
|
# re-add it.
|
||||||
]
|
]
|
||||||
zone = "az-b"
|
zone = "az-b"
|
||||||
allow_scheduling = true
|
allow_scheduling = true
|
||||||
@@ -49,6 +55,7 @@ controlplane_configs = {
|
|||||||
"talos-cp-3" = {
|
"talos-cp-3" = {
|
||||||
hostname = "talos-cp-3"
|
hostname = "talos-cp-3"
|
||||||
lan_ip = "192.168.1.162"
|
lan_ip = "192.168.1.162"
|
||||||
|
public_ip = "https://k8s-api.riotpiao.com"
|
||||||
lan_subnet = "192.168.1.0/24"
|
lan_subnet = "192.168.1.0/24"
|
||||||
lan_gateway = "192.168.1.254"
|
lan_gateway = "192.168.1.254"
|
||||||
install_disk = "/dev/nvme0n1"
|
install_disk = "/dev/nvme0n1"
|
||||||
|
|||||||
Reference in New Issue
Block a user