diff --git a/k8s/infra/longhorn/longhorn-add-disks-job.yaml b/k8s/infra/longhorn/longhorn-add-disks-job.yaml index bff5aad..782bac0 100644 --- a/k8s/infra/longhorn/longhorn-add-disks-job.yaml +++ b/k8s/infra/longhorn/longhorn-add-disks-job.yaml @@ -41,13 +41,24 @@ spec: # - /var/lib/longhorn-disk1 (sdb1, 600GB) # - /var/lib/longhorn-disk2 (sdc1, 500GB) # - /var/lib/longhorn-disk3 (sdd1, 160GB) - # - /var/lib/longhorn-disk4 (sde1, 2TB) + # + # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED + # on 2026-10-12 and must not be re-added. The device returned SCSI I/O + # errors at ~7,000/minute, the XFS filesystem on it hung, and every + # process that touched it blocked in unkillable D state. Those unkillable + # processes held Longhorn's per-node rebuild slots forever, so no new + # replica could start on talos-cp-2 and every new volume there failed to + # attach. It is removed from the Terraform disk list too, so a re-render + # does not put it back. echo "Checking talos-cp-2..." CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length') echo " Current disk count: $CURRENT_DISKS" - if [ "$CURRENT_DISKS" -lt 5 ]; then + # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4 + # existed; leaving the old threshold would re-apply the patch on every + # run once disk4 is gone. + if [ "$CURRENT_DISKS" -lt 4 ]; then echo " Adding extra disks to talos-cp-2..." kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{ "spec": { @@ -75,14 +86,6 @@ spec: "path": "/var/lib/longhorn-disk3", "storageReserved": 5368709120, "tags": [] - }, - "disk4": { - "allowScheduling": true, - "diskType": "filesystem", - "evictionRequested": false, - "path": "/var/lib/longhorn-disk4", - "storageReserved": 21474836480, - "tags": [] } } } diff --git a/terraform/terraform.tfvars.local.bak b/terraform/terraform.tfvars.local.bak index a06e2e0..0e639f3 100644 --- a/terraform/terraform.tfvars.local.bak +++ b/terraform/terraform.tfvars.local.bak @@ -20,6 +20,7 @@ controlplane_configs = { "talos-cp-1" = { hostname = "talos-cp-1" lan_ip = "192.168.1.166" + public_ip = "https://k8s-api.riotpiao.com" lan_subnet = "192.168.1.0/24" lan_gateway = "192.168.1.254" install_disk = "/dev/nvme0n1" @@ -32,6 +33,7 @@ controlplane_configs = { "talos-cp-2" = { hostname = "talos-cp-2" lan_ip = "192.168.1.214" + public_ip = "https://k8s-api.riotpiao.com" lan_subnet = "192.168.1.0/24" lan_gateway = "192.168.1.254" install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d" @@ -39,7 +41,11 @@ controlplane_configs = { { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" }, { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" }, { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" }, - { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" }, + # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was + # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous + # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica + # rebuilds on talos-cp-2 with unkillable D-state processes. Do not + # re-add it. ] zone = "az-b" allow_scheduling = true @@ -49,6 +55,7 @@ controlplane_configs = { "talos-cp-3" = { hostname = "talos-cp-3" lan_ip = "192.168.1.162" + public_ip = "https://k8s-api.riotpiao.com" lan_subnet = "192.168.1.0/24" lan_gateway = "192.168.1.254" install_disk = "/dev/nvme0n1"