homelab/k8s/infra/longhorn/longhorn-add-disks-job.yaml

# PostSync hook Job that adds extra disks to Longhorn nodes.
# talos-cp-2 has 4 extra disks mounted at /var/lib/longhorn-disk{1,2,3,4}
# that are NOT auto-discovered by Longhorn.
#
# talos-cp-3 additionally gets a tagged disk for paperless-ngx media, backed by
# the 4TB USB HDD (/dev/sdg) — tagged "paperless-media" so only the dedicated
# longhorn-paperless-media StorageClass (diskSelector match) can place replicas
# there, keeping it out of the default 3-replica pool. This patch is inert
# until the Terraform machine-config change mounts the disk at
# /var/lib/longhorn-paperless-media (pending — see terraform.tfvars.local,
# not present in this checkout); Longhorn just reports the disk not-ready
# until the path exists, no harm in applying it early.
apiVersion: batch/v1
kind: Job
metadata:
  name: longhorn-add-disks
  namespace: longhorn-system
  annotations:
    argocd.argoproj.io/hook: PostSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
  backoffLimit: 3
  template:
    metadata:
      name: longhorn-add-disks
    spec:
      restartPolicy: Never
      serviceAccountName: longhorn-expand-replicas
      containers:
      - name: add-disks
        image: bitnami/kubectl:latest
        command:
        - /bin/bash
        - -c
        - |
          set -euo pipefail
          
          echo "Adding extra disks to Longhorn nodes..."
          
          # talos-cp-2 has 4 extra disks:
          # - /var/lib/longhorn-disk1 (sdb1, 600GB)
          # - /var/lib/longhorn-disk2 (sdc1, 500GB)
          # - /var/lib/longhorn-disk3 (sdd1, 160GB)
          #
          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
          # process that touched it blocked in unkillable D state. Those unkillable
          # processes held Longhorn's per-node rebuild slots forever, so no new
          # replica could start on talos-cp-2 and every new volume there failed to
          # attach. It is removed from the Terraform disk list too, so a re-render
          # does not put it back.
          
          echo "Checking talos-cp-2..."
          CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
          echo "  Current disk count: $CURRENT_DISKS"
          
          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
          # existed; leaving the old threshold would re-apply the patch on every
          # run once disk4 is gone.
          if [ "$CURRENT_DISKS" -lt 4 ]; then
            echo "  Adding extra disks to talos-cp-2..."
            kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
              "spec": {
                "disks": {
                  "disk1": {
                    "allowScheduling": true,
                    "diskType": "filesystem",
                    "evictionRequested": false,
                    "path": "/var/lib/longhorn-disk1",
                    "storageReserved": 10737418240,
                    "tags": []
                  },
                  "disk2": {
                    "allowScheduling": true,
                    "diskType": "filesystem",
                    "evictionRequested": false,
                    "path": "/var/lib/longhorn-disk2",
                    "storageReserved": 10737418240,
                    "tags": []
                  },
                  "disk3": {
                    "allowScheduling": true,
                    "diskType": "filesystem",
                    "evictionRequested": false,
                    "path": "/var/lib/longhorn-disk3",
                    "storageReserved": 5368709120,
                    "tags": []
                  }
                }
              }
            }'
            echo "  ✓ Disks added to talos-cp-2"
          else
            echo "  ✓ talos-cp-2 already has $CURRENT_DISKS disks configured"
          fi
          
          echo "Checking talos-cp-3..."
          CP3_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-3 -o json | jq -r '.spec.disks | keys | length')
          echo "  Current disk count: $CP3_DISKS"

          if [ "$CP3_DISKS" -lt 2 ]; then
            echo "  Adding paperless-media disk to talos-cp-3..."
            kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-3 --type merge -p '{
              "spec": {
                "disks": {
                  "paperless-media": {
                    "allowScheduling": true,
                    "diskType": "filesystem",
                    "evictionRequested": false,
                    "path": "/var/lib/longhorn-paperless-media",
                    "storageReserved": 0,
                    "tags": ["paperless-media"]
                  }
                }
              }
            }'
            echo "  ✓ paperless-media disk added to talos-cp-3"
          else
            echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
          fi

          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
          # clear eviction flag that was set when scheduling was disabled.
          echo "Configuring worker-1 disk..."
          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
            "spec": {
              "disks": {
                "default-disk-1030400000000": {
                  "allowScheduling": true,
                  "evictionRequested": false,
                  "storageReserved": 429496729600
                }
              }
            }
          }'
          echo "  ✓ worker-1 disk configured (400Gi reserved)"

          echo
          echo "Waiting for disks to be ready..."
          sleep 10
          
          echo
          echo "Final storage summary:"
          kubectl -n longhorn-system get nodes.longhorn.io -o json | \
            jq -r '.items[] | "\(.metadata.name): \(.status.diskStatus | to_entries | map("\(.key): \(.value.storageAvailable/1073741824 | floor)GB avail") | join(", "))"'
          
          echo
          echo "Done."