2026-11-02 04:32:36 +00:00
1 changed files with 90 additions and 0 deletions
@@ -0,0 +1,90 @@
 # One-shot destructive health scan of a candidate Longhorn disk.
 #
 # WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
 #   It writes over the entire raw device. An ArgoCD PostSync hook would run it
 #   on every sync and destroy whatever the volume holds. Apply it by hand,
 #   once, on a disk that has no partitions and no mounts.
 #
 # WHY NOT TALOS
 #   Talos declares disks by mounting them (machine.disks in the machine
 #   config). It has no primitive for "run this command on the host", so a scan
 #   cannot live in the Talos config. This Job is the IaC-correct home.
 #
 # ORDER OF OPERATIONS
 #   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
 #   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
 #      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
 #   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
 #
 # What it does
 #   fio writes a CRC32C-verified pattern across the whole device, then reads
 #   every block back and compares. Any unreadable or mismatched sector fails
 #   the Job. This is a full write and read pass, not a sample.
 #
 # Expected duration
 #   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
 #   6 h to verify. Budget 12 hours. Check progress with:
 #     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
 apiVersion: batch/v1
 kind: Job
 metadata:
  name: disk-scan-cp1-sdb
  namespace: llm-io
  labels:
    app: disk-scan
 spec:
  backoffLimit: 0
  # Do not let a stuck scan retry silently.
  template:
    metadata:
      labels:
        app: disk-scan
    spec:
      restartPolicy: Never
      nodeSelector:
        kubernetes.io/hostname: talos-cp-1
      containers:
        - name: scan
          image: ubuntu:22.04
          securityContext:
            privileged: true
          command:
            - /bin/bash
            - -c
            - |
              set -euo pipefail
              apt-get update -qq
              apt-get install -y -qq smartmontools fio
              echo "=== device identity ==="
              smartctl -i /dev/sdb | tail -12
              echo "=== SMART before ==="
              smartctl -H /dev/sdb | tail -2
              echo "=== full write + verify (destructive) ==="
              fio --name=devverify \
                  --filename=/dev/sdb \
                  --rw=write \
                  --bs=1M \
                  --iodepth=4 \
                  --ioengine=libaio \
                  --direct=1 \
                  --verify=crc32c \
                  --do_verify=1 \
                  --verify_fatal=1 \
                  --eta=always \
                  --output-format=normal
              echo "=== SMART after ==="
              smartctl -H /dev/sdb | tail -2
              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
              echo "=== SCAN PASSED ==="
          volumeMounts:
            - name: dev
              mountPath: /dev
      volumes:
        - name: dev
          hostPath:
            path: /dev