From 5fbbe4559337a3da9ae90db986ca8b335a148820 Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Mon, 2 Nov 2026 12:28:25 +0800
Subject: [PATCH] feat(longhorn): add one-shot disk health scan job

SMART self-assessment is not proof a disk is usable. This job writes a
CRC32C-verified pattern across the whole raw device with fio, then reads
every block back and compares. Any unreadable or mismatched sector fails
the Job.

Deliberately NOT an ArgoCD hook: it is destructive, and a PostSync hook
would re-run on every sync and wipe the volume. Apply by hand, once, on a
disk with no partitions and no mounts.

Talos cannot host this. Talos declares disks by mounting them
(machine.disks); it has no primitive to execute a command on the host, so a
scan cannot live in the machine config. A Kubernetes Job is the IaC-correct
home.

Order of operations:
  1. scan the raw device (this job) - disk must be unmounted
  2. add the disk to terraform.tfvars.local longhorn_disks, make apply-cp
  3. register it with Longhorn via longhorn-add-disks-job.yaml

First target: talos-cp-1 /dev/sdb, a 2 TB ST2000LM003 (wwid
naa.50004cf20f5c4c39), currently with no partitions and not mounted.
Measured 124 MB/s write, so budget ~8-9 h for write plus verify.
---
 k8s/infra/longhorn/disk-scan-job.yaml | 90 +++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 k8s/infra/longhorn/disk-scan-job.yaml

diff --git a/k8s/infra/longhorn/disk-scan-job.yaml b/k8s/infra/longhorn/disk-scan-job.yaml
new file mode 100644
index 0000000..788611b
--- /dev/null
+++ b/k8s/infra/longhorn/disk-scan-job.yaml
@@ -0,0 +1,90 @@
+# One-shot destructive health scan of a candidate Longhorn disk.
+#
+# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
+#   It writes over the entire raw device. An ArgoCD PostSync hook would run it
+#   on every sync and destroy whatever the volume holds. Apply it by hand,
+#   once, on a disk that has no partitions and no mounts.
+#
+# WHY NOT TALOS
+#   Talos declares disks by mounting them (machine.disks in the machine
+#   config). It has no primitive for "run this command on the host", so a scan
+#   cannot live in the Talos config. This Job is the IaC-correct home.
+#
+# ORDER OF OPERATIONS
+#   1. Scan the raw device with this Job        <- the disk must be UNMOUNTED
+#   2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
+#      with device /dev/disk/by-id/<wwid> and run `make apply-cp`
+#   3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
+#
+# What it does
+#   fio writes a CRC32C-verified pattern across the whole device, then reads
+#   every block back and compares. Any unreadable or mismatched sector fails
+#   the Job. This is a full write and read pass, not a sample.
+#
+# Expected duration
+#   2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
+#   6 h to verify. Budget 12 hours. Check progress with:
+#     kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: disk-scan-cp1-sdb
+  namespace: llm-io
+  labels:
+    app: disk-scan
+spec:
+  backoffLimit: 0
+  # Do not let a stuck scan retry silently.
+  template:
+    metadata:
+      labels:
+        app: disk-scan
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      containers:
+        - name: scan
+          image: ubuntu:22.04
+          securityContext:
+            privileged: true
+          command:
+            - /bin/bash
+            - -c
+            - |
+              set -euo pipefail
+              apt-get update -qq
+              apt-get install -y -qq smartmontools fio
+
+              echo "=== device identity ==="
+              smartctl -i /dev/sdb | tail -12
+
+              echo "=== SMART before ==="
+              smartctl -H /dev/sdb | tail -2
+
+              echo "=== full write + verify (destructive) ==="
+              fio --name=devverify \
+                  --filename=/dev/sdb \
+                  --rw=write \
+                  --bs=1M \
+                  --iodepth=4 \
+                  --ioengine=libaio \
+                  --direct=1 \
+                  --verify=crc32c \
+                  --do_verify=1 \
+                  --verify_fatal=1 \
+                  --eta=always \
+                  --output-format=normal
+
+              echo "=== SMART after ==="
+              smartctl -H /dev/sdb | tail -2
+              smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print "  "$0}'
+
+              echo "=== SCAN PASSED ==="
+          volumeMounts:
+            - name: dev
+              mountPath: /dev
+      volumes:
+        - name: dev
+          hostPath:
+            path: /dev