diff --git a/k8s/infra/longhorn/disk-scan-job.yaml b/k8s/infra/longhorn/disk-scan-job.yaml new file mode 100644 index 0000000..788611b --- /dev/null +++ b/k8s/infra/longhorn/disk-scan-job.yaml @@ -0,0 +1,90 @@ +# One-shot destructive health scan of a candidate Longhorn disk. +# +# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK +# It writes over the entire raw device. An ArgoCD PostSync hook would run it +# on every sync and destroy whatever the volume holds. Apply it by hand, +# once, on a disk that has no partitions and no mounts. +# +# WHY NOT TALOS +# Talos declares disks by mounting them (machine.disks in the machine +# config). It has no primitive for "run this command on the host", so a scan +# cannot live in the Talos config. This Job is the IaC-correct home. +# +# ORDER OF OPERATIONS +# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED +# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks` +# with device /dev/disk/by-id/ and run `make apply-cp` +# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml +# +# What it does +# fio writes a CRC32C-verified pattern across the whole device, then reads +# every block back and compares. Any unreadable or mismatched sector fails +# the Job. This is a full write and read pass, not a sample. +# +# Expected duration +# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and +# 6 h to verify. Budget 12 hours. Check progress with: +# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f +apiVersion: batch/v1 +kind: Job +metadata: + name: disk-scan-cp1-sdb + namespace: llm-io + labels: + app: disk-scan +spec: + backoffLimit: 0 + # Do not let a stuck scan retry silently. + template: + metadata: + labels: + app: disk-scan + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: talos-cp-1 + containers: + - name: scan + image: ubuntu:22.04 + securityContext: + privileged: true + command: + - /bin/bash + - -c + - | + set -euo pipefail + apt-get update -qq + apt-get install -y -qq smartmontools fio + + echo "=== device identity ===" + smartctl -i /dev/sdb | tail -12 + + echo "=== SMART before ===" + smartctl -H /dev/sdb | tail -2 + + echo "=== full write + verify (destructive) ===" + fio --name=devverify \ + --filename=/dev/sdb \ + --rw=write \ + --bs=1M \ + --iodepth=4 \ + --ioengine=libaio \ + --direct=1 \ + --verify=crc32c \ + --do_verify=1 \ + --verify_fatal=1 \ + --eta=always \ + --output-format=normal + + echo "=== SMART after ===" + smartctl -H /dev/sdb | tail -2 + smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}' + + echo "=== SCAN PASSED ===" + volumeMounts: + - name: dev + mountPath: /dev + volumes: + - name: dev + hostPath: + path: /dev