91 lines
3.0 KiB
YAML
91 lines
3.0 KiB
YAML
# One-shot destructive health scan of a candidate Longhorn disk.
|
|
#
|
|
# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
|
|
# It writes over the entire raw device. An ArgoCD PostSync hook would run it
|
|
# on every sync and destroy whatever the volume holds. Apply it by hand,
|
|
# once, on a disk that has no partitions and no mounts.
|
|
#
|
|
# WHY NOT TALOS
|
|
# Talos declares disks by mounting them (machine.disks in the machine
|
|
# config). It has no primitive for "run this command on the host", so a scan
|
|
# cannot live in the Talos config. This Job is the IaC-correct home.
|
|
#
|
|
# ORDER OF OPERATIONS
|
|
# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED
|
|
# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
|
|
# with device /dev/disk/by-id/<wwid> and run `make apply-cp`
|
|
# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
|
|
#
|
|
# What it does
|
|
# fio writes a CRC32C-verified pattern across the whole device, then reads
|
|
# every block back and compares. Any unreadable or mismatched sector fails
|
|
# the Job. This is a full write and read pass, not a sample.
|
|
#
|
|
# Expected duration
|
|
# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
|
|
# 6 h to verify. Budget 12 hours. Check progress with:
|
|
# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: disk-scan-cp1-sdb
|
|
namespace: llm-io
|
|
labels:
|
|
app: disk-scan
|
|
spec:
|
|
backoffLimit: 0
|
|
# Do not let a stuck scan retry silently.
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: disk-scan
|
|
spec:
|
|
restartPolicy: Never
|
|
nodeSelector:
|
|
kubernetes.io/hostname: talos-cp-1
|
|
containers:
|
|
- name: scan
|
|
image: ubuntu:22.04
|
|
securityContext:
|
|
privileged: true
|
|
command:
|
|
- /bin/bash
|
|
- -c
|
|
- |
|
|
set -euo pipefail
|
|
apt-get update -qq
|
|
apt-get install -y -qq smartmontools fio
|
|
|
|
echo "=== device identity ==="
|
|
smartctl -i /dev/sdb | tail -12
|
|
|
|
echo "=== SMART before ==="
|
|
smartctl -H /dev/sdb | tail -2
|
|
|
|
echo "=== full write + verify (destructive) ==="
|
|
fio --name=devverify \
|
|
--filename=/dev/sdb \
|
|
--rw=write \
|
|
--bs=1M \
|
|
--iodepth=4 \
|
|
--ioengine=libaio \
|
|
--direct=1 \
|
|
--verify=crc32c \
|
|
--do_verify=1 \
|
|
--verify_fatal=1 \
|
|
--eta=always \
|
|
--output-format=normal
|
|
|
|
echo "=== SMART after ==="
|
|
smartctl -H /dev/sdb | tail -2
|
|
smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}'
|
|
|
|
echo "=== SCAN PASSED ==="
|
|
volumeMounts:
|
|
- name: dev
|
|
mountPath: /dev
|
|
volumes:
|
|
- name: dev
|
|
hostPath:
|
|
path: /dev
|