diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml index 4b05d85..47f4d2e 100644 --- a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml +++ b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml @@ -161,9 +161,9 @@ ingress: # The database was never the bottleneck. A real action_runner UPDATE measured # 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself. # -# requests.cpu rises to 750m because a request below real usage earns the pod -# little scheduling weight during contention. limits.cpu rises to 3000m -# because diff computation is bursty and needs room to run without throttling. +# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a +# temporary measure while the Actions stall was diagnosed; the real fix landed +# elsewhere, so the limit is back to the original. # # Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea # throttles again at 1000m, move the workload off the control plane rather diff --git a/k8s/infra/longhorn/disk-scan-job.yaml b/k8s/infra/longhorn/disk-scan-job.yaml new file mode 100644 index 0000000..788611b --- /dev/null +++ b/k8s/infra/longhorn/disk-scan-job.yaml @@ -0,0 +1,90 @@ +# One-shot destructive health scan of a candidate Longhorn disk. +# +# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK +# It writes over the entire raw device. An ArgoCD PostSync hook would run it +# on every sync and destroy whatever the volume holds. Apply it by hand, +# once, on a disk that has no partitions and no mounts. +# +# WHY NOT TALOS +# Talos declares disks by mounting them (machine.disks in the machine +# config). It has no primitive for "run this command on the host", so a scan +# cannot live in the Talos config. This Job is the IaC-correct home. +# +# ORDER OF OPERATIONS +# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED +# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks` +# with device /dev/disk/by-id/ and run `make apply-cp` +# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml +# +# What it does +# fio writes a CRC32C-verified pattern across the whole device, then reads +# every block back and compares. Any unreadable or mismatched sector fails +# the Job. This is a full write and read pass, not a sample. +# +# Expected duration +# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and +# 6 h to verify. Budget 12 hours. Check progress with: +# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f +apiVersion: batch/v1 +kind: Job +metadata: + name: disk-scan-cp1-sdb + namespace: llm-io + labels: + app: disk-scan +spec: + backoffLimit: 0 + # Do not let a stuck scan retry silently. + template: + metadata: + labels: + app: disk-scan + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: talos-cp-1 + containers: + - name: scan + image: ubuntu:22.04 + securityContext: + privileged: true + command: + - /bin/bash + - -c + - | + set -euo pipefail + apt-get update -qq + apt-get install -y -qq smartmontools fio + + echo "=== device identity ===" + smartctl -i /dev/sdb | tail -12 + + echo "=== SMART before ===" + smartctl -H /dev/sdb | tail -2 + + echo "=== full write + verify (destructive) ===" + fio --name=devverify \ + --filename=/dev/sdb \ + --rw=write \ + --bs=1M \ + --iodepth=4 \ + --ioengine=libaio \ + --direct=1 \ + --verify=crc32c \ + --do_verify=1 \ + --verify_fatal=1 \ + --eta=always \ + --output-format=normal + + echo "=== SMART after ===" + smartctl -H /dev/sdb | tail -2 + smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}' + + echo "=== SCAN PASSED ===" + volumeMounts: + - name: dev + mountPath: /dev + volumes: + - name: dev + hostPath: + path: /dev