Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b7de85e01e | ||
|
|
aa992872cb | ||
|
|
1fd5f25932 | ||
|
|
8f9bf6ee8d | ||
|
|
ce4434286c |
@@ -97,10 +97,25 @@ gitea:
|
||||
# USER and PASSWD injected via env vars below (GITEA__MAILER__USER, GITEA__MAILER__PASSWD)
|
||||
|
||||
# Persistence (shared storage for repos)
|
||||
#
|
||||
# 60Gi, not the original 20Gi. The container registry stores its blobs here,
|
||||
# and a single candle-vllm sm_70 image is 14GB uncompressed. At 20Gi the volume
|
||||
# filled, and every blob upload failed with:
|
||||
# Package registry API internal error: 500 write /data/tmp/package-upload/...:
|
||||
# no space left on device
|
||||
# The push client surfaces that as a bare `received unexpected HTTP status: 500`
|
||||
# after retrying every layer, which looks like a network fault and is not one.
|
||||
#
|
||||
# Note for whoever expands this again: Longhorn's admission webhook checks the
|
||||
# PHYSICAL free space of the disk holding each replica, not the thin volume
|
||||
# size. A request for 100Gi was rejected while the disk had 23.5Gi free:
|
||||
# disk ... does not have sufficient physical space for expansion:
|
||||
# physical free space would drop below minimal: left=14596177920 < minimal=49931236556
|
||||
# 60Gi fit. Raise it in steps and watch the webhook message.
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: longhorn
|
||||
size: 20Gi
|
||||
size: 60Gi
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
|
||||
@@ -146,19 +161,19 @@ ingress:
|
||||
# The database was never the bottleneck. A real action_runner UPDATE measured
|
||||
# 2.133ms with zero lock waits, so this is CPU starvation in Gitea itself.
|
||||
#
|
||||
# requests.cpu rises to 750m because a request below real usage earns the pod
|
||||
# little scheduling weight during contention. limits.cpu rises to 3000m
|
||||
# because diff computation is bursty and needs room to run without throttling.
|
||||
# requests.cpu stays 200m and limits.cpu stays 1000m. The 3000m limit was a
|
||||
# temporary measure while the Actions stall was diagnosed; the real fix landed
|
||||
# elsewhere, so the limit is back to the original.
|
||||
#
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. 3000m is 25% of
|
||||
# that node's 11950m allocatable CPU. If Gitea sustains near this limit, move
|
||||
# the workload off the control plane rather than raising the limit again.
|
||||
# Caution: this pod runs on talos-cp-3, a control-plane node. If Gitea
|
||||
# throttles again at 1000m, move the workload off the control plane rather
|
||||
# than raising the limit.
|
||||
resources:
|
||||
requests:
|
||||
cpu: 750m
|
||||
cpu: 200m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
cpu: 3000m
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
|
||||
# Tolerations for control-plane
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# One-shot destructive health scan of a candidate Longhorn disk.
|
||||
#
|
||||
# WHY THIS IS A MANUAL JOB AND NOT AN ARGOCD HOOK
|
||||
# It writes over the entire raw device. An ArgoCD PostSync hook would run it
|
||||
# on every sync and destroy whatever the volume holds. Apply it by hand,
|
||||
# once, on a disk that has no partitions and no mounts.
|
||||
#
|
||||
# WHY NOT TALOS
|
||||
# Talos declares disks by mounting them (machine.disks in the machine
|
||||
# config). It has no primitive for "run this command on the host", so a scan
|
||||
# cannot live in the Talos config. This Job is the IaC-correct home.
|
||||
#
|
||||
# ORDER OF OPERATIONS
|
||||
# 1. Scan the raw device with this Job <- the disk must be UNMOUNTED
|
||||
# 2. If it passes, add the disk to terraform.tfvars.local `longhorn_disks`
|
||||
# with device /dev/disk/by-id/<wwid> and run `make apply-cp`
|
||||
# 3. Register it with Longhorn via the pattern in longhorn-add-disks-job.yaml
|
||||
#
|
||||
# What it does
|
||||
# fio writes a CRC32C-verified pattern across the whole device, then reads
|
||||
# every block back and compares. Any unreadable or mismatched sector fails
|
||||
# the Job. This is a full write and read pass, not a sample.
|
||||
#
|
||||
# Expected duration
|
||||
# 2 TB on a 5400 rpm 2.5" drive at roughly 90 MB/s is about 6 h to write and
|
||||
# 6 h to verify. Budget 12 hours. Check progress with:
|
||||
# kubectl -n llm-io logs job/disk-scan-cp1-sdb -f
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: disk-scan-cp1-sdb
|
||||
namespace: llm-io
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
# Do not let a stuck scan retry silently.
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: disk-scan
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: talos-cp-1
|
||||
containers:
|
||||
- name: scan
|
||||
image: ubuntu:22.04
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -euo pipefail
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq smartmontools fio
|
||||
|
||||
echo "=== device identity ==="
|
||||
smartctl -i /dev/sdb | tail -12
|
||||
|
||||
echo "=== SMART before ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
|
||||
echo "=== full write + verify (destructive) ==="
|
||||
fio --name=devverify \
|
||||
--filename=/dev/sdb \
|
||||
--rw=write \
|
||||
--bs=1M \
|
||||
--iodepth=4 \
|
||||
--ioengine=libaio \
|
||||
--direct=1 \
|
||||
--verify=crc32c \
|
||||
--do_verify=1 \
|
||||
--verify_fatal=1 \
|
||||
--eta=always \
|
||||
--output-format=normal
|
||||
|
||||
echo "=== SMART after ==="
|
||||
smartctl -H /dev/sdb | tail -2
|
||||
smartctl -A /dev/sdb | awk '/Reallocated_Sector|Current_Pending|Offline_Uncorrect|UDMA_CRC/{print " "$0}'
|
||||
|
||||
echo "=== SCAN PASSED ==="
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
Reference in New Issue
Block a user