Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 rock
|
d21ee9630f | ||
 rock
|
e4caa7e841 | ||
|
harness
|
1ce9f86635 |
@@ -0,0 +1,27 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: etcd-defrag-creds
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app.kubernetes.io/name: etcd-defrag
|
||||
type: Opaque
|
||||
stringData:
|
||||
ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
|
||||
client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
|
||||
client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
|
||||
sops:
|
||||
age:
|
||||
- enc: |
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
|
||||
R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
|
||||
a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
|
||||
MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
|
||||
8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
|
||||
encrypted_regex: ^(data|stringData)$
|
||||
lastmodified: "2026-10-07T05:37:26Z"
|
||||
mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
|
||||
version: 3.13.2
|
||||
@@ -34,3 +34,4 @@ files:
|
||||
- forgejo-smtp-secrets.enc.yaml
|
||||
- paperless-ai-secrets.enc.yaml
|
||||
- pi-sandbox-secrets.enc.yaml
|
||||
- etcd-defrag-creds.enc.yaml
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
# etcd backend maintenance: compact, defrag, disarm NOSPACE.
|
||||
#
|
||||
# Why this exists
|
||||
# ---------------
|
||||
# etcd's backend file only grows. The apiserver compacts every 5m, but
|
||||
# compaction marks superseded revisions free *inside* the file; only `defrag`
|
||||
# returns that space to the filesystem. Nothing ran defrag, so the file grew
|
||||
# until it crossed the 2 GiB quota:
|
||||
#
|
||||
# 2026-10-07 backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
|
||||
# -> etcd raised NOSPACE on those two members
|
||||
# -> etcd rejected ALL writes cluster-wide
|
||||
# -> kubelets could not post node status: 3 nodes NotReady
|
||||
# -> apiserver writes returned 500, the tunnel API path 502
|
||||
#
|
||||
# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
|
||||
# dead pages. Recovery was `compact <current-revision>` then `defrag` per
|
||||
# member then `alarm disarm`. Compaction had also stalled while the alarm was
|
||||
# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
|
||||
# worked. Order matters: compact first, then defrag.
|
||||
#
|
||||
# The earlier revision of this file never ran, for two independent reasons:
|
||||
# 1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
|
||||
# `cluster-maintenance` tracks `main` (targetRevision: main), so the
|
||||
# cluster never had the CronJob or the Secret.
|
||||
# 2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
|
||||
# `exec: "/bin/sh": no such file or directory`). Any container that runs
|
||||
# a shell script needs an image that ships one.
|
||||
# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
|
||||
# 3.6.4, matching the 3.6.11 servers) pinned by digest.
|
||||
#
|
||||
# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
|
||||
# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
|
||||
# and the script exits unless the UTC epoch day is a multiple of 15. That gives
|
||||
# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
|
||||
# on the 1st, 16th and 31st, which puts two runs one day apart).
|
||||
#
|
||||
# Safety
|
||||
# ------
|
||||
# 1. Compaction uses the MINIMUM revision across members. Compacting to a
|
||||
# revision a member has not reached wedges that member - measured on
|
||||
# 2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
|
||||
# needed a reboot to rejoin.
|
||||
# 2. A member whose raft applied index lags the leader by more than 100000 is
|
||||
# skipped and reported. Defragging a frozen member reclaims nothing.
|
||||
# 3. Members are defragged one at a time. A defrag pauses the member's apply
|
||||
# loop for the duration of the file copy.
|
||||
#
|
||||
# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
|
||||
# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
|
||||
# Its client certificate is signed by the cluster's etcd CA and expires
|
||||
# 2036-10; rotate it before then by re-running the mint and re-encrypting.
|
||||
#
|
||||
# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
|
||||
# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: etcd-defrag
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app.kubernetes.io/name: etcd-defrag
|
||||
app.kubernetes.io/part-of: cluster-maintenance
|
||||
spec:
|
||||
schedule: "0 3 * * *" # daily; the script acts only every 15th day
|
||||
concurrencyPolicy: Forbid
|
||||
startingDeadlineSeconds: 600
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: defrag
|
||||
image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
|
||||
env:
|
||||
- name: ETCDCTL_API
|
||||
value: "3"
|
||||
- name: ETCDCTL_CACERT
|
||||
value: /etc/etcd/ca.crt
|
||||
- name: ETCDCTL_CERT
|
||||
value: /etc/etcd/client.crt
|
||||
- name: ETCDCTL_KEY
|
||||
value: /etc/etcd/client.key
|
||||
- name: EPS
|
||||
value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
|
||||
command: ["/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -u
|
||||
|
||||
DAY=$(( $(date -u +%s) / 86400 ))
|
||||
if [ $(( DAY % 15 )) -ne 0 ]; then
|
||||
echo "skip: UTC epoch day $DAY is not a 15-day boundary"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "== before =="
|
||||
for ep in $EPS; do
|
||||
etcdctl --endpoints="$ep" endpoint status -w table || true
|
||||
done
|
||||
|
||||
MINREV=""
|
||||
MAXAPPLIED=0
|
||||
for ep in $EPS; do
|
||||
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
|
||||
rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
|
||||
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
|
||||
echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
|
||||
if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
|
||||
MINREV="$rev"
|
||||
fi
|
||||
if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
|
||||
MAXAPPLIED="$app"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$MINREV" ]; then
|
||||
echo "FATAL: no member returned a revision; nothing attempted"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "== compact to revision $MINREV =="
|
||||
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
|
||||
|
||||
# Physical deletion of superseded keys runs in the background
|
||||
# after the compaction commit; give it a moment to drain.
|
||||
sleep 20
|
||||
|
||||
echo "== defrag each member (sequential, one at a time) =="
|
||||
rc=0
|
||||
for ep in $EPS; do
|
||||
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
|
||||
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
|
||||
if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
|
||||
echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
|
||||
rc=1
|
||||
continue
|
||||
fi
|
||||
echo "-- $ep"
|
||||
if ! etcdctl --endpoints="$ep" defrag; then
|
||||
echo "DEFRAG-FAILED $ep"
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "== disarm NOSPACE =="
|
||||
for ep in $EPS; do
|
||||
etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
|
||||
etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
|
||||
done
|
||||
|
||||
echo "== after =="
|
||||
for ep in $EPS; do
|
||||
etcdctl --endpoints="$ep" endpoint status -w table || true
|
||||
done
|
||||
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
|
||||
exit $rc
|
||||
resources:
|
||||
requests: { cpu: 10m, memory: 32Mi }
|
||||
limits: { cpu: 200m, memory: 128Mi }
|
||||
volumeMounts:
|
||||
- name: creds
|
||||
mountPath: /etc/etcd
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: creds
|
||||
secret:
|
||||
secretName: etcd-defrag-creds
|
||||
defaultMode: 0444
|
||||
@@ -41,13 +41,24 @@ spec:
|
||||
# - /var/lib/longhorn-disk1 (sdb1, 600GB)
|
||||
# - /var/lib/longhorn-disk2 (sdc1, 500GB)
|
||||
# - /var/lib/longhorn-disk3 (sdd1, 160GB)
|
||||
# - /var/lib/longhorn-disk4 (sde1, 2TB)
|
||||
#
|
||||
# /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
|
||||
# on 2026-10-12 and must not be re-added. The device returned SCSI I/O
|
||||
# errors at ~7,000/minute, the XFS filesystem on it hung, and every
|
||||
# process that touched it blocked in unkillable D state. Those unkillable
|
||||
# processes held Longhorn's per-node rebuild slots forever, so no new
|
||||
# replica could start on talos-cp-2 and every new volume there failed to
|
||||
# attach. It is removed from the Terraform disk list too, so a re-render
|
||||
# does not put it back.
|
||||
|
||||
echo "Checking talos-cp-2..."
|
||||
CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
|
||||
echo " Current disk count: $CURRENT_DISKS"
|
||||
|
||||
if [ "$CURRENT_DISKS" -lt 5 ]; then
|
||||
# 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
|
||||
# existed; leaving the old threshold would re-apply the patch on every
|
||||
# run once disk4 is gone.
|
||||
if [ "$CURRENT_DISKS" -lt 4 ]; then
|
||||
echo " Adding extra disks to talos-cp-2..."
|
||||
kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
|
||||
"spec": {
|
||||
@@ -75,14 +86,6 @@ spec:
|
||||
"path": "/var/lib/longhorn-disk3",
|
||||
"storageReserved": 5368709120,
|
||||
"tags": []
|
||||
},
|
||||
"disk4": {
|
||||
"allowScheduling": true,
|
||||
"diskType": "filesystem",
|
||||
"evictionRequested": false,
|
||||
"path": "/var/lib/longhorn-disk4",
|
||||
"storageReserved": 21474836480,
|
||||
"tags": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ controlplane_configs = {
|
||||
"talos-cp-1" = {
|
||||
hostname = "talos-cp-1"
|
||||
lan_ip = "192.168.1.166"
|
||||
public_ip = "https://k8s-api.riotpiao.com"
|
||||
lan_subnet = "192.168.1.0/24"
|
||||
lan_gateway = "192.168.1.254"
|
||||
install_disk = "/dev/nvme0n1"
|
||||
@@ -32,6 +33,7 @@ controlplane_configs = {
|
||||
"talos-cp-2" = {
|
||||
hostname = "talos-cp-2"
|
||||
lan_ip = "192.168.1.214"
|
||||
public_ip = "https://k8s-api.riotpiao.com"
|
||||
lan_subnet = "192.168.1.0/24"
|
||||
lan_gateway = "192.168.1.254"
|
||||
install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
|
||||
@@ -39,7 +41,11 @@ controlplane_configs = {
|
||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
|
||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
|
||||
{ device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
|
||||
{ device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
|
||||
# /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
|
||||
# retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
|
||||
# SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
|
||||
# rebuilds on talos-cp-2 with unkillable D-state processes. Do not
|
||||
# re-add it.
|
||||
]
|
||||
zone = "az-b"
|
||||
allow_scheduling = true
|
||||
@@ -49,6 +55,7 @@ controlplane_configs = {
|
||||
"talos-cp-3" = {
|
||||
hostname = "talos-cp-3"
|
||||
lan_ip = "192.168.1.162"
|
||||
public_ip = "https://k8s-api.riotpiao.com"
|
||||
lan_subnet = "192.168.1.0/24"
|
||||
lan_gateway = "192.168.1.254"
|
||||
install_disk = "/dev/nvme0n1"
|
||||
|
||||
Reference in New Issue
Block a user