rock
e4caa7e841
The committed CronJob could never run, for two independent reasons: - quay.io/coreos/etcd ships no /bin/sh, so command: [/bin/sh, -c] died with exec: "/bin/sh": no such file or directory - it existed only on branch fix/etcd-defrag-cron, while the ArgoCD application cluster-maintenance tracks main (targetRevision: main), so the cluster never had the CronJob or its Secret This revision fixes both and records the 2026-10-07 incident: - image bitnamilegacy/etcd:3.6.4 pinned by digest (ships bash and etcdctl 3.6.4, matching the 3.6.11 servers) - 15-day cadence by UTC epoch-day modulo; */15 in day-of-month fires on the 1st, 16th and 31st, which puts two runs one day apart - compacts to the MINIMUM revision across members, so a lagging member is never compacted past its own revision (measured: talos-cp-2 froze ~1.2M entries behind the leader and needed a reboot to rejoin) - skips defrag on a member whose applied index lags the leader by 100k+ - documents that compact must precede defrag: compaction had stalled while the NOSPACE alarm was up, so dbSizeInUse read 100% of the file and defrag alone would have reclaimed nothing
178 lines
7.9 KiB
YAML
178 lines
7.9 KiB
YAML
# etcd backend maintenance: compact, defrag, disarm NOSPACE.
|
|
#
|
|
# Why this exists
|
|
# ---------------
|
|
# etcd's backend file only grows. The apiserver compacts every 5m, but
|
|
# compaction marks superseded revisions free *inside* the file; only `defrag`
|
|
# returns that space to the filesystem. Nothing ran defrag, so the file grew
|
|
# until it crossed the 2 GiB quota:
|
|
#
|
|
# 2026-10-07 backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
|
|
# -> etcd raised NOSPACE on those two members
|
|
# -> etcd rejected ALL writes cluster-wide
|
|
# -> kubelets could not post node status: 3 nodes NotReady
|
|
# -> apiserver writes returned 500, the tunnel API path 502
|
|
#
|
|
# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
|
|
# dead pages. Recovery was `compact <current-revision>` then `defrag` per
|
|
# member then `alarm disarm`. Compaction had also stalled while the alarm was
|
|
# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
|
|
# worked. Order matters: compact first, then defrag.
|
|
#
|
|
# The earlier revision of this file never ran, for two independent reasons:
|
|
# 1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
|
|
# `cluster-maintenance` tracks `main` (targetRevision: main), so the
|
|
# cluster never had the CronJob or the Secret.
|
|
# 2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
|
|
# `exec: "/bin/sh": no such file or directory`). Any container that runs
|
|
# a shell script needs an image that ships one.
|
|
# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
|
|
# 3.6.4, matching the 3.6.11 servers) pinned by digest.
|
|
#
|
|
# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
|
|
# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
|
|
# and the script exits unless the UTC epoch day is a multiple of 15. That gives
|
|
# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
|
|
# on the 1st, 16th and 31st, which puts two runs one day apart).
|
|
#
|
|
# Safety
|
|
# ------
|
|
# 1. Compaction uses the MINIMUM revision across members. Compacting to a
|
|
# revision a member has not reached wedges that member - measured on
|
|
# 2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
|
|
# needed a reboot to rejoin.
|
|
# 2. A member whose raft applied index lags the leader by more than 100000 is
|
|
# skipped and reported. Defragging a frozen member reclaims nothing.
|
|
# 3. Members are defragged one at a time. A defrag pauses the member's apply
|
|
# loop for the duration of the file copy.
|
|
#
|
|
# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
|
|
# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
|
|
# Its client certificate is signed by the cluster's etcd CA and expires
|
|
# 2036-10; rotate it before then by re-running the mint and re-encrypting.
|
|
#
|
|
# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
|
|
# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: etcd-defrag
|
|
namespace: kube-system
|
|
labels:
|
|
app.kubernetes.io/name: etcd-defrag
|
|
app.kubernetes.io/part-of: cluster-maintenance
|
|
spec:
|
|
schedule: "0 3 * * *" # daily; the script acts only every 15th day
|
|
concurrencyPolicy: Forbid
|
|
startingDeadlineSeconds: 600
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
tolerations:
|
|
- key: node-role.kubernetes.io/control-plane
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: defrag
|
|
image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
|
|
env:
|
|
- name: ETCDCTL_API
|
|
value: "3"
|
|
- name: ETCDCTL_CACERT
|
|
value: /etc/etcd/ca.crt
|
|
- name: ETCDCTL_CERT
|
|
value: /etc/etcd/client.crt
|
|
- name: ETCDCTL_KEY
|
|
value: /etc/etcd/client.key
|
|
- name: EPS
|
|
value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -u
|
|
|
|
DAY=$(( $(date -u +%s) / 86400 ))
|
|
if [ $(( DAY % 15 )) -ne 0 ]; then
|
|
echo "skip: UTC epoch day $DAY is not a 15-day boundary"
|
|
exit 0
|
|
fi
|
|
|
|
echo "== before =="
|
|
for ep in $EPS; do
|
|
etcdctl --endpoints="$ep" endpoint status -w table || true
|
|
done
|
|
|
|
MINREV=""
|
|
MAXAPPLIED=0
|
|
for ep in $EPS; do
|
|
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
|
|
rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
|
|
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
|
|
echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
|
|
if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
|
|
MINREV="$rev"
|
|
fi
|
|
if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
|
|
MAXAPPLIED="$app"
|
|
fi
|
|
done
|
|
|
|
if [ -z "$MINREV" ]; then
|
|
echo "FATAL: no member returned a revision; nothing attempted"
|
|
exit 1
|
|
fi
|
|
|
|
echo "== compact to revision $MINREV =="
|
|
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
|
|
|
|
# Physical deletion of superseded keys runs in the background
|
|
# after the compaction commit; give it a moment to drain.
|
|
sleep 20
|
|
|
|
echo "== defrag each member (sequential, one at a time) =="
|
|
rc=0
|
|
for ep in $EPS; do
|
|
json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
|
|
app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
|
|
if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
|
|
echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
|
|
rc=1
|
|
continue
|
|
fi
|
|
echo "-- $ep"
|
|
if ! etcdctl --endpoints="$ep" defrag; then
|
|
echo "DEFRAG-FAILED $ep"
|
|
rc=1
|
|
fi
|
|
done
|
|
|
|
echo "== disarm NOSPACE =="
|
|
for ep in $EPS; do
|
|
etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
|
|
etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
|
|
done
|
|
|
|
echo "== after =="
|
|
for ep in $EPS; do
|
|
etcdctl --endpoints="$ep" endpoint status -w table || true
|
|
done
|
|
etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
|
|
exit $rc
|
|
resources:
|
|
requests: { cpu: 10m, memory: 32Mi }
|
|
limits: { cpu: 200m, memory: 128Mi }
|
|
volumeMounts:
|
|
- name: creds
|
|
mountPath: /etc/etcd
|
|
readOnly: true
|
|
volumes:
|
|
- name: creds
|
|
secret:
|
|
secretName: etcd-defrag-creds
|
|
defaultMode: 0444
|