102 lines
4.0 KiB
YAML
102 lines
4.0 KiB
YAML
|
# etcd defrag maintenance.
|
||
|
|
#
|
||
|
|
# Why this exists: the apiserver compacts etcd every 5m, but compaction only marks
|
||
|
|
# superseded revisions free INSIDE the backend file -- the file never shrinks.
|
||
|
|
# Only `etcdctl defrag` returns space to the filesystem, and nothing runs it
|
||
|
|
# automatically. On 2026-10-07 the 2 GiB quota was reached after ~73 days
|
||
|
|
# (backend 2.1 GiB) -> etcd raised the NOSPACE alarm -> etcd refused ALL writes
|
||
|
|
# -> kubelets could not post node status -> 3 nodes went NotReady and the tunnel
|
||
|
|
# origin pods on them vanished (502s). Recovery = compact + defrag + alarm disarm.
|
||
|
|
#
|
||
|
|
# This CronJob keeps the backend well under quota: compact to the current
|
||
|
|
# revision, defrag each member sequentially, then disarm any NOSPACE alarm.
|
||
|
|
#
|
||
|
|
# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
|
||
|
|
# etcd_server_quota_in_bytes > 0.8, and a NOSPACE alarm alert.
|
||
|
|
#
|
||
|
|
# Credentials: secret etcd-defrag-creds (kube-system), SOPS-managed in
|
||
|
|
# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml. Its client cert is signed by the
|
||
|
|
# cluster's etcd CA and expires 2036-10 -- rotate it before then by re-running the
|
||
|
|
# mint + re-encrypt (see the repo's sops workflow).
|
||
|
|
apiVersion: batch/v1
|
||
|
|
kind: CronJob
|
||
|
|
metadata:
|
||
|
|
name: etcd-defrag
|
||
|
|
namespace: kube-system
|
||
|
|
labels:
|
||
|
|
app.kubernetes.io/name: etcd-defrag
|
||
|
|
app.kubernetes.io/part-of: cluster-maintenance
|
||
|
|
spec:
|
||
|
|
schedule: "0 3 * * 0" # Sundays 03:00, low traffic
|
||
|
|
concurrencyPolicy: Forbid
|
||
|
|
startingDeadlineSeconds: 600
|
||
|
|
successfulJobsHistoryLimit: 3
|
||
|
|
failedJobsHistoryLimit: 3
|
||
|
|
jobTemplate:
|
||
|
|
spec:
|
||
|
|
backoffLimit: 2
|
||
|
|
template:
|
||
|
|
spec:
|
||
|
|
restartPolicy: OnFailure
|
||
|
|
tolerations:
|
||
|
|
- key: node-role.kubernetes.io/control-plane
|
||
|
|
operator: Exists
|
||
|
|
effect: NoSchedule
|
||
|
|
containers:
|
||
|
|
- name: defrag
|
||
|
|
image: quay.io/coreos/etcd:v3.5.17
|
||
|
|
env:
|
||
|
|
- name: ETCDCTL_API
|
||
|
|
value: "3"
|
||
|
|
- name: ETCDCTL_CACERT
|
||
|
|
value: /etc/etcd/ca.crt
|
||
|
|
- name: ETCDCTL_CERT
|
||
|
|
value: /etc/etcd/client.crt
|
||
|
|
- name: ETCDCTL_KEY
|
||
|
|
value: /etc/etcd/client.key
|
||
|
|
- name: EPS
|
||
|
|
value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
|
||
|
|
command: ["/bin/sh", "-c"]
|
||
|
|
args:
|
||
|
|
- |
|
||
|
|
set -u
|
||
|
|
PRIMARY=${EPS%% *}
|
||
|
|
echo "== before =="
|
||
|
|
etcdctl --endpoints=$EPS endpoint status -w table || true
|
||
|
|
|
||
|
|
REV=$(etcdctl --endpoints=$PRIMARY endpoint status -w json \
|
||
|
|
| sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
|
||
|
|
if [ -n "$REV" ]; then
|
||
|
|
echo "== compact to revision $REV =="
|
||
|
|
etcdctl --endpoints=$PRIMARY compact "$REV" || true
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo "== defrag each member (sequential, one at a time) =="
|
||
|
|
rc=0
|
||
|
|
for ep in $EPS; do
|
||
|
|
echo "-- $ep"
|
||
|
|
etcdctl --endpoints="$ep" defrag || { echo "DEFRAG-FAILED $ep"; rc=1; }
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "== disarm NOSPACE =="
|
||
|
|
for ep in $EPS; do
|
||
|
|
etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
|
||
|
|
done
|
||
|
|
|
||
|
|
echo "== after =="
|
||
|
|
etcdctl --endpoints=$EPS endpoint status -w table || true
|
||
|
|
etcdctl --endpoints=$PRIMARY alarm list || true
|
||
|
|
exit $rc
|
||
|
|
resources:
|
||
|
|
requests: { cpu: 10m, memory: 32Mi }
|
||
|
|
limits: { cpu: 200m, memory: 128Mi }
|
||
|
|
volumeMounts:
|
||
|
|
- name: creds
|
||
|
|
mountPath: /etc/etcd
|
||
|
|
readOnly: true
|
||
|
|
volumes:
|
||
|
|
- name: creds
|
||
|
|
secret:
|
||
|
|
secretName: etcd-defrag-creds
|
||
|
|
defaultMode: 0444
|