diff --git a/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml b/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml new file mode 100644 index 0000000..c8215a3 --- /dev/null +++ b/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Secret +metadata: + name: etcd-defrag-creds + namespace: kube-system + labels: + app.kubernetes.io/name: etcd-defrag +type: Opaque +stringData: + ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str] + client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str] + client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o + R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr + a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz + MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p + 8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g== + -----END AGE ENCRYPTED FILE----- + recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla + encrypted_regex: ^(data|stringData)$ + lastmodified: "2026-10-07T05:37:26Z" + mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str] + version: 3.13.2 diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml index a3dc165..92a4eaa 100644 --- a/k8s/argocd/secrets/secret-generator.yaml +++ b/k8s/argocd/secrets/secret-generator.yaml @@ -37,3 +37,4 @@ files: - oauth2-credentials.enc.yaml - forgejo-registry-token.enc.yaml - pi-sandbox-secrets.enc.yaml + - etcd-defrag-creds.enc.yaml diff --git a/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml new file mode 100644 index 0000000..c75fb3d --- /dev/null +++ b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml @@ -0,0 +1,177 @@ +# etcd backend maintenance: compact, defrag, disarm NOSPACE. +# +# Why this exists +# --------------- +# etcd's backend file only grows. The apiserver compacts every 5m, but +# compaction marks superseded revisions free *inside* the file; only `defrag` +# returns that space to the filesystem. Nothing ran defrag, so the file grew +# until it crossed the 2 GiB quota: +# +# 2026-10-07 backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3 +# -> etcd raised NOSPACE on those two members +# -> etcd rejected ALL writes cluster-wide +# -> kubelets could not post node status: 3 nodes NotReady +# -> apiserver writes returned 500, the tunnel API path 502 +# +# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was +# dead pages. Recovery was `compact ` then `defrag` per +# member then `alarm disarm`. Compaction had also stalled while the alarm was +# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have +# worked. Order matters: compact first, then defrag. +# +# The earlier revision of this file never ran, for two independent reasons: +# 1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application +# `cluster-maintenance` tracks `main` (targetRevision: main), so the +# cluster never had the CronJob or the Secret. +# 2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified: +# `exec: "/bin/sh": no such file or directory`). Any container that runs +# a shell script needs an image that ships one. +# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl +# 3.6.4, matching the 3.6.11 servers) pinned by digest. +# +# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle +# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily +# and the script exits unless the UTC epoch day is a multiple of 15. That gives +# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires +# on the 1st, 16th and 31st, which puts two runs one day apart). +# +# Safety +# ------ +# 1. Compaction uses the MINIMUM revision across members. Compacting to a +# revision a member has not reached wedges that member - measured on +# 2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and +# needed a reboot to rejoin. +# 2. A member whose raft applied index lags the leader by more than 100000 is +# skipped and reported. Defragging a frozen member reclaims nothing. +# 3. Members are defragged one at a time. A defrag pauses the member's apply +# loop for the duration of the file copy. +# +# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in +# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets). +# Its client certificate is signed by the cluster's etcd CA and expires +# 2036-10; rotate it before then by re-running the mint and re-encrypting. +# +# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes / +# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: etcd-defrag + namespace: kube-system + labels: + app.kubernetes.io/name: etcd-defrag + app.kubernetes.io/part-of: cluster-maintenance +spec: + schedule: "0 3 * * *" # daily; the script acts only every 15th day + concurrencyPolicy: Forbid + startingDeadlineSeconds: 600 + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: defrag + image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730 + env: + - name: ETCDCTL_API + value: "3" + - name: ETCDCTL_CACERT + value: /etc/etcd/ca.crt + - name: ETCDCTL_CERT + value: /etc/etcd/client.crt + - name: ETCDCTL_KEY + value: /etc/etcd/client.key + - name: EPS + value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379" + command: ["/bin/bash", "-c"] + args: + - | + set -u + + DAY=$(( $(date -u +%s) / 86400 )) + if [ $(( DAY % 15 )) -ne 0 ]; then + echo "skip: UTC epoch day $DAY is not a 15-day boundary" + exit 0 + fi + + echo "== before ==" + for ep in $EPS; do + etcdctl --endpoints="$ep" endpoint status -w table || true + done + + MINREV="" + MAXAPPLIED=0 + for ep in $EPS; do + json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue + rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1) + app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1) + echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}" + if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then + MINREV="$rev" + fi + if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then + MAXAPPLIED="$app" + fi + done + + if [ -z "$MINREV" ]; then + echo "FATAL: no member returned a revision; nothing attempted" + exit 1 + fi + + echo "== compact to revision $MINREV ==" + etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)" + + # Physical deletion of superseded keys runs in the background + # after the compaction commit; give it a moment to drain. + sleep 20 + + echo "== defrag each member (sequential, one at a time) ==" + rc=0 + for ep in $EPS; do + json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; } + app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1) + if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then + echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)" + rc=1 + continue + fi + echo "-- $ep" + if ! etcdctl --endpoints="$ep" defrag; then + echo "DEFRAG-FAILED $ep" + rc=1 + fi + done + + echo "== disarm NOSPACE ==" + for ep in $EPS; do + etcdctl --endpoints="$ep" alarm list 2>/dev/null || true + etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true + done + + echo "== after ==" + for ep in $EPS; do + etcdctl --endpoints="$ep" endpoint status -w table || true + done + etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true + exit $rc + resources: + requests: { cpu: 10m, memory: 32Mi } + limits: { cpu: 200m, memory: 128Mi } + volumeMounts: + - name: creds + mountPath: /etc/etcd + readOnly: true + volumes: + - name: creds + secret: + secretName: etcd-defrag-creds + defaultMode: 0444