# etcd defrag maintenance.
#
# Why this exists: the apiserver compacts etcd every 5m, but compaction only marks
# superseded revisions free INSIDE the backend file -- the file never shrinks.
# Only `etcdctl defrag` returns space to the filesystem, and nothing runs it
# automatically. On 2026-10-07 the 2 GiB quota was reached after ~73 days
# (backend 2.1 GiB) -> etcd raised the NOSPACE alarm -> etcd refused ALL writes
# -> kubelets could not post node status -> 3 nodes went NotReady and the tunnel
# origin pods on them vanished (502s). Recovery = compact + defrag + alarm disarm.
#
# This CronJob keeps the backend well under quota: compact to the current
# revision, defrag each member sequentially, then disarm any NOSPACE alarm.
#
# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
# etcd_server_quota_in_bytes > 0.8, and a NOSPACE alarm alert.
#
# Credentials: secret etcd-defrag-creds (kube-system), SOPS-managed in
# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml. Its client cert is signed by the
# cluster's etcd CA and expires 2036-10 -- rotate it before then by re-running the
# mint + re-encrypt (see the repo's sops workflow).
apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-defrag
  namespace: kube-system
  labels:
    app.kubernetes.io/name: etcd-defrag
    app.kubernetes.io/part-of: cluster-maintenance
spec:
  schedule: "0 3 * * 0"            # Sundays 03:00, low traffic
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 600
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          restartPolicy: OnFailure
          tolerations:
            - key: node-role.kubernetes.io/control-plane
              operator: Exists
              effect: NoSchedule
          containers:
            - name: defrag
              image: quay.io/coreos/etcd:v3.5.17
              env:
                - name: ETCDCTL_API
                  value: "3"
                - name: ETCDCTL_CACERT
                  value: /etc/etcd/ca.crt
                - name: ETCDCTL_CERT
                  value: /etc/etcd/client.crt
                - name: ETCDCTL_KEY
                  value: /etc/etcd/client.key
                - name: EPS
                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
              command: ["/bin/sh", "-c"]
              args:
                - |
                  set -u
                  PRIMARY=${EPS%% *}
                  echo "== before =="
                  etcdctl --endpoints=$EPS endpoint status -w table || true

                  REV=$(etcdctl --endpoints=$PRIMARY endpoint status -w json \
                        | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
                  if [ -n "$REV" ]; then
                    echo "== compact to revision $REV =="
                    etcdctl --endpoints=$PRIMARY compact "$REV" || true
                  fi

                  echo "== defrag each member (sequential, one at a time) =="
                  rc=0
                  for ep in $EPS; do
                    echo "-- $ep"
                    etcdctl --endpoints="$ep" defrag || { echo "DEFRAG-FAILED $ep"; rc=1; }
                  done

                  echo "== disarm NOSPACE =="
                  for ep in $EPS; do
                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
                  done

                  echo "== after =="
                  etcdctl --endpoints=$EPS endpoint status -w table || true
                  etcdctl --endpoints=$PRIMARY alarm list || true
                  exit $rc
              resources:
                requests: { cpu: 10m, memory: 32Mi }
                limits:   { cpu: 200m, memory: 128Mi }
              volumeMounts:
                - name: creds
                  mountPath: /etc/etcd
                  readOnly: true
          volumes:
            - name: creds
              secret:
                secretName: etcd-defrag-creds
                defaultMode: 0444