From 136bfaf0f275a50cda303430f36274b16680442c Mon Sep 17 00:00:00 2001 From: rock Date: Sun, 30 Aug 2026 07:27:50 -0700 Subject: [PATCH] feat(ops): add cluster-wide stale job/pod cleanup CronJob Daily 04:00 UTC sweeper in kube-system: - Delete failed Jobs older than 24h (any namespace) - Delete completed standalone Jobs older than 72h (no CronJob owner) - Delete orphan Error/Evicted pods older than 1h - Self-cleans via ttlSecondsAfterFinished --- k8s/argocd/apps/00-substrate.yaml | 21 +++ .../stale-job-cleanup.yaml | 137 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 k8s/infra/cluster-maintenance/stale-job-cleanup.yaml diff --git a/k8s/argocd/apps/00-substrate.yaml b/k8s/argocd/apps/00-substrate.yaml index 3662bf7..f7009e0 100644 --- a/k8s/argocd/apps/00-substrate.yaml +++ b/k8s/argocd/apps/00-substrate.yaml @@ -138,3 +138,24 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cluster-maintenance + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + project: homelab + source: + repoURL: https://forgejo.riotpiao.com/rock/homelab.git + targetRevision: main + path: k8s/infra/cluster-maintenance + destination: + server: https://kubernetes.default.svc + namespace: kube-system + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml b/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml new file mode 100644 index 0000000..c09951d --- /dev/null +++ b/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml @@ -0,0 +1,137 @@ +# Cluster-wide cleanup of stale failed/completed Jobs and Pods. +# Runs daily at 04:00 UTC. Deletes: +# - Failed Jobs older than 24h (any namespace) +# - Completed Jobs older than 72h with no owning CronJob +# - Orphan pods in Error/Failed/Evicted state older than 1h +# +# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit, +# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL +# and linger forever. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: stale-job-cleanup + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: stale-job-cleanup +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: stale-job-cleanup +subjects: + - kind: ServiceAccount + name: stale-job-cleanup + namespace: kube-system +roleRef: + kind: ClusterRole + name: stale-job-cleanup + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: stale-job-cleanup + namespace: kube-system + labels: + app: stale-job-cleanup +spec: + schedule: "0 4 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 86400 # self-cleanup after 24h + backoffLimit: 1 + activeDeadlineSeconds: 300 + template: + spec: + serviceAccountName: stale-job-cleanup + restartPolicy: Never + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: cleanup + image: alpine/k8s:1.31.0 + command: + - sh + - -c + - | + set -e + NOW=$(date +%s) + + echo "=== Cleaning failed Jobs older than 24h ===" + kubectl get jobs --all-namespaces -o json | \ + jq -r '.items[] | + select(.status.conditions[]?.type == "Failed") | + select(.status.completionTime or .status.startTime) | + "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \ + while read -r NS NAME TS; do + JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) + AGE_H=$(( (NOW - JOB_EPOCH) / 3600 )) + if [ "$AGE_H" -ge 24 ]; then + echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)" + kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true + fi + done + + echo "" + echo "=== Cleaning completed standalone Jobs older than 72h ===" + kubectl get jobs --all-namespaces -o json | \ + jq -r '.items[] | + select(.status.succeeded >= 1) | + select((.metadata.ownerReferences // []) | length == 0) | + "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \ + while read -r NS NAME TS; do + JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) + AGE_H=$(( (NOW - JOB_EPOCH) / 3600 )) + if [ "$AGE_H" -ge 72 ]; then + echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)" + kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true + fi + done + + echo "" + echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ===" + # Evicted pods show as Failed with reason Evicted + kubectl get pods --all-namespaces -o json | \ + jq -r '.items[] | + select( + .status.phase == "Failed" or + (.status.reason // "") == "Evicted" or + (.status.containerStatuses // [] | any(.state.terminated.reason == "Error")) + ) | + select((.metadata.ownerReferences // []) | all(.kind != "Job")) | + "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \ + while read -r NS NAME TS; do + POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) + AGE_H=$(( (NOW - POD_EPOCH) / 3600 )) + if [ "$AGE_H" -ge 1 ]; then + echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)" + kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true + fi + done + + echo "" + echo "Cleanup complete" + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi