feat(ops): add cluster-wide stale job/pod cleanup CronJob
Daily 04:00 UTC sweeper in kube-system: - Delete failed Jobs older than 24h (any namespace) - Delete completed standalone Jobs older than 72h (no CronJob owner) - Delete orphan Error/Evicted pods older than 1h - Self-cleans via ttlSecondsAfterFinished
This commit is contained in:
@@ -138,3 +138,24 @@ spec:
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: cluster-maintenance
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: homelab
|
||||
source:
|
||||
repoURL: https://forgejo.riotpiao.com/rock/homelab.git
|
||||
targetRevision: main
|
||||
path: k8s/infra/cluster-maintenance
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: kube-system
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
# Cluster-wide cleanup of stale failed/completed Jobs and Pods.
|
||||
# Runs daily at 04:00 UTC. Deletes:
|
||||
# - Failed Jobs older than 24h (any namespace)
|
||||
# - Completed Jobs older than 72h with no owning CronJob
|
||||
# - Orphan pods in Error/Failed/Evicted state older than 1h
|
||||
#
|
||||
# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
|
||||
# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
|
||||
# and linger forever.
|
||||
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: stale-job-cleanup
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: stale-job-cleanup
|
||||
rules:
|
||||
- apiGroups: ["batch"]
|
||||
resources: ["jobs"]
|
||||
verbs: ["get", "list", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "delete"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: stale-job-cleanup
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: stale-job-cleanup
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: stale-job-cleanup
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: stale-job-cleanup
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app: stale-job-cleanup
|
||||
spec:
|
||||
schedule: "0 4 * * *"
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
ttlSecondsAfterFinished: 86400 # self-cleanup after 24h
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 300
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: stale-job-cleanup
|
||||
restartPolicy: Never
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/control-plane
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: cleanup
|
||||
image: alpine/k8s:1.31.0
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -e
|
||||
NOW=$(date +%s)
|
||||
|
||||
echo "=== Cleaning failed Jobs older than 24h ==="
|
||||
kubectl get jobs --all-namespaces -o json | \
|
||||
jq -r '.items[] |
|
||||
select(.status.conditions[]?.type == "Failed") |
|
||||
select(.status.completionTime or .status.startTime) |
|
||||
"\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
|
||||
while read -r NS NAME TS; do
|
||||
JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
|
||||
AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
|
||||
if [ "$AGE_H" -ge 24 ]; then
|
||||
echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
|
||||
kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Cleaning completed standalone Jobs older than 72h ==="
|
||||
kubectl get jobs --all-namespaces -o json | \
|
||||
jq -r '.items[] |
|
||||
select(.status.succeeded >= 1) |
|
||||
select((.metadata.ownerReferences // []) | length == 0) |
|
||||
"\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
|
||||
while read -r NS NAME TS; do
|
||||
JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
|
||||
AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
|
||||
if [ "$AGE_H" -ge 72 ]; then
|
||||
echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
|
||||
kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
|
||||
# Evicted pods show as Failed with reason Evicted
|
||||
kubectl get pods --all-namespaces -o json | \
|
||||
jq -r '.items[] |
|
||||
select(
|
||||
.status.phase == "Failed" or
|
||||
(.status.reason // "") == "Evicted" or
|
||||
(.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
|
||||
) |
|
||||
select((.metadata.ownerReferences // []) | all(.kind != "Job")) |
|
||||
"\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
|
||||
while read -r NS NAME TS; do
|
||||
POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
|
||||
AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
|
||||
if [ "$AGE_H" -ge 1 ]; then
|
||||
echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
|
||||
kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Cleanup complete"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 128Mi
|
||||
Reference in New Issue
Block a user