homelab/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml

# Cluster-wide cleanup of stale failed/completed Jobs and Pods.
# Runs daily at 04:00 UTC. Deletes:
#   - Failed Jobs older than 24h (any namespace)
#   - Completed Jobs older than 72h with no owning CronJob
#   - Orphan pods in Error/Failed/Evicted state older than 1h
#
# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
# and linger forever.

apiVersion: v1
kind: ServiceAccount
metadata:
  name: stale-job-cleanup
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: stale-job-cleanup
rules:
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["get", "list", "delete"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: stale-job-cleanup
subjects:
  - kind: ServiceAccount
    name: stale-job-cleanup
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: stale-job-cleanup
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: stale-job-cleanup
  namespace: kube-system
  labels:
    app: stale-job-cleanup
spec:
  schedule: "0 4 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      ttlSecondsAfterFinished: 86400  # self-cleanup after 24h
      backoffLimit: 1
      activeDeadlineSeconds: 300
      template:
        spec:
          serviceAccountName: stale-job-cleanup
          restartPolicy: Never
          tolerations:
            - key: node-role.kubernetes.io/control-plane
              operator: Exists
              effect: NoSchedule
          containers:
            - name: cleanup
              image: alpine/k8s:1.31.0
              command:
                - sh
                - -c
                - |
                  set -e
                  NOW=$(date +%s)

                  echo "=== Cleaning failed Jobs older than 24h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.conditions[]?.type == "Failed") |
                      select(.status.completionTime or .status.startTime) |
                      "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
                    if [ "$AGE_H" -ge 24 ]; then
                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.succeeded >= 1) |
                      select((.metadata.ownerReferences // []) | length == 0) |
                      "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
                    if [ "$AGE_H" -ge 72 ]; then
                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
                  # Evicted pods show as Failed with reason Evicted
                  kubectl get pods --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(
                        .status.phase == "Failed" or
                        (.status.reason // "") == "Evicted" or
                        (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                      ) |
                      select((.metadata.ownerReferences // []) | all(.kind != "Job")) |
                      "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
                    if [ "$AGE_H" -ge 1 ]; then
                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
                      kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "Cleanup complete"
              resources:
                requests:
                  cpu: 50m
                  memory: 64Mi
                limits:
                  cpu: 250m
                  memory: 128Mi