# Cluster-wide cleanup of stale failed/completed Jobs and Pods.
# Runs every 2 minutes. Deletes:
#   - Failed Jobs older than 2m (any namespace)
#   - Completed standalone Jobs older than 2m with no owning CronJob
#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
#
# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
# and linger forever. Aggressive schedule keeps the cluster clean.

apiVersion: v1
kind: ServiceAccount
metadata:
  name: stale-job-cleanup
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: stale-job-cleanup
rules:
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["get", "list", "delete"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: stale-job-cleanup
subjects:
  - kind: ServiceAccount
    name: stale-job-cleanup
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: stale-job-cleanup
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: stale-job-cleanup
  namespace: kube-system
  labels:
    app: stale-job-cleanup
spec:
  schedule: "*/2 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      ttlSecondsAfterFinished: 86400  # self-cleanup after 24h
      backoffLimit: 1
      activeDeadlineSeconds: 300
      template:
        spec:
          serviceAccountName: stale-job-cleanup
          restartPolicy: Never
          tolerations:
            - key: node-role.kubernetes.io/control-plane
              operator: Exists
              effect: NoSchedule
          containers:
            - name: cleanup
              image: alpine/k8s:1.31.0
              command:
                - sh
                - -c
                - |
                  set -e
                  NOW=$(date +%s)

                  TTL=120  # 2 minutes in seconds

                  echo "=== Cleaning failed Jobs older than 2m ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.conditions[]?.type == "Failed") |
                      select(.status.completionTime or .status.startTime) |
                      "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE=$(( NOW - JOB_EPOCH ))
                    if [ "$AGE" -ge "$TTL" ]; then
                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
                  kubectl get jobs --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(.status.succeeded >= 1) |
                      select((.metadata.ownerReferences // []) | length == 0) |
                      "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE=$(( NOW - JOB_EPOCH ))
                    if [ "$AGE" -ge "$TTL" ]; then
                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
                  kubectl get pods --all-namespaces -o json | \
                    jq -r '.items[] |
                      select(
                        .status.phase == "Failed" or
                        .status.phase == "Succeeded" or
                        (.status.reason // "") == "Evicted" or
                        (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                      ) |
                      select((.metadata.ownerReferences // []) | all(.kind != "Job")) |
                      "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                  while read -r NS NAME TS; do
                    POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
                    AGE=$(( NOW - POD_EPOCH ))
                    if [ "$AGE" -ge "$TTL" ]; then
                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
                      kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                    fi
                  done

                  echo ""
                  echo "Cleanup complete"
              resources:
                requests:
                  cpu: 50m
                  memory: 64Mi
                limits:
                  cpu: 250m
                  memory: 128Mi