feat(ops): add cluster-wide stale job/pod cleanup CronJob

Daily 04:00 UTC sweeper in kube-system:
- Delete failed Jobs older than 24h (any namespace)
- Delete completed standalone Jobs older than 72h (no CronJob owner)
- Delete orphan Error/Evicted pods older than 1h
- Self-cleans via ttlSecondsAfterFinished

k8s/argocd/apps/00-substrate.yaml

@@ -138,3 +138,24 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: cluster-maintenance
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/rock/homelab.git
+    targetRevision: main
+    path: k8s/infra/cluster-maintenance
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/infra/cluster-maintenance/stale-job-cleanup.yaml

@@ -0,0 +1,137 @@
+# Cluster-wide cleanup of stale failed/completed Jobs and Pods.
+# Runs daily at 04:00 UTC. Deletes:
+#   - Failed Jobs older than 24h (any namespace)
+#   - Completed Jobs older than 72h with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted state older than 1h
+#
+# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
+# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
+# and linger forever.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stale-job-cleanup
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: stale-job-cleanup
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: stale-job-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: stale-job-cleanup
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: stale-job-cleanup
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: stale-job-cleanup
+  namespace: kube-system
+  labels:
+    app: stale-job-cleanup
+spec:
+  schedule: "0 4 * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 86400  # self-cleanup after 24h
+      backoffLimit: 1
+      activeDeadlineSeconds: 300
+      template:
+        spec:
+          serviceAccountName: stale-job-cleanup
+          restartPolicy: Never
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: cleanup
+              image: alpine/k8s:1.31.0
+              command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  NOW=$(date +%s)
+
+                  echo "=== Cleaning failed Jobs older than 24h ==="
+                  kubectl get jobs --all-namespaces -o json | \
+                    jq -r '.items[] |
+                      select(.status.conditions[]?.type == "Failed") |
+                      select(.status.completionTime or .status.startTime) |
+                      "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
+                  while read -r NS NAME TS; do
+                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 24 ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
+                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
+                    fi
+                  done
+
+                  echo ""
+                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
+                  kubectl get jobs --all-namespaces -o json | \
+                    jq -r '.items[] |
+                      select(.status.succeeded >= 1) |
+                      select((.metadata.ownerReferences // []) | length == 0) |
+                      "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
+                  while read -r NS NAME TS; do
+                    JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
+                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 72 ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
+                      kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
+                    fi
+                  done
+
+                  echo ""
+                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
+                  # Evicted pods show as Failed with reason Evicted
+                  kubectl get pods --all-namespaces -o json | \
+                    jq -r '.items[] |
+                      select(
+                        .status.phase == "Failed" or
+                        (.status.reason // "") == "Evicted" or
+                        (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
+                      ) |
+                      select((.metadata.ownerReferences // []) | all(.kind != "Job")) |
+                      "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
+                  while read -r NS NAME TS; do
+                    POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
+                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
+                    if [ "$AGE_H" -ge 1 ]; then
+                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
+                      kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
+                    fi
+                  done
+
+                  echo ""
+                  echo "Cleanup complete"
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 250m
+                  memory: 128Mi