From 41e894f79ee55b4c4735c026d7a04e890d8ba4ef Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Mon, 14 Sep 2026 07:56:05 +0900
Subject: [PATCH] fix: aggressive 2m cleanup for completed jobs/pods, add
 PAPERLESS_USERNAME to paperless-ai

- stale-job-cleanup: run every 2m, delete completed/failed jobs+pods after 2m
- paperless-ai: add PAPERLESS_USERNAME=admin (required to find own user ID)
---
 .../stale-job-cleanup.yaml                    | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml b/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml
index c09951d..c13e1bc 100644
--- a/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml
+++ b/k8s/infra/cluster-maintenance/stale-job-cleanup.yaml
@@ -1,12 +1,12 @@
 # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
-# Runs daily at 04:00 UTC. Deletes:
-#   - Failed Jobs older than 24h (any namespace)
-#   - Completed Jobs older than 72h with no owning CronJob
-#   - Orphan pods in Error/Failed/Evicted state older than 1h
+# Runs every 2 minutes. Deletes:
+#   - Failed Jobs older than 2m (any namespace)
+#   - Completed standalone Jobs older than 2m with no owning CronJob
+#   - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
 #
 # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
-# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL
-# and linger forever.
+# but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
+# and linger forever. Aggressive schedule keeps the cluster clean.
 
 apiVersion: v1
 kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
   labels:
     app: stale-job-cleanup
 spec:
-  schedule: "0 4 * * *"
+  schedule: "*/2 * * * *"
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 3
   failedJobsHistoryLimit: 3
@@ -74,7 +74,9 @@ spec:
                   set -e
                   NOW=$(date +%s)
 
-                  echo "=== Cleaning failed Jobs older than 24h ==="
+                  TTL=120  # 2 minutes in seconds
+
+                  echo "=== Cleaning failed Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.conditions[]?.type == "Failed") |
@@ -82,15 +84,15 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 24 ]; then
-                      echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)"
+                    AGE=$(( NOW - JOB_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning completed standalone Jobs older than 72h ==="
+                  echo "=== Cleaning completed standalone Jobs older than 2m ==="
                   kubectl get jobs --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(.status.succeeded >= 1) |
@@ -98,20 +100,20 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - JOB_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 72 ]; then
-                      echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)"
+                    AGE=$(( NOW - JOB_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
                       kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
                     fi
                   done
 
                   echo ""
-                  echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ==="
-                  # Evicted pods show as Failed with reason Evicted
+                  echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
                   kubectl get pods --all-namespaces -o json | \
                     jq -r '.items[] |
                       select(
                         .status.phase == "Failed" or
+                        .status.phase == "Succeeded" or
                         (.status.reason // "") == "Evicted" or
                         (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
                       ) |
@@ -119,9 +121,9 @@ spec:
                       "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
                   while read -r NS NAME TS; do
                     POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
-                    AGE_H=$(( (NOW - POD_EPOCH) / 3600 ))
-                    if [ "$AGE_H" -ge 1 ]; then
-                      echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)"
+                    AGE=$(( NOW - POD_EPOCH ))
+                    if [ "$AGE" -ge "$TTL" ]; then
+                      echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
                       kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
                     fi
                   done