riotpiao-poimen/homelab
2
0
Fork 0
Code Issues 23 Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

fix: aggressive 2m cleanup for completed jobs/pods, add PAPERLESS_USERNAME to paperless-ai #50

Merged
rock merged 1 commits from fix/aggressive-cleanup-and-paperless-ai into main 2026-09-13 23:00:09 +00:00
Conversation 0 Commits 1 Files Changed 1
+22 -20
1 changed files with 22 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/infra/cluster-maintenance/stale-job-cleanup.yaml
+22 -20
View File
@@ -1,12 +1,12 @@
# Cluster-wide cleanup of stale failed/completed Jobs and Pods. # Cluster-wide cleanup of stale failed/completed Jobs and Pods.
# Runs daily at 04:00 UTC. Deletes: # Runs every 2 minutes. Deletes:
# - Failed Jobs older than 24h (any namespace) # - Failed Jobs older than 2m (any namespace)
# - Completed Jobs older than 72h with no owning CronJob # - Completed standalone Jobs older than 2m with no owning CronJob
# - Orphan pods in Error/Failed/Evicted state older than 1h # - Orphan pods in Error/Failed/Evicted/Completed state older than 2m
# #
# CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit, # CronJob-owned Jobs are managed by failedJobsHistoryLimit/successfulJobsHistoryLimit,
# but standalone Jobs (helm hooks, one-off runs, longhorn maintenance) have no TTL # but standalone Jobs (helm hooks, one-off runs, CI TaskRuns) have no TTL
# and linger forever. # and linger forever. Aggressive schedule keeps the cluster clean.
apiVersion: v1 apiVersion: v1
kind: ServiceAccount kind: ServiceAccount
@@ -47,7 +47,7 @@ metadata:
labels: labels:
app: stale-job-cleanup app: stale-job-cleanup
spec: spec:
schedule: "0 4 * * *" schedule: "*/2 * * * *"
concurrencyPolicy: Forbid concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3 successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3 failedJobsHistoryLimit: 3
@@ -74,7 +74,9 @@ spec:
set -e set -e
NOW=$(date +%s) NOW=$(date +%s)
echo "=== Cleaning failed Jobs older than 24h ===" TTL=120 # 2 minutes in seconds
echo "=== Cleaning failed Jobs older than 2m ==="
kubectl get jobs --all-namespaces -o json | \ kubectl get jobs --all-namespaces -o json | \
jq -r '.items[] | jq -r '.items[] |
select(.status.conditions[]?.type == "Failed") | select(.status.conditions[]?.type == "Failed") |
@@ -82,15 +84,15 @@ spec:
"\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \ "\(.metadata.namespace) \(.metadata.name) \(.status.startTime // .status.completionTime // .metadata.creationTimestamp)"' | \
while read -r NS NAME TS; do while read -r NS NAME TS; do
JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
AGE_H=$(( (NOW - JOB_EPOCH) / 3600 )) AGE=$(( NOW - JOB_EPOCH ))
if [ "$AGE_H" -ge 24 ]; then if [ "$AGE" -ge "$TTL" ]; then
echo "[delete] $NS/$NAME (failed ${AGE_H}h ago)" echo "[delete] $NS/$NAME (failed ${AGE}s ago)"
kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
fi fi
done done
echo "" echo ""
echo "=== Cleaning completed standalone Jobs older than 72h ===" echo "=== Cleaning completed standalone Jobs older than 2m ==="
kubectl get jobs --all-namespaces -o json | \ kubectl get jobs --all-namespaces -o json | \
jq -r '.items[] | jq -r '.items[] |
select(.status.succeeded >= 1) | select(.status.succeeded >= 1) |
@@ -98,20 +100,20 @@ spec:
"\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \ "\(.metadata.namespace) \(.metadata.name) \(.status.completionTime // .metadata.creationTimestamp)"' | \
while read -r NS NAME TS; do while read -r NS NAME TS; do
JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) JOB_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
AGE_H=$(( (NOW - JOB_EPOCH) / 3600 )) AGE=$(( NOW - JOB_EPOCH ))
if [ "$AGE_H" -ge 72 ]; then if [ "$AGE" -ge "$TTL" ]; then
echo "[delete] $NS/$NAME (completed ${AGE_H}h ago, no owner)" echo "[delete] $NS/$NAME (completed ${AGE}s ago, no owner)"
kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true kubectl delete job "$NAME" -n "$NS" --cascade=foreground 2>/dev/null || true
fi fi
done done
echo "" echo ""
echo "=== Cleaning orphan Error/Failed/Evicted pods older than 1h ===" echo "=== Cleaning orphan Error/Failed/Evicted/Completed pods older than 2m ==="
# Evicted pods show as Failed with reason Evicted
kubectl get pods --all-namespaces -o json | \ kubectl get pods --all-namespaces -o json | \
jq -r '.items[] | jq -r '.items[] |
select( select(
.status.phase == "Failed" or .status.phase == "Failed" or
.status.phase == "Succeeded" or
(.status.reason // "") == "Evicted" or (.status.reason // "") == "Evicted" or
(.status.containerStatuses // [] | any(.state.terminated.reason == "Error")) (.status.containerStatuses // [] | any(.state.terminated.reason == "Error"))
) | ) |
@@ -119,9 +121,9 @@ spec:
"\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \ "\(.metadata.namespace) \(.metadata.name) \(.metadata.creationTimestamp)"' | \
while read -r NS NAME TS; do while read -r NS NAME TS; do
POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0) POD_EPOCH=$(date -d "$TS" +%s 2>/dev/null || echo 0)
AGE_H=$(( (NOW - POD_EPOCH) / 3600 )) AGE=$(( NOW - POD_EPOCH ))
if [ "$AGE_H" -ge 1 ]; then if [ "$AGE" -ge "$TTL" ]; then
echo "[delete] $NS/$NAME (error/evicted ${AGE_H}h ago)" echo "[delete] $NS/$NAME (stale ${AGE}s ago)"
kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true kubectl delete pod "$NAME" -n "$NS" --force 2>/dev/null || true
fi fi
done done