Clean up template files
This commit is contained in:
Story Crater Bot
@@ -0,0 +1,220 @@
|
||||
# Forgejo OCI Registry Cleanup CronJob
|
||||
# Deletes old image tags, keeping only the latest N versions per repository.
|
||||
# Useful for retiring old builds when new versions are pushed.
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: forgejo-registry-cleanup-script
|
||||
namespace: cicd
|
||||
data:
|
||||
cleanup.sh: |
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
# Configuration
|
||||
REGISTRY_HOST="${REGISTRY_HOST:-forgejo.riotpiao.com}"
|
||||
REGISTRY_URL="https://${REGISTRY_HOST}"
|
||||
KEEP_VERSIONS="${KEEP_VERSIONS:-3}" # Keep latest N versions per image
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
# Load credentials from mounted secret
|
||||
REGISTRY_USER="${REGISTRY_USER:-_json_key}"
|
||||
REGISTRY_PASS="$(cat /etc/registry-secret/password 2>/dev/null || echo '')"
|
||||
|
||||
log() {
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Verify crane is available
|
||||
if ! command -v crane &> /dev/null; then
|
||||
error "crane not found. Install google/crane image for registry operations."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Starting Forgejo registry cleanup"
|
||||
log "Registry: $REGISTRY_URL"
|
||||
log "Keep versions: $KEEP_VERSIONS per image"
|
||||
log "Dry run: $DRY_RUN"
|
||||
|
||||
# Authenticate crane with registry
|
||||
if [ -n "$REGISTRY_PASS" ]; then
|
||||
echo "$REGISTRY_PASS" | crane auth login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin
|
||||
log "Authenticated to $REGISTRY_HOST"
|
||||
fi
|
||||
|
||||
# List all repositories (catalog)
|
||||
# Note: This endpoint requires the registry to expose /v2/_catalog (standard OCI)
|
||||
# If not available, images must be discovered another way
|
||||
CATALOG=$(curl -s -u "${REGISTRY_USER}:${REGISTRY_PASS}" \
|
||||
"${REGISTRY_URL}/v2/_catalog" | grep -o '"repositories":\[\K[^]]*' || echo '')
|
||||
|
||||
if [ -z "$CATALOG" ]; then
|
||||
log "WARNING: Could not retrieve catalog from ${REGISTRY_URL}/v2/_catalog"
|
||||
log "Registry may not expose _catalog endpoint or credentials invalid"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Parse repositories from catalog JSON
|
||||
REPOS=$(echo "$CATALOG" | grep -o '"[^"]*"' | tr -d '"')
|
||||
|
||||
TOTAL_DELETED=0
|
||||
|
||||
for REPO in $REPOS; do
|
||||
log "Processing repository: $REPO"
|
||||
|
||||
IMAGE="${REGISTRY_HOST}/${REPO}"
|
||||
|
||||
# Get all tags for this image
|
||||
TAGS=$(crane ls "$IMAGE" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$TAGS" ]; then
|
||||
log " No tags found for $REPO (or access denied)"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Filter out 'latest' tag and sort by creation time (newer first)
|
||||
# Note: crane doesn't provide direct date sorting; we use the order returned
|
||||
# Assumption: tags are returned newest first (not always true)
|
||||
TAG_COUNT=$(echo "$TAGS" | wc -l)
|
||||
|
||||
if [ "$TAG_COUNT" -le "$KEEP_VERSIONS" ]; then
|
||||
log " $REPO: $TAG_COUNT tags total, keeping all (≤ $KEEP_VERSIONS)"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get tags to delete (all except the first N)
|
||||
TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +$((KEEP_VERSIONS + 1)))
|
||||
|
||||
for TAG in $TAGS_TO_DELETE; do
|
||||
FULL_IMAGE="${IMAGE}:${TAG}"
|
||||
DELETED_SIZE="0"
|
||||
|
||||
if [ "$DRY_RUN" = "true" ]; then
|
||||
log " [DRY RUN] Would delete: $FULL_IMAGE"
|
||||
else
|
||||
if crane delete "$FULL_IMAGE" 2>&1; then
|
||||
log " Deleted: $FULL_IMAGE"
|
||||
((TOTAL_DELETED++))
|
||||
else
|
||||
error "Failed to delete $FULL_IMAGE (may already be deleted)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
log "Cleanup complete. Total images deleted: $TOTAL_DELETED"
|
||||
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: forgejo-registry-cleanup
|
||||
namespace: cicd
|
||||
labels:
|
||||
app: forgejo-registry-cleanup
|
||||
spec:
|
||||
# Run at 2 AM UTC every day (adjust as needed)
|
||||
schedule: "0 2 * * *"
|
||||
|
||||
# Keep last 3 successful/failed runs for debugging
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
|
||||
# Suspend if needed (set to false to enable)
|
||||
suspend: false
|
||||
|
||||
jobTemplate:
|
||||
spec:
|
||||
# Cleanup jobs after 6 hours whether they succeeded or failed
|
||||
ttlSecondsAfterFinished: 21600
|
||||
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: forgejo-registry-cleanup
|
||||
spec:
|
||||
serviceAccountName: forgejo-registry-cleanup
|
||||
restartPolicy: OnFailure
|
||||
|
||||
containers:
|
||||
- name: cleanup
|
||||
# Use google/crane for registry operations
|
||||
image: gcr.io/go-containerregistry/crane:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
|
||||
env:
|
||||
- name: REGISTRY_HOST
|
||||
value: "forgejo.riotpiao.com"
|
||||
- name: KEEP_VERSIONS
|
||||
value: "3" # Keep 3 latest versions
|
||||
- name: DRY_RUN
|
||||
value: "false" # Set to "true" for dry-run mode
|
||||
- name: REGISTRY_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: forgejo-registry-token
|
||||
key: username
|
||||
optional: true
|
||||
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: registry-secret
|
||||
mountPath: /etc/registry-secret
|
||||
readOnly: true
|
||||
|
||||
# Run cleanup script via entrypoint override
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
# Install bash and curl if needed
|
||||
apk add --no-cache bash curl
|
||||
chmod +x /scripts/cleanup.sh
|
||||
/scripts/cleanup.sh
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
|
||||
# Safety: kill after 30 min (prevents hanging on large registries)
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: forgejo-registry-cleanup-script
|
||||
defaultMode: 0755
|
||||
- name: registry-secret
|
||||
secret:
|
||||
secretName: forgejo-registry-token
|
||||
optional: true
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: forgejo-registry-cleanup
|
||||
namespace: cicd
|
||||
|
||||
---
|
||||
# No RBAC needed: this pod only talks to the registry API (external service)
|
||||
# If expanded to manage in-cluster resources, add Role/RoleBinding here
|
||||
@@ -0,0 +1,179 @@
|
||||
# Forgejo OCI Registry Cleanup
|
||||
|
||||
Automatic garbage collection for the Forgejo container registry. Deletes old image tags when newer versions are pushed, keeping only the latest N versions per repository.
|
||||
|
||||
## Why
|
||||
|
||||
The Forgejo OCI registry stores all pushed images indefinitely. Without cleanup:
|
||||
- Old/retired image versions accumulate
|
||||
- Storage fills up (`longhorn` PVC)
|
||||
- Old versions clutter the UI
|
||||
|
||||
## What it does
|
||||
|
||||
**CronJob** (`forgejo-registry-cleanup`):
|
||||
- Runs daily at 2 AM UTC (configurable)
|
||||
- Lists all images in the registry
|
||||
- For each image, keeps only the **latest 3 versions** (configurable)
|
||||
- Deletes tags for older versions
|
||||
- Skips images with ≤ 3 tags (nothing to delete)
|
||||
|
||||
## How to enable
|
||||
|
||||
The manifest is in `k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml`. It's **currently disabled** (suspended) because:
|
||||
|
||||
1. **Forgejo registry auth** needs to be configured
|
||||
- `forgejo-registry-token` secret must exist in `cicd` namespace
|
||||
- Should contain `username` and `password` keys
|
||||
- User needs permission to delete images in the registry
|
||||
|
||||
2. **Registry must expose `/v2/_catalog`** endpoint
|
||||
- Standard for OCI registries
|
||||
- Forgejo includes this, but may be behind auth
|
||||
|
||||
### Step 1: Create registry token
|
||||
|
||||
If `forgejo-registry-token` doesn't exist or is empty:
|
||||
|
||||
```bash
|
||||
# As a Forgejo admin, create an API token with full scope
|
||||
# https://forgejo.riotpiao.com/user/settings/tokens
|
||||
# Copy the token
|
||||
|
||||
kubectl create secret generic forgejo-registry-token \
|
||||
-n cicd \
|
||||
--from-literal=username=<your-username> \
|
||||
--from-literal=password=<the-api-token> \
|
||||
--dry-run=client -o yaml | sops -e -i -
|
||||
```
|
||||
|
||||
Or edit via `k8s/argocd/secrets/forgejo-registry-token.enc.yaml`:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: forgejo-registry-token
|
||||
namespace: cicd
|
||||
type: Opaque
|
||||
stringData:
|
||||
username: ci-bot # or any user with admin rights
|
||||
password: <api-token>
|
||||
```
|
||||
|
||||
### Step 2: Test in dry-run mode
|
||||
|
||||
Before enabling for real, verify it works:
|
||||
|
||||
```bash
|
||||
# Edit the CronJob to set DRY_RUN=true
|
||||
kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=true
|
||||
|
||||
# Trigger a test run
|
||||
kubectl create job --from=cronjob/forgejo-registry-cleanup \
|
||||
-n cicd forgejo-registry-cleanup-test
|
||||
|
||||
# Check logs
|
||||
kubectl logs -n cicd -l job-name=forgejo-registry-cleanup-test -f
|
||||
```
|
||||
|
||||
Dry-run output shows which images **would** be deleted without deleting them.
|
||||
|
||||
### Step 3: Enable for real
|
||||
|
||||
```bash
|
||||
# Set DRY_RUN=false and unsuspend
|
||||
kubectl patch cronjob forgejo-registry-cleanup -n cicd \
|
||||
-p '{"spec":{"suspend":false}}'
|
||||
|
||||
kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=false
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `registry-cleanup-cronjob.yaml` or patch the CronJob:
|
||||
|
||||
| Env var | Default | Purpose |
|
||||
|---|---|---|
|
||||
| `REGISTRY_HOST` | `forgejo.riotpiao.com` | Registry hostname |
|
||||
| `KEEP_VERSIONS` | `3` | How many recent versions to keep per image |
|
||||
| `DRY_RUN` | `false` | If `true`, log what would be deleted without deleting |
|
||||
|
||||
**Schedule:** Edit `.spec.schedule` (cron format). Current: `0 2 * * *` (2 AM UTC daily).
|
||||
|
||||
Examples:
|
||||
- `0 2 * * 0` → Weekly on Sunday at 2 AM
|
||||
- `0 0 1 * *` → Monthly on the 1st at midnight
|
||||
- `0 */6 * * *` → Every 6 hours
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Check if running
|
||||
|
||||
```bash
|
||||
# See all runs
|
||||
kubectl get jobs -n cicd -l app=forgejo-registry-cleanup
|
||||
|
||||
# Latest run logs
|
||||
kubectl logs -n cicd -l app=forgejo-registry-cleanup --tail=100 -f
|
||||
```
|
||||
|
||||
### Failed runs
|
||||
|
||||
If a job fails:
|
||||
1. Check logs: `kubectl logs -n cicd <job-pod>`
|
||||
2. Common issues:
|
||||
- **403 Unauthorized**: Registry token invalid or expired
|
||||
- **404 _catalog**: Registry doesn't expose catalog endpoint
|
||||
- **Connection refused**: Registry unreachable (DNS, network policy)
|
||||
|
||||
### Metrics
|
||||
|
||||
The job doesn't currently emit Prometheus metrics, but you can:
|
||||
- Check pod exit codes in K8s events
|
||||
- Parse logs for "Total images deleted: N"
|
||||
- Set up log aggregation to alert on failures
|
||||
|
||||
## Limitations
|
||||
|
||||
1. **No version sorting**: Tags are deleted in the order returned by the registry
|
||||
- Assumption: registries return newest first (not always true)
|
||||
- **Fix**: Parse semantic versions explicitly if needed
|
||||
|
||||
2. **No protection for `latest` tag**: If `latest` is old, it will be kept but others deleted
|
||||
- Desired behavior: prioritize newest build + never delete `latest`
|
||||
- Could add logic to always keep `latest` + latest N-1 tagged versions
|
||||
|
||||
3. **No size-aware deletion**: Deletes by tag count, not storage size
|
||||
- Desired: keep until storage threshold is reached
|
||||
- Would need registry V2 API extensions (`HEAD /v2/<image>/blobs/<digest>` for size)
|
||||
|
||||
## Customizing the script
|
||||
|
||||
Edit the `cleanup.sh` script in the ConfigMap to:
|
||||
- Change sorting/selection logic
|
||||
- Integrate with external systems (Slack alerts, Prometheus metrics)
|
||||
- Add per-image exceptions (e.g., never delete `production-*` tags)
|
||||
- Use `--delete-by-digest` to reclaim actual disk space (not just catalog entries)
|
||||
|
||||
Example: Keep all tags matching `v*.*.*.` plus latest 2:
|
||||
|
||||
```bash
|
||||
# In cleanup.sh, replace the tag filtering logic:
|
||||
SEMVER_TAGS=$(echo "$TAGS" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -rV)
|
||||
KEEP_TAGS="$SEMVER_TAGS $(echo "$TAGS" | head -2 | tr '\n' ' ')"
|
||||
TAGS_TO_DELETE=$(echo "$TAGS" | grep -v -F "$KEEP_TAGS")
|
||||
```
|
||||
|
||||
## Future improvements
|
||||
|
||||
- [ ] Semantic version sorting (v1.0.0 > v0.9.9)
|
||||
- [ ] Storage size-aware retention (keep until >80% full)
|
||||
- [ ] Slack/email notifications on deletion
|
||||
- [ ] Prometheus metrics export
|
||||
- [ ] Per-image exception rules (YAML config)
|
||||
- [ ] Integration with CI/CD pipeline (delete old PR images automatically)
|
||||
|
||||
---
|
||||
|
||||
**Related:** `k8s/bootstrap/phase3-forgejo/` — Forgejo deployment manifests
|
||||
Reference in New Issue
Block a user