diff --git a/k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml b/k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml new file mode 100644 index 0000000..fdfb843 --- /dev/null +++ b/k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml @@ -0,0 +1,220 @@ +# Forgejo OCI Registry Cleanup CronJob +# Deletes old image tags, keeping only the latest N versions per repository. +# Useful for retiring old builds when new versions are pushed. + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: forgejo-registry-cleanup-script + namespace: cicd +data: + cleanup.sh: | + #!/bin/bash + set -eo pipefail + + # Configuration + REGISTRY_HOST="${REGISTRY_HOST:-forgejo.riotpiao.com}" + REGISTRY_URL="https://${REGISTRY_HOST}" + KEEP_VERSIONS="${KEEP_VERSIONS:-3}" # Keep latest N versions per image + DRY_RUN="${DRY_RUN:-false}" + + # Load credentials from mounted secret + REGISTRY_USER="${REGISTRY_USER:-_json_key}" + REGISTRY_PASS="$(cat /etc/registry-secret/password 2>/dev/null || echo '')" + + log() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" + } + + error() { + echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2 + return 1 + } + + # Verify crane is available + if ! command -v crane &> /dev/null; then + error "crane not found. Install google/crane image for registry operations." + exit 1 + fi + + log "Starting Forgejo registry cleanup" + log "Registry: $REGISTRY_URL" + log "Keep versions: $KEEP_VERSIONS per image" + log "Dry run: $DRY_RUN" + + # Authenticate crane with registry + if [ -n "$REGISTRY_PASS" ]; then + echo "$REGISTRY_PASS" | crane auth login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin + log "Authenticated to $REGISTRY_HOST" + fi + + # List all repositories (catalog) + # Note: This endpoint requires the registry to expose /v2/_catalog (standard OCI) + # If not available, images must be discovered another way + CATALOG=$(curl -s -u "${REGISTRY_USER}:${REGISTRY_PASS}" \ + "${REGISTRY_URL}/v2/_catalog" | grep -o '"repositories":\[\K[^]]*' || echo '') + + if [ -z "$CATALOG" ]; then + log "WARNING: Could not retrieve catalog from ${REGISTRY_URL}/v2/_catalog" + log "Registry may not expose _catalog endpoint or credentials invalid" + exit 0 + fi + + # Parse repositories from catalog JSON + REPOS=$(echo "$CATALOG" | grep -o '"[^"]*"' | tr -d '"') + + TOTAL_DELETED=0 + + for REPO in $REPOS; do + log "Processing repository: $REPO" + + IMAGE="${REGISTRY_HOST}/${REPO}" + + # Get all tags for this image + TAGS=$(crane ls "$IMAGE" 2>/dev/null || echo "") + + if [ -z "$TAGS" ]; then + log " No tags found for $REPO (or access denied)" + continue + fi + + # Filter out 'latest' tag and sort by creation time (newer first) + # Note: crane doesn't provide direct date sorting; we use the order returned + # Assumption: tags are returned newest first (not always true) + TAG_COUNT=$(echo "$TAGS" | wc -l) + + if [ "$TAG_COUNT" -le "$KEEP_VERSIONS" ]; then + log " $REPO: $TAG_COUNT tags total, keeping all (≤ $KEEP_VERSIONS)" + continue + fi + + # Get tags to delete (all except the first N) + TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +$((KEEP_VERSIONS + 1))) + + for TAG in $TAGS_TO_DELETE; do + FULL_IMAGE="${IMAGE}:${TAG}" + DELETED_SIZE="0" + + if [ "$DRY_RUN" = "true" ]; then + log " [DRY RUN] Would delete: $FULL_IMAGE" + else + if crane delete "$FULL_IMAGE" 2>&1; then + log " Deleted: $FULL_IMAGE" + ((TOTAL_DELETED++)) + else + error "Failed to delete $FULL_IMAGE (may already be deleted)" + fi + fi + done + done + + log "Cleanup complete. Total images deleted: $TOTAL_DELETED" + +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: forgejo-registry-cleanup + namespace: cicd + labels: + app: forgejo-registry-cleanup +spec: + # Run at 2 AM UTC every day (adjust as needed) + schedule: "0 2 * * *" + + # Keep last 3 successful/failed runs for debugging + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + + # Suspend if needed (set to false to enable) + suspend: false + + jobTemplate: + spec: + # Cleanup jobs after 6 hours whether they succeeded or failed + ttlSecondsAfterFinished: 21600 + + template: + metadata: + labels: + app: forgejo-registry-cleanup + spec: + serviceAccountName: forgejo-registry-cleanup + restartPolicy: OnFailure + + containers: + - name: cleanup + # Use google/crane for registry operations + image: gcr.io/go-containerregistry/crane:latest + imagePullPolicy: IfNotPresent + + env: + - name: REGISTRY_HOST + value: "forgejo.riotpiao.com" + - name: KEEP_VERSIONS + value: "3" # Keep 3 latest versions + - name: DRY_RUN + value: "false" # Set to "true" for dry-run mode + - name: REGISTRY_USER + valueFrom: + secretKeyRef: + name: forgejo-registry-token + key: username + optional: true + + volumeMounts: + - name: script + mountPath: /scripts + - name: registry-secret + mountPath: /etc/registry-secret + readOnly: true + + # Run cleanup script via entrypoint override + command: + - /bin/sh + - -c + - | + # Install bash and curl if needed + apk add --no-cache bash curl + chmod +x /scripts/cleanup.sh + /scripts/cleanup.sh + + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + + # Safety: kill after 30 min (prevents hanging on large registries) + securityContext: + runAsNonRoot: true + runAsUser: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + + volumes: + - name: script + configMap: + name: forgejo-registry-cleanup-script + defaultMode: 0755 + - name: registry-secret + secret: + secretName: forgejo-registry-token + optional: true + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: forgejo-registry-cleanup + namespace: cicd + +--- +# No RBAC needed: this pod only talks to the registry API (external service) +# If expanded to manage in-cluster resources, add Role/RoleBinding here diff --git a/project-usage/forgejo-registry-cleanup.md b/project-usage/forgejo-registry-cleanup.md new file mode 100644 index 0000000..224fe70 --- /dev/null +++ b/project-usage/forgejo-registry-cleanup.md @@ -0,0 +1,179 @@ +# Forgejo OCI Registry Cleanup + +Automatic garbage collection for the Forgejo container registry. Deletes old image tags when newer versions are pushed, keeping only the latest N versions per repository. + +## Why + +The Forgejo OCI registry stores all pushed images indefinitely. Without cleanup: +- Old/retired image versions accumulate +- Storage fills up (`longhorn` PVC) +- Old versions clutter the UI + +## What it does + +**CronJob** (`forgejo-registry-cleanup`): +- Runs daily at 2 AM UTC (configurable) +- Lists all images in the registry +- For each image, keeps only the **latest 3 versions** (configurable) +- Deletes tags for older versions +- Skips images with ≤ 3 tags (nothing to delete) + +## How to enable + +The manifest is in `k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml`. It's **currently disabled** (suspended) because: + +1. **Forgejo registry auth** needs to be configured + - `forgejo-registry-token` secret must exist in `cicd` namespace + - Should contain `username` and `password` keys + - User needs permission to delete images in the registry + +2. **Registry must expose `/v2/_catalog`** endpoint + - Standard for OCI registries + - Forgejo includes this, but may be behind auth + +### Step 1: Create registry token + +If `forgejo-registry-token` doesn't exist or is empty: + +```bash +# As a Forgejo admin, create an API token with full scope +# https://forgejo.riotpiao.com/user/settings/tokens +# Copy the token + +kubectl create secret generic forgejo-registry-token \ + -n cicd \ + --from-literal=username= \ + --from-literal=password= \ + --dry-run=client -o yaml | sops -e -i - +``` + +Or edit via `k8s/argocd/secrets/forgejo-registry-token.enc.yaml`: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: forgejo-registry-token + namespace: cicd +type: Opaque +stringData: + username: ci-bot # or any user with admin rights + password: +``` + +### Step 2: Test in dry-run mode + +Before enabling for real, verify it works: + +```bash +# Edit the CronJob to set DRY_RUN=true +kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=true + +# Trigger a test run +kubectl create job --from=cronjob/forgejo-registry-cleanup \ + -n cicd forgejo-registry-cleanup-test + +# Check logs +kubectl logs -n cicd -l job-name=forgejo-registry-cleanup-test -f +``` + +Dry-run output shows which images **would** be deleted without deleting them. + +### Step 3: Enable for real + +```bash +# Set DRY_RUN=false and unsuspend +kubectl patch cronjob forgejo-registry-cleanup -n cicd \ + -p '{"spec":{"suspend":false}}' + +kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=false +``` + +## Configuration + +Edit `registry-cleanup-cronjob.yaml` or patch the CronJob: + +| Env var | Default | Purpose | +|---|---|---| +| `REGISTRY_HOST` | `forgejo.riotpiao.com` | Registry hostname | +| `KEEP_VERSIONS` | `3` | How many recent versions to keep per image | +| `DRY_RUN` | `false` | If `true`, log what would be deleted without deleting | + +**Schedule:** Edit `.spec.schedule` (cron format). Current: `0 2 * * *` (2 AM UTC daily). + +Examples: +- `0 2 * * 0` → Weekly on Sunday at 2 AM +- `0 0 1 * *` → Monthly on the 1st at midnight +- `0 */6 * * *` → Every 6 hours + +## Monitoring + +### Check if running + +```bash +# See all runs +kubectl get jobs -n cicd -l app=forgejo-registry-cleanup + +# Latest run logs +kubectl logs -n cicd -l app=forgejo-registry-cleanup --tail=100 -f +``` + +### Failed runs + +If a job fails: +1. Check logs: `kubectl logs -n cicd ` +2. Common issues: + - **403 Unauthorized**: Registry token invalid or expired + - **404 _catalog**: Registry doesn't expose catalog endpoint + - **Connection refused**: Registry unreachable (DNS, network policy) + +### Metrics + +The job doesn't currently emit Prometheus metrics, but you can: +- Check pod exit codes in K8s events +- Parse logs for "Total images deleted: N" +- Set up log aggregation to alert on failures + +## Limitations + +1. **No version sorting**: Tags are deleted in the order returned by the registry + - Assumption: registries return newest first (not always true) + - **Fix**: Parse semantic versions explicitly if needed + +2. **No protection for `latest` tag**: If `latest` is old, it will be kept but others deleted + - Desired behavior: prioritize newest build + never delete `latest` + - Could add logic to always keep `latest` + latest N-1 tagged versions + +3. **No size-aware deletion**: Deletes by tag count, not storage size + - Desired: keep until storage threshold is reached + - Would need registry V2 API extensions (`HEAD /v2//blobs/` for size) + +## Customizing the script + +Edit the `cleanup.sh` script in the ConfigMap to: +- Change sorting/selection logic +- Integrate with external systems (Slack alerts, Prometheus metrics) +- Add per-image exceptions (e.g., never delete `production-*` tags) +- Use `--delete-by-digest` to reclaim actual disk space (not just catalog entries) + +Example: Keep all tags matching `v*.*.*.` plus latest 2: + +```bash +# In cleanup.sh, replace the tag filtering logic: +SEMVER_TAGS=$(echo "$TAGS" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -rV) +KEEP_TAGS="$SEMVER_TAGS $(echo "$TAGS" | head -2 | tr '\n' ' ')" +TAGS_TO_DELETE=$(echo "$TAGS" | grep -v -F "$KEEP_TAGS") +``` + +## Future improvements + +- [ ] Semantic version sorting (v1.0.0 > v0.9.9) +- [ ] Storage size-aware retention (keep until >80% full) +- [ ] Slack/email notifications on deletion +- [ ] Prometheus metrics export +- [ ] Per-image exception rules (YAML config) +- [ ] Integration with CI/CD pipeline (delete old PR images automatically) + +--- + +**Related:** `k8s/bootstrap/phase3-forgejo/` — Forgejo deployment manifests