Clean up template files

k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml

@@ -0,0 +1,220 @@
+# Forgejo OCI Registry Cleanup CronJob
+# Deletes old image tags, keeping only the latest N versions per repository.
+# Useful for retiring old builds when new versions are pushed.
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: forgejo-registry-cleanup-script
+  namespace: cicd
+data:
+  cleanup.sh: |
+    #!/bin/bash
+    set -eo pipefail
+    
+    # Configuration
+    REGISTRY_HOST="${REGISTRY_HOST:-forgejo.riotpiao.com}"
+    REGISTRY_URL="https://${REGISTRY_HOST}"
+    KEEP_VERSIONS="${KEEP_VERSIONS:-3}"  # Keep latest N versions per image
+    DRY_RUN="${DRY_RUN:-false}"
+    
+    # Load credentials from mounted secret
+    REGISTRY_USER="${REGISTRY_USER:-_json_key}"
+    REGISTRY_PASS="$(cat /etc/registry-secret/password 2>/dev/null || echo '')"
+    
+    log() {
+      echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*"
+    }
+    
+    error() {
+      echo "[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
+      return 1
+    }
+    
+    # Verify crane is available
+    if ! command -v crane &> /dev/null; then
+      error "crane not found. Install google/crane image for registry operations."
+      exit 1
+    fi
+    
+    log "Starting Forgejo registry cleanup"
+    log "Registry: $REGISTRY_URL"
+    log "Keep versions: $KEEP_VERSIONS per image"
+    log "Dry run: $DRY_RUN"
+    
+    # Authenticate crane with registry
+    if [ -n "$REGISTRY_PASS" ]; then
+      echo "$REGISTRY_PASS" | crane auth login "$REGISTRY_HOST" -u "$REGISTRY_USER" --password-stdin
+      log "Authenticated to $REGISTRY_HOST"
+    fi
+    
+    # List all repositories (catalog)
+    # Note: This endpoint requires the registry to expose /v2/_catalog (standard OCI)
+    # If not available, images must be discovered another way
+    CATALOG=$(curl -s -u "${REGISTRY_USER}:${REGISTRY_PASS}" \
+      "${REGISTRY_URL}/v2/_catalog" | grep -o '"repositories":\[\K[^]]*' || echo '')
+    
+    if [ -z "$CATALOG" ]; then
+      log "WARNING: Could not retrieve catalog from ${REGISTRY_URL}/v2/_catalog"
+      log "Registry may not expose _catalog endpoint or credentials invalid"
+      exit 0
+    fi
+    
+    # Parse repositories from catalog JSON
+    REPOS=$(echo "$CATALOG" | grep -o '"[^"]*"' | tr -d '"')
+    
+    TOTAL_DELETED=0
+    
+    for REPO in $REPOS; do
+      log "Processing repository: $REPO"
+      
+      IMAGE="${REGISTRY_HOST}/${REPO}"
+      
+      # Get all tags for this image
+      TAGS=$(crane ls "$IMAGE" 2>/dev/null || echo "")
+      
+      if [ -z "$TAGS" ]; then
+        log "  No tags found for $REPO (or access denied)"
+        continue
+      fi
+      
+      # Filter out 'latest' tag and sort by creation time (newer first)
+      # Note: crane doesn't provide direct date sorting; we use the order returned
+      # Assumption: tags are returned newest first (not always true)
+      TAG_COUNT=$(echo "$TAGS" | wc -l)
+      
+      if [ "$TAG_COUNT" -le "$KEEP_VERSIONS" ]; then
+        log "  $REPO: $TAG_COUNT tags total, keeping all (≤ $KEEP_VERSIONS)"
+        continue
+      fi
+      
+      # Get tags to delete (all except the first N)
+      TAGS_TO_DELETE=$(echo "$TAGS" | tail -n +$((KEEP_VERSIONS + 1)))
+      
+      for TAG in $TAGS_TO_DELETE; do
+        FULL_IMAGE="${IMAGE}:${TAG}"
+        DELETED_SIZE="0"
+        
+        if [ "$DRY_RUN" = "true" ]; then
+          log "  [DRY RUN] Would delete: $FULL_IMAGE"
+        else
+          if crane delete "$FULL_IMAGE" 2>&1; then
+            log "  Deleted: $FULL_IMAGE"
+            ((TOTAL_DELETED++))
+          else
+            error "Failed to delete $FULL_IMAGE (may already be deleted)"
+          fi
+        fi
+      done
+    done
+    
+    log "Cleanup complete. Total images deleted: $TOTAL_DELETED"
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: forgejo-registry-cleanup
+  namespace: cicd
+  labels:
+    app: forgejo-registry-cleanup
+spec:
+  # Run at 2 AM UTC every day (adjust as needed)
+  schedule: "0 2 * * *"
+  
+  # Keep last 3 successful/failed runs for debugging
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  
+  # Suspend if needed (set to false to enable)
+  suspend: false
+  
+  jobTemplate:
+    spec:
+      # Cleanup jobs after 6 hours whether they succeeded or failed
+      ttlSecondsAfterFinished: 21600
+      
+      template:
+        metadata:
+          labels:
+            app: forgejo-registry-cleanup
+        spec:
+          serviceAccountName: forgejo-registry-cleanup
+          restartPolicy: OnFailure
+          
+          containers:
+          - name: cleanup
+            # Use google/crane for registry operations
+            image: gcr.io/go-containerregistry/crane:latest
+            imagePullPolicy: IfNotPresent
+            
+            env:
+            - name: REGISTRY_HOST
+              value: "forgejo.riotpiao.com"
+            - name: KEEP_VERSIONS
+              value: "3"  # Keep 3 latest versions
+            - name: DRY_RUN
+              value: "false"  # Set to "true" for dry-run mode
+            - name: REGISTRY_USER
+              valueFrom:
+                secretKeyRef:
+                  name: forgejo-registry-token
+                  key: username
+                  optional: true
+            
+            volumeMounts:
+            - name: script
+              mountPath: /scripts
+            - name: registry-secret
+              mountPath: /etc/registry-secret
+              readOnly: true
+            
+            # Run cleanup script via entrypoint override
+            command:
+            - /bin/sh
+            - -c
+            - |
+              # Install bash and curl if needed
+              apk add --no-cache bash curl
+              chmod +x /scripts/cleanup.sh
+              /scripts/cleanup.sh
+            
+            resources:
+              requests:
+                cpu: 100m
+                memory: 128Mi
+              limits:
+                cpu: 500m
+                memory: 512Mi
+            
+            # Safety: kill after 30 min (prevents hanging on large registries)
+            securityContext:
+              runAsNonRoot: true
+              runAsUser: 65534
+              allowPrivilegeEscalation: false
+              readOnlyRootFilesystem: false
+              capabilities:
+                drop:
+                - ALL
+          
+          volumes:
+          - name: script
+            configMap:
+              name: forgejo-registry-cleanup-script
+              defaultMode: 0755
+          - name: registry-secret
+            secret:
+              secretName: forgejo-registry-token
+              optional: true
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: forgejo-registry-cleanup
+  namespace: cicd
+
+---
+# No RBAC needed: this pod only talks to the registry API (external service)
+# If expanded to manage in-cluster resources, add Role/RoleBinding here

project-usage/forgejo-registry-cleanup.md

@@ -0,0 +1,179 @@
+# Forgejo OCI Registry Cleanup
+
+Automatic garbage collection for the Forgejo container registry. Deletes old image tags when newer versions are pushed, keeping only the latest N versions per repository.
+
+## Why
+
+The Forgejo OCI registry stores all pushed images indefinitely. Without cleanup:
+- Old/retired image versions accumulate
+- Storage fills up (`longhorn` PVC)
+- Old versions clutter the UI
+
+## What it does
+
+**CronJob** (`forgejo-registry-cleanup`):
+- Runs daily at 2 AM UTC (configurable)
+- Lists all images in the registry
+- For each image, keeps only the **latest 3 versions** (configurable)
+- Deletes tags for older versions
+- Skips images with ≤ 3 tags (nothing to delete)
+
+## How to enable
+
+The manifest is in `k8s/bootstrap/phase3-forgejo/registry-cleanup-cronjob.yaml`. It's **currently disabled** (suspended) because:
+
+1. **Forgejo registry auth** needs to be configured
+   - `forgejo-registry-token` secret must exist in `cicd` namespace
+   - Should contain `username` and `password` keys
+   - User needs permission to delete images in the registry
+
+2. **Registry must expose `/v2/_catalog`** endpoint
+   - Standard for OCI registries
+   - Forgejo includes this, but may be behind auth
+
+### Step 1: Create registry token
+
+If `forgejo-registry-token` doesn't exist or is empty:
+
+```bash
+# As a Forgejo admin, create an API token with full scope
+# https://forgejo.riotpiao.com/user/settings/tokens
+# Copy the token
+
+kubectl create secret generic forgejo-registry-token \
+  -n cicd \
+  --from-literal=username=<your-username> \
+  --from-literal=password=<the-api-token> \
+  --dry-run=client -o yaml | sops -e -i -
+```
+
+Or edit via `k8s/argocd/secrets/forgejo-registry-token.enc.yaml`:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: forgejo-registry-token
+  namespace: cicd
+type: Opaque
+stringData:
+  username: ci-bot         # or any user with admin rights
+  password: <api-token>
+```
+
+### Step 2: Test in dry-run mode
+
+Before enabling for real, verify it works:
+
+```bash
+# Edit the CronJob to set DRY_RUN=true
+kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=true
+
+# Trigger a test run
+kubectl create job --from=cronjob/forgejo-registry-cleanup \
+  -n cicd forgejo-registry-cleanup-test
+
+# Check logs
+kubectl logs -n cicd -l job-name=forgejo-registry-cleanup-test -f
+```
+
+Dry-run output shows which images **would** be deleted without deleting them.
+
+### Step 3: Enable for real
+
+```bash
+# Set DRY_RUN=false and unsuspend
+kubectl patch cronjob forgejo-registry-cleanup -n cicd \
+  -p '{"spec":{"suspend":false}}'
+
+kubectl set env cronjob/forgejo-registry-cleanup -n cicd DRY_RUN=false
+```
+
+## Configuration
+
+Edit `registry-cleanup-cronjob.yaml` or patch the CronJob:
+
+| Env var | Default | Purpose |
+|---|---|---|
+| `REGISTRY_HOST` | `forgejo.riotpiao.com` | Registry hostname |
+| `KEEP_VERSIONS` | `3` | How many recent versions to keep per image |
+| `DRY_RUN` | `false` | If `true`, log what would be deleted without deleting |
+
+**Schedule:** Edit `.spec.schedule` (cron format). Current: `0 2 * * *` (2 AM UTC daily).
+
+Examples:
+- `0 2 * * 0` → Weekly on Sunday at 2 AM
+- `0 0 1 * *` → Monthly on the 1st at midnight
+- `0 */6 * * *` → Every 6 hours
+
+## Monitoring
+
+### Check if running
+
+```bash
+# See all runs
+kubectl get jobs -n cicd -l app=forgejo-registry-cleanup
+
+# Latest run logs
+kubectl logs -n cicd -l app=forgejo-registry-cleanup --tail=100 -f
+```
+
+### Failed runs
+
+If a job fails:
+1. Check logs: `kubectl logs -n cicd <job-pod>`
+2. Common issues:
+   - **403 Unauthorized**: Registry token invalid or expired
+   - **404 _catalog**: Registry doesn't expose catalog endpoint
+   - **Connection refused**: Registry unreachable (DNS, network policy)
+
+### Metrics
+
+The job doesn't currently emit Prometheus metrics, but you can:
+- Check pod exit codes in K8s events
+- Parse logs for "Total images deleted: N"
+- Set up log aggregation to alert on failures
+
+## Limitations
+
+1. **No version sorting**: Tags are deleted in the order returned by the registry
+   - Assumption: registries return newest first (not always true)
+   - **Fix**: Parse semantic versions explicitly if needed
+
+2. **No protection for `latest` tag**: If `latest` is old, it will be kept but others deleted
+   - Desired behavior: prioritize newest build + never delete `latest`
+   - Could add logic to always keep `latest` + latest N-1 tagged versions
+
+3. **No size-aware deletion**: Deletes by tag count, not storage size
+   - Desired: keep until storage threshold is reached
+   - Would need registry V2 API extensions (`HEAD /v2/<image>/blobs/<digest>` for size)
+
+## Customizing the script
+
+Edit the `cleanup.sh` script in the ConfigMap to:
+- Change sorting/selection logic
+- Integrate with external systems (Slack alerts, Prometheus metrics)
+- Add per-image exceptions (e.g., never delete `production-*` tags)
+- Use `--delete-by-digest` to reclaim actual disk space (not just catalog entries)
+
+Example: Keep all tags matching `v*.*.*.` plus latest 2:
+
+```bash
+# In cleanup.sh, replace the tag filtering logic:
+SEMVER_TAGS=$(echo "$TAGS" | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -rV)
+KEEP_TAGS="$SEMVER_TAGS $(echo "$TAGS" | head -2 | tr '\n' ' ')"
+TAGS_TO_DELETE=$(echo "$TAGS" | grep -v -F "$KEEP_TAGS")
+```
+
+## Future improvements
+
+- [ ] Semantic version sorting (v1.0.0 > v0.9.9)
+- [ ] Storage size-aware retention (keep until >80% full)
+- [ ] Slack/email notifications on deletion
+- [ ] Prometheus metrics export
+- [ ] Per-image exception rules (YAML config)
+- [ ] Integration with CI/CD pipeline (delete old PR images automatically)
+
+---
+
+**Related:** `k8s/bootstrap/phase3-forgejo/` — Forgejo deployment manifests