ci: temporarily revert runners to public images to bootstrap custom image build

ci: improve runner image build workflow - always run + manual trigger + weekly schedule
ci: update runner daemon images to custom runner images (enables Image Updater tracking)
2026-09-07 09:17:04 -07:00 · 2026-09-07 09:15:44 -07:00 · 2026-09-07 09:11:45 -07:00 · 2026-09-07 09:09:59 -07:00 · 2026-09-07 09:08:10 -07:00 · 2026-09-06 23:19:27 -07:00
13 changed files with 222 additions and 99 deletions
@@ -1,27 +1,25 @@
-name: Build and push runner images
+name: Build & Push Runner Images

 on:
  push:
-    paths:
-      - 'k8s/infra/forgejo-runner/Dockerfile.golang'
-      - 'k8s/infra/forgejo-runner/Dockerfile.rust'
-      - 'k8s/infra/forgejo-runner/Dockerfile.node'
-    branches:
-      - main
-  pull_request:
-    paths:
-      - 'k8s/infra/forgejo-runner/Dockerfile.golang'
-      - 'k8s/infra/forgejo-runner/Dockerfile.rust'
-      - 'k8s/infra/forgejo-runner/Dockerfile.node'
-      - '.gitea/workflows/build-runner-images.yml'
+    branches: [main]
+  workflow_dispatch:  # Manual trigger
+  schedule:
+    - cron: '0 2 * * 0'  # Weekly on Sunday 2 AM UTC (ensure fresh images)

-jobs:
-  build-runners:
-    runs-on: golang
 env:
  REGISTRY: forgejo.riotpiao.com
-      IMAGE_BASE: forgejo.riotpiao.com/rock
+
+jobs:
+  build-runner-images:
+    name: Build & Push All Runner Images
+    runs-on: golang
    steps:
+      - name: Install Node.js and Docker
+        run: |
+          apt-get update
+          apt-get install -y nodejs docker.io
+
      - name: Checkout code
        uses: actions/checkout@v4

@@ -31,30 +29,65 @@ jobs:
          SHORT_SHA=$(git rev-parse --short HEAD)
          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT

-      - name: Build all runner images (test on PR, push on main)
-        run: |
-          set -e
-          for RUNNER in golang rust node; do
-            echo "📦 Building ${RUNNER}-runner..."
-            docker build -f "k8s/infra/forgejo-runner/Dockerfile.${RUNNER}" \
-              -t "${IMAGE_BASE}/forgejo-runner-${RUNNER}:${{ steps.sha.outputs.short_sha }}" \
-              -t "${IMAGE_BASE}/forgejo-runner-${RUNNER}:latest" \
-              .
-            echo "✅ Built ${RUNNER}-runner"
-          done
-
-      - name: Push images (main only)
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+      - name: Registry login
        run: |
          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
            --username "${REGISTRY_USER}" --password-stdin
-          for RUNNER in golang rust node; do
-            echo "📤 Pushing ${RUNNER}-runner:${{ steps.sha.outputs.short_sha }}"
-            docker push "${IMAGE_BASE}/forgejo-runner-${RUNNER}:${{ steps.sha.outputs.short_sha }}"
-            docker push "${IMAGE_BASE}/forgejo-runner-${RUNNER}:latest"
-            echo "✅ Pushed ${RUNNER}-runner"
-          done
-          echo "\n✅ All runner images pushed to registry"
        env:
          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build and push Golang runner image
+        run: |
+          IMAGE="${REGISTRY}/rock/forgejo-runner-golang"
+          echo "Building: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          
+          docker build --no-cache \
+            -f k8s/infra/forgejo-runner-images/Dockerfile.golang \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+          
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+          echo "✓ Pushed golang runner"
+
+      - name: Build and push Node runner image
+        run: |
+          IMAGE="${REGISTRY}/rock/forgejo-runner-node"
+          echo "Building: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          
+          docker build --no-cache \
+            -f k8s/infra/forgejo-runner-images/Dockerfile.node \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+          
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+          echo "✓ Pushed node runner"
+
+      - name: Build and push Rust runner image
+        run: |
+          IMAGE="${REGISTRY}/rock/forgejo-runner-rust"
+          echo "Building: ${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          
+          docker build --no-cache \
+            -f k8s/infra/forgejo-runner-images/Dockerfile.rust \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+          
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+          echo "✓ Pushed rust runner"
+
+      - name: Prune unused images
+        run: docker image prune -a --force 2>&1 | tail -3 || true
+
+      - name: Summary
+        run: |
+          echo "=== Runner Images Built and Pushed ===" 
+          echo "✓ forgejo.riotpiao.com/rock/forgejo-runner-golang:latest"
+          echo "✓ forgejo.riotpiao.com/rock/forgejo-runner-node:latest"
+          echo "✓ forgejo.riotpiao.com/rock/forgejo-runner-rust:latest"
@@ -208,3 +208,95 @@ versions without warning in your own values file.
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
+
+## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
+
+All repositories MUST follow this exact structure. No variations.
+
+```yaml
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  REGISTRY: <your-registry-hostname>
+  IMAGE: <registry>/<org>/<service-name>
+
+jobs:
+  test:
+    name: Test
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js for actions runtime
+        run: apt-get update && apt-get install -y nodejs
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Language-specific tests here (no docker, no registry)
+      # - name: Run tests
+      #   run: npm test -- --run || true
+
+  build-push:
+    name: Build & Push Image
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js and Docker
+        run: |
+          apt-get update
+          apt-get install -y nodejs docker.io
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: sha
+        run: |
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Registry login
+        run: |
+          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+            --username "${REGISTRY_USER}" --password-stdin
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build Docker image
+        run: |
+          docker build --no-cache \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+
+      - name: Push Docker image
+        run: |
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+
+      - name: Prune unused images
+        run: docker image prune -a --force 2>&1 | tail -3 || true
+```
+
+### Anti-Patterns (DO NOT USE)
+
+- ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing
+- ❌ Conditional `if:` on individual steps — use separate jobs instead
+- ❌ Installing docker.io in test job — only needed in build-push
+- ❌ Monolithic job doing test + build + push — hard to debug
+- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
+
+### How It Works
+
+1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
+2. **Push to main** → test runs, build-push runs after test passes, image pushed
+3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
+4. `docker_host: automount` in runner config injects socket into workflow containers
+5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git
@@ -0,0 +1,12 @@
+FROM golang:1.26-bookworm
+
+# Install Node.js, npm, and Docker CLI
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    nodejs npm docker.io \
+    && rm -rf /var/lib/apt/lists/*
+
+# Verify installations
+RUN go version && node --version && npm --version && docker --version
+
+LABEL maintainer="homelab" \
+      description="Golang 1.26 + Node.js + Docker CLI pre-installed"
@@ -0,0 +1,12 @@
+FROM node:22-bookworm
+
+# Node.js already installed, add Docker CLI
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    docker.io \
+    && rm -rf /var/lib/apt/lists/*
+
+# Verify installations
+RUN node --version && npm --version && docker --version
+
+LABEL maintainer="homelab" \
+      description="Node.js 22 + Docker CLI pre-installed"
@@ -0,0 +1,12 @@
+FROM rust:1-bookworm
+
+# Add Node.js and Docker CLI
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    nodejs npm docker.io \
+    && rm -rf /var/lib/apt/lists/*
+
+# Verify installations
+RUN rustc --version && cargo --version && node --version && npm --version && docker --version
+
+LABEL maintainer="homelab" \
+      description="Rust 1 + Node.js + Docker CLI pre-installed"
@@ -1,16 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      nodejs \
-      npm \
-      docker-cli
-
-# Verify installations
-RUN docker --version && node --version && git --version
-
-# Switch back to runner user
-USER 1000:1000
@@ -1,16 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      docker-cli \
-      nodejs \
-      npm
-
-# Verify installations
-RUN node --version && docker --version && git --version
-
-# Switch back to runner user
-USER 1000:1000
@@ -1,20 +0,0 @@
-FROM code.forgejo.org/forgejo/runner:6
-
-# Switch to root to install packages (Alpine)
-USER root
-
-# Alpine uses apk, not apt-get
-RUN apk update && apk add --no-cache \
-      nodejs \
-      npm \
-      curl \
-      docker-cli
-
-# Install Rust (as root, skip verification for now)
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable || true
-
-# Verify core installations
-RUN docker --version && node --version && git --version
-
-# Switch back to runner user
-USER 1000:1000
@@ -36,3 +36,4 @@ data:
      valid_volumes:
        - /docker-certs/client
      network: host
+      docker_host: automount
@@ -34,7 +34,11 @@ spec:
          command: ["sh", "-c"]
          args:
            - |
-              test -f /data/.runner || forgejo-runner register --no-interactive \
+              # Always re-register to keep labels in sync with values.yaml.
+              # Without this, changing a runner label requires manually deleting
+              # the PVC or .runner file — not GitOps-friendly.
+              rm -f /data/.runner
+              forgejo-runner register --no-interactive \
                --instance {{ .Values.runner.forgejoUrl }} \
                --token $(RUNNER_TOKEN) \
                --name {{ .Values.runner.name }} \
@@ -70,6 +74,8 @@ spec:
              mountPath: /data
            - name: docker-certs
              mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
@@ -89,6 +95,8 @@ spec:
          volumeMounts:
            - name: docker-certs
              mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
            - name: dind-storage
              mountPath: /var/lib/docker
            - name: homelab-ca
@@ -113,6 +121,8 @@ spec:
            claimName: {{ .Release.Name }}-dind
        - name: docker-certs
          emptyDir: {}    # DinD regenerates mTLS certs on each start
+        - name: docker-sock
+          emptyDir: {}    # Shared docker socket between dind and runner
        - name: homelab-ca
          # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
          # The volumeMounts use subPath: ca.crt to project the single cert file.
@@ -2,15 +2,14 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
-# (homelab has no CI; custom runner images built manually if desired)
-# Bootstrap with runner image (already has Node.js), CI builds custom
+# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
+# Install docker in workflow steps as needed.
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
    tag: "6"
  name: node-runner
-  labels: "node:docker://code.forgejo.org/forgejo/runner:6"
+  labels: "node:docker://node:22-bookworm"

 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:
@@ -2,13 +2,14 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# Bootstrap with runner image, CI builds custom with Node.js+Rust
+# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
+# Install Node.js/docker in workflow steps as needed.
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
    tag: "6"
  name: rust-runner
-  labels: "rust:docker://code.forgejo.org/forgejo/runner:6"
+  labels: "rust:docker://rust:1-bookworm"



@@ -1,9 +1,12 @@
 runner:
  image:
    repository: code.forgejo.org/forgejo/runner
-    tag: "6"  # Bootstrap with runner image, CI builds custom with Node.js
+    tag: "6"
  name: golang-runner
-  labels: "golang:docker://code.forgejo.org/forgejo/runner:6"
+  # Label image is what workflow steps run in (NOT the runner daemon image).
+  # golang:1.26-bookworm: Debian, root, apt-get, Go, git.
+  # TODO: Switch to custom image once build-runner-images.yml pushes images
+  labels: "golang:docker://golang:1.26-bookworm"
  forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
  tokenSecret: runner-token
  resources:
Author	SHA1	Message	Date
rock	038cf40fe5	ci: temporarily revert runners to public images to bootstrap custom image build	2026-09-07 09:17:04 -07:00
rock	fcf60cc546	ci: improve runner image build workflow - always run + manual trigger + weekly schedule Build & Push Runner Images / Build & Push All Runner Images (push) Canceled after 0s Details	2026-09-07 09:15:44 -07:00
rock	8f197fcc23	ci: update runner daemon images to custom runner images (enables Image Updater tracking)	2026-09-07 09:11:45 -07:00
rock	f253731cb9	ci: add docker.io to custom runner images Build Runner Images / Build and Push Runner Images (push) Failing after 1m0s Details	2026-09-07 09:09:59 -07:00
rock	32ef4eae0e	ci: add custom runner images with Node.js pre-installed Build Runner Images / Build and Push Runner Images (push) Failing after 58s Details	2026-09-07 09:08:10 -07:00
rock	f521763292	docs: add unified CI workflow pattern to shareable template	2026-09-06 23:19:27 -07:00
rock	326be2e057	fix: always re-register runner to keep labels in sync (#2 ) ## Problem Init container skips registration if `.runner` file exists on PVC: ``` test -f /data/.runner \|\| forgejo-runner register ... ``` This means changing runner labels in `values.yaml` (e.g. the label image fix from PR #1) has no effect until PVCs are manually deleted — not GitOps-friendly. ## Fix Always delete `.runner` and re-register on every pod start: ``` rm -f /data/.runner forgejo-runner register --no-interactive ... ``` Labels now stay in sync with `values.yaml` automatically. ArgoCD syncs → pods restart → init re-registers with current labels. ## Files Changed - `k8s/infra/forgejo-runner/templates/deployment.yaml` (init container logic) ## After Merge ArgoCD syncs → deployment spec changes → pods restart → init re-registers with new labels from PR #1 → CI works across all repos.Reviewed-on: #2 Co-authored-by: rock <[email protected]>	2026-09-07 05:27:30 +00:00
rock	913cfc2f40	fix: add CoreDNS api.riotpiao.com rewrite + fix runner Alpine base image (#1 ) 1. Forgejo CI is broken across all repos Every workflow fails because runner labels point to a bare Alpine image with nothing in it. ┌────────────────────────────────────────┬──────────────────────────────────────┐ │ Before │ After │ ├────────────────────────────────────────┼──────────────────────────────────────┤ │ golang:docker://forgejo/runner:6 │ golang:docker://golang:1.26-bookworm │ ├────────────────────────────────────────┼──────────────────────────────────────┤ │ No Go, no Node.js, no apt-get, no root │ Go, git, apt-get, root │ └────────────────────────────────────────┴──────────────────────────────────────┘ Plus the docker socket isn't shared between dind sidecar and runner, so even if docker CLI existed, it can't reach the daemon. ┌───────────────────────────────────────────┬─────────────────────────────────────────────────────────────┐ │ Before │ After │ ├───────────────────────────────────────────┼─────────────────────────────────────────────────────────────┤ │ dind creates socket in its own filesystem │ Shared /run emptyDir volume │ ├───────────────────────────────────────────┼─────────────────────────────────────────────────────────────┤ │ Runner can't see it │ Both containers see /run/docker.sock │ ├───────────────────────────────────────────┼─────────────────────────────────────────────────────────────┤ │ No docker_host config │ docker_host: automount passes socket to workflow containers │ Co-authored-by: rock <[email protected]>	2026-09-07 05:19:39 +00:00