fix: wait for dind to be ready before starting runner daemon

Job containers spawned by the runner run inside dind. With TCP+TLS
(tcp://localhost:2376), localhost inside those containers doesn't
reach the dind sidecar. Unix socket at /run/docker.sock works because
both runner and dind share the /run emptyDir.

Also disables DOCKER_TLS_CERTDIR so dind creates the socket instead
of only listening on TLS TCP.

.forgejo/workflows/cluster-ci.yaml

@@ -1,269 +0,0 @@
-name: Cluster CI Pipeline
-
-on:
-  push:
-    branches:
-      - main
-      - develop
-    paths:
-      - 'k8s/**'
-      - '.forgejo/workflows/cluster-ci.yaml'
-  pull_request:
-    paths:
-      - 'k8s/**'
-
-jobs:
-  ci:
-    runs-on: docker
-    steps:
-      # === Checkout ===
-      - name: Checkout
-        run: |
-          REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
-          CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
-          git clone --depth 1 "$CLONE_URL" .
-          git fetch origin main
-          git checkout main
-
-      # === Install Tools ===
-      - name: Install Tools
-        run: |
-          unset GITHUB_TOKEN
-          apt-get update && apt-get install -y \
-            yamllint \
-            python3-pip \
-            curl \
-            jq
-
-          # kubeval
-          curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
-          mv -f kubeval /usr/local/bin/
-
-          # kustomize
-          rm -f kustomize
-          curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
-          mv -f kustomize /usr/local/bin/
-
-          # argocd
-          curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
-          chmod +x /usr/local/bin/argocd
-
-          # trivy
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
-
-          # polaris
-          curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
-          chmod +x /usr/local/bin/polaris
-
-      # === YAML Lint ===
-      - name: YAML Lint
-        run: |
-          echo "=== Linting YAML files ==="
-          yamllint k8s/ -c .yamllint.yaml || true
-
-      # === Kubeval - Validate K8s Syntax ===
-      - name: Kubeval - Validate K8s Syntax
-        run: |
-          echo "=== Validating Kubernetes manifests ==="
-          find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
-            echo "Validating $file..."
-            kubeval "$file" -d 2>/dev/null || true
-          done
-
-      # === Kustomize Build - All overlays ===
-      - name: Kustomize Build - Infrastructure
-        run: |
-          echo "=== Building k8s/infrastructure/ ==="
-          kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
-          echo "✓ Infrastructure built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
-
-      - name: Kustomize Build - Bootstrap
-        run: |
-          echo "=== Building k8s/bootstrap/ ==="
-          kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
-          echo "✓ Bootstrap built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
-
-      - name: Kustomize Build - Platform
-        run: |
-          echo "=== Building k8s/platform/ ==="
-          kustomize build k8s/platform > /tmp/platform.yaml
-          echo "✓ Platform built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
-
-      - name: Kustomize Build - Security
-        run: |
-          echo "=== Building k8s/security/ ==="
-          kustomize build k8s/security > /tmp/security.yaml
-          echo "✓ Security built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
-
-      - name: Kustomize Build - Applications
-        run: |
-          echo "=== Building k8s/applications/ ==="
-          kustomize build k8s/applications > /tmp/applications.yaml
-          echo "✓ Applications built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
-
-      - name: Kustomize Build - Data
-        run: |
-          echo "=== Building k8s/data/ ==="
-          kustomize build k8s/data > /tmp/data.yaml
-          echo "✓ Data built successfully"
-          echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
-
-      - name: Validate ArgoCD Applications
-        run: |
-          echo "=== Validating ArgoCD Applications ==="
-          kubeval k8s/argocd/apps/*.yaml
-
-      # === Trivy - Scan Dockerfile ===
-      - name: Trivy - Scan Dockerfile
-        run: |
-          if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
-            echo "=== Scanning Dockerfiles with Trivy ==="
-            find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
-          else
-            echo "No Dockerfiles found"
-          fi
-
-      # === Trivy - Scan Helm Charts ===
-      - name: Trivy - Scan Helm Charts
-        run: |
-          if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
-            echo "=== Scanning Helm charts with Trivy ==="
-            find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
-              echo "Scanning $chart..."
-              trivy config "$chart" || true
-            done
-          else
-            echo "No Helm charts found"
-          fi
-
-      # === Polaris - K8s Security Audit ===
-      - name: Polaris - K8s Security Audit
-        run: |
-          echo "=== Running Polaris K8s security audit ==="
-          polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
-
-          if [ -f /tmp/polaris-audit.json ]; then
-            echo "Security issues found:"
-            jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
-          fi
-
-      # === Check for Secrets in Code ===
-      - name: Check for Secrets in Code
-        run: |
-          echo "=== Scanning for hardcoded secrets ==="
-          # BLOCKING. This step used to only count findings and then exit 0, so a
-          # plaintext deploy key rode through it into a public remote. Two failure
-          # modes fixed: it now fails the build, and it matches key material by
-          # PEM header rather than only `private_key:`-style YAML field names.
-          # Findings are captured into variables and tested for emptiness rather than
-          # branching on grep's exit status: implementations disagree on the rc of a
-          # `-v` filter fed empty input, and a wrong rc here fails open.
-          # NOTE: --include must precede `--`; after `--` grep treats it as a filename
-          # and silently scans nothing.
-          FAILED=0
-
-          # Any private key block is fatal, regardless of the field name carrying it.
-          KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
-                   -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
-                 | grep -v "\.enc\.yaml" || true)
-          if [ -n "$KEYS" ]; then
-            echo "❌ Unencrypted private key material found:"
-            echo "$KEYS"
-            FAILED=1
-          fi
-
-          # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
-          # so encrypted files never trip this.
-          VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
-                   -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
-                 | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
-          if [ -n "$VALS" ]; then
-            echo "❌ Plaintext secret value found:"
-            echo "$VALS"
-            FAILED=1
-          fi
-
-          if [ "$FAILED" -ne 0 ]; then
-            echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
-            exit 1
-          fi
-          echo "✓ No hardcoded secrets found"
-
-      # === Check K8s Security Best Practices ===
-      - name: Check K8s Security Best Practices
-        run: |
-          echo "=== Checking K8s security best practices ==="
-
-          if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found privileged containers"
-          fi
-
-          if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
-            echo "⚠️  Found hostNetwork usage"
-          fi
-
-          echo "Checking for missing resource limits..."
-          MISSING=0
-          find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
-            if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
-              if ! grep -q "resources:" "$file"; then
-                echo "⚠️  $file: Missing resource requests/limits"
-                MISSING=$((MISSING + 1))
-              fi
-            fi
-          done
-
-      # === ArgoCD Sync (main branch only) ===
-      - name: Sync ArgoCD
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Syncing homelab-root ==="
-          argocd app sync homelab-root --force
-          argocd app wait homelab-root --timeout 5m
-
-      - name: Check Sync Status
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== ArgoCD Applications Status ==="
-          argocd app list -o table
-
-          STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
-          if [ "$STATUS" != "Synced" ]; then
-            echo "❌ Root app sync failed: $STATUS"
-            exit 1
-          fi
-          echo "✓ Root app synced successfully"
-
-      - name: Health Check
-        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-        env:
-          ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
-          ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
-        run: |
-          echo "=== Checking Application Health ==="
-          argocd app get homelab-root -o wide
-
-      # === Summary ===
-      - name: Summary
-        if: always()
-        run: |
-          echo "=== CI Pipeline Summary ==="
-          echo "✓ YAML linted"
-          echo "✓ Manifests validated"
-          echo "✓ Kustomizations built"
-          echo "✓ Security scans completed"
-          echo "✓ Secrets check passed"
-          echo "✓ Best practices verified"
-          echo ""
-          echo "✓ All checks passed"

CLAUDE.example.md

@@ -208,3 +208,95 @@ versions without warning in your own values file.
 Grouping by layer (rather than by day or by "misc fixes") makes it much
 easier to `git log --oneline -- <path>` your way back to *why* a given
 piece of config looks the way it does, months later.
+
+## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
+
+All repositories MUST follow this exact structure. No variations.
+
+```yaml
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  REGISTRY: <your-registry-hostname>
+  IMAGE: <registry>/<org>/<service-name>
+
+jobs:
+  test:
+    name: Test
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js for actions runtime
+        run: apt-get update && apt-get install -y nodejs
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Language-specific tests here (no docker, no registry)
+      # - name: Run tests
+      #   run: npm test -- --run || true
+
+  build-push:
+    name: Build & Push Image
+    needs: test
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: [golang|node|rust]
+    steps:
+      - name: Install Node.js and Docker
+        run: |
+          apt-get update
+          apt-get install -y nodejs docker.io
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get short SHA
+        id: sha
+        run: |
+          SHORT_SHA=$(git rev-parse --short HEAD)
+          echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
+
+      - name: Registry login
+        run: |
+          echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+            --username "${REGISTRY_USER}" --password-stdin
+        env:
+          REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
+          REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
+
+      - name: Build Docker image
+        run: |
+          docker build --no-cache \
+            -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
+            -t "${IMAGE}:latest" \
+            .
+
+      - name: Push Docker image
+        run: |
+          docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
+          docker push "${IMAGE}:latest"
+
+      - name: Prune unused images
+        run: docker image prune -a --force 2>&1 | tail -3 || true
+```
+
+### Anti-Patterns (DO NOT USE)
+
+- ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing
+- ❌ Conditional `if:` on individual steps — use separate jobs instead
+- ❌ Installing docker.io in test job — only needed in build-push
+- ❌ Monolithic job doing test + build + push — hard to debug
+- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
+
+### How It Works
+
+1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
+2. **Push to main** → test runs, build-push runs after test passes, image pushed
+3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
+4. `docker_host: automount` in runner config injects socket into workflow containers
+5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git

k8s/argocd/apps/30-security.yaml

@@ -152,6 +152,13 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -173,6 +180,13 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -197,6 +211,13 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
+    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
+    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
+    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
+    argocd-image-updater.argoproj.io/write-back-method: git
+    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:

k8s/infra/forgejo-runner/templates/configmap.yaml

@@ -36,3 +36,4 @@ data:
       valid_volumes:
         - /docker-certs/client
       network: host
+      docker_host: automount

k8s/infra/forgejo-runner/templates/deployment.yaml

@@ -34,7 +34,11 @@ spec:
           command: ["sh", "-c"]
           args:
             - |
-              test -f /data/.runner || forgejo-runner register --no-interactive \
+              # Always re-register to keep labels in sync with values.yaml.
+              # Without this, changing a runner label requires manually deleting
+              # the PVC or .runner file — not GitOps-friendly.
+              rm -f /data/.runner
+              forgejo-runner register --no-interactive \
                 --instance {{ .Values.runner.forgejoUrl }} \
                 --token $(RUNNER_TOKEN) \
                 --name {{ .Values.runner.name }} \
@@ -56,20 +60,18 @@ spec:
       containers:
         - name: runner
           image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
-          command: ["sh", "-c", "forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
+          command: ["sh", "-c", "while ! wget -q -O- http://localhost:2375/_ping >/dev/null 2>&1; do echo 'waiting for dind...'; sleep 2; done; echo 'dind ready'; forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
           workingDir: /data
           env:
             - name: DOCKER_HOST
-              value: tcp://localhost:2376
+              value: tcp://localhost:2375
-            - name: DOCKER_TLS_VERIFY
-              value: "1"
-            - name: DOCKER_CERT_PATH
-              value: /docker-certs/client
           volumeMounts:
             - name: runner-data
               mountPath: /data
             - name: docker-certs
               mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
             - name: homelab-ca
               mountPath: /etc/ssl/certs/homelab-ca.pem
               subPath: ca.crt
@@ -85,10 +87,12 @@ spec:
             privileged: true    # required for DinD; cicd namespace is labelled privileged
           env:
             - name: DOCKER_TLS_CERTDIR
-              value: /docker-certs
+              value: ""
           volumeMounts:
             - name: docker-certs
               mountPath: /docker-certs
+            - name: docker-sock
+              mountPath: /run
             - name: dind-storage
               mountPath: /var/lib/docker
             - name: homelab-ca
@@ -113,6 +117,8 @@ spec:
             claimName: {{ .Release.Name }}-dind
         - name: docker-certs
           emptyDir: {}    # DinD regenerates mTLS certs on each start
+        - name: docker-sock
+          emptyDir: {}    # Shared docker socket between dind and runner
         - name: homelab-ca
           # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
           # The volumeMounts use subPath: ca.crt to project the single cert file.

k8s/infra/forgejo-runner/values-node.yaml

@@ -2,9 +2,12 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
+# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
-# (homelab has no CI; custom runner images built manually if desired)
+# Install docker in workflow steps as needed.
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
   name: node-runner
   labels: "node:docker://node:22-bookworm"
 

k8s/infra/forgejo-runner/values-rust.yaml

@@ -2,20 +2,16 @@
 # runner instance. Only runner.name and runner.labels differ -- everything
 # else (image, dind, persistence, tolerations, nodeSelector) is shared.
 #
-# Use docker:27-cli instead of rust:1.83-bookworm because:
+# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
-#   - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.)
+# Install Node.js/docker in workflow steps as needed.
-#   - Has docker CLI + git + full dev tools
-#   - rust:1.83-bookworm lacks Node.js (causes action failures)
-#
-# docker:27-cli verified: docker manifest inspect docker:27-cli ✓
 runner:
+  image:
+    repository: code.forgejo.org/forgejo/runner
+    tag: "6"
   name: rust-runner
-  labels: "rust:docker://docker:27-cli"
+  labels: "rust:docker://rust:1-bookworm"
+
 
-persistence:
-  reg:
-    storageClass: longhorn
-    size: 20Gi
 
 # GC CronJob renders only from the default (golang) values to avoid duplicates
 gc:

k8s/infra/forgejo-runner/values.yaml

@@ -1,20 +1,13 @@
 runner:
   image:
     repository: code.forgejo.org/forgejo/runner
-    tag: "6"          # pin exact release before apply
+    tag: "6"
   name: golang-runner
-  # Use docker:27-cli instead of golang:1.26-bookworm because:
+  # Label image is what workflow steps run in (NOT the runner daemon image).
-  #   - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.)
+  # golang:1.26-bookworm: Debian, root, apt-get, Go, git.
-  #   - Has docker CLI + git + golang + all build tools
+  # TODO: Switch to custom image once build-runner-images.yml pushes images
-  #   - golang:1.26-bookworm lacks Node.js (causes action failures)
+  labels: "golang:docker://golang:1.26-bookworm"
-  # docker:27-cli includes Go toolchain via base debian + additional packages
-  # Verified tag: docker manifest inspect docker:27-cli ✓
-  labels: "golang:docker://docker:27-cli"
-  # In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
-  # (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
   forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
-  # tokenSecret: name of the K8s Secret that holds the runner registration token
-  # created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
   tokenSecret: runner-token
   resources:
     requests:

terraform/files/coredns/Corefile

@@ -35,6 +35,7 @@
     rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
 
     kubernetes cluster.local in-addr.arpa ip6.arpa {