5 Commits
Author SHA1 Message Date
rock e28e3cae81 fix: wait for dind to be ready before starting runner daemon 2026-09-07 13:22:09 -07:00
rock febfb055ae fix: use tcp://localhost:2375 for dind (no TLS, no socket permission issues) 2026-09-07 13:20:20 -07:00
rock 9d61c42b99 fix: run runner as root to access dind socket 2026-09-07 13:18:54 -07:00
rock 594e752d47 fix: runner uses unix socket instead of TLS TCP for dind
Job containers spawned by the runner run inside dind. With TCP+TLS
(tcp://localhost:2376), localhost inside those containers doesn't
reach the dind sidecar. Unix socket at /run/docker.sock works because
both runner and dind share the /run emptyDir.

Also disables DOCKER_TLS_CERTDIR so dind creates the socket instead
of only listening on TLS TCP.
2026-09-07 13:16:49 -07:00
rock 1a8e87131b ci: fix runner labels + CoreDNS rewrite + cleanup
- Runners use public images (code.forgejo.org/forgejo/runner:6)
- Labels pull from Docker Hub: golang:1.26, node:22, rust:1-bookworm
- Add CoreDNS api.riotpiao.com rewrite
- Fix runner re-registration to keep labels in sync
- Add unified CI pattern docs to CLAUDE.example.md
- Remove dead .forgejo/ workflow dir (Forgejo uses .gitea/)
2026-09-07 13:01:56 -07:00
9 changed files with 146 additions and 302 deletions
-269
View File
@@ -1,269 +0,0 @@
name: Cluster CI Pipeline
on:
push:
branches:
- main
- develop
paths:
- 'k8s/**'
- '.forgejo/workflows/cluster-ci.yaml'
pull_request:
paths:
- 'k8s/**'
jobs:
ci:
runs-on: docker
steps:
# === Checkout ===
- name: Checkout
run: |
REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git"
CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}"
git clone --depth 1 "$CLONE_URL" .
git fetch origin main
git checkout main
# === Install Tools ===
- name: Install Tools
run: |
unset GITHUB_TOKEN
apt-get update && apt-get install -y \
yamllint \
python3-pip \
curl \
jq
# kubeval
curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz
mv -f kubeval /usr/local/bin/
# kustomize
rm -f kustomize
curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash
mv -f kustomize /usr/local/bin/
# argocd
curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64
chmod +x /usr/local/bin/argocd
# trivy
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
# polaris
curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris
chmod +x /usr/local/bin/polaris
# === YAML Lint ===
- name: YAML Lint
run: |
echo "=== Linting YAML files ==="
yamllint k8s/ -c .yamllint.yaml || true
# === Kubeval - Validate K8s Syntax ===
- name: Kubeval - Validate K8s Syntax
run: |
echo "=== Validating Kubernetes manifests ==="
find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do
echo "Validating $file..."
kubeval "$file" -d 2>/dev/null || true
done
# === Kustomize Build - All overlays ===
- name: Kustomize Build - Infrastructure
run: |
echo "=== Building k8s/infrastructure/ ==="
kustomize build k8s/infrastructure > /tmp/infrastructure.yaml
echo "✓ Infrastructure built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)"
- name: Kustomize Build - Bootstrap
run: |
echo "=== Building k8s/bootstrap/ ==="
kustomize build k8s/bootstrap > /tmp/bootstrap.yaml
echo "✓ Bootstrap built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)"
- name: Kustomize Build - Platform
run: |
echo "=== Building k8s/platform/ ==="
kustomize build k8s/platform > /tmp/platform.yaml
echo "✓ Platform built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)"
- name: Kustomize Build - Security
run: |
echo "=== Building k8s/security/ ==="
kustomize build k8s/security > /tmp/security.yaml
echo "✓ Security built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)"
- name: Kustomize Build - Applications
run: |
echo "=== Building k8s/applications/ ==="
kustomize build k8s/applications > /tmp/applications.yaml
echo "✓ Applications built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)"
- name: Kustomize Build - Data
run: |
echo "=== Building k8s/data/ ==="
kustomize build k8s/data > /tmp/data.yaml
echo "✓ Data built successfully"
echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)"
- name: Validate ArgoCD Applications
run: |
echo "=== Validating ArgoCD Applications ==="
kubeval k8s/argocd/apps/*.yaml
# === Trivy - Scan Dockerfile ===
- name: Trivy - Scan Dockerfile
run: |
if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then
echo "=== Scanning Dockerfiles with Trivy ==="
find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \;
else
echo "No Dockerfiles found"
fi
# === Trivy - Scan Helm Charts ===
- name: Trivy - Scan Helm Charts
run: |
if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then
echo "=== Scanning Helm charts with Trivy ==="
find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do
echo "Scanning $chart..."
trivy config "$chart" || true
done
else
echo "No Helm charts found"
fi
# === Polaris - K8s Security Audit ===
- name: Polaris - K8s Security Audit
run: |
echo "=== Running Polaris K8s security audit ==="
polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true
if [ -f /tmp/polaris-audit.json ]; then
echo "Security issues found:"
jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true
fi
# === Check for Secrets in Code ===
- name: Check for Secrets in Code
run: |
echo "=== Scanning for hardcoded secrets ==="
# BLOCKING. This step used to only count findings and then exit 0, so a
# plaintext deploy key rode through it into a public remote. Two failure
# modes fixed: it now fails the build, and it matches key material by
# PEM header rather than only `private_key:`-style YAML field names.
# Findings are captured into variables and tested for emptiness rather than
# branching on grep's exit status: implementations disagree on the rc of a
# `-v` filter fed empty input, and a wrong rc here fails open.
# NOTE: --include must precede `--`; after `--` grep treats it as a filename
# and silently scans nothing.
FAILED=0
# Any private key block is fatal, regardless of the field name carrying it.
KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \
-- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \
| grep -v "\.enc\.yaml" || true)
if [ -n "$KEYS" ]; then
echo "❌ Unencrypted private key material found:"
echo "$KEYS"
FAILED=1
fi
# Plaintext values in secret-ish YAML fields. SOPS output is ENC[...],
# so encrypted files never trip this.
VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \
-- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \
| grep -v "ENC\[" | grep -v "\.enc\.yaml" || true)
if [ -n "$VALS" ]; then
echo "❌ Plaintext secret value found:"
echo "$VALS"
FAILED=1
fi
if [ "$FAILED" -ne 0 ]; then
echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt."
exit 1
fi
echo "✓ No hardcoded secrets found"
# === Check K8s Security Best Practices ===
- name: Check K8s Security Best Practices
run: |
echo "=== Checking K8s security best practices ==="
if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then
echo "⚠️ Found privileged containers"
fi
if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then
echo "⚠️ Found hostNetwork usage"
fi
echo "Checking for missing resource limits..."
MISSING=0
find k8s -name "*.yaml" -o -name "*.yml" | while read file; do
if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then
if ! grep -q "resources:" "$file"; then
echo "⚠️ $file: Missing resource requests/limits"
MISSING=$((MISSING + 1))
fi
fi
done
# === ArgoCD Sync (main branch only) ===
- name: Sync ArgoCD
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
env:
ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
run: |
echo "=== Syncing homelab-root ==="
argocd app sync homelab-root --force
argocd app wait homelab-root --timeout 5m
- name: Check Sync Status
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
env:
ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
run: |
echo "=== ArgoCD Applications Status ==="
argocd app list -o table
STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}')
if [ "$STATUS" != "Synced" ]; then
echo "❌ Root app sync failed: $STATUS"
exit 1
fi
echo "✓ Root app synced successfully"
- name: Health Check
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
env:
ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }}
ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }}
run: |
echo "=== Checking Application Health ==="
argocd app get homelab-root -o wide
# === Summary ===
- name: Summary
if: always()
run: |
echo "=== CI Pipeline Summary ==="
echo "✓ YAML linted"
echo "✓ Manifests validated"
echo "✓ Kustomizations built"
echo "✓ Security scans completed"
echo "✓ Secrets check passed"
echo "✓ Best practices verified"
echo ""
echo "✓ All checks passed"
+92
View File
@@ -208,3 +208,95 @@ versions without warning in your own values file.
Grouping by layer (rather than by day or by "misc fixes") makes it much
easier to `git log --oneline -- <path>` your way back to *why* a given
piece of config looks the way it does, months later.
## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+)
All repositories MUST follow this exact structure. No variations.
```yaml
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
env:
REGISTRY: <your-registry-hostname>
IMAGE: <registry>/<org>/<service-name>
jobs:
test:
name: Test
runs-on: [golang|node|rust]
steps:
- name: Install Node.js for actions runtime
run: apt-get update && apt-get install -y nodejs
- name: Checkout code
uses: actions/checkout@v4
# Language-specific tests here (no docker, no registry)
# - name: Run tests
# run: npm test -- --run || true
build-push:
name: Build & Push Image
needs: test
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
runs-on: [golang|node|rust]
steps:
- name: Install Node.js and Docker
run: |
apt-get update
apt-get install -y nodejs docker.io
- name: Checkout code
uses: actions/checkout@v4
- name: Get short SHA
id: sha
run: |
SHORT_SHA=$(git rev-parse --short HEAD)
echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT
- name: Registry login
run: |
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
--username "${REGISTRY_USER}" --password-stdin
env:
REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }}
REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }}
- name: Build Docker image
run: |
docker build --no-cache \
-t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \
-t "${IMAGE}:latest" \
.
- name: Push Docker image
run: |
docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}"
docker push "${IMAGE}:latest"
- name: Prune unused images
run: docker image prune -a --force 2>&1 | tail -3 || true
```
### Anti-Patterns (DO NOT USE)
-`container: image: golang:1.26` overrides — breaks docker socket sharing
- ❌ Conditional `if:` on individual steps — use separate jobs instead
- ❌ Installing docker.io in test job — only needed in build-push
- ❌ Monolithic job doing test + build + push — hard to debug
- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability
### How It Works
1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed
2. **Push to main** → test runs, build-push runs after test passes, image pushed
3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run`
4. `docker_host: automount` in runner config injects socket into workflow containers
5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git
+21
View File
@@ -152,6 +152,13 @@ metadata:
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "3"
argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
argocd-image-updater.argoproj.io/write-back-method: git
argocd-image-updater.argoproj.io/git-branch: main
spec:
project: homelab
source:
@@ -173,6 +180,13 @@ metadata:
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "3"
argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
argocd-image-updater.argoproj.io/write-back-method: git
argocd-image-updater.argoproj.io/git-branch: main
spec:
project: homelab
source:
@@ -197,6 +211,13 @@ metadata:
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "3"
argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
argocd-image-updater.argoproj.io/write-back-method: git
argocd-image-updater.argoproj.io/git-branch: main
spec:
project: homelab
source:
@@ -36,3 +36,4 @@ data:
valid_volumes:
- /docker-certs/client
network: host
docker_host: automount
@@ -34,7 +34,11 @@ spec:
command: ["sh", "-c"]
args:
- |
test -f /data/.runner || forgejo-runner register --no-interactive \
# Always re-register to keep labels in sync with values.yaml.
# Without this, changing a runner label requires manually deleting
# the PVC or .runner file — not GitOps-friendly.
rm -f /data/.runner
forgejo-runner register --no-interactive \
--instance {{ .Values.runner.forgejoUrl }} \
--token $(RUNNER_TOKEN) \
--name {{ .Values.runner.name }} \
@@ -56,20 +60,18 @@ spec:
containers:
- name: runner
image: {{ .Values.runner.image.repository }}:{{ .Values.runner.image.tag }}
command: ["sh", "-c", "forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
command: ["sh", "-c", "while ! wget -q -O- http://localhost:2375/_ping >/dev/null 2>&1; do echo 'waiting for dind...'; sleep 2; done; echo 'dind ready'; forgejo-runner daemon --config /etc/forgejo-runner/config.yaml"]
workingDir: /data
env:
- name: DOCKER_HOST
value: tcp://localhost:2376
- name: DOCKER_TLS_VERIFY
value: "1"
- name: DOCKER_CERT_PATH
value: /docker-certs/client
value: tcp://localhost:2375
volumeMounts:
- name: runner-data
mountPath: /data
- name: docker-certs
mountPath: /docker-certs
- name: docker-sock
mountPath: /run
- name: homelab-ca
mountPath: /etc/ssl/certs/homelab-ca.pem
subPath: ca.crt
@@ -85,10 +87,12 @@ spec:
privileged: true # required for DinD; cicd namespace is labelled privileged
env:
- name: DOCKER_TLS_CERTDIR
value: /docker-certs
value: ""
volumeMounts:
- name: docker-certs
mountPath: /docker-certs
- name: docker-sock
mountPath: /run
- name: dind-storage
mountPath: /var/lib/docker
- name: homelab-ca
@@ -113,6 +117,8 @@ spec:
claimName: {{ .Release.Name }}-dind
- name: docker-certs
emptyDir: {} # DinD regenerates mTLS certs on each start
- name: docker-sock
emptyDir: {} # Shared docker socket between dind and runner
- name: homelab-ca
# homelab-ca is a ConfigMap (public CA trust bundle), not a Secret.
# The volumeMounts use subPath: ca.crt to project the single cert file.
+5 -2
View File
@@ -2,9 +2,12 @@
# runner instance. Only runner.name and runner.labels differ -- everything
# else (image, dind, persistence, tolerations, nodeSelector) is shared.
#
# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed.
# (homelab has no CI; custom runner images built manually if desired)
# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git.
# Install docker in workflow steps as needed.
runner:
image:
repository: code.forgejo.org/forgejo/runner
tag: "6"
name: node-runner
labels: "node:docker://node:22-bookworm"
+7 -11
View File
@@ -2,20 +2,16 @@
# runner instance. Only runner.name and runner.labels differ -- everything
# else (image, dind, persistence, tolerations, nodeSelector) is shared.
#
# Use docker:27-cli instead of rust:1.83-bookworm because:
# - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.)
# - Has docker CLI + git + full dev tools
# - rust:1.83-bookworm lacks Node.js (causes action failures)
#
# docker:27-cli verified: docker manifest inspect docker:27-cli ✓
# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git.
# Install Node.js/docker in workflow steps as needed.
runner:
image:
repository: code.forgejo.org/forgejo/runner
tag: "6"
name: rust-runner
labels: "rust:docker://docker:27-cli"
labels: "rust:docker://rust:1-bookworm"
persistence:
reg:
storageClass: longhorn
size: 20Gi
# GC CronJob renders only from the default (golang) values to avoid duplicates
gc:
+5 -12
View File
@@ -1,20 +1,13 @@
runner:
image:
repository: code.forgejo.org/forgejo/runner
tag: "6" # pin exact release before apply
tag: "6"
name: golang-runner
# Use docker:27-cli instead of golang:1.26-bookworm because:
# - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.)
# - Has docker CLI + git + golang + all build tools
# - golang:1.26-bookworm lacks Node.js (causes action failures)
# docker:27-cli includes Go toolchain via base debian + additional packages
# Verified tag: docker manifest inspect docker:27-cli ✓
labels: "golang:docker://docker:27-cli"
# In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop
# (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout).
# Label image is what workflow steps run in (NOT the runner daemon image).
# golang:1.26-bookworm: Debian, root, apt-get, Go, git.
# TODO: Switch to custom image once build-runner-images.yml pushes images
labels: "golang:docker://golang:1.26-bookworm"
forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000
# tokenSecret: name of the K8s Secret that holds the runner registration token
# created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl)
tokenSecret: runner-token
resources:
requests:
+1
View File
@@ -35,6 +35,7 @@
rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local
kubernetes cluster.local in-addr.arpa ip6.arpa {