From 1a8e87131bb3e7e76e43f56e51e5a4621129be50 Mon Sep 17 00:00:00 2001 From: rock Date: Mon, 7 Sep 2026 13:01:56 -0700 Subject: [PATCH] ci: fix runner labels + CoreDNS rewrite + cleanup - Runners use public images (code.forgejo.org/forgejo/runner:6) - Labels pull from Docker Hub: golang:1.26, node:22, rust:1-bookworm - Add CoreDNS api.riotpiao.com rewrite - Fix runner re-registration to keep labels in sync - Add unified CI pattern docs to CLAUDE.example.md - Remove dead .forgejo/ workflow dir (Forgejo uses .gitea/) --- .forgejo/workflows/cluster-ci.yaml | 269 ------------------ CLAUDE.example.md | 92 ++++++ k8s/argocd/apps/30-security.yaml | 21 ++ .../forgejo-runner/templates/configmap.yaml | 1 + .../forgejo-runner/templates/deployment.yaml | 12 +- k8s/infra/forgejo-runner/values-node.yaml | 7 +- k8s/infra/forgejo-runner/values-rust.yaml | 18 +- k8s/infra/forgejo-runner/values.yaml | 17 +- terraform/files/coredns/Corefile | 1 + 9 files changed, 143 insertions(+), 295 deletions(-) delete mode 100644 .forgejo/workflows/cluster-ci.yaml diff --git a/.forgejo/workflows/cluster-ci.yaml b/.forgejo/workflows/cluster-ci.yaml deleted file mode 100644 index 916619e..0000000 --- a/.forgejo/workflows/cluster-ci.yaml +++ /dev/null @@ -1,269 +0,0 @@ -name: Cluster CI Pipeline - -on: - push: - branches: - - main - - develop - paths: - - 'k8s/**' - - '.forgejo/workflows/cluster-ci.yaml' - pull_request: - paths: - - 'k8s/**' - -jobs: - ci: - runs-on: docker - steps: - # === Checkout === - - name: Checkout - run: | - REPO_URL="${{ gitea.server_url }}/${{ gitea.repository }}.git" - CLONE_URL="https://${{ secrets.CI_RUNNER }}:${{ secrets.CI_RUNNER_SECRET }}@${REPO_URL#https://}" - git clone --depth 1 "$CLONE_URL" . - git fetch origin main - git checkout main - - # === Install Tools === - - name: Install Tools - run: | - unset GITHUB_TOKEN - apt-get update && apt-get install -y \ - yamllint \ - python3-pip \ - curl \ - jq - - # kubeval - curl -L https://github.com/instrumenta/kubeval/releases/latest/download/kubeval-linux-amd64.tar.gz | tar xz - mv -f kubeval /usr/local/bin/ - - # kustomize - rm -f kustomize - curl -s https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh | bash - mv -f kustomize /usr/local/bin/ - - # argocd - curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/latest/download/argocd-linux-amd64 - chmod +x /usr/local/bin/argocd - - # trivy - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin - - # polaris - curl -L https://github.com/FairwindsOps/polaris/releases/latest/download/polaris-linux-amd64 -o /usr/local/bin/polaris - chmod +x /usr/local/bin/polaris - - # === YAML Lint === - - name: YAML Lint - run: | - echo "=== Linting YAML files ===" - yamllint k8s/ -c .yamllint.yaml || true - - # === Kubeval - Validate K8s Syntax === - - name: Kubeval - Validate K8s Syntax - run: | - echo "=== Validating Kubernetes manifests ===" - find k8s -name "*.yaml" -o -name "*.yml" | grep -v "\.archive" | while read file; do - echo "Validating $file..." - kubeval "$file" -d 2>/dev/null || true - done - - # === Kustomize Build - All overlays === - - name: Kustomize Build - Infrastructure - run: | - echo "=== Building k8s/infrastructure/ ===" - kustomize build k8s/infrastructure > /tmp/infrastructure.yaml - echo "✓ Infrastructure built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/infrastructure.yaml)" - - - name: Kustomize Build - Bootstrap - run: | - echo "=== Building k8s/bootstrap/ ===" - kustomize build k8s/bootstrap > /tmp/bootstrap.yaml - echo "✓ Bootstrap built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/bootstrap.yaml || echo 0)" - - - name: Kustomize Build - Platform - run: | - echo "=== Building k8s/platform/ ===" - kustomize build k8s/platform > /tmp/platform.yaml - echo "✓ Platform built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/platform.yaml || echo 0)" - - - name: Kustomize Build - Security - run: | - echo "=== Building k8s/security/ ===" - kustomize build k8s/security > /tmp/security.yaml - echo "✓ Security built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/security.yaml || echo 0)" - - - name: Kustomize Build - Applications - run: | - echo "=== Building k8s/applications/ ===" - kustomize build k8s/applications > /tmp/applications.yaml - echo "✓ Applications built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/applications.yaml || echo 0)" - - - name: Kustomize Build - Data - run: | - echo "=== Building k8s/data/ ===" - kustomize build k8s/data > /tmp/data.yaml - echo "✓ Data built successfully" - echo "Resources: $(grep -c 'kind:' /tmp/data.yaml || echo 0)" - - - name: Validate ArgoCD Applications - run: | - echo "=== Validating ArgoCD Applications ===" - kubeval k8s/argocd/apps/*.yaml - - # === Trivy - Scan Dockerfile === - - name: Trivy - Scan Dockerfile - run: | - if find . -name "Dockerfile" 2>/dev/null | grep -v node_modules | head -1 | grep -q .; then - echo "=== Scanning Dockerfiles with Trivy ===" - find . -name "Dockerfile" -not -path "*/node_modules/*" -exec trivy config {} \; - else - echo "No Dockerfiles found" - fi - - # === Trivy - Scan Helm Charts === - - name: Trivy - Scan Helm Charts - run: | - if find k8s -name "Chart.yaml" 2>/dev/null | head -1 | grep -q .; then - echo "=== Scanning Helm charts with Trivy ===" - find k8s -name "Chart.yaml" -exec dirname {} \; | while read chart; do - echo "Scanning $chart..." - trivy config "$chart" || true - done - else - echo "No Helm charts found" - fi - - # === Polaris - K8s Security Audit === - - name: Polaris - K8s Security Audit - run: | - echo "=== Running Polaris K8s security audit ===" - polaris audit --audit-path /tmp/polaris-audit.json k8s/ || true - - if [ -f /tmp/polaris-audit.json ]; then - echo "Security issues found:" - jq '.results[] | select(.pass == false)' /tmp/polaris-audit.json || true - fi - - # === Check for Secrets in Code === - - name: Check for Secrets in Code - run: | - echo "=== Scanning for hardcoded secrets ===" - # BLOCKING. This step used to only count findings and then exit 0, so a - # plaintext deploy key rode through it into a public remote. Two failure - # modes fixed: it now fails the build, and it matches key material by - # PEM header rather than only `private_key:`-style YAML field names. - # Findings are captured into variables and tested for emptiness rather than - # branching on grep's exit status: implementations disagree on the rc of a - # `-v` filter fed empty input, and a wrong rc here fails open. - # NOTE: --include must precede `--`; after `--` grep treats it as a filename - # and silently scans nothing. - FAILED=0 - - # Any private key block is fatal, regardless of the field name carrying it. - KEYS=$(grep -rIE --include="*.yaml" --include="*.yml" \ - -- "-----BEGIN ([A-Z]+ )?PRIVATE KEY-----" k8s/ \ - | grep -v "\.enc\.yaml" || true) - if [ -n "$KEYS" ]; then - echo "❌ Unencrypted private key material found:" - echo "$KEYS" - FAILED=1 - fi - - # Plaintext values in secret-ish YAML fields. SOPS output is ENC[...], - # so encrypted files never trip this. - VALS=$(grep -rInE --include="*.yaml" --include="*.yml" \ - -- "^[[:space:]]*(password|token|apiKey|api_key|sshPrivateKey|client_secret):[[:space:]]*[\"']?[^\"'[:space:]{\$]{8,}" k8s/ \ - | grep -v "ENC\[" | grep -v "\.enc\.yaml" || true) - if [ -n "$VALS" ]; then - echo "❌ Plaintext secret value found:" - echo "$VALS" - FAILED=1 - fi - - if [ "$FAILED" -ne 0 ]; then - echo "Encrypt with SOPS (see .sops.yaml) — *.enc.yaml files are exempt." - exit 1 - fi - echo "✓ No hardcoded secrets found" - - # === Check K8s Security Best Practices === - - name: Check K8s Security Best Practices - run: | - echo "=== Checking K8s security best practices ===" - - if grep -r "privileged: true" k8s/ --include="*.yaml" --include="*.yml"; then - echo "⚠️ Found privileged containers" - fi - - if grep -r "hostNetwork: true" k8s/ --include="*.yaml" --include="*.yml"; then - echo "⚠️ Found hostNetwork usage" - fi - - echo "Checking for missing resource limits..." - MISSING=0 - find k8s -name "*.yaml" -o -name "*.yml" | while read file; do - if grep -q "kind: Deployment\|kind: StatefulSet\|kind: DaemonSet" "$file"; then - if ! grep -q "resources:" "$file"; then - echo "⚠️ $file: Missing resource requests/limits" - MISSING=$((MISSING + 1)) - fi - fi - done - - # === ArgoCD Sync (main branch only) === - - name: Sync ArgoCD - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - env: - ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }} - ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }} - run: | - echo "=== Syncing homelab-root ===" - argocd app sync homelab-root --force - argocd app wait homelab-root --timeout 5m - - - name: Check Sync Status - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - env: - ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }} - ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }} - run: | - echo "=== ArgoCD Applications Status ===" - argocd app list -o table - - STATUS=$(argocd app get homelab-root -o jsonpath='{.status.syncStatus}') - if [ "$STATUS" != "Synced" ]; then - echo "❌ Root app sync failed: $STATUS" - exit 1 - fi - echo "✓ Root app synced successfully" - - - name: Health Check - if: github.ref == 'refs/heads/main' && github.event_name == 'push' - env: - ARGOCD_SERVER: ${{ secrets.ARGOCD_SERVER }} - ARGOCD_AUTH_TOKEN: ${{ secrets.ARGOCD_AUTH_TOKEN }} - run: | - echo "=== Checking Application Health ===" - argocd app get homelab-root -o wide - - # === Summary === - - name: Summary - if: always() - run: | - echo "=== CI Pipeline Summary ===" - echo "✓ YAML linted" - echo "✓ Manifests validated" - echo "✓ Kustomizations built" - echo "✓ Security scans completed" - echo "✓ Secrets check passed" - echo "✓ Best practices verified" - echo "" - echo "✓ All checks passed" diff --git a/CLAUDE.example.md b/CLAUDE.example.md index 4870b5e..b50dcf6 100644 --- a/CLAUDE.example.md +++ b/CLAUDE.example.md @@ -208,3 +208,95 @@ versions without warning in your own values file. Grouping by layer (rather than by day or by "misc fixes") makes it much easier to `git log --oneline -- ` your way back to *why* a given piece of config looks the way it does, months later. + +## Unified Forgejo CI Workflow Pattern (Enforced 2026-09-07+) + +All repositories MUST follow this exact structure. No variations. + +```yaml +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +env: + REGISTRY: + IMAGE: // + +jobs: + test: + name: Test + runs-on: [golang|node|rust] + steps: + - name: Install Node.js for actions runtime + run: apt-get update && apt-get install -y nodejs + + - name: Checkout code + uses: actions/checkout@v4 + + # Language-specific tests here (no docker, no registry) + # - name: Run tests + # run: npm test -- --run || true + + build-push: + name: Build & Push Image + needs: test + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: [golang|node|rust] + steps: + - name: Install Node.js and Docker + run: | + apt-get update + apt-get install -y nodejs docker.io + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get short SHA + id: sha + run: | + SHORT_SHA=$(git rev-parse --short HEAD) + echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT + + - name: Registry login + run: | + echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \ + --username "${REGISTRY_USER}" --password-stdin + env: + REGISTRY_USER: ${{ secrets.FORGEJO_REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.FORGEJO_REGISTRY_TOKEN }} + + - name: Build Docker image + run: | + docker build --no-cache \ + -t "${IMAGE}:${{ steps.sha.outputs.short_sha }}" \ + -t "${IMAGE}:latest" \ + . + + - name: Push Docker image + run: | + docker push "${IMAGE}:${{ steps.sha.outputs.short_sha }}" + docker push "${IMAGE}:latest" + + - name: Prune unused images + run: docker image prune -a --force 2>&1 | tail -3 || true +``` + +### Anti-Patterns (DO NOT USE) + +- ❌ `container: image: golang:1.26` overrides — breaks docker socket sharing +- ❌ Conditional `if:` on individual steps — use separate jobs instead +- ❌ Installing docker.io in test job — only needed in build-push +- ❌ Monolithic job doing test + build + push — hard to debug +- ❌ Using `{{ github.sha }}` for image tag — use short commit SHA for readability + +### How It Works + +1. **PR to feature branch** → test job runs, build-push skipped, nothing pushed +2. **Push to main** → test runs, build-push runs after test passes, image pushed +3. Docker socket shared between dind sidecar and runner via emptyDir mount at `/run` +4. `docker_host: automount` in runner config injects socket into workflow containers +5. Secrets (FORGEJO_REGISTRY_USER, TOKEN) set in Forgejo repo settings, NOT in git diff --git a/k8s/argocd/apps/30-security.yaml b/k8s/argocd/apps/30-security.yaml index 53e9d96..a1d4d3d 100644 --- a/k8s/argocd/apps/30-security.yaml +++ b/k8s/argocd/apps/30-security.yaml @@ -152,6 +152,13 @@ metadata: namespace: argocd annotations: argocd.argoproj.io/sync-wave: "3" + argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang + argocd-image-updater.argoproj.io/runner.update-strategy: newest-build + argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$ + argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository + argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag + argocd-image-updater.argoproj.io/write-back-method: git + argocd-image-updater.argoproj.io/git-branch: main spec: project: homelab source: @@ -173,6 +180,13 @@ metadata: namespace: argocd annotations: argocd.argoproj.io/sync-wave: "3" + argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node + argocd-image-updater.argoproj.io/runner.update-strategy: newest-build + argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$ + argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository + argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag + argocd-image-updater.argoproj.io/write-back-method: git + argocd-image-updater.argoproj.io/git-branch: main spec: project: homelab source: @@ -197,6 +211,13 @@ metadata: namespace: argocd annotations: argocd.argoproj.io/sync-wave: "3" + argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust + argocd-image-updater.argoproj.io/runner.update-strategy: newest-build + argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$ + argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository + argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag + argocd-image-updater.argoproj.io/write-back-method: git + argocd-image-updater.argoproj.io/git-branch: main spec: project: homelab source: diff --git a/k8s/infra/forgejo-runner/templates/configmap.yaml b/k8s/infra/forgejo-runner/templates/configmap.yaml index c73d992..30334e9 100644 --- a/k8s/infra/forgejo-runner/templates/configmap.yaml +++ b/k8s/infra/forgejo-runner/templates/configmap.yaml @@ -36,3 +36,4 @@ data: valid_volumes: - /docker-certs/client network: host + docker_host: automount diff --git a/k8s/infra/forgejo-runner/templates/deployment.yaml b/k8s/infra/forgejo-runner/templates/deployment.yaml index 6b11c26..2ef614f 100644 --- a/k8s/infra/forgejo-runner/templates/deployment.yaml +++ b/k8s/infra/forgejo-runner/templates/deployment.yaml @@ -34,7 +34,11 @@ spec: command: ["sh", "-c"] args: - | - test -f /data/.runner || forgejo-runner register --no-interactive \ + # Always re-register to keep labels in sync with values.yaml. + # Without this, changing a runner label requires manually deleting + # the PVC or .runner file — not GitOps-friendly. + rm -f /data/.runner + forgejo-runner register --no-interactive \ --instance {{ .Values.runner.forgejoUrl }} \ --token $(RUNNER_TOKEN) \ --name {{ .Values.runner.name }} \ @@ -70,6 +74,8 @@ spec: mountPath: /data - name: docker-certs mountPath: /docker-certs + - name: docker-sock + mountPath: /run - name: homelab-ca mountPath: /etc/ssl/certs/homelab-ca.pem subPath: ca.crt @@ -89,6 +95,8 @@ spec: volumeMounts: - name: docker-certs mountPath: /docker-certs + - name: docker-sock + mountPath: /run - name: dind-storage mountPath: /var/lib/docker - name: homelab-ca @@ -113,6 +121,8 @@ spec: claimName: {{ .Release.Name }}-dind - name: docker-certs emptyDir: {} # DinD regenerates mTLS certs on each start + - name: docker-sock + emptyDir: {} # Shared docker socket between dind and runner - name: homelab-ca # homelab-ca is a ConfigMap (public CA trust bundle), not a Secret. # The volumeMounts use subPath: ca.crt to project the single cert file. diff --git a/k8s/infra/forgejo-runner/values-node.yaml b/k8s/infra/forgejo-runner/values-node.yaml index c7861e8..e75a0aa 100644 --- a/k8s/infra/forgejo-runner/values-node.yaml +++ b/k8s/infra/forgejo-runner/values-node.yaml @@ -2,9 +2,12 @@ # runner instance. Only runner.name and runner.labels differ -- everything # else (image, dind, persistence, tolerations, nodeSelector) is shared. # -# node:22-bookworm ships Node natively. Docker client installed via workflow step if needed. -# (homelab has no CI; custom runner images built manually if desired) +# Label image: node:22-bookworm — Debian, root, apt-get, Node.js, npm, git. +# Install docker in workflow steps as needed. runner: + image: + repository: code.forgejo.org/forgejo/runner + tag: "6" name: node-runner labels: "node:docker://node:22-bookworm" diff --git a/k8s/infra/forgejo-runner/values-rust.yaml b/k8s/infra/forgejo-runner/values-rust.yaml index b0c00d2..b645cf3 100644 --- a/k8s/infra/forgejo-runner/values-rust.yaml +++ b/k8s/infra/forgejo-runner/values-rust.yaml @@ -2,20 +2,16 @@ # runner instance. Only runner.name and runner.labels differ -- everything # else (image, dind, persistence, tolerations, nodeSelector) is shared. # -# Use docker:27-cli instead of rust:1.83-bookworm because: -# - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.) -# - Has docker CLI + git + full dev tools -# - rust:1.83-bookworm lacks Node.js (causes action failures) -# -# docker:27-cli verified: docker manifest inspect docker:27-cli ✓ +# Label image: rust:1-bookworm — Debian, root, apt-get, Rust, cargo, git. +# Install Node.js/docker in workflow steps as needed. runner: + image: + repository: code.forgejo.org/forgejo/runner + tag: "6" name: rust-runner - labels: "rust:docker://docker:27-cli" + labels: "rust:docker://rust:1-bookworm" + -persistence: - reg: - storageClass: longhorn - size: 20Gi # GC CronJob renders only from the default (golang) values to avoid duplicates gc: diff --git a/k8s/infra/forgejo-runner/values.yaml b/k8s/infra/forgejo-runner/values.yaml index 88de7d1..c8e7052 100644 --- a/k8s/infra/forgejo-runner/values.yaml +++ b/k8s/infra/forgejo-runner/values.yaml @@ -1,20 +1,13 @@ runner: image: repository: code.forgejo.org/forgejo/runner - tag: "6" # pin exact release before apply + tag: "6" name: golang-runner - # Use docker:27-cli instead of golang:1.26-bookworm because: - # - Needs Node.js for GitHub Actions (actions/checkout@v4, etc.) - # - Has docker CLI + git + golang + all build tools - # - golang:1.26-bookworm lacks Node.js (causes action failures) - # docker:27-cli includes Go toolchain via base debian + additional packages - # Verified tag: docker manifest inspect docker:27-cli ✓ - labels: "golang:docker://docker:27-cli" - # In-cluster Service (:3000) — direct, avoids the ingress/public-hostname hop - # (the public URL is :443 which forgejo doesn't serve; runner got i/o timeout). + # Label image is what workflow steps run in (NOT the runner daemon image). + # golang:1.26-bookworm: Debian, root, apt-get, Go, git. + # TODO: Switch to custom image once build-runner-images.yml pushes images + labels: "golang:docker://golang:1.26-bookworm" forgejoUrl: http://forgejo-gitea-http.cicd.svc.cluster.local:3000 - # tokenSecret: name of the K8s Secret that holds the runner registration token - # created automatically by the helmfile presync hook (see helmfile.yaml.gotmpl) tokenSecret: runner-token resources: requests: diff --git a/terraform/files/coredns/Corefile b/terraform/files/coredns/Corefile index 6df1c75..03b2fbf 100644 --- a/terraform/files/coredns/Corefile +++ b/terraform/files/coredns/Corefile @@ -35,6 +35,7 @@ rewrite name longhorn.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name paperless.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name img.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local + rewrite name api.riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local rewrite name riotpiao.com ingress-nginx-controller.ingress-nginx.svc.cluster.local kubernetes cluster.local in-addr.arpa ip6.arpa {