# k8s/forge/runner.yaml # Forgejo Actions runner with Docker-in-Docker (DinD) sidecar. # Phase 3.2 of talos_version_control.html. # # Pod layout (two containers, one pod): # register initContainer — registers with Forgejo once; skips if .runner exists # runner main container — long-running daemon that polls Forgejo for jobs # dind sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376 # # Prerequisites (Phase 3.1): # TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \ # gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n') # kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN" # # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that # # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a # # stale CA from before the "unified certificate" migration. # kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \ # | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin # # CA trust for ephemeral job containers (Phase 3.3): # The homelab-ca secret above only reaches the runner/register/dind containers. # It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint, # node:22-bookworm) — those are fresh sibling containers with their own trust # store. Without this, any git/curl/apk call to forgejo.riotpiao.homelab.com # from inside a job fails with "SSL certificate problem: unable to get local # issuer certificate". Build a merged bundle (public roots + homelab CA) and # bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job # container via forgejo-runner's config.yaml container.options. # # IMPORTANT: source the CA from the live cluster secret, NOT from # k8s/forge/pki/ca.crt — that repo file is a stale CA from before the # "unified certificate" migration (different key, fails verification # against the cert actually served by forgejo.riotpiao.homelab.com). The # org-wide CA that signs the live ingress cert lives in # cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already # synced from it. # docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt # kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt # kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt # Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml). # # Apply: # kubectl apply -f k8s/forge/runner.yaml # kubectl -n cicd rollout status deploy/forgejo-runner # kubectl -n cicd logs deploy/forgejo-runner -c runner -f # # expect: "runner: daemon started" / "connected to Forgejo" # ── PVCs ────────────────────────────────────────────────────────────────────── # runner-reg — persists the .runner registration file so the runner doesn't # re-register on every pod restart (token is one-use-per-registration) # runner-dind — persists the Docker layer cache across pod restarts; keeps # rebuilds fast — images don't need to be re-pulled every time apiVersion: v1 kind: PersistentVolumeClaim metadata: name: runner-reg namespace: cicd spec: accessModes: [ReadWriteOnce] storageClassName: longhorn resources: requests: storage: 1Gi --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: runner-dind namespace: cicd spec: accessModes: [ReadWriteOnce] storageClassName: longhorn resources: requests: storage: 30Gi --- # ── DinD TLS certs, issued by the homelab's unified CA ─────────────────────── # DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA # + server/client cert pair on every container start if none is supplied. Its # server cert's SAN list only ever covers "docker", the pod hostname, and # "localhost" - so anything reaching it via a stable Service DNS name (added # below for story-crater-backend's release.yaml to build/push images) fails # TLS hostname verification, even though the handshake itself succeeds. # # Fix: supply our own server+client cert pair, both issued by the same # ClusterIssuer (homelab-ca) that already signs the live ingress cert, so # they share one trust root. dockerd-entrypoint.sh skips its own generation # step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem # already present and no CA private key alongside them (confirmed by reading # the script directly: `kubectl exec -n cicd -c dind -- cat # /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA" # path it's designed for. apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: forgejo-runner-dind-server namespace: cicd spec: secretName: forgejo-runner-dind-server-tls issuerRef: name: homelab-ca kind: ClusterIssuer commonName: docker:dind server dnsNames: - forgejo-runner-dind.cicd.svc.cluster.local - forgejo-runner-dind.cicd.svc - forgejo-runner-dind - docker - localhost usages: - server auth - digital signature - key encipherment --- apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: forgejo-runner-dind-client namespace: cicd spec: secretName: forgejo-runner-dind-client-tls issuerRef: name: homelab-ca kind: ClusterIssuer commonName: docker:dind client usages: - client auth - digital signature - key encipherment --- # Stable address for the dind sidecar's docker API (2376, mTLS) - lets # CI workflows (e.g. story-crater-backend's release.yaml) build/push images # by reaching this runner's own already-working outer dind directly, instead # of the per-job `services:` sidecar pattern (confirmed broken: act-runner # never registers a DNS alias for service containers - job container's # /etc/hosts has no entry for it, `docker info` fails with a DNS lookup # error, not a TLS/connection error). apiVersion: v1 kind: Service metadata: name: forgejo-runner-dind namespace: cicd spec: selector: app: forgejo-runner ports: - port: 2376 targetPort: 2376 --- # ── Runner config ───────────────────────────────────────────────────────────── # container.options is appended to every `docker run` DinD issues for a job # container, so this is what actually gets the merged CA bundle (ca-bundle # secret, see header comment) trusted inside golangci-lint, node, etc. - and # now also what propagates the homelab-CA-signed client cert (above) into # job containers that need to talk back to dind themselves (e.g. building # and pushing images). # Source paths resolve against the dind container's filesystem (it's the # daemon creating these containers), so both are mounted into dind below. apiVersion: v1 kind: ConfigMap metadata: name: forgejo-runner-config namespace: cicd data: config.yaml: | container: options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro # Without this, sanitizeConfig() in forgejo/act silently drops any bind # mount whose source isn't allowlisted here — including ones injected # via container.options above, not just workflow-declared volumes. valid_volumes: - /etc/forgejo-ca/ca-certificates.crt - /docker-certs/client --- # ── Deployment ──────────────────────────────────────────────────────────────── apiVersion: apps/v1 kind: Deployment metadata: name: forgejo-runner namespace: cicd spec: replicas: 1 # RWO PVCs mean only one pod can mount them at a time. # Recreate ensures the old pod fully terminates before the new one starts. strategy: type: Recreate selector: matchLabels: app: forgejo-runner template: metadata: labels: app: forgejo-runner spec: # runner/register containers run as uid 1000 (image default); fsGroup # makes kubelet chown+chmod the Longhorn PVC's group to 1000 with # write access, otherwise writes to /data (.runner config) fail with # "permission denied" since the volume is root:root 755 by default. securityContext: fsGroup: 1000 tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule initContainers: # Registers the runner with Forgejo exactly once. # test -f /data/.runner makes it idempotent — re-registration would # consume the one-time token and break the runner. - name: register image: code.forgejo.org/forgejo/runner:6 command: ["sh", "-c"] args: - | test -f /data/.runner && echo "already registered, skipping" && exit 0 forgejo-runner register --no-interactive \ --instance https://forgejo.riotpiao.homelab.com \ --token "$RUNNER_TOKEN" \ --name talos-runner \ --labels "docker:docker://node:22-bookworm" env: - name: RUNNER_TOKEN valueFrom: secretKeyRef: name: runner-token key: token volumeMounts: - name: runner-data mountPath: /data # CA cert so the register call can verify Forgejo's TLS cert - name: homelab-ca mountPath: /etc/ssl/certs/homelab-ca.pem subPath: ca.crt workingDir: /data containers: # ── Runner daemon ──────────────────────────────────────────────────── # Polls Forgejo for pending jobs and executes them inside DinD. # The `until docker info` loop waits for the DinD sidecar to finish # its TLS setup before starting the daemon — without this the runner # starts before Docker is ready and immediately errors out. - name: runner image: code.forgejo.org/forgejo/runner:6 command: ["sh", "-c"] args: - | until nc -z localhost 2376 >/dev/null 2>&1; do echo "waiting for docker daemon..."; sleep 2 done forgejo-runner daemon --config /data/config.yaml workingDir: /data env: # Connect to the DinD sidecar via mTLS on localhost - name: DOCKER_HOST value: tcp://localhost:2376 - name: DOCKER_TLS_VERIFY value: "1" - name: DOCKER_CERT_PATH value: /docker-certs/client volumeMounts: - name: runner-data mountPath: /data - name: docker-certs mountPath: /docker-certs - name: homelab-ca mountPath: /etc/ssl/certs/homelab-ca.pem subPath: ca.crt # forgejo-runner's container.options, read from this file, is what # propagates the CA bundle into per-job containers (see ca-bundle # secret + dind mount below) - name: runner-config mountPath: /data/config.yaml subPath: config.yaml # Homelab-CA-signed client cert (overlays whatever's in the # docker-certs emptyDir at this subpath) - matches the server # cert dind now presents, see Certificates above. - name: dind-client-tls mountPath: /docker-certs/client resources: requests: cpu: 100m memory: 256Mi limits: cpu: "2" memory: 4Gi # ── DinD sidecar ────────────────────────────────────────────────────── # Full Docker daemon running inside the pod. # privileged: true is required for DinD — the cicd namespace is labelled # pod-security.kubernetes.io/enforce=privileged to allow this. # DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs # on startup; the runner reads the client certs from /docker-certs/client. # runner-dind PVC mounts /var/lib/docker so the layer cache persists # across pod restarts. - name: dind image: docker:27-dind securityContext: privileged: true env: - name: DOCKER_TLS_CERTDIR value: /docker-certs volumeMounts: - name: docker-certs mountPath: /docker-certs - name: dind-storage mountPath: /var/lib/docker # Trust the homelab CA so DinD can pull from Forgejo's OCI registry - name: homelab-ca mountPath: /etc/ssl/certs/homelab-ca.pem subPath: ca.crt # Merged CA bundle (public roots + homelab CA), bind-mounted from # here into every job container by container.options above — # this path is resolved against dind's filesystem since dind is # the daemon actually creating those containers. - name: ca-bundle mountPath: /etc/forgejo-ca/ca-certificates.crt subPath: ca-certificates.crt # Homelab-CA-signed server+client certs (see Certificates above), # overlaying the matching subpaths of the docker-certs emptyDir. # dockerd-entrypoint.sh detects these and skips its own # self-signed generation entirely (no CA private key is supplied # alongside them, so it can't regenerate even if it wanted to). - name: dind-server-tls mountPath: /docker-certs/server - name: dind-client-tls mountPath: /docker-certs/client resources: requests: cpu: 100m memory: 256Mi limits: cpu: "2" memory: 4Gi volumes: - name: runner-data persistentVolumeClaim: claimName: runner-reg - name: dind-storage persistentVolumeClaim: claimName: runner-dind # emptyDir parent mount for /docker-certs - the server/ and client/ # subpaths are now overlaid by the homelab-CA-signed dind-server-tls/ # dind-client-tls secret mounts below (dockerd-entrypoint.sh no # longer self-generates once it finds those present). This emptyDir # just needs to exist as the parent directory; nothing writes # directly to it anymore. - name: docker-certs emptyDir: {} - name: homelab-ca secret: secretName: homelab-ca - name: ca-bundle secret: secretName: ca-bundle - name: runner-config configMap: name: forgejo-runner-config # cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to # the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects # under $DOCKER_TLS_CERTDIR/{server,client}/. - name: dind-server-tls secret: secretName: forgejo-runner-dind-server-tls items: - key: ca.crt path: ca.pem - key: tls.crt path: cert.pem - key: tls.key path: key.pem - name: dind-client-tls secret: secretName: forgejo-runner-dind-client-tls items: - key: ca.crt path: ca.pem - key: tls.crt path: cert.pem - key: tls.key path: key.pem --- # ── NetworkPolicy ───────────────────────────────────────────────────────────── # Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and # the public internet for action dependencies and base images. # LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to # prevent a compromised CI job from pivoting into the cluster or LAN. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: runner-egress namespace: cicd spec: podSelector: matchLabels: app: forgejo-runner policyTypes: [Egress] egress: # Forgejo (same namespace — git push, OCI registry push/pull) - to: - podSelector: {} # ingress-nginx (the runner talks to Forgejo via its public hostname, # https://forgejo.riotpiao.homelab.com, which resolves to the ingress # controller's ClusterIP — a different namespace on the pod network) - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: ingress-nginx ports: - protocol: TCP port: 443 - protocol: TCP port: 80 # CoreDNS (DNS resolution for action deps and Forgejo hostname) - to: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system ports: - protocol: UDP port: 53 - protocol: TCP port: 53 # Public internet for action dependencies and base images # LAN and pod network are explicitly excluded - to: - ipBlock: cidr: 0.0.0.0/0 except: - 192.168.1.0/24 - 10.244.0.0/16