homelab/k8s/applications/forge/runner.yaml

# k8s/forge/runner.yaml
# Forgejo Actions runner with Docker-in-Docker (DinD) sidecar.
# Phase 3.2 of talos_version_control.html.
#
# Pod layout (two containers, one pod):
#   register   initContainer — registers with Forgejo once; skips if .runner exists
#   runner     main container — long-running daemon that polls Forgejo for jobs
#   dind       sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376
#
# Prerequisites (Phase 3.1):
#   TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \
#     gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n')
#   kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN"
#   # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that
#   # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a
#   # stale CA from before the "unified certificate" migration.
#   kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \
#     | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin
#
# CA trust for ephemeral job containers (Phase 3.3):
#   The homelab-ca secret above only reaches the runner/register/dind containers.
#   It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint,
#   node:22-bookworm) — those are fresh sibling containers with their own trust
#   store. Without this, any git/curl/apk call to forgejo.riotpiao.com
#   from inside a job fails with "SSL certificate problem: unable to get local
#   issuer certificate". Build a merged bundle (public roots + homelab CA) and
#   bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job
#   container via forgejo-runner's config.yaml container.options.
#
#   IMPORTANT: source the CA from the live cluster secret, NOT from
#   k8s/forge/pki/ca.crt — that repo file is a stale CA from before the
#   "unified certificate" migration (different key, fails verification
#   against the cert actually served by forgejo.riotpiao.com). The
#   org-wide CA that signs the live ingress cert lives in
#   cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already
#   synced from it.
#     docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt
#     kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt
#     kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt
#   Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml).
#
# Apply:
#   kubectl apply -f k8s/forge/runner.yaml
#   kubectl -n cicd rollout status deploy/forgejo-runner
#   kubectl -n cicd logs deploy/forgejo-runner -c runner -f
#   # expect: "runner: daemon started" / "connected to Forgejo"

# ── PVCs ──────────────────────────────────────────────────────────────────────
# runner-reg  — persists the .runner registration file so the runner doesn't
#               re-register on every pod restart (token is one-use-per-registration)
# runner-dind — persists the Docker layer cache across pod restarts; keeps
#               rebuilds fast — images don't need to be re-pulled every time
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: runner-reg
  namespace: cicd
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: longhorn
  resources:
    requests:
      storage: 1Gi

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: runner-dind
  namespace: cicd
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: longhorn
  resources:
    requests:
      storage: 30Gi

---
# ── DinD TLS certs, issued by the homelab's unified CA ───────────────────────
# DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA
# + server/client cert pair on every container start if none is supplied. Its
# server cert's SAN list only ever covers "docker", the pod hostname, and
# "localhost" - so anything reaching it via a stable Service DNS name (added
# below for story-crater-backend's release.yaml to build/push images) fails
# TLS hostname verification, even though the handshake itself succeeds.
#
# Fix: supply our own server+client cert pair, both issued by the same
# ClusterIssuer (homelab-ca) that already signs the live ingress cert, so
# they share one trust root. dockerd-entrypoint.sh skips its own generation
# step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem
# already present and no CA private key alongside them (confirmed by reading
# the script directly: `kubectl exec -n cicd <pod> -c dind -- cat
# /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA"
# path it's designed for.
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: forgejo-runner-dind-server
  namespace: cicd
spec:
  secretName: forgejo-runner-dind-server-tls
  issuerRef:
    name: homelab-ca
    kind: ClusterIssuer
  commonName: docker:dind server
  dnsNames:
    - forgejo-runner-dind.cicd.svc.cluster.local
    - forgejo-runner-dind.cicd.svc
    - forgejo-runner-dind
    - docker
    - localhost
  usages:
    - server auth
    - digital signature
    - key encipherment

---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: forgejo-runner-dind-client
  namespace: cicd
spec:
  secretName: forgejo-runner-dind-client-tls
  issuerRef:
    name: homelab-ca
    kind: ClusterIssuer
  commonName: docker:dind client
  usages:
    - client auth
    - digital signature
    - key encipherment

---
# Stable address for the dind sidecar's docker API (2376, mTLS) - lets
# CI workflows (e.g. story-crater-backend's release.yaml) build/push images
# by reaching this runner's own already-working outer dind directly, instead
# of the per-job `services:` sidecar pattern (confirmed broken: act-runner
# never registers a DNS alias for service containers - job container's
# /etc/hosts has no entry for it, `docker info` fails with a DNS lookup
# error, not a TLS/connection error).
apiVersion: v1
kind: Service
metadata:
  name: forgejo-runner-dind
  namespace: cicd
spec:
  selector:
    app: forgejo-runner
  ports:
    - port: 2376
      targetPort: 2376

---
# ── Runner config ─────────────────────────────────────────────────────────────
# container.options is appended to every `docker run` DinD issues for a job
# container, so this is what actually gets the merged CA bundle (ca-bundle
# secret, see header comment) trusted inside golangci-lint, node, etc. - and
# now also what propagates the homelab-CA-signed client cert (above) into
# job containers that need to talk back to dind themselves (e.g. building
# and pushing images).
# Source paths resolve against the dind container's filesystem (it's the
# daemon creating these containers), so both are mounted into dind below.
apiVersion: v1
kind: ConfigMap
metadata:
  name: forgejo-runner-config
  namespace: cicd
data:
  config.yaml: |
    container:
      options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro
      # Without this, sanitizeConfig() in forgejo/act silently drops any bind
      # mount whose source isn't allowlisted here — including ones injected
      # via container.options above, not just workflow-declared volumes.
      valid_volumes:
        - /etc/forgejo-ca/ca-certificates.crt
        - /docker-certs/client

---
# ── Deployment ────────────────────────────────────────────────────────────────
apiVersion: apps/v1
kind: Deployment
metadata:
  name: forgejo-runner
  namespace: cicd
spec:
  replicas: 1
  # RWO PVCs mean only one pod can mount them at a time.
  # Recreate ensures the old pod fully terminates before the new one starts.
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: forgejo-runner
  template:
    metadata:
      labels:
        app: forgejo-runner
    spec:
      # runner/register containers run as uid 1000 (image default); fsGroup
      # makes kubelet chown+chmod the Longhorn PVC's group to 1000 with
      # write access, otherwise writes to /data (.runner config) fail with
      # "permission denied" since the volume is root:root 755 by default.
      securityContext:
        fsGroup: 1000
      tolerations:
        - key: node-role.kubernetes.io/control-plane
          operator: Exists
          effect: NoSchedule

      initContainers:
        # Registers the runner with Forgejo exactly once.
        # test -f /data/.runner makes it idempotent — re-registration would
        # consume the one-time token and break the runner.
        - name: register
          image: code.forgejo.org/forgejo/runner:6
          command: ["sh", "-c"]
          args:
            - |
              test -f /data/.runner && echo "already registered, skipping" && exit 0
              forgejo-runner register --no-interactive \
                --instance https://forgejo.riotpiao.com \
                --token "$RUNNER_TOKEN" \
                --name talos-runner \
                --labels "docker:docker://node:22-bookworm"
          env:
            - name: RUNNER_TOKEN
              valueFrom:
                secretKeyRef:
                  name: runner-token
                  key: token
          volumeMounts:
            - name: runner-data
              mountPath: /data
            # CA cert so the register call can verify Forgejo's TLS cert
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
          workingDir: /data

      containers:
        # ── Runner daemon ────────────────────────────────────────────────────
        # Polls Forgejo for pending jobs and executes them inside DinD.
        # The `until docker info` loop waits for the DinD sidecar to finish
        # its TLS setup before starting the daemon — without this the runner
        # starts before Docker is ready and immediately errors out.
        - name: runner
          image: code.forgejo.org/forgejo/runner:6
          command: ["sh", "-c"]
          args:
            - |
              until nc -z localhost 2376 >/dev/null 2>&1; do
                echo "waiting for docker daemon..."; sleep 2
              done
              forgejo-runner daemon --config /data/config.yaml
          workingDir: /data
          env:
            # Connect to the DinD sidecar via mTLS on localhost
            - name: DOCKER_HOST
              value: tcp://localhost:2376
            - name: DOCKER_TLS_VERIFY
              value: "1"
            - name: DOCKER_CERT_PATH
              value: /docker-certs/client
          volumeMounts:
            - name: runner-data
              mountPath: /data
            - name: docker-certs
              mountPath: /docker-certs
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
            # forgejo-runner's container.options, read from this file, is what
            # propagates the CA bundle into per-job containers (see ca-bundle
            # secret + dind mount below)
            - name: runner-config
              mountPath: /data/config.yaml
              subPath: config.yaml
            # Homelab-CA-signed client cert (overlays whatever's in the
            # docker-certs emptyDir at this subpath) - matches the server
            # cert dind now presents, see Certificates above.
            - name: dind-client-tls
              mountPath: /docker-certs/client
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: "2"
              memory: 4Gi

        # ── DinD sidecar ──────────────────────────────────────────────────────
        # Full Docker daemon running inside the pod.
        # privileged: true is required for DinD — the cicd namespace is labelled
        # pod-security.kubernetes.io/enforce=privileged to allow this.
        # DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs
        # on startup; the runner reads the client certs from /docker-certs/client.
        # runner-dind PVC mounts /var/lib/docker so the layer cache persists
        # across pod restarts.
        - name: dind
          image: docker:27-dind
          securityContext:
            privileged: true
          env:
            - name: DOCKER_TLS_CERTDIR
              value: /docker-certs
          volumeMounts:
            - name: docker-certs
              mountPath: /docker-certs
            - name: dind-storage
              mountPath: /var/lib/docker
            # Trust the homelab CA so DinD can pull from Forgejo's OCI registry
            - name: homelab-ca
              mountPath: /etc/ssl/certs/homelab-ca.pem
              subPath: ca.crt
            # Merged CA bundle (public roots + homelab CA), bind-mounted from
            # here into every job container by container.options above —
            # this path is resolved against dind's filesystem since dind is
            # the daemon actually creating those containers.
            - name: ca-bundle
              mountPath: /etc/forgejo-ca/ca-certificates.crt
              subPath: ca-certificates.crt
            # Homelab-CA-signed server+client certs (see Certificates above),
            # overlaying the matching subpaths of the docker-certs emptyDir.
            # dockerd-entrypoint.sh detects these and skips its own
            # self-signed generation entirely (no CA private key is supplied
            # alongside them, so it can't regenerate even if it wanted to).
            - name: dind-server-tls
              mountPath: /docker-certs/server
            - name: dind-client-tls
              mountPath: /docker-certs/client
          resources:
            requests:
              cpu: 100m
              memory: 256Mi
            limits:
              cpu: "2"
              memory: 4Gi

      volumes:
        - name: runner-data
          persistentVolumeClaim:
            claimName: runner-reg
        - name: dind-storage
          persistentVolumeClaim:
            claimName: runner-dind
        # emptyDir parent mount for /docker-certs - the server/ and client/
        # subpaths are now overlaid by the homelab-CA-signed dind-server-tls/
        # dind-client-tls secret mounts below (dockerd-entrypoint.sh no
        # longer self-generates once it finds those present). This emptyDir
        # just needs to exist as the parent directory; nothing writes
        # directly to it anymore.
        - name: docker-certs
          emptyDir: {}
        - name: homelab-ca
          secret:
            secretName: homelab-ca
        - name: ca-bundle
          secret:
            secretName: ca-bundle
        - name: runner-config
          configMap:
            name: forgejo-runner-config
        # cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to
        # the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects
        # under $DOCKER_TLS_CERTDIR/{server,client}/.
        - name: dind-server-tls
          secret:
            secretName: forgejo-runner-dind-server-tls
            items:
              - key: ca.crt
                path: ca.pem
              - key: tls.crt
                path: cert.pem
              - key: tls.key
                path: key.pem
        - name: dind-client-tls
          secret:
            secretName: forgejo-runner-dind-client-tls
            items:
              - key: ca.crt
                path: ca.pem
              - key: tls.crt
                path: cert.pem
              - key: tls.key
                path: key.pem

---
# ── NetworkPolicy ─────────────────────────────────────────────────────────────
# Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and
# the public internet for action dependencies and base images.
# LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to
# prevent a compromised CI job from pivoting into the cluster or LAN.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: runner-egress
  namespace: cicd
spec:
  podSelector:
    matchLabels:
      app: forgejo-runner
  policyTypes: [Egress]
  egress:
    # Forgejo (same namespace — git push, OCI registry push/pull)
    - to:
        - podSelector: {}
    # ingress-nginx (the runner talks to Forgejo via its public hostname,
    # https://forgejo.riotpiao.com, which resolves to the ingress
    # controller's ClusterIP — a different namespace on the pod network)
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: ingress-nginx
      ports:
        - protocol: TCP
          port: 443
        - protocol: TCP
          port: 80
    # CoreDNS (DNS resolution for action deps and Forgejo hostname)
    - to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: kube-system
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53
    # Public internet for action dependencies and base images
    # LAN and pod network are explicitly excluded
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
            except:
              - 192.168.1.0/24
              - 10.244.0.0/16