rock/homelab
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

k8s/aux: add cert-manager longhorn dashboard forge dev-tools and shadowsocks

Browse Source 
- cert-manager ClusterIssuers (LetsEncrypt + homelab-ca)
- Longhorn storage dashboard
- Portainer dashboard config
- Forgejo git service
- Claude terminal remote access
- Shadowsocks tunnel for remote access
This commit is contained in:
Story Crater Bot
2026-07-11 19:19:59 -07:00
parent 18f2f94f8e
commit e1d0cfd70b
18 changed files with 1201 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/forge/runner-gc-cronjob.yaml
+95
View File
@@ -0,0 +1,95 @@
# k8s/forge/runner-gc-cronjob.yaml
# Garbage-collects the forgejo-runner's DinD layer cache (runner-dind PVC,
# 30Gi). Every CI build/pull only adds images and build-cache layers — there
# is no automatic pruning, so without this the PVC fills up and breaks builds.
#
# Runs `docker image prune` / `docker builder prune` inside the live dind
# container via `kubectl exec`, rather than a sidecar in the runner pod itself,
# so it can run on its own schedule independent of runner restarts.
apiVersion: v1
kind: ServiceAccount
metadata:
name: runner-gc
namespace: cicd
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: runner-gc
namespace: cicd
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["pods/exec"]
verbs: ["create"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: runner-gc
namespace: cicd
subjects:
- kind: ServiceAccount
name: runner-gc
namespace: cicd
roleRef:
kind: Role
name: runner-gc
apiGroup: rbac.authorization.k8s.io
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: forgejo-runner-image-gc
namespace: cicd
spec:
schedule: "0 3 * * *" # daily 03:00
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 1
activeDeadlineSeconds: 600
template:
spec:
serviceAccountName: runner-gc
restartPolicy: Never
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
containers:
- name: gc
image: alpine/k8s:1.31.0
command:
- sh
- -c
- |
set -e
POD=$(kubectl -n cicd get pod -l app=forgejo-runner -o jsonpath='{.items[0].metadata.name}')
if [ -z "$POD" ]; then
echo "no forgejo-runner pod found, skipping"
exit 0
fi
echo "before:"
kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
echo "pruning images unused for >72h on $POD"
kubectl -n cicd exec "$POD" -c dind -- docker image prune -af --filter "until=72h"
echo "pruning build cache unused for >72h on $POD"
kubectl -n cicd exec "$POD" -c dind -- docker builder prune -af --filter "until=72h"
echo "after:"
kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 250m
memory: 128Mi
k8s/forge/runner.yaml
+438
View File
@@ -0,0 +1,438 @@
# k8s/forge/runner.yaml
# Forgejo Actions runner with Docker-in-Docker (DinD) sidecar.
# Phase 3.2 of talos_version_control.html.
#
# Pod layout (two containers, one pod):
# register initContainer — registers with Forgejo once; skips if .runner exists
# runner main container — long-running daemon that polls Forgejo for jobs
# dind sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376
#
# Prerequisites (Phase 3.1):
# TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \
# gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n')
# kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN"
# # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that
# # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a
# # stale CA from before the "unified certificate" migration.
# kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \
# | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin
#
# CA trust for ephemeral job containers (Phase 3.3):
# The homelab-ca secret above only reaches the runner/register/dind containers.
# It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint,
# node:22-bookworm) — those are fresh sibling containers with their own trust
# store. Without this, any git/curl/apk call to forgejo.riotpiao.homelab.com
# from inside a job fails with "SSL certificate problem: unable to get local
# issuer certificate". Build a merged bundle (public roots + homelab CA) and
# bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job
# container via forgejo-runner's config.yaml container.options.
#
# IMPORTANT: source the CA from the live cluster secret, NOT from
# k8s/forge/pki/ca.crt — that repo file is a stale CA from before the
# "unified certificate" migration (different key, fails verification
# against the cert actually served by forgejo.riotpiao.homelab.com). The
# org-wide CA that signs the live ingress cert lives in
# cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already
# synced from it.
# docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt
# kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt
# kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt
# Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml).
#
# Apply:
# kubectl apply -f k8s/forge/runner.yaml
# kubectl -n cicd rollout status deploy/forgejo-runner
# kubectl -n cicd logs deploy/forgejo-runner -c runner -f
# # expect: "runner: daemon started" / "connected to Forgejo"
# ── PVCs ──────────────────────────────────────────────────────────────────────
# runner-reg — persists the .runner registration file so the runner doesn't
# re-register on every pod restart (token is one-use-per-registration)
# runner-dind — persists the Docker layer cache across pod restarts; keeps
# rebuilds fast — images don't need to be re-pulled every time
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: runner-reg
namespace: cicd
spec:
accessModes: [ReadWriteOnce]
storageClassName: longhorn
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: runner-dind
namespace: cicd
spec:
accessModes: [ReadWriteOnce]
storageClassName: longhorn
resources:
requests:
storage: 30Gi
---
# ── DinD TLS certs, issued by the homelab's unified CA ───────────────────────
# DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA
# + server/client cert pair on every container start if none is supplied. Its
# server cert's SAN list only ever covers "docker", the pod hostname, and
# "localhost" - so anything reaching it via a stable Service DNS name (added
# below for story-crater-backend's release.yaml to build/push images) fails
# TLS hostname verification, even though the handshake itself succeeds.
#
# Fix: supply our own server+client cert pair, both issued by the same
# ClusterIssuer (homelab-ca) that already signs the live ingress cert, so
# they share one trust root. dockerd-entrypoint.sh skips its own generation
# step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem
# already present and no CA private key alongside them (confirmed by reading
# the script directly: `kubectl exec -n cicd <pod> -c dind -- cat
# /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA"
# path it's designed for.
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: forgejo-runner-dind-server
namespace: cicd
spec:
secretName: forgejo-runner-dind-server-tls
issuerRef:
name: homelab-ca
kind: ClusterIssuer
commonName: docker:dind server
dnsNames:
- forgejo-runner-dind.cicd.svc.cluster.local
- forgejo-runner-dind.cicd.svc
- forgejo-runner-dind
- docker
- localhost
usages:
- server auth
- digital signature
- key encipherment
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: forgejo-runner-dind-client
namespace: cicd
spec:
secretName: forgejo-runner-dind-client-tls
issuerRef:
name: homelab-ca
kind: ClusterIssuer
commonName: docker:dind client
usages:
- client auth
- digital signature
- key encipherment
---
# Stable address for the dind sidecar's docker API (2376, mTLS) - lets
# CI workflows (e.g. story-crater-backend's release.yaml) build/push images
# by reaching this runner's own already-working outer dind directly, instead
# of the per-job `services:` sidecar pattern (confirmed broken: act-runner
# never registers a DNS alias for service containers - job container's
# /etc/hosts has no entry for it, `docker info` fails with a DNS lookup
# error, not a TLS/connection error).
apiVersion: v1
kind: Service
metadata:
name: forgejo-runner-dind
namespace: cicd
spec:
selector:
app: forgejo-runner
ports:
- port: 2376
targetPort: 2376
---
# ── Runner config ─────────────────────────────────────────────────────────────
# container.options is appended to every `docker run` DinD issues for a job
# container, so this is what actually gets the merged CA bundle (ca-bundle
# secret, see header comment) trusted inside golangci-lint, node, etc. - and
# now also what propagates the homelab-CA-signed client cert (above) into
# job containers that need to talk back to dind themselves (e.g. building
# and pushing images).
# Source paths resolve against the dind container's filesystem (it's the
# daemon creating these containers), so both are mounted into dind below.
apiVersion: v1
kind: ConfigMap
metadata:
name: forgejo-runner-config
namespace: cicd
data:
config.yaml: |
container:
options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro
# Without this, sanitizeConfig() in forgejo/act silently drops any bind
# mount whose source isn't allowlisted here — including ones injected
# via container.options above, not just workflow-declared volumes.
valid_volumes:
- /etc/forgejo-ca/ca-certificates.crt
- /docker-certs/client
---
# ── Deployment ────────────────────────────────────────────────────────────────
apiVersion: apps/v1
kind: Deployment
metadata:
name: forgejo-runner
namespace: cicd
spec:
replicas: 1
# RWO PVCs mean only one pod can mount them at a time.
# Recreate ensures the old pod fully terminates before the new one starts.
strategy:
type: Recreate
selector:
matchLabels:
app: forgejo-runner
template:
metadata:
labels:
app: forgejo-runner
spec:
# runner/register containers run as uid 1000 (image default); fsGroup
# makes kubelet chown+chmod the Longhorn PVC's group to 1000 with
# write access, otherwise writes to /data (.runner config) fail with
# "permission denied" since the volume is root:root 755 by default.
securityContext:
fsGroup: 1000
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
initContainers:
# Registers the runner with Forgejo exactly once.
# test -f /data/.runner makes it idempotent — re-registration would
# consume the one-time token and break the runner.
- name: register
image: code.forgejo.org/forgejo/runner:6
command: ["sh", "-c"]
args:
- |
test -f /data/.runner && echo "already registered, skipping" && exit 0
forgejo-runner register --no-interactive \
--instance https://forgejo.riotpiao.homelab.com \
--token "$RUNNER_TOKEN" \
--name talos-runner \
--labels "docker:docker://node:22-bookworm"
env:
- name: RUNNER_TOKEN
valueFrom:
secretKeyRef:
name: runner-token
key: token
volumeMounts:
- name: runner-data
mountPath: /data
# CA cert so the register call can verify Forgejo's TLS cert
- name: homelab-ca
mountPath: /etc/ssl/certs/homelab-ca.pem
subPath: ca.crt
workingDir: /data
containers:
# ── Runner daemon ────────────────────────────────────────────────────
# Polls Forgejo for pending jobs and executes them inside DinD.
# The `until docker info` loop waits for the DinD sidecar to finish
# its TLS setup before starting the daemon — without this the runner
# starts before Docker is ready and immediately errors out.
- name: runner
image: code.forgejo.org/forgejo/runner:6
command: ["sh", "-c"]
args:
- |
until nc -z localhost 2376 >/dev/null 2>&1; do
echo "waiting for docker daemon..."; sleep 2
done
forgejo-runner daemon --config /data/config.yaml
workingDir: /data
env:
# Connect to the DinD sidecar via mTLS on localhost
- name: DOCKER_HOST
value: tcp://localhost:2376
- name: DOCKER_TLS_VERIFY
value: "1"
- name: DOCKER_CERT_PATH
value: /docker-certs/client
volumeMounts:
- name: runner-data
mountPath: /data
- name: docker-certs
mountPath: /docker-certs
- name: homelab-ca
mountPath: /etc/ssl/certs/homelab-ca.pem
subPath: ca.crt
# forgejo-runner's container.options, read from this file, is what
# propagates the CA bundle into per-job containers (see ca-bundle
# secret + dind mount below)
- name: runner-config
mountPath: /data/config.yaml
subPath: config.yaml
# Homelab-CA-signed client cert (overlays whatever's in the
# docker-certs emptyDir at this subpath) - matches the server
# cert dind now presents, see Certificates above.
- name: dind-client-tls
mountPath: /docker-certs/client
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "2"
memory: 4Gi
# ── DinD sidecar ──────────────────────────────────────────────────────
# Full Docker daemon running inside the pod.
# privileged: true is required for DinD — the cicd namespace is labelled
# pod-security.kubernetes.io/enforce=privileged to allow this.
# DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs
# on startup; the runner reads the client certs from /docker-certs/client.
# runner-dind PVC mounts /var/lib/docker so the layer cache persists
# across pod restarts.
- name: dind
image: docker:27-dind
securityContext:
privileged: true
env:
- name: DOCKER_TLS_CERTDIR
value: /docker-certs
volumeMounts:
- name: docker-certs
mountPath: /docker-certs
- name: dind-storage
mountPath: /var/lib/docker
# Trust the homelab CA so DinD can pull from Forgejo's OCI registry
- name: homelab-ca
mountPath: /etc/ssl/certs/homelab-ca.pem
subPath: ca.crt
# Merged CA bundle (public roots + homelab CA), bind-mounted from
# here into every job container by container.options above —
# this path is resolved against dind's filesystem since dind is
# the daemon actually creating those containers.
- name: ca-bundle
mountPath: /etc/forgejo-ca/ca-certificates.crt
subPath: ca-certificates.crt
# Homelab-CA-signed server+client certs (see Certificates above),
# overlaying the matching subpaths of the docker-certs emptyDir.
# dockerd-entrypoint.sh detects these and skips its own
# self-signed generation entirely (no CA private key is supplied
# alongside them, so it can't regenerate even if it wanted to).
- name: dind-server-tls
mountPath: /docker-certs/server
- name: dind-client-tls
mountPath: /docker-certs/client
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: "2"
memory: 4Gi
volumes:
- name: runner-data
persistentVolumeClaim:
claimName: runner-reg
- name: dind-storage
persistentVolumeClaim:
claimName: runner-dind
# emptyDir parent mount for /docker-certs - the server/ and client/
# subpaths are now overlaid by the homelab-CA-signed dind-server-tls/
# dind-client-tls secret mounts below (dockerd-entrypoint.sh no
# longer self-generates once it finds those present). This emptyDir
# just needs to exist as the parent directory; nothing writes
# directly to it anymore.
- name: docker-certs
emptyDir: {}
- name: homelab-ca
secret:
secretName: homelab-ca
- name: ca-bundle
secret:
secretName: ca-bundle
- name: runner-config
configMap:
name: forgejo-runner-config
# cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to
# the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects
# under $DOCKER_TLS_CERTDIR/{server,client}/.
- name: dind-server-tls
secret:
secretName: forgejo-runner-dind-server-tls
items:
- key: ca.crt
path: ca.pem
- key: tls.crt
path: cert.pem
- key: tls.key
path: key.pem
- name: dind-client-tls
secret:
secretName: forgejo-runner-dind-client-tls
items:
- key: ca.crt
path: ca.pem
- key: tls.crt
path: cert.pem
- key: tls.key
path: key.pem
---
# ── NetworkPolicy ─────────────────────────────────────────────────────────────
# Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and
# the public internet for action dependencies and base images.
# LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to
# prevent a compromised CI job from pivoting into the cluster or LAN.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: runner-egress
namespace: cicd
spec:
podSelector:
matchLabels:
app: forgejo-runner
policyTypes: [Egress]
egress:
# Forgejo (same namespace — git push, OCI registry push/pull)
- to:
- podSelector: {}
# ingress-nginx (the runner talks to Forgejo via its public hostname,
# https://forgejo.riotpiao.homelab.com, which resolves to the ingress
# controller's ClusterIP — a different namespace on the pod network)
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: ingress-nginx
ports:
- protocol: TCP
port: 443
- protocol: TCP
port: 80
# CoreDNS (DNS resolution for action deps and Forgejo hostname)
- to:
- namespaceSelector:
matchLabels:
kubernetes.io/metadata.name: kube-system
ports:
- protocol: UDP
port: 53
- protocol: TCP
port: 53
# Public internet for action dependencies and base images
# LAN and pod network are explicitly excluded
- to:
- ipBlock:
cidr: 0.0.0.0/0
except:
- 192.168.1.0/24
- 10.244.0.0/16