k8s/aux: add cert-manager longhorn dashboard forge dev-tools and shadowsocks

- cert-manager ClusterIssuers (LetsEncrypt + homelab-ca) - Longhorn storage dashboard - Portainer dashboard config - Forgejo git service - Claude terminal remote access - Shadowsocks tunnel for remote access
2026-07-11 19:19:59 -07:00
parent 18f2f94f8e
commit e1d0cfd70b
18 changed files with 1201 additions and 0 deletions
@@ -0,0 +1,10 @@
+# k8s/cert-manager/cert-manager-values.yaml
+# cert-manager issues every per-hostname cert off homelab-ca and renews the
+# wildcard cert nginx serves — certmanager_certificate_expiration_timestamp_seconds
+# is the early-warning signal before any service's TLS breaks.
+
+prometheus:
+  enabled: true
+  servicemonitor:
+    enabled: true
+    interval: 60s
@@ -0,0 +1,25 @@
+# Ingress for Portainer container UI — routes to OAuth2-Proxy
+# TLS terminated here; oauth2-proxy handles OIDC auth
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: portainer
+  namespace: dashboard
+spec:
+  ingressClassName: nginx
+  tls:
+  - secretName: portainer-tls
+    hosts:
+    - portainer.riotpiao.homelab.com
+  rules:
+  - host: portainer.riotpiao.homelab.com
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: oauth2-proxy-portainer
+            port:
+              number: 4180
@@ -0,0 +1,106 @@
+# OAuth2-Proxy for Portainer container management UI
+# Protects container orchestration interface with Authentik OIDC
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oauth2-proxy
+  namespace: dashboard
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy-portainer
+  namespace: dashboard
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: oauth2-proxy-portainer
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy-portainer
+      annotations:
+        secret.reloader.stakater.com/reload: "portainer-oidc"
+    spec:
+      serviceAccountName: oauth2-proxy
+      containers:
+      - name: oauth2-proxy
+        image: quay.io/oauth2-proxy/oauth2-proxy:v7.5.1
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 4180
+          protocol: TCP
+        env:
+        - name: OAUTH2_PROXY_PROVIDER
+          value: "oidc"
+        - name: OAUTH2_PROXY_OIDC_ISSUER_URL
+          value: "https://authentik.riotpiao.homelab.com/application/o/portainer/"
+        - name: OAUTH2_PROXY_CLIENT_ID
+          value: "portainer"
+        - name: OAUTH2_PROXY_CLIENT_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: portainer-oidc
+              key: clientSecret
+        - name: OAUTH2_PROXY_COOKIE_SECRET
+          valueFrom:
+            secretKeyRef:
+              name: portainer-oidc
+              key: cookieSecret
+        - name: OAUTH2_PROXY_REDIRECT_URL
+          value: "https://portainer.riotpiao.homelab.com/oauth2/callback"
+        - name: OAUTH2_PROXY_UPSTREAM
+          value: "http://portainer:9000"
+        - name: OAUTH2_PROXY_COOKIE_SECURE
+          value: "true"
+        - name: OAUTH2_PROXY_COOKIE_HTTPONLY
+          value: "true"
+        - name: OAUTH2_PROXY_COOKIE_SAMESITE
+          value: "Lax"
+        - name: OAUTH2_PROXY_EMAIL_DOMAIN
+          value: "*"
+        - name: OAUTH2_PROXY_SKIP_AUTH_REGEX
+          value: "^/health|^/api/status"
+        - name: OAUTH2_PROXY_PASS_AUTHORIZATION_HEADER
+          value: "true"
+        - name: OAUTH2_PROXY_REVERSE_PROXY
+          value: "true"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 128Mi
+          limits:
+            cpu: 200m
+            memory: 256Mi
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 5
+          periodSeconds: 5
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy-portainer
+  namespace: dashboard
+spec:
+  type: ClusterIP
+  ports:
+  - port: 4180
+    targetPort: http
+    protocol: TCP
+    name: http
+  selector:
+    app: oauth2-proxy-portainer
@@ -0,0 +1,6 @@
+apiVersion: v2
+name: claude-terminal
+description: Persistent Claude CLI terminal running in tmux with web access via gotty
+type: application
+version: 1.0.0
+appVersion: "1.0"
@@ -0,0 +1,31 @@
+FROM --platform=linux/amd64 ubuntu:24.04
+
+RUN apt-get update && apt-get install -y \
+    tmux \
+    curl \
+    git \
+    build-essential \
+    nodejs \
+    npm \
+    bash \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install gotty (web terminal access)
+RUN curl -sL https://github.com/sorenisanerd/gotty/releases/download/v1.5.0/gotty_linux_amd64.tar.gz | \
+    tar xz -C /usr/local/bin && chmod +x /usr/local/bin/gotty
+
+# Install Claude CLI
+RUN npm install -g claude-code-cli 2>&1 || echo "Note: Claude CLI will be available after NPM package is published"
+
+WORKDIR /root
+
+# Create persistent storage dir
+RUN mkdir -p /root/.claude /root/.config /root/.cache
+
+# Entrypoint: start tmux session and gotty
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 8080
+
+ENTRYPOINT ["/entrypoint.sh"]
@@ -0,0 +1,49 @@
+# Claude Terminal — Persistent Remote Dev Environment
+
+Runs Claude CLI in a persistent tmux session with web-based terminal access via gotty.
+
+## Building the Image
+
+Build for `linux/amd64`:
+
+```bash
+cd homelab
+docker buildx build --platform linux/amd64 \
+  -t forgejo.riotpiao.homelab.com/rock/claude-terminal:latest \
+  -f k8s/dev-tools/Dockerfile \
+  k8s/dev-tools
+
+# Log in to Forgejo registry
+docker login forgejo.riotpiao.homelab.com \
+  --username ci-bot \
+  --password "$(talos get cluster/iam/agents/ci-bot --key token)"
+
+# Push
+docker push forgejo.riotpiao.homelab.com/rock/claude-terminal:latest
+```
+
+Or use the provided build script:
+
+```bash
+./k8s/dev-tools/build.sh
+```
+
+## Deployment
+
+Update `values.yaml` if needed, then deploy via helmfile:
+
+```bash
+helmfile apply -l name=claude-terminal
+```
+
+Access the terminal at: **https://claude.riotpiao.homelab.com**
+
+## Persistent Storage
+
+- All Claude configuration stored in `/root/.claude` (persistent PVC, 10Gi Longhorn)
+- Survives pod restarts and node reboots
+- Accessible immediately after reconnecting
+
+## SSH Access (Optional)
+
+To add SSH access, extend the Dockerfile to include openssh-server and mount the PVC as home directory.
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+
+REGISTRY="forgejo.riotpiao.homelab.com"
+IMAGE_NAME="rock/claude-terminal"
+TAG="latest"
+FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${TAG}"
+
+echo "🔨 Building Claude Terminal image for linux/amd64..."
+docker buildx build --platform linux/amd64 \
+  -t "${FULL_IMAGE}" \
+  -f Dockerfile \
+  . || { echo "❌ Build failed"; exit 1; }
+
+echo "🔓 Logging in to Forgejo registry..."
+REGISTRY_TOKEN=$(talos get cluster/iam/agents/ci-bot --key token)
+echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" \
+  --username ci-bot \
+  --password-stdin || { echo "❌ Login failed"; exit 1; }
+
+echo "📤 Pushing image to registry..."
+docker push "${FULL_IMAGE}" || { echo "❌ Push failed"; exit 1; }
+
+echo "✅ Successfully pushed ${FULL_IMAGE}"
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Start tmux server in background
+tmux new-session -d -s claude -c /root "bash"
+
+# Give tmux a moment to stabilize
+sleep 1
+
+# Start gotty serving the tmux session
+# -w: allow write (make terminal interactive)
+# -p 8080: listen on port 8080
+exec gotty -p 8080 -w tmux attach-session -t claude
@@ -0,0 +1,49 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "claude-terminal.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "claude-terminal.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "claude-terminal.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "claude-terminal.labels" -}}
+helm.sh/chart: {{ include "claude-terminal.chart" . }}
+{{ include "claude-terminal.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "claude-terminal.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "claude-terminal.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "claude-terminal.fullname" . }}
+  labels:
+    {{- include "claude-terminal.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "claude-terminal.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "claude-terminal.selectorLabels" . | nindent 8 }}
+    spec:
+      containers:
+      - name: claude-terminal
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /
+            port: http
+          initialDelaySeconds: 30
+          periodSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /
+            port: http
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        resources:
+          {{- toYaml .Values.resources | nindent 12 }}
+        volumeMounts:
+        - name: claude-storage
+          mountPath: {{ .Values.persistence.mountPath }}
+      volumes:
+      - name: claude-storage
+        persistentVolumeClaim:
+          claimName: {{ include "claude-terminal.fullname" . }}-pvc
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
@@ -0,0 +1,41 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "claude-terminal.fullname" . }}
+  labels:
+    {{- include "claude-terminal.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.ingress.className }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            pathType: {{ .pathType }}
+            backend:
+              service:
+                name: {{ include "claude-terminal.fullname" $ }}
+                port:
+                  number: {{ $.Values.service.port }}
+          {{- end }}
+    {{- end }}
+{{- end }}
@@ -0,0 +1,15 @@
+{{- if .Values.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "claude-terminal.fullname" . }}-pvc
+  labels:
+    {{- include "claude-terminal.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: {{ .Values.persistence.storageClass }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size }}
+{{- end }}
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "claude-terminal.fullname" . }}
+  labels:
+    {{- include "claude-terminal.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "claude-terminal.selectorLabels" . | nindent 4 }}
@@ -0,0 +1,45 @@
+replicaCount: 1
+
+image:
+  repository: localhost:5000/claude-terminal
+  pullPolicy: IfNotPresent
+  tag: latest
+
+service:
+  type: ClusterIP
+  port: 8080
+
+ingress:
+  enabled: true
+  className: nginx
+  annotations:
+    cert-manager.io/cluster-issuer: homelab-ca
+  hosts:
+    - host: claude.riotpiao.homelab.com
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - secretName: claude-terminal-tls
+      hosts:
+        - claude.riotpiao.homelab.com
+
+persistence:
+  enabled: true
+  storageClass: longhorn
+  size: 10Gi
+  mountPath: /root/.claude
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 512Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
@@ -0,0 +1,95 @@
+# k8s/forge/runner-gc-cronjob.yaml
+# Garbage-collects the forgejo-runner's DinD layer cache (runner-dind PVC,
+# 30Gi). Every CI build/pull only adds images and build-cache layers — there
+# is no automatic pruning, so without this the PVC fills up and breaks builds.
+#
+# Runs `docker image prune` / `docker builder prune` inside the live dind
+# container via `kubectl exec`, rather than a sidecar in the runner pod itself,
+# so it can run on its own schedule independent of runner restarts.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: runner-gc
+  namespace: cicd
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: runner-gc
+  namespace: cicd
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: runner-gc
+  namespace: cicd
+subjects:
+  - kind: ServiceAccount
+    name: runner-gc
+    namespace: cicd
+roleRef:
+  kind: Role
+  name: runner-gc
+  apiGroup: rbac.authorization.k8s.io
+
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: forgejo-runner-image-gc
+  namespace: cicd
+spec:
+  schedule: "0 3 * * *"   # daily 03:00
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      activeDeadlineSeconds: 600
+      template:
+        spec:
+          serviceAccountName: runner-gc
+          restartPolicy: Never
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: gc
+              image: alpine/k8s:1.31.0
+              command:
+                - sh
+                - -c
+                - |
+                  set -e
+                  POD=$(kubectl -n cicd get pod -l app=forgejo-runner -o jsonpath='{.items[0].metadata.name}')
+                  if [ -z "$POD" ]; then
+                    echo "no forgejo-runner pod found, skipping"
+                    exit 0
+                  fi
+                  echo "before:"
+                  kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
+                  echo "pruning images unused for >72h on $POD"
+                  kubectl -n cicd exec "$POD" -c dind -- docker image prune -af --filter "until=72h"
+                  echo "pruning build cache unused for >72h on $POD"
+                  kubectl -n cicd exec "$POD" -c dind -- docker builder prune -af --filter "until=72h"
+                  echo "after:"
+                  kubectl -n cicd exec "$POD" -c dind -- df -h /var/lib/docker
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 64Mi
+                limits:
+                  cpu: 250m
+                  memory: 128Mi
@@ -0,0 +1,438 @@
+# k8s/forge/runner.yaml
+# Forgejo Actions runner with Docker-in-Docker (DinD) sidecar.
+# Phase 3.2 of talos_version_control.html.
+#
+# Pod layout (two containers, one pod):
+#   register   initContainer — registers with Forgejo once; skips if .runner exists
+#   runner     main container — long-running daemon that polls Forgejo for jobs
+#   dind       sidecar — Docker daemon the runner talks to via mTLS on tcp://localhost:2376
+#
+# Prerequisites (Phase 3.1):
+#   TOKEN=$(kubectl -n cicd exec deploy/forgejo-gitea -- \
+#     gitea actions generate-runner-token 2>/dev/null | tr -d '\r\n')
+#   kubectl -n cicd create secret generic runner-token --from-literal=token="$TOKEN"
+#   # CA must come from cert-manager's homelab-ca-secret (the org-wide CA that
+#   # signs the live ingress cert), NOT k8s/forge/pki/ca.crt — that file is a
+#   # stale CA from before the "unified certificate" migration.
+#   kubectl get secret homelab-ca-secret -n cert-manager -o jsonpath='{.data.tls\.crt}' \
+#     | base64 -d | kubectl -n cicd create secret generic homelab-ca --from-file=ca.crt=/dev/stdin
+#
+# CA trust for ephemeral job containers (Phase 3.3):
+#   The homelab-ca secret above only reaches the runner/register/dind containers.
+#   It does NOT reach the per-job containers DinD spawns (e.g. golangci-lint,
+#   node:22-bookworm) — those are fresh sibling containers with their own trust
+#   store. Without this, any git/curl/apk call to forgejo.riotpiao.homelab.com
+#   from inside a job fails with "SSL certificate problem: unable to get local
+#   issuer certificate". Build a merged bundle (public roots + homelab CA) and
+#   bind-mount it over /etc/ssl/certs/ca-certificates.crt in every job
+#   container via forgejo-runner's config.yaml container.options.
+#
+#   IMPORTANT: source the CA from the live cluster secret, NOT from
+#   k8s/forge/pki/ca.crt — that repo file is a stale CA from before the
+#   "unified certificate" migration (different key, fails verification
+#   against the cert actually served by forgejo.riotpiao.homelab.com). The
+#   org-wide CA that signs the live ingress cert lives in
+#   cert-manager/homelab-ca-secret, and cicd/homelab-ca above is already
+#   synced from it.
+#     docker run --rm docker:27-dind cat /etc/ssl/certs/ca-certificates.crt > /tmp/ca-bundle.crt
+#     kubectl -n cicd get secret homelab-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >> /tmp/ca-bundle.crt
+#     kubectl -n cicd create secret generic ca-bundle --from-file=ca-certificates.crt=/tmp/ca-bundle.crt
+#   Re-run this whenever the homelab CA rotates (see talos-forge-trust.yaml).
+#
+# Apply:
+#   kubectl apply -f k8s/forge/runner.yaml
+#   kubectl -n cicd rollout status deploy/forgejo-runner
+#   kubectl -n cicd logs deploy/forgejo-runner -c runner -f
+#   # expect: "runner: daemon started" / "connected to Forgejo"
+
+# ── PVCs ──────────────────────────────────────────────────────────────────────
+# runner-reg  — persists the .runner registration file so the runner doesn't
+#               re-register on every pod restart (token is one-use-per-registration)
+# runner-dind — persists the Docker layer cache across pod restarts; keeps
+#               rebuilds fast — images don't need to be re-pulled every time
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: runner-reg
+  namespace: cicd
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 1Gi
+
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: runner-dind
+  namespace: cicd
+spec:
+  accessModes: [ReadWriteOnce]
+  storageClassName: longhorn
+  resources:
+    requests:
+      storage: 30Gi
+
+---
+# ── DinD TLS certs, issued by the homelab's unified CA ───────────────────────
+# DinD's own entrypoint (dockerd-entrypoint.sh) self-generates a throwaway CA
+# + server/client cert pair on every container start if none is supplied. Its
+# server cert's SAN list only ever covers "docker", the pod hostname, and
+# "localhost" - so anything reaching it via a stable Service DNS name (added
+# below for story-crater-backend's release.yaml to build/push images) fails
+# TLS hostname verification, even though the handshake itself succeeds.
+#
+# Fix: supply our own server+client cert pair, both issued by the same
+# ClusterIssuer (homelab-ca) that already signs the live ingress cert, so
+# they share one trust root. dockerd-entrypoint.sh skips its own generation
+# step entirely once it finds $DOCKER_TLS_CERTDIR/server/{ca,cert,key}.pem
+# already present and no CA private key alongside them (confirmed by reading
+# the script directly: `kubectl exec -n cicd <pod> -c dind -- cat
+# /usr/local/bin/dockerd-entrypoint.sh`) - exactly the "bring your own CA"
+# path it's designed for.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: forgejo-runner-dind-server
+  namespace: cicd
+spec:
+  secretName: forgejo-runner-dind-server-tls
+  issuerRef:
+    name: homelab-ca
+    kind: ClusterIssuer
+  commonName: docker:dind server
+  dnsNames:
+    - forgejo-runner-dind.cicd.svc.cluster.local
+    - forgejo-runner-dind.cicd.svc
+    - forgejo-runner-dind
+    - docker
+    - localhost
+  usages:
+    - server auth
+    - digital signature
+    - key encipherment
+
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: forgejo-runner-dind-client
+  namespace: cicd
+spec:
+  secretName: forgejo-runner-dind-client-tls
+  issuerRef:
+    name: homelab-ca
+    kind: ClusterIssuer
+  commonName: docker:dind client
+  usages:
+    - client auth
+    - digital signature
+    - key encipherment
+
+---
+# Stable address for the dind sidecar's docker API (2376, mTLS) - lets
+# CI workflows (e.g. story-crater-backend's release.yaml) build/push images
+# by reaching this runner's own already-working outer dind directly, instead
+# of the per-job `services:` sidecar pattern (confirmed broken: act-runner
+# never registers a DNS alias for service containers - job container's
+# /etc/hosts has no entry for it, `docker info` fails with a DNS lookup
+# error, not a TLS/connection error).
+apiVersion: v1
+kind: Service
+metadata:
+  name: forgejo-runner-dind
+  namespace: cicd
+spec:
+  selector:
+    app: forgejo-runner
+  ports:
+    - port: 2376
+      targetPort: 2376
+
+---
+# ── Runner config ─────────────────────────────────────────────────────────────
+# container.options is appended to every `docker run` DinD issues for a job
+# container, so this is what actually gets the merged CA bundle (ca-bundle
+# secret, see header comment) trusted inside golangci-lint, node, etc. - and
+# now also what propagates the homelab-CA-signed client cert (above) into
+# job containers that need to talk back to dind themselves (e.g. building
+# and pushing images).
+# Source paths resolve against the dind container's filesystem (it's the
+# daemon creating these containers), so both are mounted into dind below.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: forgejo-runner-config
+  namespace: cicd
+data:
+  config.yaml: |
+    container:
+      options: -v /etc/forgejo-ca/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro -v /docker-certs/client:/docker-certs/client:ro
+      # Without this, sanitizeConfig() in forgejo/act silently drops any bind
+      # mount whose source isn't allowlisted here — including ones injected
+      # via container.options above, not just workflow-declared volumes.
+      valid_volumes:
+        - /etc/forgejo-ca/ca-certificates.crt
+        - /docker-certs/client
+
+---
+# ── Deployment ────────────────────────────────────────────────────────────────
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: forgejo-runner
+  namespace: cicd
+spec:
+  replicas: 1
+  # RWO PVCs mean only one pod can mount them at a time.
+  # Recreate ensures the old pod fully terminates before the new one starts.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: forgejo-runner
+  template:
+    metadata:
+      labels:
+        app: forgejo-runner
+    spec:
+      # runner/register containers run as uid 1000 (image default); fsGroup
+      # makes kubelet chown+chmod the Longhorn PVC's group to 1000 with
+      # write access, otherwise writes to /data (.runner config) fail with
+      # "permission denied" since the volume is root:root 755 by default.
+      securityContext:
+        fsGroup: 1000
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+
+      initContainers:
+        # Registers the runner with Forgejo exactly once.
+        # test -f /data/.runner makes it idempotent — re-registration would
+        # consume the one-time token and break the runner.
+        - name: register
+          image: code.forgejo.org/forgejo/runner:6
+          command: ["sh", "-c"]
+          args:
+            - |
+              test -f /data/.runner && echo "already registered, skipping" && exit 0
+              forgejo-runner register --no-interactive \
+                --instance https://forgejo.riotpiao.homelab.com \
+                --token "$RUNNER_TOKEN" \
+                --name talos-runner \
+                --labels "docker:docker://node:22-bookworm"
+          env:
+            - name: RUNNER_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: runner-token
+                  key: token
+          volumeMounts:
+            - name: runner-data
+              mountPath: /data
+            # CA cert so the register call can verify Forgejo's TLS cert
+            - name: homelab-ca
+              mountPath: /etc/ssl/certs/homelab-ca.pem
+              subPath: ca.crt
+          workingDir: /data
+
+      containers:
+        # ── Runner daemon ────────────────────────────────────────────────────
+        # Polls Forgejo for pending jobs and executes them inside DinD.
+        # The `until docker info` loop waits for the DinD sidecar to finish
+        # its TLS setup before starting the daemon — without this the runner
+        # starts before Docker is ready and immediately errors out.
+        - name: runner
+          image: code.forgejo.org/forgejo/runner:6
+          command: ["sh", "-c"]
+          args:
+            - |
+              until nc -z localhost 2376 >/dev/null 2>&1; do
+                echo "waiting for docker daemon..."; sleep 2
+              done
+              forgejo-runner daemon --config /data/config.yaml
+          workingDir: /data
+          env:
+            # Connect to the DinD sidecar via mTLS on localhost
+            - name: DOCKER_HOST
+              value: tcp://localhost:2376
+            - name: DOCKER_TLS_VERIFY
+              value: "1"
+            - name: DOCKER_CERT_PATH
+              value: /docker-certs/client
+          volumeMounts:
+            - name: runner-data
+              mountPath: /data
+            - name: docker-certs
+              mountPath: /docker-certs
+            - name: homelab-ca
+              mountPath: /etc/ssl/certs/homelab-ca.pem
+              subPath: ca.crt
+            # forgejo-runner's container.options, read from this file, is what
+            # propagates the CA bundle into per-job containers (see ca-bundle
+            # secret + dind mount below)
+            - name: runner-config
+              mountPath: /data/config.yaml
+              subPath: config.yaml
+            # Homelab-CA-signed client cert (overlays whatever's in the
+            # docker-certs emptyDir at this subpath) - matches the server
+            # cert dind now presents, see Certificates above.
+            - name: dind-client-tls
+              mountPath: /docker-certs/client
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+
+        # ── DinD sidecar ──────────────────────────────────────────────────────
+        # Full Docker daemon running inside the pod.
+        # privileged: true is required for DinD — the cicd namespace is labelled
+        # pod-security.kubernetes.io/enforce=privileged to allow this.
+        # DOCKER_TLS_CERTDIR causes DinD to generate mTLS certs in /docker-certs
+        # on startup; the runner reads the client certs from /docker-certs/client.
+        # runner-dind PVC mounts /var/lib/docker so the layer cache persists
+        # across pod restarts.
+        - name: dind
+          image: docker:27-dind
+          securityContext:
+            privileged: true
+          env:
+            - name: DOCKER_TLS_CERTDIR
+              value: /docker-certs
+          volumeMounts:
+            - name: docker-certs
+              mountPath: /docker-certs
+            - name: dind-storage
+              mountPath: /var/lib/docker
+            # Trust the homelab CA so DinD can pull from Forgejo's OCI registry
+            - name: homelab-ca
+              mountPath: /etc/ssl/certs/homelab-ca.pem
+              subPath: ca.crt
+            # Merged CA bundle (public roots + homelab CA), bind-mounted from
+            # here into every job container by container.options above —
+            # this path is resolved against dind's filesystem since dind is
+            # the daemon actually creating those containers.
+            - name: ca-bundle
+              mountPath: /etc/forgejo-ca/ca-certificates.crt
+              subPath: ca-certificates.crt
+            # Homelab-CA-signed server+client certs (see Certificates above),
+            # overlaying the matching subpaths of the docker-certs emptyDir.
+            # dockerd-entrypoint.sh detects these and skips its own
+            # self-signed generation entirely (no CA private key is supplied
+            # alongside them, so it can't regenerate even if it wanted to).
+            - name: dind-server-tls
+              mountPath: /docker-certs/server
+            - name: dind-client-tls
+              mountPath: /docker-certs/client
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: "2"
+              memory: 4Gi
+
+      volumes:
+        - name: runner-data
+          persistentVolumeClaim:
+            claimName: runner-reg
+        - name: dind-storage
+          persistentVolumeClaim:
+            claimName: runner-dind
+        # emptyDir parent mount for /docker-certs - the server/ and client/
+        # subpaths are now overlaid by the homelab-CA-signed dind-server-tls/
+        # dind-client-tls secret mounts below (dockerd-entrypoint.sh no
+        # longer self-generates once it finds those present). This emptyDir
+        # just needs to exist as the parent directory; nothing writes
+        # directly to it anymore.
+        - name: docker-certs
+          emptyDir: {}
+        - name: homelab-ca
+          secret:
+            secretName: homelab-ca
+        - name: ca-bundle
+          secret:
+            secretName: ca-bundle
+        - name: runner-config
+          configMap:
+            name: forgejo-runner-config
+        # cert-manager issues these as tls.crt/tls.key/ca.crt - remapped to
+        # the ca.pem/cert.pem/key.pem filenames dockerd-entrypoint.sh expects
+        # under $DOCKER_TLS_CERTDIR/{server,client}/.
+        - name: dind-server-tls
+          secret:
+            secretName: forgejo-runner-dind-server-tls
+            items:
+              - key: ca.crt
+                path: ca.pem
+              - key: tls.crt
+                path: cert.pem
+              - key: tls.key
+                path: key.pem
+        - name: dind-client-tls
+          secret:
+            secretName: forgejo-runner-dind-client-tls
+            items:
+              - key: ca.crt
+                path: ca.pem
+              - key: tls.crt
+                path: cert.pem
+              - key: tls.key
+                path: key.pem
+
+---
+# ── NetworkPolicy ─────────────────────────────────────────────────────────────
+# Restrict runner egress: it may only reach Forgejo (cicd ns), CoreDNS, and
+# the public internet for action dependencies and base images.
+# LAN (192.168.1.0/24) and the pod network (10.244.0.0/16) are blocked to
+# prevent a compromised CI job from pivoting into the cluster or LAN.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: runner-egress
+  namespace: cicd
+spec:
+  podSelector:
+    matchLabels:
+      app: forgejo-runner
+  policyTypes: [Egress]
+  egress:
+    # Forgejo (same namespace — git push, OCI registry push/pull)
+    - to:
+        - podSelector: {}
+    # ingress-nginx (the runner talks to Forgejo via its public hostname,
+    # https://forgejo.riotpiao.homelab.com, which resolves to the ingress
+    # controller's ClusterIP — a different namespace on the pod network)
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: ingress-nginx
+      ports:
+        - protocol: TCP
+          port: 443
+        - protocol: TCP
+          port: 80
+    # CoreDNS (DNS resolution for action deps and Forgejo hostname)
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    # Public internet for action dependencies and base images
+    # LAN and pod network are explicitly excluded
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 192.168.1.0/24
+              - 10.244.0.0/16
@@ -0,0 +1,23 @@
+# k8s/longhorn/longhorn-servicemonitor.yaml
+# Longhorn is deployed via raw manifest (cluster-config/longhorn_bootstrap.sh), not
+# Helm, so there's no chart values toggle for metrics — the manager already exposes
+# them on the existing longhorn-backend service (port 9500); this just wires Prometheus
+# to it. longhorn_volume_robustness is the availability signal (degraded/faulted).
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: longhorn-manager
+  namespace: longhorn-system
+  labels:
+    app: longhorn-manager
+spec:
+  selector:
+    matchLabels:
+      app: longhorn-manager
+  namespaceSelector:
+    matchNames:
+      - longhorn-system
+  endpoints:
+    - port: manager
+      interval: 30s
+      scrapeTimeout: 10s
@@ -0,0 +1,159 @@
+# k8s/shadowsocks/shadowsocks.yaml
+# Personal Shadowsocks proxy (for Shadowrocket/other SS clients) — an
+# alternative tunnel to the WireGuard setup in cluster-config/, useful when
+# a network blocks/throttles WireGuard but not generic TLS-looking traffic.
+#
+# Everything that varies between deployments (port, password, method) lives
+# in the shadowsocks-config Secret below — the Deployment/Service never
+# hardcode a value, so re-pointing this at a new port or rotating the
+# password is a Secret edit + rollout restart, no YAML edit.
+#
+# Prerequisites:
+#   talos put cluster/SHADOWSOCKS_PASSWORD SHADOWSOCKS_PASSWORD="$(openssl rand -base64 24)"
+#   talos put cluster/SHADOWSOCKS_PORT SHADOWSOCKS_PORT="8388"
+#
+# Apply:
+#   kubectl create namespace vpn --dry-run=client -o yaml | kubectl apply -f -
+#   kubectl -n vpn create secret generic shadowsocks-config \
+#     --from-literal=SERVER_PORT="$(talos get cluster/SHADOWSOCKS_PORT --key SHADOWSOCKS_PORT)" \
+#     --from-literal=PASSWORD="$(talos get cluster/SHADOWSOCKS_PASSWORD --key SHADOWSOCKS_PASSWORD)" \
+#     --from-literal=METHOD="aes-256-gcm" \
+#     --from-literal=TIMEOUT="300"
+#   kubectl apply -f k8s/shadowsocks/shadowsocks.yaml
+#
+# Rotate password (or change port) later:
+#   kubectl -n vpn delete secret shadowsocks-config && <recreate with new values>
+#   kubectl -n vpn rollout restart deploy/shadowsocks
+#
+# Client config: SERVER_PORT/METHOD/PASSWORD above feed directly into the
+# Shadowrocket/SS client's server, method, and password fields. SERVER_ADDR
+# for the client is the LB IP below (192.168.1.166), or your router's WAN
+# address/DDNS hostname (riotpiao.duckdns.org) with port-forwarding to it —
+# same pattern as the wg1 WireGuard peer in cluster-config/phone_config.conf.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: vpn
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shadowsocks
+  namespace: vpn
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: shadowsocks
+  template:
+    metadata:
+      labels:
+        app: shadowsocks
+    spec:
+      containers:
+        - name: shadowsocks
+          image: shadowsocks/shadowsocks-libev:latest
+          env:
+            - name: SERVER_ADDR
+              value: "0.0.0.0"
+            - name: SERVER_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: shadowsocks-config
+                  key: SERVER_PORT
+            - name: PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: shadowsocks-config
+                  key: PASSWORD
+            - name: METHOD
+              valueFrom:
+                secretKeyRef:
+                  name: shadowsocks-config
+                  key: METHOD
+            - name: TIMEOUT
+              valueFrom:
+                secretKeyRef:
+                  name: shadowsocks-config
+                  key: TIMEOUT
+          # containerPort is informational only (no portRange support for
+          # env-driven SERVER_PORT) — the Service below is what actually
+          # routes traffic, matched on the same Secret key via downward API
+          # isn't available for Service ports, so targetPort uses the literal
+          # port name instead; see Service ports comment.
+          ports:
+            - containerPort: 8388
+              protocol: TCP
+            - containerPort: 8388
+              protocol: UDP
+          resources:
+            requests:
+              cpu: 50m
+              memory: 32Mi
+            limits:
+              cpu: 500m
+              memory: 128Mi
+
+---
+# LoadBalancer via Cilium LB-IPAM (see k8s/cilium/lb-ipam-pool.yaml) — pinned
+# to .166 so router port-forwarding and the DDNS hostname stay stable across
+# pod/service recreates, same pattern forgejo uses at .165.
+#
+# NOTE: SERVER_PORT in the Secret must match port/targetPort/nodePort here.
+# If you change the port, update both the Secret and this Service together.
+apiVersion: v1
+kind: Service
+metadata:
+  name: shadowsocks
+  namespace: vpn
+  annotations:
+    io.cilium/lb-ipam-ips: "192.168.1.166"
+spec:
+  type: LoadBalancer
+  selector:
+    app: shadowsocks
+  ports:
+    - name: tcp
+      protocol: TCP
+      port: 8388
+      targetPort: 8388
+    - name: udp
+      protocol: UDP
+      port: 8388
+      targetPort: 8388
+
+---
+# Restrict egress like the forgejo-runner pattern (k8s/forge/runner.yaml) —
+# a proxy server is, by design, an open relay to the internet for whoever
+# holds the password; LAN/pod-network egress is blocked so a compromised
+# password can't be used to pivot into the cluster or LAN. CoreDNS is
+# explicitly allowed — shadowsocks-libev resolves client-requested hostnames
+# itself, so blanket-blocking the service subnet would break that.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: shadowsocks-egress
+  namespace: vpn
+spec:
+  podSelector:
+    matchLabels:
+      app: shadowsocks
+  policyTypes: [Egress]
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except:
+              - 192.168.1.0/24
+              - 10.244.0.0/16
+              - 10.96.0.0/12