infra: add helmfile and talos cluster configuration

- helmfile: 18 releases across 22 namespaces - Pod disruption budgets for zero-downtime drain - Nginx ingress with LoadBalancer + Cilium LB-IPAM - Cluster bootstrap hooks
2026-07-11 19:16:41 -07:00
parent 5b00616f41
commit 11898733e8
3 changed files with 1076 additions and 0 deletions
@@ -0,0 +1,86 @@
+# cluster-config/cilium-values.yaml
+# Cilium CNI — installed via talosctl (not helmfile) during cluster bootstrap.
+# Applied once: `helm install cilium cilium/cilium -n kube-system -f cilium-values.yaml`
+#
+# Why Cilium: Talos Linux does not ship kube-proxy. Cilium's eBPF dataplane
+# replaces it entirely (kubeProxyReplacement: true) and also handles LB-IPAM
+# so LoadBalancer services get real IPs without MetalLB.
+
+# ── cgroup ────────────────────────────────────────────────────────────────────
+# Talos mounts cgroups at boot before any container runtime starts.
+# autoMount: false tells Cilium to use the existing mount rather than trying
+# to mount its own — double-mounting on Talos causes init failures.
+cgroup:
+  autoMount:
+    enabled: false
+  hostRoot: /sys/fs/cgroup   # where Talos exposes the cgroup v2 hierarchy
+
+# ── IPAM ──────────────────────────────────────────────────────────────────────
+# kubernetes mode: Cilium allocates pod IPs from the pod CIDR that Talos
+# configured for each node (--pod-cidr in the kubelet). Alternative is
+# Cilium's own cluster-pool IPAM, but that requires extra config and
+# conflicts with the Talos node CIDR assignment.
+ipam:
+  mode: kubernetes
+
+# ── Operator ──────────────────────────────────────────────────────────────────
+# Single replica is fine for a 3-node homelab. The operator manages CiliumNode
+# objects and LB-IPAM pools — it does not sit in the data path.
+operator:
+  replicas: 1
+
+# ── kube-proxy replacement ────────────────────────────────────────────────────
+# Talos is deliberately installed without kube-proxy (machineConfig
+# install.extensions excludes it). Cilium must replace it completely —
+# partial replacement would leave Service ClusterIPs unreachable.
+kubeProxyReplacement: true
+
+# ── L2 announcements ──────────────────────────────────────────────────────────
+# Without this, LB-IPAM (k8s/cilium/lb-ipam-pool.yaml) assigns real IPs to
+# LoadBalancer Services, but nothing ARPs for them on the LAN — the IP shows
+# up in `kubectl get svc` but is 100% unreachable from outside the cluster
+# (confirmed: forgejo's .165 and shadowsocks' .166 both had incomplete ARP
+# entries and 100% ping loss before this). This flag is what actually makes
+# k8s/cilium/l2-announcement-policy.yaml take effect instead of being inert.
+l2announcements:
+  enabled: true
+
+# ── API server endpoint ───────────────────────────────────────────────────────
+# Cilium needs to talk to the Kubernetes API to watch Nodes/Services/Endpoints.
+# On Talos the API server listens on 127.0.0.1:7445 locally (the external
+# port 6443 requires the node's external cert, which may not be available
+# during early bootstrap). This is the standard Talos Cilium bootstrap config.
+k8sServiceHost: 127.0.0.1
+k8sServicePort: 7445
+
+# ── Security context / capabilities ──────────────────────────────────────────
+# Cilium's eBPF programs run in the kernel and require elevated capabilities.
+# These are the minimum set needed — removing any of them breaks networking.
+#
+#   NET_ADMIN / NET_RAW  — manipulate iptables/nftables and raw sockets
+#   IPC_LOCK             — lock eBPF maps in memory (prevents paging out BPF state)
+#   SYS_ADMIN            — call bpf() syscall and mount BPF filesystem
+#   SYS_RESOURCE         — raise RLIMIT_MEMLOCK for BPF map memory
+#   DAC_OVERRIDE / FOWNER / SETGID / SETUID — file permission ops during init
+#   CHOWN / KILL         — container lifecycle management
+#
+# cleanCiliumState runs as a one-shot init container to wipe stale eBPF state
+# on upgrades — it needs NET_ADMIN, SYS_ADMIN, SYS_RESOURCE only.
+securityContext:
+  capabilities:
+    ciliumAgent:
+    - CHOWN
+    - KILL
+    - NET_ADMIN
+    - NET_RAW
+    - IPC_LOCK
+    - SYS_ADMIN
+    - SYS_RESOURCE
+    - DAC_OVERRIDE
+    - FOWNER
+    - SETGID
+    - SETUID
+    cleanCiliumState:
+    - NET_ADMIN
+    - SYS_ADMIN
+    - SYS_RESOURCE
@@ -0,0 +1,19 @@
+# bootstrap.sh
+#!/bin/bash
+
+# Wait for cluster to be ready
+kubectl wait --for=condition=Ready nodes --all --timeout=300s
+
+# Longhorn requires privileged pods and hostPath volumes
+kubectl create namespace longhorn-system --dry-run=client -o yaml | kubectl apply -f -
+kubectl label namespace longhorn-system \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/enforce-version=latest \
+  --overwrite
+
+# Install Longhorn
+kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.7.0/deploy/longhorn.yaml
+
+# Set as default StorageClass
+kubectl patch storageclass longhorn \
+  -p '{"metadata":{"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'