rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(bootstrap): correct cluster config + complete Phase4 ArgoCD bootstrap permanent fixes

Browse Source
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:03 -07:00
parent aea48deb99
commit 5b3307ffee
19 changed files with 520 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+128 -209
View File
@@ -1,21 +1,21 @@
# ── Node IPs ──────────────────────────────────────────────────────────────────
# CP_IP has a default. All W{N}_IP variables are expected to be exported from
# ~/.zshrc (e.g. export W1_IP=192.168.1.162). No guards — assumed always set.
CP_IP ?= 192.168.1.213
export CP_IP
# ── Node IPs (3-CP HA topology) ───────────────────────────────────────────────
CP1_IP := 192.168.1.166 # talos-cp-1
CP2_IP := 192.168.1.213 # talos-cp-2 (storage: 3 disks)
CP3_IP := 192.168.1.162 # talos-cp-3
CP_VIP := 192.168.1.166 # controlplane VIP (currently .166)
# ── Paths ─────────────────────────────────────────────────────────────────────
TALOSCONFIG := cluster-config/coreconfig
CP_CONFIG := cluster-config/controlplane.yaml
SECRETS := cluster-config/secrets.yaml
TALOSCONFIG := cluster-config/talosconfig
CP1_CONFIG := cluster-config/talos-cp-1.yaml
CP2_CONFIG := cluster-config/talos-cp-2.yaml
CP3_CONFIG := cluster-config/talos-cp-3.yaml
KUBECONFIG := cluster-config/kubeconfig
CLUSTER_NAME := homelab-cluster
CP_ENDPOINT := https://$(CP_IP):6443
TALOS_IMAGE := factory.core.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.13.3
CLUSTER_NAME := homelab
CP_ENDPOINT := https://$(CP_VIP):6443
TALOSCTL := corectl --coreconfig $(TALOSCONFIG)
# Use talosctl (not corectl). Needs TALOSCONFIG env var pointing to talosconfig file.
TALOSCTL := talosctl
KUBECTL := kubectl --kubeconfig $(KUBECONFIG)
# Derive IP and config from worker number N (used by generic targets).
@@ -28,236 +28,157 @@ W_CONFIG = cluster-config/worker-$(N).yaml
# ── Help ──────────────────────────────────────────────────────────────────────
.PHONY: help
help:
@echo "Homelab cluster — available targets"
@echo "Homelab cluster (3-CP HA: .166/.213/.163) — available targets"
@echo ""
@echo " Status"
@echo " Status & Services"
@echo " nodes kubectl get nodes"
@echo " status-cp core node overview (control plane)"
@echo " status-w1 core node overview (worker-1)"
@echo " services-cp list core services (control plane)"
@echo " services-w1 list core services (worker-1)"
@echo " status-all etcd members on all 3 CPs"
@echo " status-cp1/2/3 etcd members on specific CP"
@echo " services-cp1/2/3 list Talos services on specific CP"
@echo ""
@echo " Logs"
@echo " logs-cp stream kubelet logs (control plane)"
@echo " logs-w1 stream kubelet logs (worker-1)"
@echo " dmesg-cp kernel dmesg (control plane)"
@echo " dmesg-w1 kernel dmesg (worker-1)"
@echo " log-svc-cp stream a service log (control plane) SVC=<name>"
@echo " log-svc-w1 stream a service log (worker-1) SVC=<name>"
@echo " logs-cp1/2/3 stream kubelet logs from CP{1,2,3}"
@echo " dmesg-cp1/2/3 stream kernel dmesg from CP{1,2,3}"
@echo " log-svc-cp1/2/3 stream service logs (SVC=<name>)"
@echo ""
@echo " Config"
@echo " gen-config regenerate controlplane.yaml + worker-N.yaml from secrets"
@echo " apply-cp apply controlplane.yaml to CP node (live cluster)"
@echo " apply-w1 apply cluster-config/worker-1.yaml to worker-1"
@echo " apply-w1-insecure first-time apply to worker-1 (no certs yet)"
@echo " apply-worker apply cluster-config/worker-N.yaml N=<num> W<N>_IP=<ip>"
@echo " apply-worker-new first-time apply (--insecure) N=<num> W<N>_IP=<ip>"
@echo " Config Apply"
@echo " apply-all apply configs to all 3 CPs (talos-cp-{1,2,3}.yaml)"
@echo " apply-cp1/2/3 apply config to specific CP"
@echo ""
@echo " Upgrade"
@echo " upgrade-cp upgrade Talos on control plane"
@echo " upgrade-w1 upgrade Talos on worker-1"
@echo " upgrade-worker upgrade any worker N=<num> W<N>_IP=<ip>"
@echo ""
@echo " Shutdown / Reboot"
@echo " shutdown-cluster graceful full shutdown (drain w1 → off w1 → off cp)"
@echo " shutdown-cp shut down control plane only"
@echo " shutdown-w1 shut down worker-1 only"
@echo " shutdown-worker shut down any worker N=<num> W<N>_IP=<ip>"
@echo " reboot-cp reboot control plane"
@echo " reboot-w1 reboot worker-1"
@echo " reboot-worker reboot any worker N=<num> W<N>_IP=<ip>"
@echo ""
@echo " Inspect (node filesystem)"
@echo " node-ls <ip> <path> list files on a node"
@echo " node-read <ip> <path> read a file on a node"
@echo ""
@echo " Maintenance"
@echo " clean-pods delete Evicted/Failed/Terminating pods cluster-wide"
@echo " Reboot"
@echo " reboot-all reboot all 3 CPs"
@echo " reboot-cp1/2/3 reboot specific CP"
@echo ""
@echo " Port-forwards"
@echo " pf-grafana localhost:3000 → Grafana"
@echo " pf-minio localhost:9001 → MinIO console / localhost:9000 → S3 API"
@echo " pf-loki localhost:3100 → Loki HTTP API"
@echo " pf-portainer localhost:9000 → Portainer UI (dashboard ns)"
@echo " pf-prometheus localhost:9090 → Prometheus UI (monitoring ns)"
@echo " pf-longhorn localhost:8080 → Longhorn UI"
@echo " pf-iam localhost:7000 → Authentik IAM (when deployed)"
@echo " pf-prometheus localhost:9090 → Prometheus UI"
@echo ""
@echo " CLI"
@echo " cli build core-cli and install to ~/.local/bin/core"
@echo ""
@echo " Variables"
@echo " CP_IP (default: 192.168.1.160)"
@echo " W1_IP (export from ~/.zshrc — e.g. export W1_IP=192.168.1.162)"
@echo " N (required for generic targets — worker number, e.g. N=2)"
@echo " W<N>_IP (export from ~/.zshrc — e.g. export W2_IP=192.168.1.163)"
@echo " SVC (required for log-svc-* targets, e.g. SVC=kubelet)"
@echo " IPs"
@echo " CP1 (talos-cp-1): $(CP1_IP) — NVMe, wg0/wg1, VIP"
@echo " CP2 (talos-cp-2): $(CP2_IP) — 3 Longhorn disks"
@echo " CP3 (talos-cp-3): $(CP3_IP) — NVMe"
# ── Status ────────────────────────────────────────────────────────────────────
.PHONY: nodes
nodes:
$(KUBECTL) get nodes -o wide
.PHONY: status-all
status-all: status-cp1 status-cp2 status-cp3
.PHONY: status-cp1
status-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) etcd members
.PHONY: status-cp2
status-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) etcd members
.PHONY: status-cp3
status-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) etcd members
.PHONY: status-cp
status-cp:
$(TALOSCTL) --nodes $(CP_IP) get members
status-cp: status-all
.PHONY: status-w1
status-w1:
$(TALOSCTL) --nodes $(W1_IP) get members
.PHONY: services-cp1
services-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) service
.PHONY: services-cp
services-cp:
$(TALOSCTL) --nodes $(CP_IP) service
.PHONY: services-cp2
services-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) service
.PHONY: services-w1
services-w1:
$(TALOSCTL) --nodes $(W1_IP) service
.PHONY: services-cp3
services-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) service
# ── Logs ──────────────────────────────────────────────────────────────────────
.PHONY: logs-cp
logs-cp:
$(TALOSCTL) --nodes $(CP_IP) logs kubelet -f
# ── Logs (3-CP) ───────────────────────────────────────────────────────────────
.PHONY: logs-cp1
logs-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs kubelet -f
.PHONY: logs-w1
logs-w1:
$(TALOSCTL) --nodes $(W1_IP) logs kubelet -f
.PHONY: logs-cp2
logs-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs kubelet -f
.PHONY: dmesg-cp
dmesg-cp:
$(TALOSCTL) --nodes $(CP_IP) dmesg --follow
.PHONY: logs-cp3
logs-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs kubelet -f
.PHONY: dmesg-w1
dmesg-w1:
$(TALOSCTL) --nodes $(W1_IP) dmesg --follow
.PHONY: dmesg-cp1
dmesg-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) dmesg --follow
# Usage: make log-svc-cp SVC=etcd
.PHONY: log-svc-cp
log-svc-cp:
.PHONY: dmesg-cp2
dmesg-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) dmesg --follow
.PHONY: dmesg-cp3
dmesg-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) dmesg --follow
# Usage: make log-svc-cp1 SVC=etcd
.PHONY: log-svc-cp1
log-svc-cp1:
ifndef SVC
$(error SVC is not set — run: make log-svc-cp SVC=<service-name>)
$(error SVC is not set — run: make log-svc-cp1 SVC=<service-name>)
endif
$(TALOSCTL) --nodes $(CP_IP) logs $(SVC) -f
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs $(SVC) -f
.PHONY: log-svc-w1
log-svc-w1:
.PHONY: log-svc-cp2
log-svc-cp2:
ifndef SVC
$(error SVC is not set — run: make log-svc-w1 SVC=<service-name>)
$(error SVC is not set — run: make log-svc-cp2 SVC=<service-name>)
endif
$(TALOSCTL) --nodes $(W1_IP) logs $(SVC) -f
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs $(SVC) -f
# ── Config generation ─────────────────────────────────────────────────────────
.PHONY: gen-config
gen-config:
corectl gen config $(CLUSTER_NAME) $(CP_ENDPOINT) \
--with-secrets $(SECRETS) \
--output-dir cluster-config/ \
--force
.PHONY: log-svc-cp3
log-svc-cp3:
ifndef SVC
$(error SVC is not set — run: make log-svc-cp3 SVC=<service-name>)
endif
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs $(SVC) -f
# ── Config Apply (3-CP) ───────────────────────────────────────────────────────
.PHONY: apply-all
apply-all: apply-cp1 apply-cp2 apply-cp3
@echo "✓ All 3 control planes configured"
.PHONY: apply-cp1
apply-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) apply-config -f $(CP1_CONFIG)
.PHONY: apply-cp2
apply-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) apply-config -f $(CP2_CONFIG)
.PHONY: apply-cp3
apply-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) apply-config -f $(CP3_CONFIG)
# ── Config apply ──────────────────────────────────────────────────────────────
.PHONY: apply-cp
apply-cp:
$(TALOSCTL) apply-config \
--nodes $(CP_IP) \
--file $(CP_CONFIG)
apply-cp: apply-all
.PHONY: apply-w1
apply-w1:
$(TALOSCTL) apply-config \
--nodes $(W1_IP) \
--file cluster-config/worker-1.yaml
.PHONY: reboot-all
reboot-all: reboot-cp1 reboot-cp2 reboot-cp3
@echo "✓ All 3 control planes rebooting"
# First-time apply to worker-1 (no certs yet)
.PHONY: apply-w1-insecure
apply-w1-insecure:
$(TALOSCTL) apply-config \
--nodes $(W1_IP) \
--file cluster-config/worker-1.yaml \
--insecure
.PHONY: reboot-cp1
reboot-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) reboot
# Generic targets — derive both IP and config from N.
# Usage: make apply-worker N=2 W2_IP=192.168.1.162
# make apply-worker N=3 W3_IP=192.168.1.163
.PHONY: apply-worker
apply-worker:
ifndef N
$(error N is not set — run: make apply-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) apply-config \
--nodes $(W_IP) \
--file $(W_CONFIG)
.PHONY: reboot-cp2
reboot-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) reboot
.PHONY: apply-worker-new
apply-worker-new:
ifndef N
$(error N is not set — run: make apply-worker-new N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) apply-config \
--nodes $(W_IP) \
--file $(W_CONFIG) \
--insecure
# ── Upgrade ───────────────────────────────────────────────────────────────────
.PHONY: upgrade-cp
upgrade-cp:
$(TALOSCTL) upgrade \
--nodes $(CP_IP) \
--image $(TALOS_IMAGE) \
--preserve
.PHONY: upgrade-w1
upgrade-w1:
$(TALOSCTL) upgrade \
--nodes $(W1_IP) \
--image $(TALOS_IMAGE) \
--preserve
# Usage: make upgrade-worker N=2 W2_IP=192.168.1.162
.PHONY: upgrade-worker
upgrade-worker:
ifndef N
$(error N is not set — run: make upgrade-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) upgrade \
--nodes $(W_IP) \
--image $(TALOS_IMAGE) \
--preserve
# ── Shutdown / Reboot ─────────────────────────────────────────────────────────
# Full cluster: drain workers first so pods stop cleanly, then workers off,
# then CP last (etcd must be the final process to stop).
.PHONY: shutdown-cluster
shutdown-cluster:
@echo "--- draining core-worker-1 ---"
$(KUBECTL) drain core-worker-1 --ignore-daemonsets --delete-emptydir-data
@echo "--- shutting down worker-1 ---"
$(TALOSCTL) --nodes $(W1_IP) shutdown
@echo "--- shutting down control plane (last) ---"
$(TALOSCTL) --nodes $(CP_IP) shutdown
.PHONY: shutdown-cp
shutdown-cp:
$(TALOSCTL) --nodes $(CP_IP) shutdown
.PHONY: shutdown-w1
shutdown-w1:
$(TALOSCTL) --nodes $(W1_IP) shutdown
# Usage: make shutdown-worker N=2 W2_IP=192.168.1.162
.PHONY: shutdown-worker
shutdown-worker:
ifndef N
$(error N is not set — run: make shutdown-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) --nodes $(W_IP) shutdown
.PHONY: reboot-cp3
reboot-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) reboot
.PHONY: reboot-cp
reboot-cp:
$(TALOSCTL) --nodes $(CP_IP) reboot
.PHONY: reboot-w1
reboot-w1:
$(TALOSCTL) --nodes $(W1_IP) reboot
reboot-cp: reboot-all
# Usage: make reboot-worker N=2 W2_IP=192.168.1.162
.PHONY: reboot-worker
@@ -268,18 +189,16 @@ endif
$(TALOSCTL) --nodes $(W_IP) reboot
# ── Inspect ───────────────────────────────────────────────────────────────────
# Positional args: make node-ls 192.168.1.160 /etc/kubernetes/manifests
# $(word 2/3, $(MAKECMDGOALS)) captures the extra words; the % rule absorbs
# them so Make doesn't error with "No rule to make target".
# Positional args: make node-ls 192.168.1.166 /etc/kubernetes/manifests
.PHONY: node-ls
node-ls:
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
.PHONY: node-read
node-read:
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
# Absorb positional arguments passed to node-ls / node-read
# Absorb positional arguments
%:
@: