Story Crater Bot
5b00616f41
- README: cluster architecture, quick start, use cases - USAGE: stack topology, custom CLI reference - TROUBLESHOOTING: operational safety rules - .env.example: configuration template - Makefile: build shortcuts
328 lines
12 KiB
Makefile
328 lines
12 KiB
Makefile
# ── Node IPs ──────────────────────────────────────────────────────────────────
|
|
# CP_IP has a default. All W{N}_IP variables are expected to be exported from
|
|
# ~/.zshrc (e.g. export W1_IP=192.168.1.162). No guards — assumed always set.
|
|
CP_IP ?= 192.168.1.213
|
|
|
|
export CP_IP
|
|
|
|
# ── Paths ─────────────────────────────────────────────────────────────────────
|
|
TALOSCONFIG := cluster-config/coreconfig
|
|
CP_CONFIG := cluster-config/controlplane.yaml
|
|
SECRETS := cluster-config/secrets.yaml
|
|
KUBECONFIG := cluster-config/kubeconfig
|
|
|
|
CLUSTER_NAME := homelab-cluster
|
|
CP_ENDPOINT := https://$(CP_IP):6443
|
|
TALOS_IMAGE := factory.core.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.13.3
|
|
|
|
TALOSCTL := corectl --coreconfig $(TALOSCONFIG)
|
|
KUBECTL := kubectl --kubeconfig $(KUBECONFIG)
|
|
|
|
# Derive IP and config from worker number N (used by generic targets).
|
|
# $(W$(N)_IP) expands to e.g. $(W2_IP) when N=2.
|
|
W_IP = $(W$(N)_IP)
|
|
W_CONFIG = cluster-config/worker-$(N).yaml
|
|
|
|
.DEFAULT_GOAL := help
|
|
|
|
# ── Help ──────────────────────────────────────────────────────────────────────
|
|
.PHONY: help
|
|
help:
|
|
@echo "Homelab cluster — available targets"
|
|
@echo ""
|
|
@echo " Status"
|
|
@echo " nodes kubectl get nodes"
|
|
@echo " status-cp core node overview (control plane)"
|
|
@echo " status-w1 core node overview (worker-1)"
|
|
@echo " services-cp list core services (control plane)"
|
|
@echo " services-w1 list core services (worker-1)"
|
|
@echo ""
|
|
@echo " Logs"
|
|
@echo " logs-cp stream kubelet logs (control plane)"
|
|
@echo " logs-w1 stream kubelet logs (worker-1)"
|
|
@echo " dmesg-cp kernel dmesg (control plane)"
|
|
@echo " dmesg-w1 kernel dmesg (worker-1)"
|
|
@echo " log-svc-cp stream a service log (control plane) SVC=<name>"
|
|
@echo " log-svc-w1 stream a service log (worker-1) SVC=<name>"
|
|
@echo ""
|
|
@echo " Config"
|
|
@echo " gen-config regenerate controlplane.yaml + worker-N.yaml from secrets"
|
|
@echo " apply-cp apply controlplane.yaml to CP node (live cluster)"
|
|
@echo " apply-w1 apply cluster-config/worker-1.yaml to worker-1"
|
|
@echo " apply-w1-insecure first-time apply to worker-1 (no certs yet)"
|
|
@echo " apply-worker apply cluster-config/worker-N.yaml N=<num> W<N>_IP=<ip>"
|
|
@echo " apply-worker-new first-time apply (--insecure) N=<num> W<N>_IP=<ip>"
|
|
@echo ""
|
|
@echo " Upgrade"
|
|
@echo " upgrade-cp upgrade Talos on control plane"
|
|
@echo " upgrade-w1 upgrade Talos on worker-1"
|
|
@echo " upgrade-worker upgrade any worker N=<num> W<N>_IP=<ip>"
|
|
@echo ""
|
|
@echo " Shutdown / Reboot"
|
|
@echo " shutdown-cluster graceful full shutdown (drain w1 → off w1 → off cp)"
|
|
@echo " shutdown-cp shut down control plane only"
|
|
@echo " shutdown-w1 shut down worker-1 only"
|
|
@echo " shutdown-worker shut down any worker N=<num> W<N>_IP=<ip>"
|
|
@echo " reboot-cp reboot control plane"
|
|
@echo " reboot-w1 reboot worker-1"
|
|
@echo " reboot-worker reboot any worker N=<num> W<N>_IP=<ip>"
|
|
@echo ""
|
|
@echo " Inspect (node filesystem)"
|
|
@echo " node-ls <ip> <path> list files on a node"
|
|
@echo " node-read <ip> <path> read a file on a node"
|
|
@echo ""
|
|
@echo " Maintenance"
|
|
@echo " clean-pods delete Evicted/Failed/Terminating pods cluster-wide"
|
|
@echo ""
|
|
@echo " Port-forwards"
|
|
@echo " pf-grafana localhost:3000 → Grafana"
|
|
@echo " pf-minio localhost:9001 → MinIO console / localhost:9000 → S3 API"
|
|
@echo " pf-loki localhost:3100 → Loki HTTP API"
|
|
@echo " pf-portainer localhost:9000 → Portainer UI (dashboard ns)"
|
|
@echo " pf-prometheus localhost:9090 → Prometheus UI (monitoring ns)"
|
|
@echo " pf-longhorn localhost:8080 → Longhorn UI"
|
|
@echo " pf-iam localhost:7000 → Authentik IAM (when deployed)"
|
|
@echo ""
|
|
@echo " CLI"
|
|
@echo " cli build core-cli and install to ~/.local/bin/core"
|
|
@echo ""
|
|
@echo " Variables"
|
|
@echo " CP_IP (default: 192.168.1.160)"
|
|
@echo " W1_IP (export from ~/.zshrc — e.g. export W1_IP=192.168.1.162)"
|
|
@echo " N (required for generic targets — worker number, e.g. N=2)"
|
|
@echo " W<N>_IP (export from ~/.zshrc — e.g. export W2_IP=192.168.1.163)"
|
|
@echo " SVC (required for log-svc-* targets, e.g. SVC=kubelet)"
|
|
|
|
# ── Status ────────────────────────────────────────────────────────────────────
|
|
.PHONY: nodes
|
|
nodes:
|
|
$(KUBECTL) get nodes -o wide
|
|
|
|
.PHONY: status-cp
|
|
status-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) get members
|
|
|
|
.PHONY: status-w1
|
|
status-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) get members
|
|
|
|
.PHONY: services-cp
|
|
services-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) service
|
|
|
|
.PHONY: services-w1
|
|
services-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) service
|
|
|
|
# ── Logs ──────────────────────────────────────────────────────────────────────
|
|
.PHONY: logs-cp
|
|
logs-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) logs kubelet -f
|
|
|
|
.PHONY: logs-w1
|
|
logs-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) logs kubelet -f
|
|
|
|
.PHONY: dmesg-cp
|
|
dmesg-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) dmesg --follow
|
|
|
|
.PHONY: dmesg-w1
|
|
dmesg-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) dmesg --follow
|
|
|
|
# Usage: make log-svc-cp SVC=etcd
|
|
.PHONY: log-svc-cp
|
|
log-svc-cp:
|
|
ifndef SVC
|
|
$(error SVC is not set — run: make log-svc-cp SVC=<service-name>)
|
|
endif
|
|
$(TALOSCTL) --nodes $(CP_IP) logs $(SVC) -f
|
|
|
|
.PHONY: log-svc-w1
|
|
log-svc-w1:
|
|
ifndef SVC
|
|
$(error SVC is not set — run: make log-svc-w1 SVC=<service-name>)
|
|
endif
|
|
$(TALOSCTL) --nodes $(W1_IP) logs $(SVC) -f
|
|
|
|
# ── Config generation ─────────────────────────────────────────────────────────
|
|
.PHONY: gen-config
|
|
gen-config:
|
|
corectl gen config $(CLUSTER_NAME) $(CP_ENDPOINT) \
|
|
--with-secrets $(SECRETS) \
|
|
--output-dir cluster-config/ \
|
|
--force
|
|
|
|
# ── Config apply ──────────────────────────────────────────────────────────────
|
|
.PHONY: apply-cp
|
|
apply-cp:
|
|
$(TALOSCTL) apply-config \
|
|
--nodes $(CP_IP) \
|
|
--file $(CP_CONFIG)
|
|
|
|
.PHONY: apply-w1
|
|
apply-w1:
|
|
$(TALOSCTL) apply-config \
|
|
--nodes $(W1_IP) \
|
|
--file cluster-config/worker-1.yaml
|
|
|
|
# First-time apply to worker-1 (no certs yet)
|
|
.PHONY: apply-w1-insecure
|
|
apply-w1-insecure:
|
|
$(TALOSCTL) apply-config \
|
|
--nodes $(W1_IP) \
|
|
--file cluster-config/worker-1.yaml \
|
|
--insecure
|
|
|
|
# Generic targets — derive both IP and config from N.
|
|
# Usage: make apply-worker N=2 W2_IP=192.168.1.162
|
|
# make apply-worker N=3 W3_IP=192.168.1.163
|
|
.PHONY: apply-worker
|
|
apply-worker:
|
|
ifndef N
|
|
$(error N is not set — run: make apply-worker N=<num> W<N>_IP=<ip>)
|
|
endif
|
|
$(TALOSCTL) apply-config \
|
|
--nodes $(W_IP) \
|
|
--file $(W_CONFIG)
|
|
|
|
.PHONY: apply-worker-new
|
|
apply-worker-new:
|
|
ifndef N
|
|
$(error N is not set — run: make apply-worker-new N=<num> W<N>_IP=<ip>)
|
|
endif
|
|
$(TALOSCTL) apply-config \
|
|
--nodes $(W_IP) \
|
|
--file $(W_CONFIG) \
|
|
--insecure
|
|
|
|
# ── Upgrade ───────────────────────────────────────────────────────────────────
|
|
.PHONY: upgrade-cp
|
|
upgrade-cp:
|
|
$(TALOSCTL) upgrade \
|
|
--nodes $(CP_IP) \
|
|
--image $(TALOS_IMAGE) \
|
|
--preserve
|
|
|
|
.PHONY: upgrade-w1
|
|
upgrade-w1:
|
|
$(TALOSCTL) upgrade \
|
|
--nodes $(W1_IP) \
|
|
--image $(TALOS_IMAGE) \
|
|
--preserve
|
|
|
|
# Usage: make upgrade-worker N=2 W2_IP=192.168.1.162
|
|
.PHONY: upgrade-worker
|
|
upgrade-worker:
|
|
ifndef N
|
|
$(error N is not set — run: make upgrade-worker N=<num> W<N>_IP=<ip>)
|
|
endif
|
|
$(TALOSCTL) upgrade \
|
|
--nodes $(W_IP) \
|
|
--image $(TALOS_IMAGE) \
|
|
--preserve
|
|
|
|
# ── Shutdown / Reboot ─────────────────────────────────────────────────────────
|
|
# Full cluster: drain workers first so pods stop cleanly, then workers off,
|
|
# then CP last (etcd must be the final process to stop).
|
|
.PHONY: shutdown-cluster
|
|
shutdown-cluster:
|
|
@echo "--- draining core-worker-1 ---"
|
|
$(KUBECTL) drain core-worker-1 --ignore-daemonsets --delete-emptydir-data
|
|
@echo "--- shutting down worker-1 ---"
|
|
$(TALOSCTL) --nodes $(W1_IP) shutdown
|
|
@echo "--- shutting down control plane (last) ---"
|
|
$(TALOSCTL) --nodes $(CP_IP) shutdown
|
|
|
|
.PHONY: shutdown-cp
|
|
shutdown-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) shutdown
|
|
|
|
.PHONY: shutdown-w1
|
|
shutdown-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) shutdown
|
|
|
|
# Usage: make shutdown-worker N=2 W2_IP=192.168.1.162
|
|
.PHONY: shutdown-worker
|
|
shutdown-worker:
|
|
ifndef N
|
|
$(error N is not set — run: make shutdown-worker N=<num> W<N>_IP=<ip>)
|
|
endif
|
|
$(TALOSCTL) --nodes $(W_IP) shutdown
|
|
|
|
.PHONY: reboot-cp
|
|
reboot-cp:
|
|
$(TALOSCTL) --nodes $(CP_IP) reboot
|
|
|
|
.PHONY: reboot-w1
|
|
reboot-w1:
|
|
$(TALOSCTL) --nodes $(W1_IP) reboot
|
|
|
|
# Usage: make reboot-worker N=2 W2_IP=192.168.1.162
|
|
.PHONY: reboot-worker
|
|
reboot-worker:
|
|
ifndef N
|
|
$(error N is not set — run: make reboot-worker N=<num> W<N>_IP=<ip>)
|
|
endif
|
|
$(TALOSCTL) --nodes $(W_IP) reboot
|
|
|
|
# ── Inspect ───────────────────────────────────────────────────────────────────
|
|
# Positional args: make node-ls 192.168.1.160 /etc/kubernetes/manifests
|
|
# $(word 2/3, $(MAKECMDGOALS)) captures the extra words; the % rule absorbs
|
|
# them so Make doesn't error with "No rule to make target".
|
|
.PHONY: node-ls
|
|
node-ls:
|
|
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
|
|
|
|
.PHONY: node-read
|
|
node-read:
|
|
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
|
|
|
|
# Absorb positional arguments passed to node-ls / node-read
|
|
%:
|
|
@:
|
|
|
|
# ── Maintenance ───────────────────────────────────────────────────────────────
|
|
.PHONY: clean-pods
|
|
clean-pods:
|
|
@echo "--- removing Failed/Evicted pods ---"
|
|
$(KUBECTL) delete pods -A --field-selector=status.phase=Failed --ignore-not-found
|
|
@echo "--- force-deleting stuck Terminating pods ---"
|
|
@$(KUBECTL) get pods -A | awk '/Terminating/{print $$1, $$2}' | \
|
|
xargs -r -n2 sh -c '$(KUBECTL) delete pod -n $$0 $$1 --force --grace-period=0' || true
|
|
|
|
# ── Port-forwards ─────────────────────────────────────────────────────────────
|
|
.PHONY: pf-query
|
|
pf-grafana:
|
|
$(KUBECTL) port-forward -n logging svc/grafana 3000:80
|
|
|
|
.PHONY: pf-minio
|
|
pf-minio:
|
|
$(KUBECTL) port-forward -n storage svc/minio 9001:9001 &
|
|
$(KUBECTL) port-forward -n storage svc/minio 9000:9000 &
|
|
|
|
.PHONY: pf-loki
|
|
pf-loki:
|
|
$(KUBECTL) port-forward -n logging svc/loki 3100:3100
|
|
|
|
.PHONY: pf-iam
|
|
pf-iam:
|
|
$(KUBECTL) port-forward -n iam svc/authentik-server 7000:80
|
|
|
|
.PHONY: pf-portainer
|
|
pf-portainer:
|
|
$(KUBECTL) port-forward -n dashboard svc/portainer 9000:9000
|
|
|
|
.PHONY: pf-prometheus
|
|
pf-prometheus:
|
|
$(KUBECTL) port-forward -n monitoring svc/prometheus-kube-prometheus-prometheus 9090:9090
|
|
|
|
.PHONY: pf-longhorn
|
|
pf-longhorn:
|
|
$(KUBECTL) port-forward -n longhorn-system svc/longhorn-frontend 8080:80
|
|
|
|
# ── CLI ───────────────────────────────────────────────────────────────────────
|
|
.PHONY: cli
|
|
cli:
|
|
$(MAKE) -C core-cli install
|