diff --git a/Makefile b/Makefile index ac5a968..0886608 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # ── Node IPs (3-CP HA topology) ─────────────────────────────────────────────── CP1_IP := 192.168.1.166 # talos-cp-1 -CP2_IP := 192.168.1.213 # talos-cp-2 (storage: 3 disks) +CP2_IP := 192.168.1.214 # talos-cp-2 (storage: 3 disks) CP3_IP := 192.168.1.162 # talos-cp-3 CP_VIP := 192.168.1.166 # controlplane VIP (currently .166) @@ -28,7 +28,7 @@ W_CONFIG = cluster-config/worker-$(N).yaml # ── Help ────────────────────────────────────────────────────────────────────── .PHONY: help help: - @echo "Homelab cluster (3-CP HA: .166/.213/.163) — available targets" + @echo "Homelab cluster (3-CP HA: .166/.214/.162) — available targets" @echo "" @echo " Status & Services" @echo " nodes kubectl get nodes" diff --git a/terraform/main.tf b/terraform/main.tf index 6c73a84..f4d5a8e 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -74,6 +74,56 @@ resource "local_file" "controlplane_configs" { }) } +# Worker machine configurations +resource "local_file" "worker_configs" { + for_each = var.worker_configs + + filename = "${path.module}/../cluster-config/worker-${each.key}.yaml" + + content = templatefile("${path.module}/templates/worker.tftpl", { + version = "v1alpha1" + hostname = each.value.hostname + token = var.machine_token + ca_crt = var.machine_ca_crt + ca_key = var.machine_ca_key + lan_ip = each.value.lan_ip + lan_subnet = each.value.lan_subnet + lan_gateway = each.value.lan_gateway + network_interface = each.value.network_interface + kubelet_image = local.kubelet_image + cluster_dns_ip = local.cluster_dns_ip + install_disk = each.value.install_disk + factory_image = each.value.factory_image != null ? each.value.factory_image : (each.value.gpu_count > 0 ? data.talos_image_factory_urls.gpu_installer.urls.installer : local.factory_image) + talos_version = var.talos_version + dns_servers = var.cluster_config.dns_servers + forgejo_registry_ip = var.forgejo_registry_ip + forgejo_hostname = var.forgejo_hostname + zone = each.value.zone + gpu_count = each.value.gpu_count + extra_disks = each.value.extra_disks + swap_size = each.value.swap_size + ephemeral_max_size = each.value.ephemeral_max_size + + # Cluster config + cluster_id = var.cluster_id + cluster_secret = var.cluster_secret + controlplane_ip = local.controlplane_ip + cluster_name = var.cluster_name + pod_subnets = var.cluster_config.pod_subnets + service_subnets = var.cluster_config.service_subnets + dns_domain = var.cluster_config.dns_domain + bootstrap_token = var.bootstrap_token + + # Kubernetes certs + kubernetes_ca_crt = var.kubernetes_ca_crt + kubernetes_ca_key = var.kubernetes_ca_key + aggregator_ca_crt = var.aggregator_ca_crt + aggregator_ca_key = var.aggregator_ca_key + service_account_key = var.service_account_key + secretbox_encryption_secret = var.secretbox_encryption_secret + }) +} + # Output paths for reference output "controlplane_config_paths" { value = { @@ -81,3 +131,10 @@ output "controlplane_config_paths" { } description = "Paths to generated controlplane configs" } + +output "worker_config_paths" { + value = { + for k, v in local_file.worker_configs : k => v.filename + } + description = "Paths to generated worker configs" +} diff --git a/terraform/talos_schematic.tf b/terraform/talos_schematic.tf index 0e7b531..502b924 100644 --- a/terraform/talos_schematic.tf +++ b/terraform/talos_schematic.tf @@ -19,13 +19,50 @@ data "talos_image_factory_urls" "longhorn_installer" { platform = "metal" } +# GPU-enabled schematic with NVIDIA drivers and toolkit (+ Longhorn deps, since +# worker nodes also run the Longhorn CSI DaemonSet). +# LTS channel (580.xx), not production (595.xx) -- Tesla V100 (Volta) is on +# NVIDIA's Legacy driver branch; the 595.xx production driver ignores it entirely +# ("NVRM: No NVIDIA GPU found"). +resource "talos_image_factory_schematic" "gpu_enabled" { + schematic = jsonencode({ + customization = { + systemExtensions = { + officialExtensions = [ + "siderolabs/iscsi-tools", + "siderolabs/util-linux-tools", + "siderolabs/nonfree-kmod-nvidia-lts", + "siderolabs/nvidia-container-toolkit-lts", + ] + } + } + }) +} + +# Generate installer image URL for GPU schematic +data "talos_image_factory_urls" "gpu_installer" { + talos_version = var.talos_version + schematic_id = talos_image_factory_schematic.gpu_enabled.id + platform = "metal" +} + # Output the schematic ID and installer URL for reference output "talos_schematic_id" { value = talos_image_factory_schematic.longhorn.id description = "Talos Image Factory schematic ID with Longhorn dependencies" } +output "talos_gpu_schematic_id" { + value = talos_image_factory_schematic.gpu_enabled.id + description = "Talos Image Factory schematic ID with GPU (NVIDIA driver + container toolkit)" +} + output "talos_installer_url" { value = data.talos_image_factory_urls.longhorn_installer.urls.installer description = "Talos installer image URL with iscsi-tools and util-linux-tools" } + +output "talos_gpu_installer_url" { + value = data.talos_image_factory_urls.gpu_installer.urls.installer + description = "Talos installer image URL with NVIDIA GPU drivers and container toolkit" +} diff --git a/terraform/templates/controlplane.tftpl b/terraform/templates/controlplane.tftpl index 13e7e1d..c31cd1e 100644 --- a/terraform/templates/controlplane.tftpl +++ b/terraform/templates/controlplane.tftpl @@ -47,7 +47,7 @@ machine: install: disk: ${install_disk} image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version} - wipe: false + wipe: true grubUseUKICmdline: true disks: %{ for disk in longhorn_disks ~} diff --git a/terraform/templates/worker.tftpl b/terraform/templates/worker.tftpl new file mode 100644 index 0000000..507ab71 --- /dev/null +++ b/terraform/templates/worker.tftpl @@ -0,0 +1,127 @@ +version: ${version} +debug: false +persist: true + +machine: + type: worker + token: ${token} + ca: + crt: ${ca_crt} + certSANs: + - ${lan_ip} + network: + hostname: ${hostname} + interfaces: + - interface: ${network_interface} + addresses: + - ${lan_ip}/24 + routes: + - network: 0.0.0.0/0 + gateway: ${lan_gateway} + dhcp: false + dhcpOptions: + ipv6: false + nameservers: +%{ for ns in dns_servers ~} + - ${ns} +%{ endfor ~} + extraHostEntries: + - ip: ${forgejo_registry_ip} + aliases: + - ${forgejo_hostname} + kubelet: + image: ${kubelet_image} + defaultRuntimeSeccompProfileEnabled: true + disableManifestsDirectory: true + clusterDNS: + - ${cluster_dns_ip} + extraArgs: + rotate-server-certificates: true + extraConfig: + memorySwap: + swapBehavior: LimitedSwap + nodeIP: + validSubnets: + - 192.168.1.0/24 + install: + disk: ${install_disk} + image: ${factory_image} + wipe: true + grubUseUKICmdline: true +%{ if gpu_count > 0 ~} + kernel: + modules: + - name: nvidia + - name: nvidia_uvm + - name: nvidia_drm + - name: nvidia_modeset +%{ endif ~} + disks: +%{ for disk in extra_disks ~} + - device: ${disk.device} + partitions: + - mountpoint: ${disk.mountpoint} +%{ endfor ~} + features: + kubePrism: + enabled: true + port: 7445 + hostDNS: + enabled: false + nodeLabels: + topology.kubernetes.io/region: homelab + topology.kubernetes.io/zone: ${zone} + node-role.kubernetes.io/gpu-node: "" +%{ if gpu_count > 0 ~} + nvidia.com/gpu: "true" + gpu-count: "${gpu_count}" +%{ endif ~} + +cluster: + id: ${cluster_id} + secret: ${cluster_secret} + controlPlane: + endpoint: https://${controlplane_ip}:6443 + clusterName: ${cluster_name} + network: + dnsDomain: ${dns_domain} + podSubnets: +%{ for subnet in pod_subnets ~} + - ${subnet} +%{ endfor ~} + serviceSubnets: +%{ for subnet in service_subnets ~} + - ${subnet} +%{ endfor ~} + cni: + name: none + token: ${bootstrap_token} + secretboxEncryptionSecret: ${secretbox_encryption_secret} + ca: + crt: ${kubernetes_ca_crt} + discovery: + enabled: true + registries: + kubernetes: + disabled: true + service: {} +%{ if swap_size != "" ~} +--- +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + diskSelector: + match: disk.transport == "nvme" + maxSize: ${ephemeral_max_size} + grow: false +--- +apiVersion: v1alpha1 +kind: SwapVolumeConfig +name: swap1 +provisioning: + diskSelector: + match: disk.transport == "nvme" + minSize: ${swap_size} + maxSize: ${swap_size} +%{ endif ~} diff --git a/terraform/tfplan-gpu b/terraform/tfplan-gpu new file mode 100644 index 0000000..e484b64 Binary files /dev/null and b/terraform/tfplan-gpu differ diff --git a/terraform/tfplan-rename b/terraform/tfplan-rename new file mode 100644 index 0000000..d10b71b Binary files /dev/null and b/terraform/tfplan-rename differ diff --git a/terraform/tfplan-worker b/terraform/tfplan-worker new file mode 100644 index 0000000..2bcfe99 Binary files /dev/null and b/terraform/tfplan-worker differ diff --git a/terraform/variables.tf b/terraform/variables.tf index 3a9b4cf..45aa573 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -121,6 +121,28 @@ variable "controlplane_configs" { description = "Control plane machine configurations, keyed by node" } +variable "worker_configs" { + type = map(object({ + hostname = string + lan_ip = string + lan_subnet = string + lan_gateway = string + install_disk = string + network_interface = optional(string, "eno1") + zone = string + gpu_count = optional(number, 0) + factory_image = optional(string) + swap_size = optional(string, "") + ephemeral_max_size = optional(string, "700GiB") + extra_disks = optional(list(object({ + device = string + mountpoint = string + })), []) + })) + default = {} + description = "Worker machine configurations, keyed by node" +} + variable "cluster_config" { type = object({ controlplane_ip = string