diff --git a/terraform/main.tf b/terraform/main.tf index 6c73a84..1f9f39d 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -74,6 +74,53 @@ resource "local_file" "controlplane_configs" { }) } +# Worker machine configurations +resource "local_file" "worker_configs" { + for_each = var.worker_configs + + filename = "${path.module}/../cluster-config/worker-${each.key}.yaml" + + content = templatefile("${path.module}/templates/worker.tftpl", { + version = "v1alpha1" + hostname = each.value.hostname + token = var.machine_token + ca_crt = var.machine_ca_crt + ca_key = var.machine_ca_key + lan_ip = each.value.lan_ip + lan_subnet = each.value.lan_subnet + lan_gateway = each.value.lan_gateway + kubelet_image = local.kubelet_image + cluster_dns_ip = local.cluster_dns_ip + install_disk = each.value.install_disk + factory_image = each.value.factory_image != null ? each.value.factory_image : (each.value.gpu_count > 0 ? data.talos_image_factory_urls.gpu_installer.urls.installer : local.factory_image) + talos_version = var.talos_version + dns_servers = var.cluster_config.dns_servers + forgejo_registry_ip = var.forgejo_registry_ip + forgejo_hostname = var.forgejo_hostname + zone = each.value.zone + gpu_count = each.value.gpu_count + extra_disks = each.value.extra_disks + + # Cluster config + cluster_id = var.cluster_id + cluster_secret = var.cluster_secret + controlplane_ip = local.controlplane_ip + cluster_name = var.cluster_name + pod_subnets = var.cluster_config.pod_subnets + service_subnets = var.cluster_config.service_subnets + dns_domain = var.cluster_config.dns_domain + bootstrap_token = var.bootstrap_token + + # Kubernetes certs + kubernetes_ca_crt = var.kubernetes_ca_crt + kubernetes_ca_key = var.kubernetes_ca_key + aggregator_ca_crt = var.aggregator_ca_crt + aggregator_ca_key = var.aggregator_ca_key + service_account_key = var.service_account_key + secretbox_encryption_secret = var.secretbox_encryption_secret + }) +} + # Output paths for reference output "controlplane_config_paths" { value = { @@ -81,3 +128,10 @@ output "controlplane_config_paths" { } description = "Paths to generated controlplane configs" } + +output "worker_config_paths" { + value = { + for k, v in local_file.worker_configs : k => v.filename + } + description = "Paths to generated worker configs" +} diff --git a/terraform/talos_schematic.tf b/terraform/talos_schematic.tf index 0e7b531..815e4ab 100644 --- a/terraform/talos_schematic.tf +++ b/terraform/talos_schematic.tf @@ -19,13 +19,44 @@ data "talos_image_factory_urls" "longhorn_installer" { platform = "metal" } +# GPU-enabled schematic with NVIDIA drivers and toolkit +resource "talos_image_factory_schematic" "gpu_enabled" { + schematic = jsonencode({ + customization = { + systemExtensions = { + officialExtensions = [ + "siderolabs/nvidia-driver", + "siderolabs/nvidia-container-toolkit", + ] + } + } + }) +} + +# Generate installer image URL for GPU schematic +data "talos_image_factory_urls" "gpu_installer" { + talos_version = var.talos_version + schematic_id = talos_image_factory_schematic.gpu_enabled.id + platform = "metal" +} + # Output the schematic ID and installer URL for reference output "talos_schematic_id" { value = talos_image_factory_schematic.longhorn.id description = "Talos Image Factory schematic ID with Longhorn dependencies" } +output "talos_gpu_schematic_id" { + value = talos_image_factory_schematic.gpu_enabled.id + description = "Talos Image Factory schematic ID with GPU (NVIDIA driver + container toolkit)" +} + output "talos_installer_url" { value = data.talos_image_factory_urls.longhorn_installer.urls.installer description = "Talos installer image URL with iscsi-tools and util-linux-tools" } + +output "talos_gpu_installer_url" { + value = data.talos_image_factory_urls.gpu_installer.urls.installer + description = "Talos installer image URL with NVIDIA GPU drivers and container toolkit" +} diff --git a/terraform/templates/worker.tftpl b/terraform/templates/worker.tftpl new file mode 100644 index 0000000..fce498b --- /dev/null +++ b/terraform/templates/worker.tftpl @@ -0,0 +1,93 @@ +version: ${version} +debug: false +persist: true + +machine: + type: worker + token: ${token} + ca: + crt: ${ca_crt} + certSANs: + - ${lan_ip} + network: + hostname: ${hostname} + interfaces: + - interface: eno1 + addresses: + - ${lan_ip}/24 + routes: + - network: 0.0.0.0/0 + gateway: ${lan_gateway} + dhcp: false + dhcpOptions: + ipv6: false + nameservers: +%{ for ns in dns_servers ~} + - ${ns} +%{ endfor ~} + extraHostEntries: + - ip: ${forgejo_registry_ip} + aliases: + - ${forgejo_hostname} + kubelet: + image: ${kubelet_image} + defaultRuntimeSeccompProfileEnabled: true + disableManifestsDirectory: true + clusterDNS: + - ${cluster_dns_ip} + extraArgs: + rotate-server-certificates: true + nodeIP: + validSubnets: + - 192.168.1.0/24 + install: + disk: ${install_disk} + image: ${factory_image} + wipe: false + grubUseUKICmdline: true + disks: +%{ for disk in extra_disks ~} + - device: ${disk.device} + partitions: + - mountpoint: ${disk.mountpoint} +%{ endfor ~} + features: + kubePrism: + enabled: true + port: 7445 + hostDNS: + enabled: false + nodeLabels: + topology.kubernetes.io/region: homelab + topology.kubernetes.io/zone: ${zone} +%{ if gpu_count > 0 ~} + nvidia.com/gpu: "true" + gpu-count: "${gpu_count}" +%{ endif ~} + +cluster: + id: ${cluster_id} + secret: ${cluster_secret} + controlPlane: + endpoint: https://${controlplane_ip}:6443 + clusterName: ${cluster_name} + network: + dnsDomain: ${dns_domain} + podSubnets: +%{ for subnet in pod_subnets ~} + - ${subnet} +%{ endfor ~} + serviceSubnets: +%{ for subnet in service_subnets ~} + - ${subnet} +%{ endfor ~} + cni: + name: none + token: ${bootstrap_token} + secretboxEncryptionSecret: ${secretbox_encryption_secret} + discovery: + enabled: true + registries: + kubernetes: + disabled: true + service: {} diff --git a/terraform/tfplan-gpu b/terraform/tfplan-gpu new file mode 100644 index 0000000..e484b64 Binary files /dev/null and b/terraform/tfplan-gpu differ diff --git a/terraform/tfplan-worker b/terraform/tfplan-worker new file mode 100644 index 0000000..2bcfe99 Binary files /dev/null and b/terraform/tfplan-worker differ diff --git a/terraform/variables.tf b/terraform/variables.tf index 3a9b4cf..11d38c8 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -121,6 +121,25 @@ variable "controlplane_configs" { description = "Control plane machine configurations, keyed by node" } +variable "worker_configs" { + type = map(object({ + hostname = string + lan_ip = string + lan_subnet = string + lan_gateway = string + install_disk = string + zone = string + gpu_count = optional(number, 0) + factory_image = optional(string) + extra_disks = optional(list(object({ + device = string + mountpoint = string + })), []) + })) + default = {} + description = "Worker machine configurations, keyed by node" +} + variable "cluster_config" { type = object({ controlplane_ip = string