rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(terraform): GPU worker node support (schematic, interface/diskSelector/swap tuning, gpu-node label, NVIDIA LTS extensions)

Browse Source
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:04 -07:00
parent efb9389093
commit cdf1cdeb18
9 changed files with 246 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -2
View File
@@ -1,6 +1,6 @@
# ── Node IPs (3-CP HA topology) ───────────────────────────────────────────────
CP1_IP := 192.168.1.166 # talos-cp-1
CP2_IP := 192.168.1.213 # talos-cp-2 (storage: 3 disks)
CP2_IP := 192.168.1.214 # talos-cp-2 (storage: 3 disks)
CP3_IP := 192.168.1.162 # talos-cp-3
CP_VIP := 192.168.1.166 # controlplane VIP (currently .166)
@@ -28,7 +28,7 @@ W_CONFIG = cluster-config/worker-$(N).yaml
# ── Help ──────────────────────────────────────────────────────────────────────
.PHONY: help
help:
@echo "Homelab cluster (3-CP HA: .166/.213/.163) — available targets"
@echo "Homelab cluster (3-CP HA: .166/.214/.162) — available targets"
@echo ""
@echo " Status & Services"
@echo " nodes kubectl get nodes"
terraform/main.tf
+57
View File
@@ -74,6 +74,56 @@ resource "local_file" "controlplane_configs" {
})
}
# Worker machine configurations
resource "local_file" "worker_configs" {
for_each = var.worker_configs
filename = "${path.module}/../cluster-config/worker-${each.key}.yaml"
content = templatefile("${path.module}/templates/worker.tftpl", {
version = "v1alpha1"
hostname = each.value.hostname
token = var.machine_token
ca_crt = var.machine_ca_crt
ca_key = var.machine_ca_key
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
network_interface = each.value.network_interface
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
factory_image = each.value.factory_image != null ? each.value.factory_image : (each.value.gpu_count > 0 ? data.talos_image_factory_urls.gpu_installer.urls.installer : local.factory_image)
talos_version = var.talos_version
dns_servers = var.cluster_config.dns_servers
forgejo_registry_ip = var.forgejo_registry_ip
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
gpu_count = each.value.gpu_count
extra_disks = each.value.extra_disks
swap_size = each.value.swap_size
ephemeral_max_size = each.value.ephemeral_max_size
# Cluster config
cluster_id = var.cluster_id
cluster_secret = var.cluster_secret
controlplane_ip = local.controlplane_ip
cluster_name = var.cluster_name
pod_subnets = var.cluster_config.pod_subnets
service_subnets = var.cluster_config.service_subnets
dns_domain = var.cluster_config.dns_domain
bootstrap_token = var.bootstrap_token
# Kubernetes certs
kubernetes_ca_crt = var.kubernetes_ca_crt
kubernetes_ca_key = var.kubernetes_ca_key
aggregator_ca_crt = var.aggregator_ca_crt
aggregator_ca_key = var.aggregator_ca_key
service_account_key = var.service_account_key
secretbox_encryption_secret = var.secretbox_encryption_secret
})
}
# Output paths for reference
output "controlplane_config_paths" {
value = {
@@ -81,3 +131,10 @@ output "controlplane_config_paths" {
}
description = "Paths to generated controlplane configs"
}
output "worker_config_paths" {
value = {
for k, v in local_file.worker_configs : k => v.filename
}
description = "Paths to generated worker configs"
}
terraform/talos_schematic.tf
+37
View File
@@ -19,13 +19,50 @@ data "talos_image_factory_urls" "longhorn_installer" {
platform = "metal"
}
# GPU-enabled schematic with NVIDIA drivers and toolkit (+ Longhorn deps, since
# worker nodes also run the Longhorn CSI DaemonSet).
# LTS channel (580.xx), not production (595.xx) -- Tesla V100 (Volta) is on
# NVIDIA's Legacy driver branch; the 595.xx production driver ignores it entirely
# ("NVRM: No NVIDIA GPU found").
resource "talos_image_factory_schematic" "gpu_enabled" {
schematic = jsonencode({
customization = {
systemExtensions = {
officialExtensions = [
"siderolabs/iscsi-tools",
"siderolabs/util-linux-tools",
"siderolabs/nonfree-kmod-nvidia-lts",
"siderolabs/nvidia-container-toolkit-lts",
]
}
}
})
}
# Generate installer image URL for GPU schematic
data "talos_image_factory_urls" "gpu_installer" {
talos_version = var.talos_version
schematic_id = talos_image_factory_schematic.gpu_enabled.id
platform = "metal"
}
# Output the schematic ID and installer URL for reference
output "talos_schematic_id" {
value = talos_image_factory_schematic.longhorn.id
description = "Talos Image Factory schematic ID with Longhorn dependencies"
}
output "talos_gpu_schematic_id" {
value = talos_image_factory_schematic.gpu_enabled.id
description = "Talos Image Factory schematic ID with GPU (NVIDIA driver + container toolkit)"
}
output "talos_installer_url" {
value = data.talos_image_factory_urls.longhorn_installer.urls.installer
description = "Talos installer image URL with iscsi-tools and util-linux-tools"
}
output "talos_gpu_installer_url" {
value = data.talos_image_factory_urls.gpu_installer.urls.installer
description = "Talos installer image URL with NVIDIA GPU drivers and container toolkit"
}
terraform/templates/controlplane.tftpl
+1 -1
View File
@@ -47,7 +47,7 @@ machine:
install:
disk: ${install_disk}
image: factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${talos_version}
wipe: false
wipe: true
grubUseUKICmdline: true
disks:
%{ for disk in longhorn_disks ~}
terraform/templates/worker.tftpl
+127
View File
@@ -0,0 +1,127 @@
version: ${version}
debug: false
persist: true
machine:
type: worker
token: ${token}
ca:
crt: ${ca_crt}
certSANs:
- ${lan_ip}
network:
hostname: ${hostname}
interfaces:
- interface: ${network_interface}
addresses:
- ${lan_ip}/24
routes:
- network: 0.0.0.0/0
gateway: ${lan_gateway}
dhcp: false
dhcpOptions:
ipv6: false
nameservers:
%{ for ns in dns_servers ~}
- ${ns}
%{ endfor ~}
extraHostEntries:
- ip: ${forgejo_registry_ip}
aliases:
- ${forgejo_hostname}
kubelet:
image: ${kubelet_image}
defaultRuntimeSeccompProfileEnabled: true
disableManifestsDirectory: true
clusterDNS:
- ${cluster_dns_ip}
extraArgs:
rotate-server-certificates: true
extraConfig:
memorySwap:
swapBehavior: LimitedSwap
nodeIP:
validSubnets:
- 192.168.1.0/24
install:
disk: ${install_disk}
image: ${factory_image}
wipe: true
grubUseUKICmdline: true
%{ if gpu_count > 0 ~}
kernel:
modules:
- name: nvidia
- name: nvidia_uvm
- name: nvidia_drm
- name: nvidia_modeset
%{ endif ~}
disks:
%{ for disk in extra_disks ~}
- device: ${disk.device}
partitions:
- mountpoint: ${disk.mountpoint}
%{ endfor ~}
features:
kubePrism:
enabled: true
port: 7445
hostDNS:
enabled: false
nodeLabels:
topology.kubernetes.io/region: homelab
topology.kubernetes.io/zone: ${zone}
node-role.kubernetes.io/gpu-node: ""
%{ if gpu_count > 0 ~}
nvidia.com/gpu: "true"
gpu-count: "${gpu_count}"
%{ endif ~}
cluster:
id: ${cluster_id}
secret: ${cluster_secret}
controlPlane:
endpoint: https://${controlplane_ip}:6443
clusterName: ${cluster_name}
network:
dnsDomain: ${dns_domain}
podSubnets:
%{ for subnet in pod_subnets ~}
- ${subnet}
%{ endfor ~}
serviceSubnets:
%{ for subnet in service_subnets ~}
- ${subnet}
%{ endfor ~}
cni:
name: none
token: ${bootstrap_token}
secretboxEncryptionSecret: ${secretbox_encryption_secret}
ca:
crt: ${kubernetes_ca_crt}
discovery:
enabled: true
registries:
kubernetes:
disabled: true
service: {}
%{ if swap_size != "" ~}
---
apiVersion: v1alpha1
kind: VolumeConfig
name: EPHEMERAL
provisioning:
diskSelector:
match: disk.transport == "nvme"
maxSize: ${ephemeral_max_size}
grow: false
---
apiVersion: v1alpha1
kind: SwapVolumeConfig
name: swap1
provisioning:
diskSelector:
match: disk.transport == "nvme"
minSize: ${swap_size}
maxSize: ${swap_size}
%{ endif ~}
terraform/tfplan-gpu
BIN
View File
Binary file not shown.
terraform/tfplan-rename
BIN
View File
Binary file not shown.
terraform/tfplan-worker
BIN
View File
Binary file not shown.
terraform/variables.tf
+22
View File
@@ -121,6 +121,28 @@ variable "controlplane_configs" {
description = "Control plane machine configurations, keyed by node"
}
variable "worker_configs" {
type = map(object({
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
network_interface = optional(string, "eno1")
zone = string
gpu_count = optional(number, 0)
factory_image = optional(string)
swap_size = optional(string, "")
ephemeral_max_size = optional(string, "700GiB")
extra_disks = optional(list(object({
device = string
mountpoint = string
})), [])
}))
default = {}
description = "Worker machine configurations, keyed by node"
}
variable "cluster_config" {
type = object({
controlplane_ip = string