feat(monitoring): enable Alertmanager (null receiver, longhorn PVC, az-a) + fix forgejo-rules ns forgejo->cicd — alerting delivery was disabled; forgejo PrometheusRule targeted a nonexistent namespace

This commit is contained in:
Story Crater Bot
2026-08-13 07:10:03 -07:00
parent 2d7127b37e
commit 3a91c19b5c
13 changed files with 145 additions and 139 deletions
@@ -0,0 +1,13 @@
# CiliumL2AnnouncementPolicy — ARPs each LoadBalancer IP (from lb-ippool) on the
# LAN so the EXTERNAL-IP is actually reachable. Without it, LB-IPAM assigns IPs
# but nothing answers ARP (100% packet loss / incomplete ARP). One node per IP
# holds the lease (leaderElection) to avoid ARP flapping. Requires kube-proxy
# replacement (enabled) and Cilium L2 announcements (default since v1.14).
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: homelab-l2-announce
spec:
loadBalancerIPs: true
interfaces:
- eno1
+15
View File
@@ -0,0 +1,15 @@
# CiliumLoadBalancerIPPool — the IPs Cilium LB-IPAM may assign to LoadBalancer
# Services. CIDR 192.168.1.160/28 covers .160.175 on the LAN.
# .160 ingress-nginx (LoadBalancer)
# .161 forgejo-ssh (LoadBalancer)
# .162.175 free
# Do NOT add a 10.6.0.0/24 block — that is the WireGuard subnet; letting LB-IPAM
# hand out a CP tunnel IP breaks the tunnel, and L2 ARP only works on the LAN
# interface (eno1) anyway. Keep this pool LAN-only.
apiVersion: cilium.io/v2alpha1
kind: CiliumLoadBalancerIPPool
metadata:
name: homelab-pool
spec:
blocks:
- cidr: "192.168.1.160/28"
+24 -20
View File
@@ -22,26 +22,28 @@ resource "local_file" "controlplane_configs" {
filename = "${path.module}/../cluster-config/${each.key}.yaml"
content = templatefile("${path.module}/templates/controlplane.tftpl", {
version = "v1alpha1"
hostname = each.value.hostname
token = var.machine_token
ca_crt = var.machine_ca_crt
ca_key = var.machine_ca_key
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
factory_image = local.factory_image
talos_version = var.talos_version
longhorn_disks = each.value.longhorn_disks
dns_servers = var.cluster_config.dns_servers
forgejo_registry_ip = var.forgejo_registry_ip
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
allow_scheduling = each.value.allow_scheduling
coredns_corefile = file("${path.module}/files/coredns/Corefile")
version = "v1alpha1"
hostname = each.value.hostname
token = var.machine_token
ca_crt = var.machine_ca_crt
ca_key = var.machine_ca_key
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
factory_image = local.factory_image
talos_version = var.talos_version
longhorn_disks = each.value.longhorn_disks
dns_servers = var.cluster_config.dns_servers
forgejo_registry_ip = var.forgejo_registry_ip
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
allow_scheduling = each.value.allow_scheduling
coredns_corefile = file("${path.module}/files/coredns/Corefile")
cilium_lb_ippool = file("${path.module}/files/cilium/lb-ippool.yaml")
cilium_l2_announcement = file("${path.module}/files/cilium/l2-announcement.yaml")
# Cloudflare Tunnel cert SANs (talos :50000 and kube-apiserver :6443)
cloudflare_talos_sans = each.value.cloudflare_talos_sans
@@ -101,6 +103,8 @@ resource "local_file" "worker_configs" {
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
gpu_count = each.value.gpu_count
node_labels = each.value.node_labels
node_taints = each.value.node_taints
extra_disks = each.value.extra_disks
swap_size = each.value.swap_size
ephemeral_max_size = each.value.ephemeral_max_size
+12
View File
@@ -164,6 +164,18 @@ cluster:
kind: Namespace
metadata:
name: kube-system
# Cilium LoadBalancer IPAM pool + L2 announcement policy. Substrate networking
# (owned here alongside the Cilium install), single source of truth in
# terraform/files/cilium/*.yaml. Provides LAN LoadBalancer IPs for ingress-nginx
# (.160) and forgejo-ssh (.161). Was previously an ArgoCD app whose empty
# kustomization never actually applied it (the live pool came from manual
# kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
- name: cilium-lb-ippool
contents: |
${indent(8, cilium_lb_ippool)}
- name: cilium-l2-announcement
contents: |
${indent(8, cilium_l2_announcement)}
# CoreDNS Corefile with homelab hostname rewrites (single source of truth in
# terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
# the nginx ingress controller so OIDC auto-discovery against
+10 -1
View File
@@ -71,11 +71,20 @@ machine:
nodeLabels:
topology.kubernetes.io/region: homelab
topology.kubernetes.io/zone: ${zone}
node-role.kubernetes.io/gpu-node: ""
%{ if gpu_count > 0 ~}
node-role.kubernetes.io/gpu-node: ""
nvidia.com/gpu: "true"
gpu-count: "${gpu_count}"
%{ endif ~}
%{ for k, v in node_labels ~}
${k}: "${v}"
%{ endfor ~}
%{ if length(node_taints) > 0 ~}
nodeTaints:
%{ for t in node_taints ~}
${t.key}: "${t.value}:${t.effect}"
%{ endfor ~}
%{ endif ~}
cluster:
id: ${cluster_id}
+17 -8
View File
@@ -123,14 +123,23 @@ variable "controlplane_configs" {
variable "worker_configs" {
type = map(object({
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
network_interface = optional(string, "eno1")
zone = string
gpu_count = optional(number, 0)
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
network_interface = optional(string, "eno1")
zone = string
gpu_count = optional(number, 0)
# Extra node labels beyond the topology/GPU defaults.
node_labels = optional(map(string), {})
# Taints make a node dedicated: only pods carrying a matching toleration
# schedule there. effect is NoSchedule | PreferNoSchedule | NoExecute.
node_taints = optional(list(object({
key = string
value = string
effect = string
})), [])
factory_image = optional(string)
swap_size = optional(string, "")
ephemeral_max_size = optional(string, "700GiB")