rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(monitoring): enable Alertmanager (null receiver, longhorn PVC, az-a) + fix forgejo-rules ns forgejo->cicd — alerting delivery was disabled; forgejo PrometheusRule targeted a nonexistent namespace

Browse Source
This commit is contained in:
Story Crater Bot
2026-08-18 15:08:04 -07:00
parent 9e0a83f8ca
commit 61b906cce1
13 changed files with 145 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s/argocd/apps/05-networking.yaml
+8 -26
View File
@@ -1,26 +1,8 @@
# Wave 0 — networking policies layered on the Cilium CNI + CoreDNS that the
# cluster bootstrap already installed (substrate). These are raw manifests only.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: cilium-policy
namespace: argocd
annotations:
argocd.argoproj.io/sync-wave: "0"
spec:
project: homelab
source:
repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
targetRevision: main
path: k8s/bootstrap/cilium
destination:
server: https://kubernetes.default.svc
namespace: kube-system
syncPolicy:
automated:
prune: true
selfHeal: true
# coredns-config Application removed: CoreDNS (incl. homelab hostname rewrites)
# is owned by Talos via an inlineManifest (terraform/files/coredns/Corefile).
# Managing the coredns ConfigMap from ArgoCD too would let the two reconcilers
# fight and revert the rewrites.
# Wave 0 — networking substrate is Talos-owned (terraform inlineManifests), not
# ArgoCD:
# - CoreDNS Corefile + hostname rewrites -> terraform/files/coredns/Corefile
# - Cilium LB-IPAM pool + L2 announcement -> terraform/files/cilium/*.yaml
# Both were previously ArgoCD apps here whose empty `resources: []`
# kustomizations never actually applied them (live objects came from manual
# kubectl). Managing them from ArgoCD too would let two reconcilers fight. This
# file intentionally defines no Applications now.
k8s/bootstrap/cilium/kustomization.yaml
-5
View File
@@ -1,5 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: kube-system
resources: []
# Cilium deployed via Helm chart
k8s/bootstrap/cilium/l2-announcement-policy.yaml
-39
View File
@@ -1,39 +0,0 @@
# k8s/cilium/l2-announcement-policy.yaml
# CiliumL2AnnouncementPolicy — without this, LB-IPAM (lb-ipam-pool.yaml)
# assigns IPs to LoadBalancer Services but nothing ARPs for them on the LAN,
# so they're unreachable from outside the cluster even though `kubectl get
# svc` shows a real EXTERNAL-IP. Confirmed both forgejo's .165 and
# shadowsocks' .166 were 100% packet loss / incomplete ARP before this.
#
# loadBalancerIPs: true makes Cilium announce every Service's LB-IPAM IP via
# ARP from whichever node currently holds the lease for it (one node per IP,
# decided by leaderElection — not all nodes simultaneously, which would
# otherwise cause ARP flapping/duplicate-IP confusion on the LAN).
#
# externalIPs/loadBalancerIPs split exists because Cilium also supports
# announcing Service externalIPs (a different field, unused in this repo);
# we only need loadBalancerIPs since every exposed Service here is type
# LoadBalancer via lb-ipam-pool.yaml.
#
# requires kube-proxy replacement (already the case — see
# k8s/talos-iam or helmfile.yaml.gotmpl kubeProxyReplacement=true) and a
# Cilium build with L2 announcements enabled (default since v1.14).
#
# Apply once after cluster bootstrap, alongside lb-ipam-pool.yaml:
# kubectl apply -f k8s/cilium/l2-announcement-policy.yaml
#
# Verify:
# kubectl get ciliuml2announcementpolicy
# ping 192.168.1.165 && ping 192.168.1.166 # both should now respond
# arp -a | grep 192.168.1.16 # should resolve to a real MAC
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: homelab-l2-announce
spec:
loadBalancerIPs: true
interfaces:
- eno1
# No nodeSelector restriction — all 3 nodes already run workloads
# (allowSchedulingOnControlPlanes: true in controlplane.yaml), and with
# 3 zone-labeled nodes, redundancy for per-IP leader election is maintained.
k8s/bootstrap/cilium/lb-ipam-pool.yaml
-36
View File
@@ -1,36 +0,0 @@
# k8s/cilium/lb-ipam-pool.yaml
# CiliumLoadBalancerIPPool — tells Cilium LB-IPAM which IPs it can assign
# to LoadBalancer services in this cluster.
#
# CIDR 192.168.1.160/28 covers .160–.175 on the LAN:
# .160 talos-cp-1 (node — not assignable to services)
# .161 reserved
# .162 talos-worker-1 (node — not assignable to services)
# .163–.175 free for LoadBalancer services
#
# Current service IP assignments (via io.cilium/lb-ipam-ips annotation):
# 192.168.1.165 forgejo-gitea-http (cicd)
# 192.168.1.165 forgejo-gitea-ssh (cicd) — same IP, different ports
# 192.168.1.166 shadowsocks (vpn)
#
# Apply once after cluster bootstrap:
# kubectl apply -f k8s/cilium/lb-ipam-pool.yaml
#
# Verify assignment:
# kubectl get svc -n cicd forgejo-gitea-http forgejo-gitea-ssh
# # EXTERNAL-IP should change from <pending> to 192.168.1.165
apiVersion: "cilium.io/v2alpha1"
kind: CiliumLoadBalancerIPPool
metadata:
name: homelab-pool
spec:
blocks:
- cidr: "192.168.1.160/28"
# DO NOT add any 10.6.0.0/24 block here. That is the WireGuard subnet
# (10.6.0.1 = talos-cp-1 tunnel IP, 10.6.0.2 = DNS — see
# cluster-config/controlplane.yaml). A 10.6.0.x block let Cilium LB-IPAM
# auto-assign the CP's own tunnel IP to a Service, which broke the
# WireGuard tunnel and locked out the default kubectl context. It also
# can't work over WireGuard anyway — L2 announcements only ARP on eno1
# (the LAN interface), not wg0. Keep this pool LAN-only.
k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
+16 -2
View File
@@ -14,6 +14,18 @@ valkey-cluster:
redis:
enabled: false
# External SSH access for git over the LAN. The chart's ssh Service becomes a
# LoadBalancer with a stable IP from the Cilium homelab-pool (192.168.1.160/28,
# L2-announced) so `git clone ssh://[email protected]:2222/...` works from the
# LAN. gitea's sshd listens on 2222 in-pod; port 2222 is exposed directly to
# avoid needing privileged :22.
service:
ssh:
type: LoadBalancer
port: 2222
annotations:
lbipam.cilium.io/ips: "192.168.1.161"
gitea:
admin:
existingSecret: forgejo-admin
@@ -22,8 +34,10 @@ gitea:
server:
DOMAIN: forgejo.riotpiao.com
ROOT_URL: https://forgejo.riotpiao.com
SSH_DOMAIN: forgejo.riotpiao.com
SSH_PORT: 22
# SSH clone URLs advertise git.riotpiao.com:2222 (the LoadBalancer above).
SSH_DOMAIN: git.riotpiao.com
SSH_PORT: 2222
SSH_LISTEN_PORT: 2222
database:
DB_TYPE: postgres
k8s/infra/monitoring/alerts/svc-forgejo-rules.yaml
+1 -1
View File
@@ -2,7 +2,7 @@ apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: forgejo-rules
namespace: forgejo
namespace: cicd
spec:
groups:
- name: forgejo.rules
k8s/infra/monitoring/prometheus-values.yaml
+29 -1
View File
@@ -20,8 +20,36 @@ grafana:
enabled: false
# ── Alertmanager ──────────────────────────────────────────────────────────────
# Enabled with a default (null) receiver — every firing PrometheusRule lands in
# the Alertmanager UI and Grafana's Alerting view; no external Slack/email/
# PagerDuty notifier is wired yet (add a receiver + route later). Storage pinned
# to az-a (sole Longhorn node) like Prometheus so the RWO PVC can attach.
alertmanager:
enabled: false
enabled: true
alertmanagerSpec:
nodeSelector:
topology.kubernetes.io/zone: az-a
tolerations:
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
storage:
volumeClaimTemplate:
spec:
storageClassName: longhorn
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 2Gi
config:
route:
group_by: ["alertname", "namespace"]
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receiver: "null"
receivers:
- name: "null"
# ── Prometheus ────────────────────────────────────────────────────────────────
prometheus:
terraform/files/cilium/l2-announcement.yaml
+13
View File
@@ -0,0 +1,13 @@
# CiliumL2AnnouncementPolicy — ARPs each LoadBalancer IP (from lb-ippool) on the
# LAN so the EXTERNAL-IP is actually reachable. Without it, LB-IPAM assigns IPs
# but nothing answers ARP (100% packet loss / incomplete ARP). One node per IP
# holds the lease (leaderElection) to avoid ARP flapping. Requires kube-proxy
# replacement (enabled) and Cilium L2 announcements (default since v1.14).
apiVersion: cilium.io/v2alpha1
kind: CiliumL2AnnouncementPolicy
metadata:
name: homelab-l2-announce
spec:
loadBalancerIPs: true
interfaces:
- eno1
terraform/files/cilium/lb-ippool.yaml
+15
View File
@@ -0,0 +1,15 @@
# CiliumLoadBalancerIPPool — the IPs Cilium LB-IPAM may assign to LoadBalancer
# Services. CIDR 192.168.1.160/28 covers .160–.175 on the LAN.
# .160 ingress-nginx (LoadBalancer)
# .161 forgejo-ssh (LoadBalancer)
# .162–.175 free
# Do NOT add a 10.6.0.0/24 block — that is the WireGuard subnet; letting LB-IPAM
# hand out a CP tunnel IP breaks the tunnel, and L2 ARP only works on the LAN
# interface (eno1) anyway. Keep this pool LAN-only.
apiVersion: cilium.io/v2alpha1
kind: CiliumLoadBalancerIPPool
metadata:
name: homelab-pool
spec:
blocks:
- cidr: "192.168.1.160/28"
terraform/main.tf
+24 -20
View File
@@ -22,26 +22,28 @@ resource "local_file" "controlplane_configs" {
filename = "${path.module}/../cluster-config/${each.key}.yaml"
content = templatefile("${path.module}/templates/controlplane.tftpl", {
version = "v1alpha1"
hostname = each.value.hostname
token = var.machine_token
ca_crt = var.machine_ca_crt
ca_key = var.machine_ca_key
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
factory_image = local.factory_image
talos_version = var.talos_version
longhorn_disks = each.value.longhorn_disks
dns_servers = var.cluster_config.dns_servers
forgejo_registry_ip = var.forgejo_registry_ip
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
allow_scheduling = each.value.allow_scheduling
coredns_corefile = file("${path.module}/files/coredns/Corefile")
version = "v1alpha1"
hostname = each.value.hostname
token = var.machine_token
ca_crt = var.machine_ca_crt
ca_key = var.machine_ca_key
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
factory_image = local.factory_image
talos_version = var.talos_version
longhorn_disks = each.value.longhorn_disks
dns_servers = var.cluster_config.dns_servers
forgejo_registry_ip = var.forgejo_registry_ip
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
allow_scheduling = each.value.allow_scheduling
coredns_corefile = file("${path.module}/files/coredns/Corefile")
cilium_lb_ippool = file("${path.module}/files/cilium/lb-ippool.yaml")
cilium_l2_announcement = file("${path.module}/files/cilium/l2-announcement.yaml")
# Cloudflare Tunnel cert SANs (talos :50000 and kube-apiserver :6443)
cloudflare_talos_sans = each.value.cloudflare_talos_sans
@@ -101,6 +103,8 @@ resource "local_file" "worker_configs" {
forgejo_hostname = var.forgejo_hostname
zone = each.value.zone
gpu_count = each.value.gpu_count
node_labels = each.value.node_labels
node_taints = each.value.node_taints
extra_disks = each.value.extra_disks
swap_size = each.value.swap_size
ephemeral_max_size = each.value.ephemeral_max_size
terraform/templates/controlplane.tftpl
+12
View File
@@ -164,6 +164,18 @@ cluster:
kind: Namespace
metadata:
name: kube-system
# Cilium LoadBalancer IPAM pool + L2 announcement policy. Substrate networking
# (owned here alongside the Cilium install), single source of truth in
# terraform/files/cilium/*.yaml. Provides LAN LoadBalancer IPs for ingress-nginx
# (.160) and forgejo-ssh (.161). Was previously an ArgoCD app whose empty
# kustomization never actually applied it (the live pool came from manual
# kubectl); moved here so LB-IPAM exists before any LoadBalancer Service syncs.
- name: cilium-lb-ippool
contents: |
${indent(8, cilium_lb_ippool)}
- name: cilium-l2-announcement
contents: |
${indent(8, cilium_l2_announcement)}
# CoreDNS Corefile with homelab hostname rewrites (single source of truth in
# terraform/files/coredns/Corefile). In-cluster pods resolve *.riotpiao.com to
# the nginx ingress controller so OIDC auto-discovery against
terraform/templates/worker.tftpl
+10 -1
View File
@@ -71,11 +71,20 @@ machine:
nodeLabels:
topology.kubernetes.io/region: homelab
topology.kubernetes.io/zone: ${zone}
node-role.kubernetes.io/gpu-node: ""
%{ if gpu_count > 0 ~}
node-role.kubernetes.io/gpu-node: ""
nvidia.com/gpu: "true"
gpu-count: "${gpu_count}"
%{ endif ~}
%{ for k, v in node_labels ~}
${k}: "${v}"
%{ endfor ~}
%{ if length(node_taints) > 0 ~}
nodeTaints:
%{ for t in node_taints ~}
${t.key}: "${t.value}:${t.effect}"
%{ endfor ~}
%{ endif ~}
cluster:
id: ${cluster_id}
terraform/variables.tf
+17 -8
View File
@@ -123,14 +123,23 @@ variable "controlplane_configs" {
variable "worker_configs" {
type = map(object({
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
network_interface = optional(string, "eno1")
zone = string
gpu_count = optional(number, 0)
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
network_interface = optional(string, "eno1")
zone = string
gpu_count = optional(number, 0)
# Extra node labels beyond the topology/GPU defaults.
node_labels = optional(map(string), {})
# Taints make a node dedicated: only pods carrying a matching toleration
# schedule there. effect is NoSchedule | PreferNoSchedule | NoExecute.
node_taints = optional(list(object({
key = string
value = string
effect = string
})), [])
factory_image = optional(string)
swap_size = optional(string, "")
ephemeral_max_size = optional(string, "700GiB")