From e93d4ba86c52a0eb8bea69469a22a5de16167ceb Mon Sep 17 00:00:00 2001 From: rock Date: Sat, 24 Oct 2026 02:30:48 -0700 Subject: [PATCH] fix(cert-manager): restore wildcard renewal and drop per-host TLS The ACME DNS-01 self-check resolved through CoreDNS and never saw the TXT record. The wildcard renewal stalled on 2026-09-23 and riotpiao-com-tls expired on 2026-10-23. Every Cloudflare Tunnel origin then failed with "x509: certificate has expired or is not yet valid". Changes: - cert-manager: pin the self-check to the public resolvers with --dns01-recursive-nameservers and --dns01-recursive-nameservers-only. Verified against Let's Encrypt staging: the challenge went valid and the certificate issued. - comfyui and forgejo: drop the per-host tls blocks and the cert-manager.io/cluster-issuer annotations. The wildcard covers both hosts, and the per-host certificates (comfyui-tls, forgejo-tls) expired with the wildcard. - riotpiao-com-cert: document the wildcard as nginx's default certificate and the zero-downtime renewal path. Renewal now runs 30 days before expiry, writes the same secret, and nginx hot-reloads without a pod restart. --- k8s/apps/comfyui/ingress.yaml | 8 +++---- .../cert-manager/cert-manager-values.yaml | 13 +++++++++++ k8s/bootstrap/ingress/riotpiao-com-cert.yaml | 22 +++++++++++++------ .../phase3-forgejo/forgejo-values.yaml | 11 +++++----- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/k8s/apps/comfyui/ingress.yaml b/k8s/apps/comfyui/ingress.yaml index 88577d6..21ad2d4 100644 --- a/k8s/apps/comfyui/ingress.yaml +++ b/k8s/apps/comfyui/ingress.yaml @@ -4,7 +4,9 @@ metadata: name: comfyui namespace: comfyui annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod + # No cert-manager annotation and no tls: block. This host is covered by + # the *.riotpiao.com wildcard, which nginx serves as its + # default-ssl-certificate (ingress-nginx/riotpiao-com-tls). nginx.ingress.kubernetes.io/proxy-read-timeout: "600" nginx.ingress.kubernetes.io/proxy-send-timeout: "600" nginx.ingress.kubernetes.io/proxy-body-size: "0" @@ -16,10 +18,6 @@ metadata: proxy_set_header Connection "upgrade"; spec: ingressClassName: nginx - tls: - - secretName: comfyui-tls - hosts: - - comfyui.riotpiao.com rules: - host: comfyui.riotpiao.com http: diff --git a/k8s/bootstrap/cert-manager/cert-manager-values.yaml b/k8s/bootstrap/cert-manager/cert-manager-values.yaml index 71b34f2..845a5f5 100644 --- a/k8s/bootstrap/cert-manager/cert-manager-values.yaml +++ b/k8s/bootstrap/cert-manager/cert-manager-values.yaml @@ -8,6 +8,19 @@ # depend on them existing first. installCRDs: true +# Point the ACME DNS-01 self-check at the public resolvers directly. +# +# Without these flags cert-manager resolves through the pod resolver +# (CoreDNS at 10.96.0.10). The self-check then never saw the TXT record and +# reported "Waiting for DNS-01 challenge propagation" forever. The wildcard +# certificate renewal stalled on 2026-09-23 and the cert expired on +# 2026-10-23, which broke every Cloudflare Tunnel origin with +# "x509: certificate has expired". +# +extraArgs: + - --dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53 + - --dns01-recursive-nameservers-only + # Tolerate the control-plane taint (talos-cp-1 is the only schedulable node). tolerations: - key: node-role.kubernetes.io/control-plane diff --git a/k8s/bootstrap/ingress/riotpiao-com-cert.yaml b/k8s/bootstrap/ingress/riotpiao-com-cert.yaml index 4b0fdbd..4bee29e 100644 --- a/k8s/bootstrap/ingress/riotpiao-com-cert.yaml +++ b/k8s/bootstrap/ingress/riotpiao-com-cert.yaml @@ -1,13 +1,21 @@ # k8s/bootstrap/ingress/riotpiao-com-cert.yaml # Public wildcard cert for *.riotpiao.com, signed by Let's Encrypt (DNS-01 via -# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml). Distinct from -# homelab-tls (self-signed homelab-ca, internal *.riotpiao.com) — -# reference riotpiao-com-tls explicitly on ingresses that expose riotpiao.com -# hosts to the internet; it is not nginx's default-ssl-certificate. +# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml). # -# issuerRef starts on letsencrypt-staging. Switch to letsencrypt-prod once the -# staging cert issues cleanly, then delete this Certificate's Secret to force -# a re-issue against prod (staging certs aren't browser-trusted). +# nginx serves this secret as its default-ssl-certificate +# (../ingress/nginx-values.yaml), so every Ingress that omits a tls: block uses +# it. Do not add per-host tls: blocks for *.riotpiao.com hosts: they create a +# second, single-host certificate and split the wildcard coverage. +# +# Renewal is automatic and does not drop traffic: +# - cert-manager starts a new order 30 days before expiry (renewBefore). +# - cert-manager writes the renewed pair into the same secret. +# - ingress-nginx watches the secret and hot-reloads. No pod restart. +# - The old pair stays valid until the new one is written, so no gap exists. +# +# The DNS-01 self-check must reach the public resolvers directly. Without the +# --dns01-recursive-nameservers flags in ../../cert-manager/cert-manager-values.yaml +# the self-check never sees the TXT record and renewal stalls until expiry. apiVersion: cert-manager.io/v1 kind: Certificate metadata: diff --git a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml index e02befc..4ad62e3 100644 --- a/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml +++ b/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml @@ -107,7 +107,9 @@ ingress: enabled: true className: nginx annotations: - cert-manager.io/cluster-issuer: letsencrypt-prod + # No cert-manager annotation. forgejo.riotpiao.com is covered by the + # *.riotpiao.com wildcard, served as nginx's default-ssl-certificate. + # # This chart Ingress and the hand-written one in # k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com. # ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's @@ -125,10 +127,9 @@ ingress: paths: - path: / pathType: Prefix - tls: - - secretName: forgejo-tls - hosts: - - forgejo.riotpiao.com + # tls: intentionally omitted. The chart would create a second, single-host + # certificate (forgejo-tls) that expired on 2026-10-23 and competed with the + # wildcard. The wildcard covers this host. # Resources resources: