fix(cert-manager): restore wildcard renewal and drop per-host TLS

The ACME DNS-01 self-check resolved through CoreDNS and never saw the TXT
record. The wildcard renewal stalled on 2026-09-23 and riotpiao-com-tls
expired on 2026-10-23. Every Cloudflare Tunnel origin then failed with
"x509: certificate has expired or is not yet valid".

Changes:

- cert-manager: pin the self-check to the public resolvers with
  --dns01-recursive-nameservers and --dns01-recursive-nameservers-only.
  Verified against Let's Encrypt staging: the challenge went valid and the
  certificate issued.
- comfyui and forgejo: drop the per-host tls blocks and the
  cert-manager.io/cluster-issuer annotations. The wildcard covers both
  hosts, and the per-host certificates (comfyui-tls, forgejo-tls) expired
  with the wildcard.
- riotpiao-com-cert: document the wildcard as nginx's default certificate
  and the zero-downtime renewal path.

Renewal now runs 30 days before expiry, writes the same secret, and nginx
hot-reloads without a pod restart.

k8s/apps/comfyui/ingress.yaml

@@ -4,7 +4,9 @@ metadata:
   name: comfyui
   namespace: comfyui
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation and no tls: block. This host is covered by
+    # the *.riotpiao.com wildcard, which nginx serves as its
+    # default-ssl-certificate (ingress-nginx/riotpiao-com-tls).
     nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
@@ -16,10 +18,6 @@ metadata:
       proxy_set_header Connection "upgrade";
 spec:
   ingressClassName: nginx
-  tls:
-    - secretName: comfyui-tls
-      hosts:
-        - comfyui.riotpiao.com
   rules:
     - host: comfyui.riotpiao.com
       http:

k8s/argocd/apps/10-storage-observability.yaml

@@ -249,3 +249,26 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: local-path-provisioner
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/local-path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: local-path-storage
+  # No CreateNamespace: the Namespace object (with its privileged PSA label,
+  # which the helper Pods need for their hostPath) is part of the manifest.
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/bootstrap/cert-manager/cert-manager-values.yaml

@@ -8,6 +8,19 @@
 # depend on them existing first.
 installCRDs: true
 
+# Point the ACME DNS-01 self-check at the public resolvers directly.
+#
+# Without these flags cert-manager resolves through the pod resolver
+# (CoreDNS at 10.96.0.10). The self-check then never saw the TXT record and
+# reported "Waiting for DNS-01 challenge propagation" forever. The wildcard
+# certificate renewal stalled on 2026-09-23 and the cert expired on
+# 2026-10-23, which broke every Cloudflare Tunnel origin with
+# "x509: certificate has expired".
+#
+extraArgs:
+  - --dns01-recursive-nameservers=1.1.1.1:53,8.8.8.8:53
+  - --dns01-recursive-nameservers-only
+
 # Tolerate the control-plane taint (talos-cp-1 is the only schedulable node).
 tolerations:
   - key: node-role.kubernetes.io/control-plane

k8s/bootstrap/ingress/riotpiao-com-cert.yaml

@@ -1,13 +1,21 @@
 # k8s/bootstrap/ingress/riotpiao-com-cert.yaml
 # Public wildcard cert for *.riotpiao.com, signed by Let's Encrypt (DNS-01 via
-# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml). Distinct from
+# Cloudflare, see ../cert-manager/letsencrypt-issuer.yaml).
-# homelab-tls (self-signed homelab-ca, internal *.riotpiao.com) —
-# reference riotpiao-com-tls explicitly on ingresses that expose riotpiao.com
-# hosts to the internet; it is not nginx's default-ssl-certificate.
 #
-# issuerRef starts on letsencrypt-staging. Switch to letsencrypt-prod once the
+# nginx serves this secret as its default-ssl-certificate
-# staging cert issues cleanly, then delete this Certificate's Secret to force
+# (../ingress/nginx-values.yaml), so every Ingress that omits a tls: block uses
-# a re-issue against prod (staging certs aren't browser-trusted).
+# it. Do not add per-host tls: blocks for *.riotpiao.com hosts: they create a
+# second, single-host certificate and split the wildcard coverage.
+#
+# Renewal is automatic and does not drop traffic:
+#   - cert-manager starts a new order 30 days before expiry (renewBefore).
+#   - cert-manager writes the renewed pair into the same secret.
+#   - ingress-nginx watches the secret and hot-reloads. No pod restart.
+#   - The old pair stays valid until the new one is written, so no gap exists.
+#
+# The DNS-01 self-check must reach the public resolvers directly. Without the
+# --dns01-recursive-nameservers flags in ../../cert-manager/cert-manager-values.yaml
+# the self-check never sees the TXT record and renewal stalls until expiry.
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -107,7 +107,9 @@ ingress:
   enabled: true
   className: nginx
   annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-prod
+    # No cert-manager annotation. forgejo.riotpiao.com is covered by the
+    # *.riotpiao.com wildcard, served as nginx's default-ssl-certificate.
+    #
     # This chart Ingress and the hand-written one in
     # k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com.
     # ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's
@@ -125,10 +127,9 @@ ingress:
       paths:
         - path: /
           pathType: Prefix
-  tls:
+  # tls: intentionally omitted. The chart would create a second, single-host
-    - secretName: forgejo-tls
+  # certificate (forgejo-tls) that expired on 2026-10-23 and competed with the
-      hosts:
+  # wildcard. The wildcard covers this host.
-        - forgejo.riotpiao.com
 
 # Resources
 resources:

k8s/infra/local-path/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No commonLabels on purpose: this is a vendored third-party manifest and
+# kustomize would also rewrite the provisioner Deployment's selector, which
+# is immutable on re-apply.
+resources:
+  - local-path-storage.yaml

k8s/infra/local-path/local-path-storage.yaml

@@ -0,0 +1,200 @@
+# Vendored from https://github.com/rancher/local-path-provisioner
+#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
+#
+# Local modifications (the only three; re-apply them on any re-vendor):
+#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
+#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
+#      baseline policy, which forbids that;
+#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
+#      filesystem read-only, so upstream's /opt path cannot be created;
+#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: local-path-provisioner-role
+  namespace: local-path-storage
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-path-provisioner-bind
+  namespace: local-path-storage
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+        - name: local-path-provisioner
+          image: docker.io/rancher/local-path-provisioner:v0.0.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - local-path-provisioner
+            - --debug
+            - start
+            - --config
+            - /etc/config/config.json
+          ports:
+            - name: health
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 12
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config/
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_MOUNT_PATH
+              value: /etc/config/
+            - name: HEALTH_PORT
+              value: "8080"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-path
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["/var/lib/local-path-provisioner"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node.kubernetes.io/disk-pressure
+          operator: Exists
+          effect: NoSchedule
+      containers:
+      - name: helper-pod
+        image: docker.io/library/busybox:1.36
+        imagePullPolicy: IfNotPresent

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -41,13 +41,24 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          # - /var/lib/longhorn-disk4 (sde1, 2TB)
+          #
+          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
+          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
+          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
+          # process that touched it blocked in unkillable D state. Those unkillable
+          # processes held Longhorn's per-node rebuild slots forever, so no new
+          # replica could start on talos-cp-2 and every new volume there failed to
+          # attach. It is removed from the Terraform disk list too, so a re-render
+          # does not put it back.
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          if [ "$CURRENT_DISKS" -lt 5 ]; then
+          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
+          # existed; leaving the old threshold would re-apply the patch on every
+          # run once disk4 is gone.
+          if [ "$CURRENT_DISKS" -lt 4 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -75,14 +86,6 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
-                  },
-                  "disk4": {
-                    "allowScheduling": true,
-                    "diskType": "filesystem",
-                    "evictionRequested": false,
-                    "path": "/var/lib/longhorn-disk4",
-                    "storageReserved": 21474836480,
-                    "tags": []
                   }
                 }
               }

terraform/terraform.tfvars.local.bak

@@ -20,6 +20,7 @@ controlplane_configs = {
   "talos-cp-1" = {
     hostname     = "talos-cp-1"
     lan_ip       = "192.168.1.166"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"
@@ -32,6 +33,7 @@ controlplane_configs = {
   "talos-cp-2" = {
     hostname     = "talos-cp-2"
     lan_ip       = "192.168.1.214"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -39,7 +41,11 @@ controlplane_configs = {
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
+        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
+        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
+        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
+        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
+        # re-add it.
       ]
     zone             = "az-b"
     allow_scheduling = true
@@ -49,6 +55,7 @@ controlplane_configs = {
   "talos-cp-3" = {
     hostname     = "talos-cp-3"
     lan_ip       = "192.168.1.162"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"