From d7b96fb29b521b42d7a33f95943b22d498a6e9e6 Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Tue, 6 Oct 2026 11:38:40 +0800
Subject: [PATCH 1/5] fix(minio): freeze the community image by digest and
 alert on pull failure

MinIO deleted the community edition images upstream. Docker Hub reports
{"message":"object not found"} for minio/minio, and an anonymous pull of
quay.io/minio/mc returns HTTP 401. talos-cp-1 dropped the cached image during
the DiskPressure event on 2026-09-22. The tenant pod was recreated on
2026-09-23 and has been in ImagePullBackOff since. MinIO served no traffic for
13 days, which put vault-0 and loki-0 into CrashLoopBackOff and left the
paperless backups stuck.

Pin the tenant image to the digest of the last community release, so a retag or
a deletion on the mirror cannot break the pull again. Repoint the two
quay.io/minio/mc references at the archived Bitnami client image, also by
digest. Alert on ImagePullBackOff and on a pod that stays not ready, because
neither case raised any alert.

Evidence: https://hub.docker.com/v2/repositories/minio/minio/tags -> {"message":"object not found"}
---
 k8s/apps/paperless/backup-cronjob.yaml         |  2 +-
 .../minio/minio-provision-paperless-job.yaml   |  4 ++--
 k8s/infra/minio/minio-tenant.yaml              |  5 ++++-
 .../monitoring/alerts/cluster-alerts.yaml      | 18 ++++++++++++++++++
 4 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/k8s/apps/paperless/backup-cronjob.yaml b/k8s/apps/paperless/backup-cronjob.yaml
index 46dcbd6..be56123 100644
--- a/k8s/apps/paperless/backup-cronjob.yaml
+++ b/k8s/apps/paperless/backup-cronjob.yaml
@@ -48,7 +48,7 @@ spec:
                   mountPath: /backup
           containers:
             - name: mc-mirror
-              image: quay.io/minio/mc:latest
+              image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
               env:
                 - name: ACCESS_KEY
                   valueFrom:
diff --git a/k8s/infra/minio/minio-provision-paperless-job.yaml b/k8s/infra/minio/minio-provision-paperless-job.yaml
index d1e849e..7329686 100644
--- a/k8s/infra/minio/minio-provision-paperless-job.yaml
+++ b/k8s/infra/minio/minio-provision-paperless-job.yaml
@@ -52,14 +52,14 @@ spec:
       restartPolicy: Never
       initContainers:
         - name: kubectl-copy
-          image: bitnami/kubectl:latest
+          image: bitnami/kubectl:latest@sha256:f7f9e4f64d9e114650c115a4ac6fd383394b3d494061a8f17b1a0f6c8d55bc25
           command: ["sh", "-c", "cp $(which kubectl) /shared/kubectl"]
           volumeMounts:
             - name: shared
               mountPath: /shared
       containers:
         - name: provision
-          image: quay.io/minio/mc:latest
+          image: docker.io/bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158
           volumeMounts:
             - name: shared
               mountPath: /shared
diff --git a/k8s/infra/minio/minio-tenant.yaml b/k8s/infra/minio/minio-tenant.yaml
index 810821e..ab61e93 100644
--- a/k8s/infra/minio/minio-tenant.yaml
+++ b/k8s/infra/minio/minio-tenant.yaml
@@ -16,7 +16,10 @@ spec:
   # backend could not resolve minio-cluster-hl. Official images now need registry auth
   # (quay.io/minio/minio returns 401), so this pins a public mirror of the same upstream
   # release; swap it for the official image once MinIO registry credentials exist.
-  image: coollabsio/minio:RELEASE.2025-10-15T17-29-55Z
+  # The digest freezes those bytes, so a retag or a deletion on the mirror cannot break
+  # the pull again. The same digest already sits in the containerd cache on talos-cp-1,
+  # so the first start needs no registry access.
+  image: docker.io/coollabsio/minio:RELEASE.2025-10-15T17-29-55Z@sha256:69b55a1c1c5dc285ce04db96689f5b2102317fc77a50680a1874ca6efd1c87f9
 
   # Disable operator auto-TLS: MinIO serves plain HTTP internally on 9000.
   # External TLS is terminated at nginx ingress (wildcard riotpiao-com-tls cert).
diff --git a/k8s/infra/monitoring/alerts/cluster-alerts.yaml b/k8s/infra/monitoring/alerts/cluster-alerts.yaml
index 040055b..2899fb2 100644
--- a/k8s/infra/monitoring/alerts/cluster-alerts.yaml
+++ b/k8s/infra/monitoring/alerts/cluster-alerts.yaml
@@ -27,6 +27,24 @@ spec:
           annotations:
             summary: "{{ $value }} pod(s) stuck in Pending state for >10m"
 
+        # ImagePullBackOff on a vendor-deleted image produced no alert for 13 days
+        - alert: PodImagePullFailure
+          expr: sum(kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"}) by (namespace, pod) > 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"
+
+        # Pod not ready
+        - alert: PodNotReady
+          expr: sum(kube_pod_status_ready{condition="false"}) by (namespace, pod) > 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "{{ $labels.namespace }}/{{ $labels.pod }} is not ready"
+
         # CrashLoopBackOff
         - alert: PodCrashLooping
           expr: sum(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}) by (namespace, pod) > 0

From cc62f74404bf022e4901765617bc552bb50087f2 Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Tue, 6 Oct 2026 12:45:12 +0800
Subject: [PATCH 2/5] feat(storage): add local-path-provisioner as the fleet's
 persistent-storage backend

---
 k8s/argocd/apps/10-storage-observability.yaml |  23 ++
 k8s/infra/local-path/kustomization.yaml       |   7 +
 k8s/infra/local-path/local-path-storage.yaml  | 200 ++++++++++++++++++
 3 files changed, 230 insertions(+)
 create mode 100644 k8s/infra/local-path/kustomization.yaml
 create mode 100644 k8s/infra/local-path/local-path-storage.yaml

diff --git a/k8s/argocd/apps/10-storage-observability.yaml b/k8s/argocd/apps/10-storage-observability.yaml
index 1b57f05..f4e0823 100644
--- a/k8s/argocd/apps/10-storage-observability.yaml
+++ b/k8s/argocd/apps/10-storage-observability.yaml
@@ -249,3 +249,26 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: local-path-provisioner
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/local-path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: local-path-storage
+  # No CreateNamespace: the Namespace object (with its privileged PSA label,
+  # which the helper Pods need for their hostPath) is part of the manifest.
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
diff --git a/k8s/infra/local-path/kustomization.yaml b/k8s/infra/local-path/kustomization.yaml
new file mode 100644
index 0000000..e315384
--- /dev/null
+++ b/k8s/infra/local-path/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No commonLabels on purpose: this is a vendored third-party manifest and
+# kustomize would also rewrite the provisioner Deployment's selector, which
+# is immutable on re-apply.
+resources:
+  - local-path-storage.yaml
diff --git a/k8s/infra/local-path/local-path-storage.yaml b/k8s/infra/local-path/local-path-storage.yaml
new file mode 100644
index 0000000..111feb7
--- /dev/null
+++ b/k8s/infra/local-path/local-path-storage.yaml
@@ -0,0 +1,200 @@
+# Vendored from https://github.com/rancher/local-path-provisioner
+#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
+#
+# Local modifications (the only three; re-apply them on any re-vendor):
+#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
+#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
+#      baseline policy, which forbids that;
+#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
+#      filesystem read-only, so upstream's /opt path cannot be created;
+#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: local-path-provisioner-role
+  namespace: local-path-storage
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-path-provisioner-bind
+  namespace: local-path-storage
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+        - name: local-path-provisioner
+          image: docker.io/rancher/local-path-provisioner:v0.0.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - local-path-provisioner
+            - --debug
+            - start
+            - --config
+            - /etc/config/config.json
+          ports:
+            - name: health
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 12
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config/
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_MOUNT_PATH
+              value: /etc/config/
+            - name: HEALTH_PORT
+              value: "8080"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-path
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["/var/lib/local-path-provisioner"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node.kubernetes.io/disk-pressure
+          operator: Exists
+          effect: NoSchedule
+      containers:
+      - name: helper-pod
+        image: docker.io/library/busybox:1.36
+        imagePullPolicy: IfNotPresent

From 1ce9f866358c8026383cbeddc2b1ef5e8f922fb6 Mon Sep 17 00:00:00 2001
From: harness <harness@local>
Date: Wed, 7 Oct 2026 13:37:45 +0800
Subject: [PATCH 3/5] feat(etcd): weekly defrag CronJob to stay under the etcd
 backend quota

etcd's 2 GiB quota was reached on 2026-10-07 (backend 2.1 GiB, ~73 days of
churn). etcd raised NOSPACE and refused all writes, so kubelets could not post
node status: three nodes went NotReady and the tunnel origin pods on them
disappeared. Recovery was compact + defrag + alarm disarm per member.

The apiserver compacts every 5m, but compaction never shrinks the backend file -
only a defrag returns space, and nothing scheduled it. This adds:

- CronJob etcd-defrag (kube-system, Sundays 03:00): compact to the current
  revision, defrag each member sequentially, disarm NOSPACE, print before/after.
- Secret etcd-defrag-creds (SOPS/ksops, k8s/argocd/secrets): etcd CA plus a
  CN=etcd-defrag client cert signed by the cluster's etcd CA (expires 2036-10).

Applied by the existing apps: cluster-maintenance (directory) and 00-secrets
(ksops generator).
---
 k8s/argocd/secrets/etcd-defrag-creds.enc.yaml |  27 +++++
 k8s/argocd/secrets/secret-generator.yaml      |   1 +
 .../etcd-defrag-cronjob.yaml                  | 101 ++++++++++++++++++
 3 files changed, 129 insertions(+)
 create mode 100644 k8s/argocd/secrets/etcd-defrag-creds.enc.yaml
 create mode 100644 k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml

diff --git a/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml b/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml
new file mode 100644
index 0000000..c8215a3
--- /dev/null
+++ b/k8s/argocd/secrets/etcd-defrag-creds.enc.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: etcd-defrag-creds
+    namespace: kube-system
+    labels:
+        app.kubernetes.io/name: etcd-defrag
+type: Opaque
+stringData:
+    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
+    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
+    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
+            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
+            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
+            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
+            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T05:37:26Z"
+    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
+    version: 3.13.2
diff --git a/k8s/argocd/secrets/secret-generator.yaml b/k8s/argocd/secrets/secret-generator.yaml
index f034421..4cf48e8 100644
--- a/k8s/argocd/secrets/secret-generator.yaml
+++ b/k8s/argocd/secrets/secret-generator.yaml
@@ -34,3 +34,4 @@ files:
   - forgejo-smtp-secrets.enc.yaml
   - paperless-ai-secrets.enc.yaml
     - pi-sandbox-secrets.enc.yaml
+    - etcd-defrag-creds.enc.yaml
diff --git a/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml
new file mode 100644
index 0000000..99a0ad1
--- /dev/null
+++ b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml
@@ -0,0 +1,101 @@
+# etcd defrag maintenance.
+#
+# Why this exists: the apiserver compacts etcd every 5m, but compaction only marks
+# superseded revisions free INSIDE the backend file -- the file never shrinks.
+# Only `etcdctl defrag` returns space to the filesystem, and nothing runs it
+# automatically. On 2026-10-07 the 2 GiB quota was reached after ~73 days
+# (backend 2.1 GiB) -> etcd raised the NOSPACE alarm -> etcd refused ALL writes
+# -> kubelets could not post node status -> 3 nodes went NotReady and the tunnel
+# origin pods on them vanished (502s). Recovery = compact + defrag + alarm disarm.
+#
+# This CronJob keeps the backend well under quota: compact to the current
+# revision, defrag each member sequentially, then disarm any NOSPACE alarm.
+#
+# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
+# etcd_server_quota_in_bytes > 0.8, and a NOSPACE alarm alert.
+#
+# Credentials: secret etcd-defrag-creds (kube-system), SOPS-managed in
+# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml. Its client cert is signed by the
+# cluster's etcd CA and expires 2036-10 -- rotate it before then by re-running the
+# mint + re-encrypt (see the repo's sops workflow).
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-defrag
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: etcd-defrag
+    app.kubernetes.io/part-of: cluster-maintenance
+spec:
+  schedule: "0 3 * * 0"            # Sundays 03:00, low traffic
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 600
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: defrag
+              image: quay.io/coreos/etcd:v3.5.17
+              env:
+                - name: ETCDCTL_API
+                  value: "3"
+                - name: ETCDCTL_CACERT
+                  value: /etc/etcd/ca.crt
+                - name: ETCDCTL_CERT
+                  value: /etc/etcd/client.crt
+                - name: ETCDCTL_KEY
+                  value: /etc/etcd/client.key
+                - name: EPS
+                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -u
+                  PRIMARY=${EPS%% *}
+                  echo "== before =="
+                  etcdctl --endpoints=$EPS endpoint status -w table || true
+
+                  REV=$(etcdctl --endpoints=$PRIMARY endpoint status -w json \
+                        | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
+                  if [ -n "$REV" ]; then
+                    echo "== compact to revision $REV =="
+                    etcdctl --endpoints=$PRIMARY compact "$REV" || true
+                  fi
+
+                  echo "== defrag each member (sequential, one at a time) =="
+                  rc=0
+                  for ep in $EPS; do
+                    echo "-- $ep"
+                    etcdctl --endpoints="$ep" defrag || { echo "DEFRAG-FAILED $ep"; rc=1; }
+                  done
+
+                  echo "== disarm NOSPACE =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
+                  done
+
+                  echo "== after =="
+                  etcdctl --endpoints=$EPS endpoint status -w table || true
+                  etcdctl --endpoints=$PRIMARY alarm list || true
+                  exit $rc
+              resources:
+                requests: { cpu: 10m, memory: 32Mi }
+                limits:   { cpu: 200m, memory: 128Mi }
+              volumeMounts:
+                - name: creds
+                  mountPath: /etc/etcd
+                  readOnly: true
+          volumes:
+            - name: creds
+              secret:
+                secretName: etcd-defrag-creds
+                defaultMode: 0444

From e4caa7e84115143a57c5237fc7718a6c939127f5 Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Wed, 7 Oct 2026 17:11:35 +0800
Subject: [PATCH 4/5] fix(etcd): make the defrag CronJob runnable and 15-day

The committed CronJob could never run, for two independent reasons:
- quay.io/coreos/etcd ships no /bin/sh, so command: [/bin/sh, -c] died with
  exec: "/bin/sh": no such file or directory
- it existed only on branch fix/etcd-defrag-cron, while the ArgoCD application
  cluster-maintenance tracks main (targetRevision: main), so the cluster never
  had the CronJob or its Secret

This revision fixes both and records the 2026-10-07 incident:
- image bitnamilegacy/etcd:3.6.4 pinned by digest (ships bash and etcdctl 3.6.4,
  matching the 3.6.11 servers)
- 15-day cadence by UTC epoch-day modulo; */15 in day-of-month fires on the 1st,
  16th and 31st, which puts two runs one day apart
- compacts to the MINIMUM revision across members, so a lagging member is never
  compacted past its own revision (measured: talos-cp-2 froze ~1.2M entries
  behind the leader and needed a reboot to rejoin)
- skips defrag on a member whose applied index lags the leader by 100k+
- documents that compact must precede defrag: compaction had stalled while the
  NOSPACE alarm was up, so dbSizeInUse read 100% of the file and defrag alone
  would have reclaimed nothing
---
 .../etcd-defrag-cronjob.yaml                  | 136 ++++++++++++++----
 1 file changed, 106 insertions(+), 30 deletions(-)

diff --git a/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml
index 99a0ad1..c75fb3d 100644
--- a/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml
+++ b/k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml
@@ -1,23 +1,58 @@
-# etcd defrag maintenance.
+# etcd backend maintenance: compact, defrag, disarm NOSPACE.
 #
-# Why this exists: the apiserver compacts etcd every 5m, but compaction only marks
-# superseded revisions free INSIDE the backend file -- the file never shrinks.
-# Only `etcdctl defrag` returns space to the filesystem, and nothing runs it
-# automatically. On 2026-10-07 the 2 GiB quota was reached after ~73 days
-# (backend 2.1 GiB) -> etcd raised the NOSPACE alarm -> etcd refused ALL writes
-# -> kubelets could not post node status -> 3 nodes went NotReady and the tunnel
-# origin pods on them vanished (502s). Recovery = compact + defrag + alarm disarm.
+# Why this exists
+# ---------------
+# etcd's backend file only grows. The apiserver compacts every 5m, but
+# compaction marks superseded revisions free *inside* the file; only `defrag`
+# returns that space to the filesystem. Nothing ran defrag, so the file grew
+# until it crossed the 2 GiB quota:
 #
-# This CronJob keeps the backend well under quota: compact to the current
-# revision, defrag each member sequentially, then disarm any NOSPACE alarm.
+#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
+#               -> etcd raised NOSPACE on those two members
+#               -> etcd rejected ALL writes cluster-wide
+#               -> kubelets could not post node status: 3 nodes NotReady
+#               -> apiserver writes returned 500, the tunnel API path 502
+#
+# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
+# dead pages. Recovery was `compact <current-revision>` then `defrag` per
+# member then `alarm disarm`. Compaction had also stalled while the alarm was
+# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
+# worked. Order matters: compact first, then defrag.
+#
+# The earlier revision of this file never ran, for two independent reasons:
+#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
+#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
+#      cluster never had the CronJob or the Secret.
+#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
+#      `exec: "/bin/sh": no such file or directory`). Any container that runs
+#      a shell script needs an image that ships one.
+# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
+# 3.6.4, matching the 3.6.11 servers) pinned by digest.
+#
+# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
+# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
+# and the script exits unless the UTC epoch day is a multiple of 15. That gives
+# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
+# on the 1st, 16th and 31st, which puts two runs one day apart).
+#
+# Safety
+# ------
+# 1. Compaction uses the MINIMUM revision across members. Compacting to a
+#    revision a member has not reached wedges that member - measured on
+#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
+#    needed a reboot to rejoin.
+# 2. A member whose raft applied index lags the leader by more than 100000 is
+#    skipped and reported. Defragging a frozen member reclaims nothing.
+# 3. Members are defragged one at a time. A defrag pauses the member's apply
+#    loop for the duration of the file copy.
+#
+# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
+# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
+# Its client certificate is signed by the cluster's etcd CA and expires
+# 2036-10; rotate it before then by re-running the mint and re-encrypting.
 #
 # Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
-# etcd_server_quota_in_bytes > 0.8, and a NOSPACE alarm alert.
-#
-# Credentials: secret etcd-defrag-creds (kube-system), SOPS-managed in
-# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml. Its client cert is signed by the
-# cluster's etcd CA and expires 2036-10 -- rotate it before then by re-running the
-# mint + re-encrypt (see the repo's sops workflow).
+# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -27,7 +62,7 @@ metadata:
     app.kubernetes.io/name: etcd-defrag
     app.kubernetes.io/part-of: cluster-maintenance
 spec:
-  schedule: "0 3 * * 0"            # Sundays 03:00, low traffic
+  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
   concurrencyPolicy: Forbid
   startingDeadlineSeconds: 600
   successfulJobsHistoryLimit: 3
@@ -44,7 +79,7 @@ spec:
               effect: NoSchedule
           containers:
             - name: defrag
-              image: quay.io/coreos/etcd:v3.5.17
+              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
               env:
                 - name: ETCDCTL_API
                   value: "3"
@@ -56,36 +91,77 @@ spec:
                   value: /etc/etcd/client.key
                 - name: EPS
                   value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
-              command: ["/bin/sh", "-c"]
+              command: ["/bin/bash", "-c"]
               args:
                 - |
                   set -u
-                  PRIMARY=${EPS%% *}
-                  echo "== before =="
-                  etcdctl --endpoints=$EPS endpoint status -w table || true
 
-                  REV=$(etcdctl --endpoints=$PRIMARY endpoint status -w json \
-                        | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
-                  if [ -n "$REV" ]; then
-                    echo "== compact to revision $REV =="
-                    etcdctl --endpoints=$PRIMARY compact "$REV" || true
+                  DAY=$(( $(date -u +%s) / 86400 ))
+                  if [ $(( DAY % 15 )) -ne 0 ]; then
+                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
+                    exit 0
                   fi
 
+                  echo "== before =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+
+                  MINREV=""
+                  MAXAPPLIED=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
+                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
+                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
+                      MINREV="$rev"
+                    fi
+                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
+                      MAXAPPLIED="$app"
+                    fi
+                  done
+
+                  if [ -z "$MINREV" ]; then
+                    echo "FATAL: no member returned a revision; nothing attempted"
+                    exit 1
+                  fi
+
+                  echo "== compact to revision $MINREV =="
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
+
+                  # Physical deletion of superseded keys runs in the background
+                  # after the compaction commit; give it a moment to drain.
+                  sleep 20
+
                   echo "== defrag each member (sequential, one at a time) =="
                   rc=0
                   for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
+                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
+                      rc=1
+                      continue
+                    fi
                     echo "-- $ep"
-                    etcdctl --endpoints="$ep" defrag || { echo "DEFRAG-FAILED $ep"; rc=1; }
+                    if ! etcdctl --endpoints="$ep" defrag; then
+                      echo "DEFRAG-FAILED $ep"
+                      rc=1
+                    fi
                   done
 
                   echo "== disarm NOSPACE =="
                   for ep in $EPS; do
+                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
                     etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
                   done
 
                   echo "== after =="
-                  etcdctl --endpoints=$EPS endpoint status -w table || true
-                  etcdctl --endpoints=$PRIMARY alarm list || true
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
                   exit $rc
               resources:
                 requests: { cpu: 10m, memory: 32Mi }

From d21ee9630ff39953ef67f96672d234147df85a0a Mon Sep 17 00:00:00 2001
From: rock <rock@riotpiao.com>
Date: Tue, 13 Oct 2026 08:09:42 +0800
Subject: [PATCH 5/5] fix(longhorn): retire the failed 2TB PERC disk on
 talos-cp-2

The disk behind /var/lib/longhorn-disk4 (sde1, 2TB PERC H710P virtual disk)
failed: 54 million SCSI I/O errors and climbing at ~7,000/minute, against 3 on
every other device on the node. Its XFS filesystem hung, so five Longhorn
replica processes and the longhorn-manager blocked in uninterruptible D state
in openat and unlinkat. D-state processes cannot be killed, so they held the
node's three replica rebuild slots permanently: no new replica could start on
talos-cp-2 and every new volume there failed to attach with DeadlineExceeded.
That is what stopped the azhi database from coming up.

Removed from all three declarations so it cannot come back:
- terraform.tfvars.local.bak, longhorn_disks for talos-cp-2
- the Longhorn add-disks Job, which would otherwise re-register it
- the Job's disk-count guard, 5 -> 4, or it would fire on every run once the
  disk is gone

The running cluster already has the disk fenced
(allowScheduling=false, evictionRequested=true); the replicas are evicted and
the device retired at the controller after that.
---
 .../longhorn/longhorn-add-disks-job.yaml      | 23 +++++++++++--------
 terraform/terraform.tfvars.local.bak          |  9 +++++++-
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/k8s/infra/longhorn/longhorn-add-disks-job.yaml b/k8s/infra/longhorn/longhorn-add-disks-job.yaml
index bff5aad..782bac0 100644
--- a/k8s/infra/longhorn/longhorn-add-disks-job.yaml
+++ b/k8s/infra/longhorn/longhorn-add-disks-job.yaml
@@ -41,13 +41,24 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          # - /var/lib/longhorn-disk4 (sde1, 2TB)
+          #
+          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
+          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
+          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
+          # process that touched it blocked in unkillable D state. Those unkillable
+          # processes held Longhorn's per-node rebuild slots forever, so no new
+          # replica could start on talos-cp-2 and every new volume there failed to
+          # attach. It is removed from the Terraform disk list too, so a re-render
+          # does not put it back.
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          if [ "$CURRENT_DISKS" -lt 5 ]; then
+          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
+          # existed; leaving the old threshold would re-apply the patch on every
+          # run once disk4 is gone.
+          if [ "$CURRENT_DISKS" -lt 4 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -75,14 +86,6 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
-                  },
-                  "disk4": {
-                    "allowScheduling": true,
-                    "diskType": "filesystem",
-                    "evictionRequested": false,
-                    "path": "/var/lib/longhorn-disk4",
-                    "storageReserved": 21474836480,
-                    "tags": []
                   }
                 }
               }
diff --git a/terraform/terraform.tfvars.local.bak b/terraform/terraform.tfvars.local.bak
index a06e2e0..0e639f3 100644
--- a/terraform/terraform.tfvars.local.bak
+++ b/terraform/terraform.tfvars.local.bak
@@ -20,6 +20,7 @@ controlplane_configs = {
   "talos-cp-1" = {
     hostname     = "talos-cp-1"
     lan_ip       = "192.168.1.166"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"
@@ -32,6 +33,7 @@ controlplane_configs = {
   "talos-cp-2" = {
     hostname     = "talos-cp-2"
     lan_ip       = "192.168.1.214"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -39,7 +41,11 @@ controlplane_configs = {
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
+        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
+        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
+        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
+        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
+        # re-add it.
       ]
     zone             = "az-b"
     allow_scheduling = true
@@ -49,6 +55,7 @@ controlplane_configs = {
   "talos-cp-3" = {
     hostname     = "talos-cp-3"
     lan_ip       = "192.168.1.162"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"