Merge pull request 'fix(longhorn): retire the failed 2TB PERC disk on talos-cp-2' (#62) from fix/retire-longhorn-disk4 into main

The disk behind /var/lib/longhorn-disk4 (sde1, 2TB PERC H710P virtual disk)
failed: 54 million SCSI I/O errors and climbing at ~7,000/minute, against 3 on
every other device on the node. Its XFS filesystem hung, so five Longhorn
replica processes and the longhorn-manager blocked in uninterruptible D state
in openat and unlinkat. D-state processes cannot be killed, so they held the
node's three replica rebuild slots permanently: no new replica could start on
talos-cp-2 and every new volume there failed to attach with DeadlineExceeded.
That is what stopped the azhi database from coming up.

Removed from all three declarations so it cannot come back:
- terraform.tfvars.local.bak, longhorn_disks for talos-cp-2
- the Longhorn add-disks Job, which would otherwise re-register it
- the Job's disk-count guard, 5 -> 4, or it would fire on every run once the
  disk is gone

The running cluster already has the disk fenced
(allowScheduling=false, evictionRequested=true); the replicas are evicted and
the device retired at the controller after that.

k8s/argocd/apps/10-storage-observability.yaml

@@ -249,3 +249,26 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: local-path-provisioner
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/local-path
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: local-path-storage
+  # No CreateNamespace: the Namespace object (with its privileged PSA label,
+  # which the helper Pods need for their hostPath) is part of the manifest.
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/apps/72-azhi-riotpiao.yaml

@@ -0,0 +1,36 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: azhi-riotpiao
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: '8'
+spec:
+  project: homelab
+  source:
+    # Private GitHub. The read-only credential is Secret azhi-infra-github in
+    # namespace argocd, created out-of-band.
+    repoURL: https://github.com/ToyoGlobal/azhi-infra.git
+    # The repository's default branch. It is not `main`: this repo's work
+    # happens on the staging slice branch, and `main` is behind it.
+    targetRevision: feat/azi-riotpiao-cnpg
+    # One directory, because ArgoCD reads a single path per source. That
+    # directory is a root kustomization over foundation, postgres and
+    # azhi-server.
+    path: k8s/overlays/azi-riotpiao
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: azi-riotpiao
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    # The two Secrets this namespace needs are credentials, so they are created
+    # by hand and NOT by this Application:
+    #   acr-pull                 — the private Alibaba ACR pull credential
+    #   azhi-riotpiao-app-env    — DB credentials, JWT secrets, AES key
+    # A missing imagePullSecret fails the pod outright and a missing app-env
+    # Secret leaves it in CreateContainerConfigError, which is the intended
+    # loud failure rather than a pod that runs against unknown values.

k8s/argocd/projects/homelab-project.yaml

@@ -16,6 +16,12 @@ spec:
     - https://github.com/Riotpiaole/Poimen-memory.git
     - https://github.com/Riotpiaole/Poimen-workflows.git
     - https://github.com/Riotpiaole/poimen*.git
+    # ToyoGlobal (GitHub, private). The azhi-infra Application reads its
+    # manifests from here, so the URL must be listed or ArgoCD rejects the
+    # Application with "repository not permitted". A read-only credential for
+    # this repo lives in Secret azhi-infra-github in namespace argocd; it is
+    # created out-of-band and is not committed, because it is a credential.
+    - https://github.com/ToyoGlobal/azhi-infra.git
     # In-cluster Forgejo repos — explicit allowlist (no wildcard)
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git

k8s/infra/local-path/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# No commonLabels on purpose: this is a vendored third-party manifest and
+# kustomize would also rewrite the provisioner Deployment's selector, which
+# is immutable on re-apply.
+resources:
+  - local-path-storage.yaml

k8s/infra/local-path/local-path-storage.yaml

@@ -0,0 +1,200 @@
+# Vendored from https://github.com/rancher/local-path-provisioner
+#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
+#
+# Local modifications (the only three; re-apply them on any re-vendor):
+#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
+#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
+#      baseline policy, which forbids that;
+#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
+#      filesystem read-only, so upstream's /opt path cannot be created;
+#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: local-path-storage
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/audit: privileged
+    pod-security.kubernetes.io/warn: privileged
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: local-path-provisioner-service-account
+  namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: local-path-provisioner-role
+  namespace: local-path-storage
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: local-path-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: local-path-provisioner-bind
+  namespace: local-path-storage
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: local-path-provisioner-bind
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: local-path-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: local-path-provisioner-service-account
+    namespace: local-path-storage
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: local-path-provisioner
+  namespace: local-path-storage
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: local-path-provisioner
+  template:
+    metadata:
+      labels:
+        app: local-path-provisioner
+    spec:
+      serviceAccountName: local-path-provisioner-service-account
+      containers:
+        - name: local-path-provisioner
+          image: docker.io/rancher/local-path-provisioner:v0.0.37
+          imagePullPolicy: IfNotPresent
+          command:
+            - local-path-provisioner
+            - --debug
+            - start
+            - --config
+            - /etc/config/config.json
+          ports:
+            - name: health
+              containerPort: 8080
+              protocol: TCP
+          startupProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            failureThreshold: 12
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: health
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: health
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/config/
+          env:
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: CONFIG_MOUNT_PATH
+              value: /etc/config/
+            - name: HEALTH_PORT
+              value: "8080"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: local-path-config
+
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: local-path
+provisioner: rancher.io/local-path
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: local-path-config
+  namespace: local-path-storage
+data:
+  config.json: |-
+    {
+            "nodePathMap":[
+            {
+                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                    "paths":["/var/lib/local-path-provisioner"]
+            }
+            ]
+    }
+  setup: |-
+    #!/bin/sh
+    set -eu
+    mkdir -m 0777 -p "$VOL_DIR"
+  teardown: |-
+    #!/bin/sh
+    set -eu
+    rm -rf "$VOL_DIR"
+  helperPod.yaml: |-
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: helper-pod
+    spec:
+      priorityClassName: system-node-critical
+      tolerations:
+        - key: node.kubernetes.io/disk-pressure
+          operator: Exists
+          effect: NoSchedule
+      containers:
+      - name: helper-pod
+        image: docker.io/library/busybox:1.36
+        imagePullPolicy: IfNotPresent

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -41,13 +41,24 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          # - /var/lib/longhorn-disk4 (sde1, 2TB)
+          #
+          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
+          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
+          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
+          # process that touched it blocked in unkillable D state. Those unkillable
+          # processes held Longhorn's per-node rebuild slots forever, so no new
+          # replica could start on talos-cp-2 and every new volume there failed to
+          # attach. It is removed from the Terraform disk list too, so a re-render
+          # does not put it back.
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          if [ "$CURRENT_DISKS" -lt 5 ]; then
+          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
+          # existed; leaving the old threshold would re-apply the patch on every
+          # run once disk4 is gone.
+          if [ "$CURRENT_DISKS" -lt 4 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -75,14 +86,6 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
-                  },
-                  "disk4": {
-                    "allowScheduling": true,
-                    "diskType": "filesystem",
-                    "evictionRequested": false,
-                    "path": "/var/lib/longhorn-disk4",
-                    "storageReserved": 21474836480,
-                    "tags": []
                   }
                 }
               }

terraform/terraform.tfvars.local.bak

@@ -20,6 +20,7 @@ controlplane_configs = {
   "talos-cp-1" = {
     hostname     = "talos-cp-1"
     lan_ip       = "192.168.1.166"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"
@@ -32,6 +33,7 @@ controlplane_configs = {
   "talos-cp-2" = {
     hostname     = "talos-cp-2"
     lan_ip       = "192.168.1.214"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -39,7 +41,11 @@ controlplane_configs = {
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
+        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
+        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
+        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
+        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
+        # re-add it.
       ]
     zone             = "az-b"
     allow_scheduling = true
@@ -49,6 +55,7 @@ controlplane_configs = {
   "talos-cp-3" = {
     hostname     = "talos-cp-3"
     lan_ip       = "192.168.1.162"
+    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"