Merge pull request 'feat(argocd): reconcile azhi-infra from its private GitHub repo' (#60) from feat/azhi-riotpiao-argocd into main

The homelab AppProject allowlist had no azhi repository, so an Application
pointing at ToyoGlobal/azhi-infra fails admission with 'repository not
permitted'. Adding the URL is the whole gate.

The Application itself targets the repository default branch
(harness/staging-slice), not main, and path k8s/overlays/azi-riotpiao. That
overlay renders postgres and the API into namespace azi-riotpiao.

The two Secrets the namespace needs stay out of Git: acr-pull carries the ACR
pull credential and azhi-riotpiao-app-env carries the database credentials,
the JWT secrets and the AES key.

k8s/argocd/apps/10-storage-observability.yaml

@@ -249,26 +249,3 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
----
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: local-path-provisioner
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/infra/local-path
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: local-path-storage
-  # No CreateNamespace: the Namespace object (with its privileged PSA label,
-  # which the helper Pods need for their hostPath) is part of the manifest.
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true

k8s/argocd/apps/72-azhi-riotpiao.yaml

@@ -0,0 +1,36 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: azhi-riotpiao
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: '8'
+spec:
+  project: homelab
+  source:
+    # Private GitHub. The read-only credential is Secret azhi-infra-github in
+    # namespace argocd, created out-of-band.
+    repoURL: https://github.com/ToyoGlobal/azhi-infra.git
+    # The repository's default branch. It is not `main`: this repo's work
+    # happens on the staging slice branch, and `main` is behind it.
+    targetRevision: harness/staging-slice
+    # One directory, because ArgoCD reads a single path per source. That
+    # directory is a root kustomization over foundation, postgres and
+    # azhi-server.
+    path: k8s/overlays/azi-riotpiao
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: azi-riotpiao
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    # The two Secrets this namespace needs are credentials, so they are created
+    # by hand and NOT by this Application:
+    #   acr-pull                 — the private Alibaba ACR pull credential
+    #   azhi-riotpiao-app-env    — DB credentials, JWT secrets, AES key
+    # A missing imagePullSecret fails the pod outright and a missing app-env
+    # Secret leaves it in CreateContainerConfigError, which is the intended
+    # loud failure rather than a pod that runs against unknown values.

k8s/argocd/apps/99-oidc-bootstrap.yaml

@@ -0,0 +1,20 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: oidc-bootstrap
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "5"
+spec:
+  project: homelab
+  source:
+    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
+    targetRevision: main
+    path: k8s/infra/oidc-bootstrap
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kube-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

k8s/argocd/projects/homelab-project.yaml

@@ -16,6 +16,12 @@ spec:
     - https://github.com/Riotpiaole/Poimen-memory.git
     - https://github.com/Riotpiaole/Poimen-workflows.git
     - https://github.com/Riotpiaole/poimen*.git
+    # ToyoGlobal (GitHub, private). The azhi-infra Application reads its
+    # manifests from here, so the URL must be listed or ArgoCD rejects the
+    # Application with "repository not permitted". A read-only credential for
+    # this repo lives in Secret azhi-infra-github in namespace argocd; it is
+    # created out-of-band and is not committed, because it is a credential.
+    - https://github.com/ToyoGlobal/azhi-infra.git
     # In-cluster Forgejo repos — explicit allowlist (no wildcard)
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git

k8s/argocd/secrets/forgejo-registry-token.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: forgejo-registry-token
+    namespace: cicd
+data:
+    username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
+    password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
+            ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
+            cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
+            WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
+            SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/memory-agent-oidc.enc.yaml

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: memory-agent-oidc
+    namespace: poimen
+data:
+    CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
+    CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
+    ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
+    TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
+            Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
+            K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
+            cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
+            es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/oauth2-credentials.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+    name: oauth2-credentials
+    namespace: iam
+data:
+    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
+    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
+    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
+    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
+    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
+            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
+            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
+            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
+            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-06T02:01:24Z"
+    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -33,5 +33,8 @@ files:
     - gotify-smtp-secrets.enc.yaml
     - forgejo-smtp-secrets.enc.yaml
     - paperless-ai-secrets.enc.yaml
+    - memory-agent-oidc.enc.yaml
+    - oauth2-credentials.enc.yaml
+    - forgejo-registry-token.enc.yaml
     - pi-sandbox-secrets.enc.yaml
     - etcd-defrag-creds.enc.yaml

k8s/infra/local-path/kustomization.yaml

@@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-# No commonLabels on purpose: this is a vendored third-party manifest and
-# kustomize would also rewrite the provisioner Deployment's selector, which
-# is immutable on re-apply.
-resources:
-  - local-path-storage.yaml

k8s/infra/local-path/local-path-storage.yaml

@@ -1,200 +0,0 @@
-# Vendored from https://github.com/rancher/local-path-provisioner
-#   deploy/local-path-storage.yaml @ v0.0.37 (release 2026-08-05)
-#
-# Local modifications (the only three; re-apply them on any re-vendor):
-#   1. the namespace carries pod-security.kubernetes.io/enforce=privileged, because the
-#      provisioner's helper Pods mount a hostPath and this cluster defaults to the
-#      baseline policy, which forbids that;
-#   2. config.json nodePathMap is /var/lib/local-path-provisioner — Talos mounts the root
-#      filesystem read-only, so upstream's /opt path cannot be created;
-#   3. the helper Pod's busybox image is pinned to a tag instead of a floating name.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: local-path-storage
-  labels:
-    pod-security.kubernetes.io/enforce: privileged
-    pod-security.kubernetes.io/audit: privileged
-    pod-security.kubernetes.io/warn: privileged
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: local-path-provisioner-service-account
-  namespace: local-path-storage
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: local-path-provisioner-role
-  namespace: local-path-storage
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: local-path-provisioner-role
-rules:
-  - apiGroups: [""]
-    resources: ["nodes", "persistentvolumeclaims", "configmaps", "pods", "pods/log"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]
-  - apiGroups: [""]
-    resources: ["events"]
-    verbs: ["create", "patch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: local-path-provisioner-bind
-  namespace: local-path-storage
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: local-path-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: local-path-provisioner-service-account
-    namespace: local-path-storage
-
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: local-path-provisioner-bind
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: local-path-provisioner-role
-subjects:
-  - kind: ServiceAccount
-    name: local-path-provisioner-service-account
-    namespace: local-path-storage
-
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: local-path-provisioner
-  namespace: local-path-storage
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: local-path-provisioner
-  template:
-    metadata:
-      labels:
-        app: local-path-provisioner
-    spec:
-      serviceAccountName: local-path-provisioner-service-account
-      containers:
-        - name: local-path-provisioner
-          image: docker.io/rancher/local-path-provisioner:v0.0.37
-          imagePullPolicy: IfNotPresent
-          command:
-            - local-path-provisioner
-            - --debug
-            - start
-            - --config
-            - /etc/config/config.json
-          ports:
-            - name: health
-              containerPort: 8080
-              protocol: TCP
-          startupProbe:
-            httpGet:
-              path: /health
-              port: health
-            initialDelaySeconds: 5
-            periodSeconds: 5
-            failureThreshold: 12
-          livenessProbe:
-            httpGet:
-              path: /health
-              port: health
-            initialDelaySeconds: 10
-            periodSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /ready
-              port: health
-            initialDelaySeconds: 5
-            periodSeconds: 5
-          volumeMounts:
-            - name: config-volume
-              mountPath: /etc/config/
-          env:
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: CONFIG_MOUNT_PATH
-              value: /etc/config/
-            - name: HEALTH_PORT
-              value: "8080"
-      volumes:
-        - name: config-volume
-          configMap:
-            name: local-path-config
-
----
-apiVersion: storage.k8s.io/v1
-kind: StorageClass
-metadata:
-  name: local-path
-provisioner: rancher.io/local-path
-volumeBindingMode: WaitForFirstConsumer
-reclaimPolicy: Delete
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: local-path-config
-  namespace: local-path-storage
-data:
-  config.json: |-
-    {
-            "nodePathMap":[
-            {
-                    "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
-                    "paths":["/var/lib/local-path-provisioner"]
-            }
-            ]
-    }
-  setup: |-
-    #!/bin/sh
-    set -eu
-    mkdir -m 0777 -p "$VOL_DIR"
-  teardown: |-
-    #!/bin/sh
-    set -eu
-    rm -rf "$VOL_DIR"
-  helperPod.yaml: |-
-    apiVersion: v1
-    kind: Pod
-    metadata:
-      name: helper-pod
-    spec:
-      priorityClassName: system-node-critical
-      tolerations:
-        - key: node.kubernetes.io/disk-pressure
-          operator: Exists
-          effect: NoSchedule
-      containers:
-      - name: helper-pod
-        image: docker.io/library/busybox:1.36
-        imagePullPolicy: IfNotPresent

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -41,24 +41,13 @@ spec:
           # - /var/lib/longhorn-disk1 (sdb1, 600GB)
           # - /var/lib/longhorn-disk2 (sdc1, 500GB)
           # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          #
-          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
-          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
-          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
-          # process that touched it blocked in unkillable D state. Those unkillable
-          # processes held Longhorn's per-node rebuild slots forever, so no new
-          # replica could start on talos-cp-2 and every new volume there failed to
-          # attach. It is removed from the Terraform disk list too, so a re-render
-          # does not put it back.
+          # - /var/lib/longhorn-disk4 (sde1, 2TB)
           
           echo "Checking talos-cp-2..."
           CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
           echo "  Current disk count: $CURRENT_DISKS"
           
-          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
-          # existed; leaving the old threshold would re-apply the patch on every
-          # run once disk4 is gone.
-          if [ "$CURRENT_DISKS" -lt 4 ]; then
+          if [ "$CURRENT_DISKS" -lt 5 ]; then
             echo "  Adding extra disks to talos-cp-2..."
             kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
               "spec": {
@@ -86,6 +75,14 @@ spec:
                     "path": "/var/lib/longhorn-disk3",
                     "storageReserved": 5368709120,
                     "tags": []
+                  },
+                  "disk4": {
+                    "allowScheduling": true,
+                    "diskType": "filesystem",
+                    "evictionRequested": false,
+                    "path": "/var/lib/longhorn-disk4",
+                    "storageReserved": 21474836480,
+                    "tags": []
                   }
                 }
               }

k8s/infra/oidc-bootstrap/bootstrap.yaml

@@ -0,0 +1,59 @@
+# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
+# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
+# tokens were never accepted -> every login got 401). Delete this whole directory after.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: oidc-bootstrap
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: oidc-bootstrap
+    namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: oidc-bootstrap
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-bootstrap-restart-apiserver-v2
+  namespace: kube-system
+spec:
+  backoffLimit: 4
+  ttlSecondsAfterFinished: 7200
+  template:
+    spec:
+      serviceAccountName: oidc-bootstrap
+      restartPolicy: Never
+      containers:
+        - name: restart
+          image: docker.io/curlimages/curl:8.22.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -u
+              TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+              CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+              for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
+                curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN"                   -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0"                   -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
+              done
+              echo "deletes requested; kubelet recreates each static pod in ~20-40s"

k8s/infra/oidc-bootstrap/restart-apiserver.yaml

@@ -0,0 +1,141 @@
+# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
+#
+# Two earlier mechanisms failed and are documented so nobody repeats them:
+#   1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
+#   2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
+#      so every pod SIGTERMed itself (exit 143) before reaching the real process.
+# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
+# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
+# three control planes by 2 minutes each, so only one API server is down at a time.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp1-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-1
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 10s before restarting the API server on talos-cp-1"
+              sleep 10
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp2-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-2
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 120s before restarting the API server on talos-cp-2"
+              sleep 120
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: oidc-restart-apiserver-cp3-v2
+  namespace: cicd
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      hostPID: true
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: talos-cp-3
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      containers:
+        - name: kill
+          image: docker.io/library/busybox:1.36
+          securityContext:
+            privileged: true
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              echo "sleeping 240s before restarting the API server on talos-cp-3"
+              sleep 240
+              self=$$
+              killed=0
+              for d in /proc/[0-9]*; do
+                pid=${d#/proc/}
+                [ "$pid" = "$self" ] && continue
+                comm=$(cat "$d/comm" 2>/dev/null)
+                [ "$comm" = "kube-apiserver" ] || continue
+                echo "SIGTERM -> pid $pid"
+                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
+              done
+              echo "signalled $killed API server process(es); kubelet recreates the static pod"
+

terraform/terraform.tfvars.local.bak

@@ -20,7 +20,6 @@ controlplane_configs = {
   "talos-cp-1" = {
     hostname     = "talos-cp-1"
     lan_ip       = "192.168.1.166"
-    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"
@@ -33,7 +32,6 @@ controlplane_configs = {
   "talos-cp-2" = {
     hostname     = "talos-cp-2"
     lan_ip       = "192.168.1.214"
-    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -41,11 +39,7 @@ controlplane_configs = {
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
         { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
-        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
-        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
-        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
-        # re-add it.
+        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
       ]
     zone             = "az-b"
     allow_scheduling = true
@@ -55,7 +49,6 @@ controlplane_configs = {
   "talos-cp-3" = {
     hostname     = "talos-cp-3"
     lan_ip       = "192.168.1.162"
-    public_ip    = "https://k8s-api.riotpiao.com"
     lan_subnet   = "192.168.1.0/24"
     lan_gateway  = "192.168.1.254"
     install_disk = "/dev/nvme0n1"