feat(storage): add local-path-provisioner as the fleet's persistent-storage backend

2026-10-06 12:46:58 +08:00
14 changed files with 37 additions and 594 deletions
@@ -1,36 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: azhi-riotpiao
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: '8'
-spec:
-  project: homelab
-  source:
-    # Private GitHub. The read-only credential is Secret azhi-infra-github in
-    # namespace argocd, created out-of-band.
-    repoURL: https://github.com/ToyoGlobal/azhi-infra.git
-    # The repository's default branch. It is not `main`: this repo's work
-    # happens on the staging slice branch, and `main` is behind it.
-    targetRevision: feat/azi-riotpiao-cnpg
-    # One directory, because ArgoCD reads a single path per source. That
-    # directory is a root kustomization over foundation, postgres and
-    # azhi-server.
-    path: k8s/overlays/azi-riotpiao
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: azi-riotpiao
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-    - CreateNamespace=true
-    # The two Secrets this namespace needs are credentials, so they are created
-    # by hand and NOT by this Application:
-    #   acr-pull                 — the private Alibaba ACR pull credential
-    #   azhi-riotpiao-app-env    — DB credentials, JWT secrets, AES key
-    # A missing imagePullSecret fails the pod outright and a missing app-env
-    # Secret leaves it in CreateContainerConfigError, which is the intended
-    # loud failure rather than a pod that runs against unknown values.
@@ -1,20 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: oidc-bootstrap
-  namespace: argocd
-  annotations:
-    argocd.argoproj.io/sync-wave: "5"
-spec:
-  project: homelab
-  source:
-    repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-    targetRevision: main
-    path: k8s/infra/oidc-bootstrap
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: kube-system
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
@@ -16,12 +16,6 @@ spec:
    - https://github.com/Riotpiaole/Poimen-memory.git
    - https://github.com/Riotpiaole/Poimen-workflows.git
    - https://github.com/Riotpiaole/poimen*.git
-    # ToyoGlobal (GitHub, private). The azhi-infra Application reads its
-    # manifests from here, so the URL must be listed or ArgoCD rejects the
-    # Application with "repository not permitted". A read-only credential for
-    # this repo lives in Secret azhi-infra-github in namespace argocd; it is
-    # created out-of-band and is not committed, because it is a credential.
-    - https://github.com/ToyoGlobal/azhi-infra.git
    # In-cluster Forgejo repos — explicit allowlist (no wildcard)
    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
    - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git
@@ -1,27 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-    name: etcd-defrag-creds
-    namespace: kube-system
-    labels:
-        app.kubernetes.io/name: etcd-defrag
-type: Opaque
-stringData:
-    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
-    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
-    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
-            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
-            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
-            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
-            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-07T05:37:26Z"
-    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
-    version: 3.13.2
@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: forgejo-registry-token
-    namespace: cicd
-data:
-    username: ENC[AES256_GCM,data:oRNecs0a7rs=,iv:/5EFdNqpSjoxe7XDmAEDMd6lXd0SMsRiMmBM9SIwDHY=,tag:Wi+ggoP+9/vgz9rHrxB9nA==,type:str]
-    password: ENC[AES256_GCM,data:HbmwnWk3ST5SDWXwIhu+M5f/FOfFbJHaG0Q/IWrO63HRXmJTmLMpbjtWIXC9IMFoHeDZ1MNAV8A=,iv:ztJZgpaS//yYCiyr6/ubs/UiXkbGMnokvy4Pe45jQMk=,tag:WMpawc/SVAV4mxsCd6XqAA==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjQmhxN2E3ZnNTdkR6U2c1
-            ZStsQktkeWUwMEdmdFIyMkRYRVk1ZGppWlZ3CmUzR2JubWw2VjdWUGt6S2lpNmQ3
-            cDdSWjJiYzRIWS9UVERpVDFJenFnaDAKLS0tIGhDSmNMSjZwcDVPR3U0dUcwNk9Q
-            WmdGS1lyS3dkSXIvOFhyRklGRll2UlUK8NGts6a6Fv10c6gxIIPqIIqfgzQ3EEyk
-            SFrMThwbdaOqLY4AMeDrg+k0Y2t1Xj/XOvdNbRCRictDwZUsptvO7A==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
-    mac: ENC[AES256_GCM,data:imebczVzG9YWd42ER4a+w+Bi3dxbN0HFm/1igxzJt6Z0HG/q6i0s59swpGVruk9dQJWrlO8KKqX739ATp2KGC5szrEuRFvPemF594XFCg6ykW+ZeP5VtTvpJg2nHv7/Pu+71ljKKi4gqOJ4SJDn8owAbSN+su6PiT323wax930A=,iv:iG/G2+qQkvbBU95w/GG2PrbAHDOaamuvjqMDBxSW6zI=,tag:wy9Vn72HBDHqCyzTcV15/Q==,type:str]
-    version: 3.13.2
@@ -1,26 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: memory-agent-oidc
-    namespace: poimen
-data:
-    CLIENT_ID: ENC[AES256_GCM,data:c/RvtV6anofX8h1BmlKu9w==,iv:3lQ1BR8kVvF1RHVfKstmb0Y7u/uX5OrvmGiCi/xTpjw=,tag:9JC2WkfQFWaiZroAEmPenw==,type:str]
-    CLIENT_SECRET: ENC[AES256_GCM,data:I258/6O43PccyllYN/wfoKvTgNgZONTivM/KFIr8TGqSl+2XQkwBXvK2VQc5Xgc5cpwlrgnjk8pjQbqr,iv:MZyA+V5z9Po2VoxqvOSlWh1fZp1lrkcbYMx7VcSIaJs=,tag:gXH0BpA8KOZiVd+4RnhRzg==,type:str]
-    ISSUER: ENC[AES256_GCM,data:GX/uQPDoeJWXXQZlzD6oOIvxo7H4LRAv4MrPAiCgmv6WxW59+KKkymuDT/T9MiDm4bNhxVPgB7nHbDcMHfPvU2zCNM2TWYUsNOgGwxNAqM8=,iv:5OEsdrUZvMLO2C8Ea2i5alsyFGtmEk3othTemk8fOk4=,tag:8CMlRY2JCe4z4fv4WuWjRQ==,type:str]
-    TOKEN_URL: ENC[AES256_GCM,data:KeU1pQzvQ2/1lQhOqNtcR7t5TGVx1J7NY8ogxCXS6IuHhVe0zWYIbC4BYFc2H9PAjsMJwCCg75N1FrIiCYBvpfK8ln0=,iv:NbPiJC/B7W1Zm+JwnEGDikE2cb3L4l6TBeQTKEb2boM=,tag:FRmllFLZzQW03lsCu9mAUA==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmK0dPSmhKT01FSitDdUNW
-            Um1XQU9FVnBUeDdoWXpCcXRzVXh0VXkvNGxNClZaQ3R5U2NkYTJjaFdHRWhUVmFS
-            K0tHZVRFRDZGREVRZzZqcWJ6dnRpUUEKLS0tIE15UUNtMStKbWwzb3dEVGEwN1VQ
-            cktlVTM2SUhpOGJZMXJOOG84RnhCQzAKOdM0pThEaN37SJtRrKIivncMu8Z91Btq
-            es6NOlJdBUhzIhnEJWfDzZ9aA6I/fYzugKbC+AfUwdGgThAgHx9crg==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
-    mac: ENC[AES256_GCM,data:/BTQ984D+hsk0LmnjD3hxopB65NAR54LhoAPNcATWA7MCTJXfdnYRu3gNxUzyiyCBd+rDaPp77TBnak4jVjMnOiYJsYLj1R7hrrhTZopz8pT+YNMupperwsirKZwK8it4Mu+fnpCyox2mtGED0hk+Nh/b6IteTRt5lS7noumo2M=,iv:RWIrVkNYj+8mqMSPiLueToexabzAtVbIR525m6bUrlg=,tag:5+wcNoNYsjGKpFlML+N70Q==,type:str]
-    version: 3.13.2
@@ -1,27 +0,0 @@
-apiVersion: v1
-kind: Secret
-type: Opaque
-metadata:
-    name: oauth2-credentials
-    namespace: iam
-data:
-    api-gw-client-secret: ENC[AES256_GCM,data:YiUCk61cCsTJE8gTLZWhqVPsCSu3b4OnG+Oh456A+mnxip0FjLTTqV+kYBdfZivIkn9ua4RcVW+rYILu,iv:K08m1EaB7/SIVNWZ3FfeRP4tbCAo6KeiNC0NpE9VO1o=,tag:ZXV9YRWUOPJ/vdoqMtvQeA==,type:str]
-    minio-client-secret: ENC[AES256_GCM,data:vcwLo1cJqbZr2MwkAKO09xPjb09ss7+s6HIvEfaLxt8ka5PUAlXQ37pTkynuktlKq/t0DvSk+x1+5G7QqVD1JJlwyDGM6yuOhBQ3MwucLNl+T4RDKtgGuA==,iv:sCdd+UNX2TfTAM6Zslrh8KrvBSzqpe67F3YMGrl4YDU=,tag:9Tpzm9NEXNuXyCd7l7GeHw==,type:str]
-    poimen-client-secret: ENC[AES256_GCM,data:dYtCiMGZaOFokihfGD2SXJDfXrhfwEQcTOPNu7JNHDUnx0hqwAdYAPAJTEN21p0OpHYDINMs5lwUpMwY,iv:ll8USuzs0881ryHImVjHGO0iFxUzqg5XtI6Ekd+EFXE=,tag:I/4ZfB9Ss4zEP57sRtRTXQ==,type:str]
-    paperless-client-secret: ENC[AES256_GCM,data:XMj9rHkI20lI37vChq9wS5lUq7l95SisRX0LChuj09O5w76Zt9NdG6OEUYQFf1z8vCfMJxBInIEEIq0e,iv:UpYy2ZEZ4AgBACEyrlxEcr13V+8Ksu4p8JHYApmyYHo=,tag:o9Z+/b6T9POgowIsks56kw==,type:str]
-    grafana-client-secret: ENC[AES256_GCM,data:V/j0+P3HoDLqYjRalWK4v1hvJTuVmzxVra47HsF1/BZvWZjIyEIH3Pt8BekT3O1dUFynxjcaxdZMBVYg,iv:9pMp+vIsp/sA7GP9wqNDf5DdZVLDow46OURW0mMRXIc=,tag:i7VJFA5o0hToZEJ5JPWIvw==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2amZPendIOU9PeStOcmJG
-            c1dzOFVvd3RoT3dXQlJUNkdCSytEZUVGVEJJCklMUkNjUytBMXh0OVUrNUlpMWw3
-            aHRFT1hBZTZHNHVtaGJCTEFyT2ZWNWsKLS0tIENxR3FKMEplV0V5Y0dsZzBkODVG
-            U0ZlT2R0VFdEQmpsakVvZ1FJRkRQYTAKESQcSlMAuCG0/tLWWZaeoPayf1EKxSaf
-            e4o8TmjmSgPEhepQ/LmNXl6obyuqUxztkh4Mm6pocqDp+gfzt7X/1g==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-06T02:01:24Z"
-    mac: ENC[AES256_GCM,data:AI/znHiUa/Rwg0YRz65qUZ+b6V73oi3ZfAG/V5uZZvrGFrA0EISzD5LaP5Kl4GUU1svo4G5EI4aftPDgpdPptz21CCa8iYkXGe9pTDmtaE5ZICQAUbRcVhGMTfuWFhrgEhMfCJdOpfHaIQVEsQJQJgrnkXQVuUe9aT75LQz7pnY=,iv:KbIaYT0VeBT2WeSU5V5ZB4e+04JICYaY+egGN1Pv5bM=,tag:pPbhDxeLUx6V1XC/LPwxTg==,type:str]
-    version: 3.13.2
@@ -7,34 +7,30 @@ metadata:
      exec:
        path: ksops
 files:
-    - agent-pod-models.enc.yaml
-    - agent-pod-ssh-key.enc.yaml
-    - authentik-secrets.enc.yaml
-    - cloudflare-secrets.enc.yaml
-    - forgejo-registry-pat.enc.yaml
-    - forgejo-registry-pat-api.enc.yaml
-    - forgejo-registry-pull.enc.yaml
-    - forgejo-runner-token.enc.yaml
-    - forgejo-secrets.enc.yaml
-    - grafana-oidc-secrets.enc.yaml
-    - grafana-secrets.enc.yaml
-    - homarr-auth-oidc.enc.yaml
-    - homarr-db-encryption.enc.yaml
-    - homarr-secrets.enc.yaml
-    - homelab-ca-secrets.enc.yaml
-    - loki-secrets.enc.yaml
-    - minio-secrets.enc.yaml
-    - paperless-secrets.enc.yaml
-    - vault-secrets.enc.yaml
-    - vault-unseal-keys.enc.yaml
-    - portfolio-secrets.enc.yaml
-    - gotify-admin-secrets.enc.yaml
-    - gotify-tokens-secrets.enc.yaml
-    - gotify-smtp-secrets.enc.yaml
-    - forgejo-smtp-secrets.enc.yaml
-    - paperless-ai-secrets.enc.yaml
-    - memory-agent-oidc.enc.yaml
-    - oauth2-credentials.enc.yaml
-    - forgejo-registry-token.enc.yaml
+  - agent-pod-models.enc.yaml
+  - agent-pod-ssh-key.enc.yaml
+  - authentik-secrets.enc.yaml
+  - cloudflare-secrets.enc.yaml
+  - forgejo-registry-pat.enc.yaml
+  - forgejo-registry-pat-api.enc.yaml
+  - forgejo-registry-pull.enc.yaml
+  - forgejo-runner-token.enc.yaml
+  - forgejo-secrets.enc.yaml
+  - grafana-oidc-secrets.enc.yaml
+  - grafana-secrets.enc.yaml
+  - homarr-auth-oidc.enc.yaml
+  - homarr-db-encryption.enc.yaml
+  - homarr-secrets.enc.yaml
+  - homelab-ca-secrets.enc.yaml
+  - loki-secrets.enc.yaml
+  - minio-secrets.enc.yaml
+  - paperless-secrets.enc.yaml
+  - vault-secrets.enc.yaml
+  - vault-unseal-keys.enc.yaml
+  - portfolio-secrets.enc.yaml
+  - gotify-admin-secrets.enc.yaml
+  - gotify-tokens-secrets.enc.yaml
+  - gotify-smtp-secrets.enc.yaml
+  - forgejo-smtp-secrets.enc.yaml
+  - paperless-ai-secrets.enc.yaml
    - pi-sandbox-secrets.enc.yaml
-    - etcd-defrag-creds.enc.yaml
@@ -1,177 +0,0 @@
-# etcd backend maintenance: compact, defrag, disarm NOSPACE.
-#
-# Why this exists
-# ---------------
-# etcd's backend file only grows. The apiserver compacts every 5m, but
-# compaction marks superseded revisions free *inside* the file; only `defrag`
-# returns that space to the filesystem. Nothing ran defrag, so the file grew
-# until it crossed the 2 GiB quota:
-#
-#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
-#               -> etcd raised NOSPACE on those two members
-#               -> etcd rejected ALL writes cluster-wide
-#               -> kubelets could not post node status: 3 nodes NotReady
-#               -> apiserver writes returned 500, the tunnel API path 502
-#
-# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
-# dead pages. Recovery was `compact <current-revision>` then `defrag` per
-# member then `alarm disarm`. Compaction had also stalled while the alarm was
-# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
-# worked. Order matters: compact first, then defrag.
-#
-# The earlier revision of this file never ran, for two independent reasons:
-#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
-#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
-#      cluster never had the CronJob or the Secret.
-#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
-#      `exec: "/bin/sh": no such file or directory`). Any container that runs
-#      a shell script needs an image that ships one.
-# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
-# 3.6.4, matching the 3.6.11 servers) pinned by digest.
-#
-# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
-# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
-# and the script exits unless the UTC epoch day is a multiple of 15. That gives
-# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
-# on the 1st, 16th and 31st, which puts two runs one day apart).
-#
-# Safety
-# ------
-# 1. Compaction uses the MINIMUM revision across members. Compacting to a
-#    revision a member has not reached wedges that member - measured on
-#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
-#    needed a reboot to rejoin.
-# 2. A member whose raft applied index lags the leader by more than 100000 is
-#    skipped and reported. Defragging a frozen member reclaims nothing.
-# 3. Members are defragged one at a time. A defrag pauses the member's apply
-#    loop for the duration of the file copy.
-#
-# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
-# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
-# Its client certificate is signed by the cluster's etcd CA and expires
-# 2036-10; rotate it before then by re-running the mint and re-encrypting.
-#
-# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
-# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: etcd-defrag
-  namespace: kube-system
-  labels:
-    app.kubernetes.io/name: etcd-defrag
-    app.kubernetes.io/part-of: cluster-maintenance
-spec:
-  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
-  concurrencyPolicy: Forbid
-  startingDeadlineSeconds: 600
-  successfulJobsHistoryLimit: 3
-  failedJobsHistoryLimit: 3
-  jobTemplate:
-    spec:
-      backoffLimit: 2
-      template:
-        spec:
-          restartPolicy: OnFailure
-          tolerations:
-            - key: node-role.kubernetes.io/control-plane
-              operator: Exists
-              effect: NoSchedule
-          containers:
-            - name: defrag
-              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
-              env:
-                - name: ETCDCTL_API
-                  value: "3"
-                - name: ETCDCTL_CACERT
-                  value: /etc/etcd/ca.crt
-                - name: ETCDCTL_CERT
-                  value: /etc/etcd/client.crt
-                - name: ETCDCTL_KEY
-                  value: /etc/etcd/client.key
-                - name: EPS
-                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
-              command: ["/bin/bash", "-c"]
-              args:
-                - |
-                  set -u
-
-                  DAY=$(( $(date -u +%s) / 86400 ))
-                  if [ $(( DAY % 15 )) -ne 0 ]; then
-                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
-                    exit 0
-                  fi
-
-                  echo "== before =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" endpoint status -w table || true
-                  done
-
-                  MINREV=""
-                  MAXAPPLIED=0
-                  for ep in $EPS; do
-                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
-                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
-                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
-                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
-                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
-                      MINREV="$rev"
-                    fi
-                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
-                      MAXAPPLIED="$app"
-                    fi
-                  done
-
-                  if [ -z "$MINREV" ]; then
-                    echo "FATAL: no member returned a revision; nothing attempted"
-                    exit 1
-                  fi
-
-                  echo "== compact to revision $MINREV =="
-                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
-
-                  # Physical deletion of superseded keys runs in the background
-                  # after the compaction commit; give it a moment to drain.
-                  sleep 20
-
-                  echo "== defrag each member (sequential, one at a time) =="
-                  rc=0
-                  for ep in $EPS; do
-                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
-                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
-                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
-                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
-                      rc=1
-                      continue
-                    fi
-                    echo "-- $ep"
-                    if ! etcdctl --endpoints="$ep" defrag; then
-                      echo "DEFRAG-FAILED $ep"
-                      rc=1
-                    fi
-                  done
-
-                  echo "== disarm NOSPACE =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
-                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
-                  done
-
-                  echo "== after =="
-                  for ep in $EPS; do
-                    etcdctl --endpoints="$ep" endpoint status -w table || true
-                  done
-                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
-                  exit $rc
-              resources:
-                requests: { cpu: 10m, memory: 32Mi }
-                limits:   { cpu: 200m, memory: 128Mi }
-              volumeMounts:
-                - name: creds
-                  mountPath: /etc/etcd
-                  readOnly: true
-          volumes:
-            - name: creds
-              secret:
-                secretName: etcd-defrag-creds
-                defaultMode: 0444
@@ -41,24 +41,13 @@ spec:
          # - /var/lib/longhorn-disk1 (sdb1, 600GB)
          # - /var/lib/longhorn-disk2 (sdc1, 500GB)
          # - /var/lib/longhorn-disk3 (sdd1, 160GB)
-          #
-          # /var/lib/longhorn-disk4 (sde1, the 2TB PERC virtual disk) was RETIRED
-          # on 2026-10-12 and must not be re-added. The device returned SCSI I/O
-          # errors at ~7,000/minute, the XFS filesystem on it hung, and every
-          # process that touched it blocked in unkillable D state. Those unkillable
-          # processes held Longhorn's per-node rebuild slots forever, so no new
-          # replica could start on talos-cp-2 and every new volume there failed to
-          # attach. It is removed from the Terraform disk list too, so a re-render
-          # does not put it back.
+          # - /var/lib/longhorn-disk4 (sde1, 2TB)
          
          echo "Checking talos-cp-2..."
          CURRENT_DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io talos-cp-2 -o json | jq -r '.spec.disks | keys | length')
          echo "  Current disk count: $CURRENT_DISKS"
          
-          # 4 = default-disk-080400000000 + disk1..disk3. It was 5 while disk4
-          # existed; leaving the old threshold would re-apply the patch on every
-          # run once disk4 is gone.
-          if [ "$CURRENT_DISKS" -lt 4 ]; then
+          if [ "$CURRENT_DISKS" -lt 5 ]; then
            echo "  Adding extra disks to talos-cp-2..."
            kubectl -n longhorn-system patch nodes.longhorn.io talos-cp-2 --type merge -p '{
              "spec": {
@@ -86,6 +75,14 @@ spec:
                    "path": "/var/lib/longhorn-disk3",
                    "storageReserved": 5368709120,
                    "tags": []
+                  },
+                  "disk4": {
+                    "allowScheduling": true,
+                    "diskType": "filesystem",
+                    "evictionRequested": false,
+                    "path": "/var/lib/longhorn-disk4",
+                    "storageReserved": 21474836480,
+                    "tags": []
                  }
                }
              }
@@ -1,59 +0,0 @@
-# One-shot bootstrap: restart the control-plane apiservers ONCE so they pick up the
-# OIDC authenticator (they last started before the Authentik provider existed, so OIDC
-# tokens were never accepted -> every login got 401). Delete this whole directory after.
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: oidc-bootstrap
-  namespace: kube-system
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: oidc-bootstrap
-  namespace: kube-system
-rules:
-  - apiGroups: [""]
-    resources: ["pods"]
-    verbs: ["get", "list", "delete"]
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: oidc-bootstrap
-  namespace: kube-system
-subjects:
-  - kind: ServiceAccount
-    name: oidc-bootstrap
-    namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: oidc-bootstrap
---
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: oidc-bootstrap-restart-apiserver-v2
-  namespace: kube-system
-spec:
-  backoffLimit: 4
-  ttlSecondsAfterFinished: 7200
-  template:
-    spec:
-      serviceAccountName: oidc-bootstrap
-      restartPolicy: Never
-      containers:
-        - name: restart
-          image: docker.io/curlimages/curl:8.22.0
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              set -u
-              TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
-              CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-              for pod in kube-apiserver-talos-cp-1 kube-apiserver-talos-cp-2 kube-apiserver-talos-cp-3; do
-                curl -sS --max-time 15 --cacert "$CA" -H "Authorization: Bearer $TOKEN"                   -X DELETE "https://kubernetes.default.svc/api/v1/namespaces/kube-system/pods/$pod?gracePeriodSeconds=0"                   -o /dev/null -w "$pod -> HTTP %{http_code}\n" || echo "$pod -> request failed"
-              done
-              echo "deletes requested; kubelet recreates each static pod in ~20-40s"
@@ -1,141 +0,0 @@
-# One-shot: restart the control-plane API servers so their OIDC authenticator initialises.
-#
-# Two earlier mechanisms failed and are documented so nobody repeats them:
-#   1. deleting the mirror pod does NOT restart a static pod (deletes returned 200, no restart);
-#   2. matching /proc/*/cmdline for 'kube-apiserver' also matched this script's own cmdline,
-#      so every pod SIGTERMed itself (exit 143) before reaching the real process.
-# This version matches /proc/<pid>/comm (the process name) and skips \$\$, so self-kill is
-# impossible. hostPID + privileged are permitted in the 'cicd' namespace. Delays stagger the
-# three control planes by 2 minutes each, so only one API server is down at a time.
---
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: oidc-restart-apiserver-cp1-v2
-  namespace: cicd
-spec:
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
-  template:
-    spec:
-      hostPID: true
-      restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: talos-cp-1
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-      containers:
-        - name: kill
-          image: docker.io/library/busybox:1.36
-          securityContext:
-            privileged: true
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              echo "sleeping 10s before restarting the API server on talos-cp-1"
-              sleep 10
-              self=$$
-              killed=0
-              for d in /proc/[0-9]*; do
-                pid=${d#/proc/}
-                [ "$pid" = "$self" ] && continue
-                comm=$(cat "$d/comm" 2>/dev/null)
-                [ "$comm" = "kube-apiserver" ] || continue
-                echo "SIGTERM -> pid $pid"
-                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
-              done
-              echo "signalled $killed API server process(es); kubelet recreates the static pod"
-
---
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: oidc-restart-apiserver-cp2-v2
-  namespace: cicd
-spec:
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
-  template:
-    spec:
-      hostPID: true
-      restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: talos-cp-2
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-      containers:
-        - name: kill
-          image: docker.io/library/busybox:1.36
-          securityContext:
-            privileged: true
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              echo "sleeping 120s before restarting the API server on talos-cp-2"
-              sleep 120
-              self=$$
-              killed=0
-              for d in /proc/[0-9]*; do
-                pid=${d#/proc/}
-                [ "$pid" = "$self" ] && continue
-                comm=$(cat "$d/comm" 2>/dev/null)
-                [ "$comm" = "kube-apiserver" ] || continue
-                echo "SIGTERM -> pid $pid"
-                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
-              done
-              echo "signalled $killed API server process(es); kubelet recreates the static pod"
-
---
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: oidc-restart-apiserver-cp3-v2
-  namespace: cicd
-spec:
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
-  template:
-    spec:
-      hostPID: true
-      restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: talos-cp-3
-      tolerations:
-        - key: node-role.kubernetes.io/control-plane
-          operator: Exists
-          effect: NoSchedule
-        - key: node-role.kubernetes.io/master
-          operator: Exists
-          effect: NoSchedule
-      containers:
-        - name: kill
-          image: docker.io/library/busybox:1.36
-          securityContext:
-            privileged: true
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              echo "sleeping 240s before restarting the API server on talos-cp-3"
-              sleep 240
-              self=$$
-              killed=0
-              for d in /proc/[0-9]*; do
-                pid=${d#/proc/}
-                [ "$pid" = "$self" ] && continue
-                comm=$(cat "$d/comm" 2>/dev/null)
-                [ "$comm" = "kube-apiserver" ] || continue
-                echo "SIGTERM -> pid $pid"
-                kill -TERM "$pid" 2>/dev/null && killed=$((killed+1))
-              done
-              echo "signalled $killed API server process(es); kubelet recreates the static pod"
-
@@ -20,7 +20,6 @@ controlplane_configs = {
  "talos-cp-1" = {
    hostname     = "talos-cp-1"
    lan_ip       = "192.168.1.166"
-    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/nvme0n1"
@@ -33,7 +32,6 @@ controlplane_configs = {
  "talos-cp-2" = {
    hostname     = "talos-cp-2"
    lan_ip       = "192.168.1.214"
-    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/disk/by-id/wwn-0x644a842029bd3f002720989b07d7143d"
@@ -41,11 +39,7 @@ controlplane_configs = {
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b60f4375d97062", mountpoint = "/var/lib/longhorn-disk1" },
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b5ffeb8bb8b47f", mountpoint = "/var/lib/longhorn-disk2" },
        { device = "/dev/disk/by-id/wwn-0x644a842029bd3f0031b6000c8dabb266", mountpoint = "/var/lib/longhorn-disk3" },
-        # /var/lib/longhorn-disk4 (wwn-0x6b083fe0c5782700321370dc23fd765b) was
-        # retired 2026-10-12: the 2TB PERC virtual disk failed with continuous
-        # SCSI I/O errors, hung its XFS filesystem, and wedged Longhorn replica
-        # rebuilds on talos-cp-2 with unkillable D-state processes. Do not
-        # re-add it.
+        { device = "/dev/disk/by-id/wwn-0x6b083fe0c5782700321370dc23fd765b", mountpoint = "/var/lib/longhorn-disk4" },
      ]
    zone             = "az-b"
    allow_scheduling = true
@@ -55,7 +49,6 @@ controlplane_configs = {
  "talos-cp-3" = {
    hostname     = "talos-cp-3"
    lan_ip       = "192.168.1.162"
-    public_ip    = "https://k8s-api.riotpiao.com"
    lan_subnet   = "192.168.1.0/24"
    lan_gateway  = "192.168.1.254"
    install_disk = "/dev/nvme0n1"