feat(argocd): let ArgoCD reconcile azhi-infra from its private GitHub repo

The homelab AppProject allowlist had no azhi repository, so an Application
pointing at ToyoGlobal/azhi-infra fails admission with 'repository not
permitted'. Adding the URL is the whole gate.

The Application itself targets the repository default branch
(harness/staging-slice), not main, and path k8s/overlays/azi-riotpiao. That
overlay renders postgres and the API into namespace azi-riotpiao.

The two Secrets the namespace needs stay out of Git: acr-pull carries the ACR
pull credential and azhi-riotpiao-app-env carries the database credentials,
the JWT secrets and the AES key.

k8s/argocd/apps/72-azhi-riotpiao.yaml

@@ -0,0 +1,36 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: azhi-riotpiao
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: '8'
+spec:
+  project: homelab
+  source:
+    # Private GitHub. The read-only credential is Secret azhi-infra-github in
+    # namespace argocd, created out-of-band.
+    repoURL: https://github.com/ToyoGlobal/azhi-infra.git
+    # The repository's default branch. It is not `main`: this repo's work
+    # happens on the staging slice branch, and `main` is behind it.
+    targetRevision: harness/staging-slice
+    # One directory, because ArgoCD reads a single path per source. That
+    # directory is a root kustomization over foundation, postgres and
+    # azhi-server.
+    path: k8s/overlays/azi-riotpiao
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: azi-riotpiao
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+    - CreateNamespace=true
+    # The two Secrets this namespace needs are credentials, so they are created
+    # by hand and NOT by this Application:
+    #   acr-pull                 — the private Alibaba ACR pull credential
+    #   azhi-riotpiao-app-env    — DB credentials, JWT secrets, AES key
+    # A missing imagePullSecret fails the pod outright and a missing app-env
+    # Secret leaves it in CreateContainerConfigError, which is the intended
+    # loud failure rather than a pod that runs against unknown values.

k8s/argocd/projects/homelab-project.yaml

@@ -16,6 +16,12 @@ spec:
     - https://github.com/Riotpiaole/Poimen-memory.git
     - https://github.com/Riotpiaole/Poimen-workflows.git
     - https://github.com/Riotpiaole/poimen*.git
+    # ToyoGlobal (GitHub, private). The azhi-infra Application reads its
+    # manifests from here, so the URL must be listed or ArgoCD rejects the
+    # Application with "repository not permitted". A read-only credential for
+    # this repo lives in Secret azhi-infra-github in namespace argocd; it is
+    # created out-of-band and is not committed, because it is a credential.
+    - https://github.com/ToyoGlobal/azhi-infra.git
     # In-cluster Forgejo repos — explicit allowlist (no wildcard)
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
     - https://forgejo.riotpiao.com/riotpiao-poimen/homelab-frontend.git

k8s/argocd/secrets/etcd-defrag-creds.enc.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: etcd-defrag-creds
+    namespace: kube-system
+    labels:
+        app.kubernetes.io/name: etcd-defrag
+type: Opaque
+stringData:
+    ca.crt: ENC[AES256_GCM,data:vso/Ouz5QRoRF2CFj9TsGYhoGkX8oGJIeiFCoS92AAQbyyGPTiZ/fPNFM03p+TpeKDb+Z+Fyo/HXuDeT5Fee5kjPnUHXf2hiUBmJLyHnPMp5i57dRzTC8X4VaNeV/1MmP2WpkWwieWOGBxJVIIrgeQhQexxmIzSlQ5Io9dVOH8F6iyakwdf7qfLFGdoer3rfrXadSOJtKRpLneFJDL2OmR1042LZ8KtNuRRXI/qrdpgr2bnyxFuCRc+tk5QY73mpwwWfKGn6FW/rVdjiEftkC+AvTUItx7RkYMrz0PlTYPUOLustEF2gIgWheYZhE5ebZDurVSG//sudjASliYJItwgSk3G5mZihOSDA7Jcus+U8p1iHQ9CQsey3CSAMmuSGRFmq42kI8135kjtBYrzVswHjX1/51qPMyA2TIeWlRJ7AJlHihG317gT1vPeM1ULl3PkuCxaBLCtjotZBzKLJ/6zXr1ig77Baz71YOVB0D0MgFvzIdao+mdxS8WLOZdiXOCwbI5Y2eo7wJHG4BnpKEdx1rjPkv7DI2NNEoXA7e5FvJWinyAxOy6YGH4Fv6dTlEVhI3Nl/1X+CeafL6Hk8YoqA04Hm+/dJCZi7SsF0r6zfPNNsZLivM8DN18h8KYM3eVY3CwtBldE7atz+oprQ6K/yGUbblHRQwuG3sH3AHrCKLPiUrMIi0jWc0IpdPZRZFL7SnMULfV05QgcvGySOJCuQU+/OYzrRs7umOjwpqvi9PyJYVXBliiYfJutvmfpFAhQj,iv:KnW9LYNzxZ6r3Q6LTeOrgSLLPgiJlibVp6YmTEKBkjE=,tag:KTL1nOWdNWDA4dLVaA4//g==,type:str]
+    client.crt: ENC[AES256_GCM,data:IEmZe505SwkW3U5vmRtFxR6PpuSFGWEHflUZwyhSODzYG1zLM8olVWDtH+yUVI54kLXxTtcpMkKrTBjpAxHG8zUTLPdTCVzPMq6U20jH4PYFMzoldnSgpRl/eiADUmE4I32CBbPdM5nkPvjVEYMmRafEGyJctjG6Shhh/sUO1yE9A+h1R7o3UCRPttWTozSKN4dPi75v1TSAOLyXqvQYl5Ea9iKW47ea7Q+XTcoKfMPXsTJp44kiZnVW0VT/7xNydR7AVWJJzYltY5380j0Ho+7PfNd+QqmdodkaZPS4b0PaoinMiZ40JnChGVhdS91j3IkUlU1g/+G2iVnPR9KB07xqRubqiASxIvpSaSCeYl3rR7+dXYX18VpWcWnJ6XoREoqxJiqmMQWYSROYDTd4G/zs4Usuh5r0wIodmCjud+g89mmXFtH0PbsMCVvZDUswjVfeT2PUBm036TXlxF/1aAy2GDqycbPgJgDLyw9tdiQUZe2iA+Z/Wx80qgQjmmqCaw77cDergLFj9KnByl3Tinwfm+IWFeQiDW+fwSzz+/vrpl55BkSVkVhw7r2Jd6zyg4VtLVXbxYhzdNz695N+fjkZHuPE5tCwr4WCzAg3vSS9hA7IHo8k93WO8s6+KlyjuwTdjPIYZghebO4ktwdqvhvuqxv8Nzwpc5J7Z2H2IChjXg==,iv:l8BwxxEfHfYrm+mM5UE6Yj42DeeU9y23jjIE1KjIiME=,tag:HwRT6/myIsxc84BYDj3K6w==,type:str]
+    client.key: ENC[AES256_GCM,data:QD/Glf3lFslBEdEoJ3CNrYzlw2HLCvGSeVuJLtxnFne2aJOuw0sk/e0h/kSAxvG4dZ+LIWJLLbqTjKHOAhkvS/ZOA7GSea4ubNkvTkw8+JBRuYwb2D2iB8kZhlER/CpiGkYJukjkgGP1CzHdvBY1M88pW9w6rH74MD3jY47qExIaNduQj5vkUQZ8ajqkd79l0nk98ZTawfPG4AFcEtAIKAPIuubT2GZ3qqNLjOI7agbtRv8xI44hhb2wj5J9xdl1GrbOWwG5EOo+hLNrcAAOCcWBpNTUCKBy7mrk2ID8jX3HsFWN76DDlQzkhcphMhlM7g==,iv:49HGJAEgllEkB3YSiZLmbWs/vlbFbY52kiKesdGVC/Y=,tag:g8YZ9at3V2l6RFRfvO2lAQ==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4Z0h0R2dhbDdRcFJydE9o
+            R0hWeCtwVzluZWZlR0VtOXcxdW9FbklOQXlzCmo3eXNyNlFjejNETkJiVVRVWFhr
+            a2pORnhIQTI0UUk2N3dWMDZJcEhYU1UKLS0tIEduaHlmMHQza1A0TGVZb0JrbTYz
+            MWJwQjcrdkkzQXhOTmhycERsM0FtNVEKG+gBRcvUsyfRzuW999EQNzyDh2/5fY6p
+            8ab2xvTOJPiMUBTBuQZsA+iDImytUO/02ROQsbEZkALijp1nqrsV/g==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    encrypted_regex: ^(data|stringData)$
+    lastmodified: "2026-10-07T05:37:26Z"
+    mac: ENC[AES256_GCM,data:vu6ltV0jYQNGV4eqIcqAp9p2TJ8ULBX5JBELUgm/nA9GX+I6Pv3fyjpcHRSWY0HbLvCwgXd22CG4IYcj4HJFtQGZR7OcQA5WM5plZH3RCPdvPNix/798vkETfpJxorQjlHoq4nlr3dEmUCbzdtE07qV2W1M6GJr4C9xKvB947bQ=,iv:aA9mjZCFAoLEhGqkl6NfyBZ3x3vWGfLYYAEe8jXuA4U=,tag:Drdiji33ckDd8ildpvSEXw==,type:str]
+    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -37,3 +37,4 @@ files:
     - oauth2-credentials.enc.yaml
     - forgejo-registry-token.enc.yaml
     - pi-sandbox-secrets.enc.yaml
+    - etcd-defrag-creds.enc.yaml

k8s/infra/cluster-maintenance/etcd-defrag-cronjob.yaml

@@ -0,0 +1,177 @@
+# etcd backend maintenance: compact, defrag, disarm NOSPACE.
+#
+# Why this exists
+# ---------------
+# etcd's backend file only grows. The apiserver compacts every 5m, but
+# compaction marks superseded revisions free *inside* the file; only `defrag`
+# returns that space to the filesystem. Nothing ran defrag, so the file grew
+# until it crossed the 2 GiB quota:
+#
+#   2026-10-07  backend 2,148,843,520 B (2.1 GB) on talos-cp-1 and talos-cp-3
+#               -> etcd raised NOSPACE on those two members
+#               -> etcd rejected ALL writes cluster-wide
+#               -> kubelets could not post node status: 3 nodes NotReady
+#               -> apiserver writes returned 500, the tunnel API path 502
+#
+# The live data was tiny: 15,943 keys, 69 MB after compaction. The rest was
+# dead pages. Recovery was `compact <current-revision>` then `defrag` per
+# member then `alarm disarm`. Compaction had also stalled while the alarm was
+# up, so `dbSizeInUse` read 100% of the file - defrag alone would NOT have
+# worked. Order matters: compact first, then defrag.
+#
+# The earlier revision of this file never ran, for two independent reasons:
+#   1. it existed only on branch fix/etcd-defrag-cron; the ArgoCD application
+#      `cluster-maintenance` tracks `main` (targetRevision: main), so the
+#      cluster never had the CronJob or the Secret.
+#   2. the image was quay.io/coreos/etcd, which has NO /bin/sh (verified:
+#      `exec: "/bin/sh": no such file or directory`). Any container that runs
+#      a shell script needs an image that ships one.
+# This revision fixes both: image bitnamilegacy/etcd (ships bash + etcdctl
+# 3.6.4, matching the 3.6.11 servers) pinned by digest.
+#
+# Cadence: every 15 days. Measured growth is ~29 MB/day, so a 15-day cycle
+# peaks near 500 MB - comfortably under the 2 GiB quota. The schedule is daily
+# and the script exits unless the UTC epoch day is a multiple of 15. That gives
+# an exact 15-day interval; `*/15` in the day-of-month field does not (it fires
+# on the 1st, 16th and 31st, which puts two runs one day apart).
+#
+# Safety
+# ------
+# 1. Compaction uses the MINIMUM revision across members. Compacting to a
+#    revision a member has not reached wedges that member - measured on
+#    2026-10-07, when talos-cp-2 froze ~1.2M entries behind the leader and
+#    needed a reboot to rejoin.
+# 2. A member whose raft applied index lags the leader by more than 100000 is
+#    skipped and reported. Defragging a frozen member reclaims nothing.
+# 3. Members are defragged one at a time. A defrag pauses the member's apply
+#    loop for the duration of the file copy.
+#
+# Credentials: Secret etcd-defrag-creds in kube-system, SOPS-managed in
+# k8s/argocd/secrets/etcd-defrag-creds.enc.yaml (ksops generator 00-secrets).
+# Its client certificate is signed by the cluster's etcd CA and expires
+# 2036-10; rotate it before then by re-running the mint and re-encrypting.
+#
+# Companion alerts (monitoring): etcd_mvcc_db_total_size_in_bytes /
+# etcd_server_quota_backend_bytes > 0.8, and an etcd NOSPACE alarm alert.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: etcd-defrag
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: etcd-defrag
+    app.kubernetes.io/part-of: cluster-maintenance
+spec:
+  schedule: "0 3 * * *"            # daily; the script acts only every 15th day
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 600
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: OnFailure
+          tolerations:
+            - key: node-role.kubernetes.io/control-plane
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: defrag
+              image: bitnamilegacy/etcd:3.6.4@sha256:c86c06f1ce6a08b6df21eb7a9655b1818778b64c85b5cbb8413ebf7534789730
+              env:
+                - name: ETCDCTL_API
+                  value: "3"
+                - name: ETCDCTL_CACERT
+                  value: /etc/etcd/ca.crt
+                - name: ETCDCTL_CERT
+                  value: /etc/etcd/client.crt
+                - name: ETCDCTL_KEY
+                  value: /etc/etcd/client.key
+                - name: EPS
+                  value: "https://192.168.1.166:2379 https://192.168.1.214:2379 https://192.168.1.162:2379"
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  set -u
+
+                  DAY=$(( $(date -u +%s) / 86400 ))
+                  if [ $(( DAY % 15 )) -ne 0 ]; then
+                    echo "skip: UTC epoch day $DAY is not a 15-day boundary"
+                    exit 0
+                  fi
+
+                  echo "== before =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+
+                  MINREV=""
+                  MAXAPPLIED=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || continue
+                    rev=$(printf '%s' "$json" | sed -n 's/.*"revision":\([0-9]*\).*/\1/p' | head -1)
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    echo "member $ep revision=${rev:-?} raftAppliedIndex=${app:-?}"
+                    if [ -n "$rev" ] && { [ -z "$MINREV" ] || [ "$rev" -lt "$MINREV" ]; }; then
+                      MINREV="$rev"
+                    fi
+                    if [ -n "$app" ] && [ "$app" -gt "$MAXAPPLIED" ]; then
+                      MAXAPPLIED="$app"
+                    fi
+                  done
+
+                  if [ -z "$MINREV" ]; then
+                    echo "FATAL: no member returned a revision; nothing attempted"
+                    exit 1
+                  fi
+
+                  echo "== compact to revision $MINREV =="
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" compact "$MINREV" || echo "COMPACT-FAILED (continuing: defrag still reclaims free pages)"
+
+                  # Physical deletion of superseded keys runs in the background
+                  # after the compaction commit; give it a moment to drain.
+                  sleep 20
+
+                  echo "== defrag each member (sequential, one at a time) =="
+                  rc=0
+                  for ep in $EPS; do
+                    json=$(etcdctl --endpoints="$ep" endpoint status -w json 2>/dev/null) || { echo "SKIP $ep (unreachable)"; rc=1; continue; }
+                    app=$(printf '%s' "$json" | sed -n 's/.*"raftAppliedIndex":\([0-9]*\).*/\1/p' | head -1)
+                    if [ -n "$app" ] && [ "$app" -lt $(( MAXAPPLIED - 100000 )) ]; then
+                      echo "SKIP $ep (applied index $app lags leader $MAXAPPLIED; defrag would reclaim nothing)"
+                      rc=1
+                      continue
+                    fi
+                    echo "-- $ep"
+                    if ! etcdctl --endpoints="$ep" defrag; then
+                      echo "DEFRAG-FAILED $ep"
+                      rc=1
+                    fi
+                  done
+
+                  echo "== disarm NOSPACE =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" alarm list 2>/dev/null || true
+                    etcdctl --endpoints="$ep" alarm disarm 2>/dev/null || true
+                  done
+
+                  echo "== after =="
+                  for ep in $EPS; do
+                    etcdctl --endpoints="$ep" endpoint status -w table || true
+                  done
+                  etcdctl --endpoints="$(echo "$EPS" | awk '{print $1}')" alarm list || true
+                  exit $rc
+              resources:
+                requests: { cpu: 10m, memory: 32Mi }
+                limits:   { cpu: 200m, memory: 128Mi }
+              volumeMounts:
+                - name: creds
+                  mountPath: /etc/etcd
+                  readOnly: true
+          volumes:
+            - name: creds
+              secret:
+                secretName: etcd-defrag-creds
+                defaultMode: 0444