fix(longhorn): enable worker-1 scheduling, 400Gi reserved

32 degraded + 3 faulted volumes from replica rebuild failures.
cp-1/cp-2 over-committed; worker-1 had 602Gi free but scheduling
disabled. Enable it with 400Gi reserved for LLM weights.

- Add worker-1 to longhorn-nodes.yaml (allowScheduling: true)
- Add worker-1 disk config to add-disks-job (storageReserved: 400Gi,
  evictionRequested: false)

k8s/argocd/apps/01-image-updater.yaml

@@ -1,33 +0,0 @@
-# ArgoCD Image Updater - auto-updates Application images from registry
-# Watches forgejo.riotpiao.com for new image tags and updates Applications
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: argocd-image-updater
-  namespace: argocd
-  finalizers:
-    - resources-finalizer.argocd.argoproj.io
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
-spec:
-  project: homelab
-  revisionHistoryLimit: 3
-  sources:
-    - repoURL: https://argoproj.github.io/argo-helm
-      chart: argocd-image-updater
-      targetRevision: "0.11.2"
-      helm:
-        valueFiles:
-          - $values/k8s/infra/argocd-image-updater/values.yaml
-    - repoURL: https://forgejo.riotpiao.com/riotpiao-poimen/homelab.git
-      targetRevision: main
-      ref: values
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: argocd
-  syncPolicy:
-    automated:
-      prune: true
-      selfHeal: true
-    syncOptions:
-      - CreateNamespace=false

k8s/argocd/apps/30-security.yaml

@@ -152,13 +152,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-golang
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -180,13 +175,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-node
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:
@@ -211,13 +201,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "3"
-    argocd-image-updater.argoproj.io/image-list: runner=forgejo.riotpiao.com/rock/forgejo-runner-rust
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/runner.update-strategy: newest-build
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/runner.allow-tags: regexp:^[0-9a-f]{7}$|^latest$|^v[0-9]+$
-    argocd-image-updater.argoproj.io/runner.helm.image-name: runner.image.repository
-    argocd-image-updater.argoproj.io/runner.helm.image-tag: runner.image.tag
-    argocd-image-updater.argoproj.io/write-back-method: git
-    argocd-image-updater.argoproj.io/git-branch: main
 spec:
   project: homelab
   source:

k8s/argocd/apps/55-api-gateway.yaml

@@ -36,11 +36,9 @@ metadata:
     app.kubernetes.io/component: gateway
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # ArgoCD Image Updater - auto-update on new image push
+    # Image promotion: Forgejo package webhook → api-gw /v1/webhooks/forgejo
-    argocd-image-updater.argoproj.io/image-list: gw=forgejo.riotpiao.com/rock/api-gateway
+    # → patches this Application's kustomize.images via k8s API.
-    argocd-image-updater.argoproj.io/gw.update-strategy: digest
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/gw.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   revisionHistoryLimit: 3

k8s/argocd/apps/60-applications.yaml

@@ -246,11 +246,8 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "8"
-    # ArgoCD Image Updater - auto-update on new image push
+    # Image promotion: Forgejo package webhook → api-gw.
-    argocd-image-updater.argoproj.io/image-list: app=forgejo.riotpiao.com/rock/portfolio
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/app.update-strategy: digest
-    argocd-image-updater.argoproj.io/app.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   source:

k8s/argocd/apps/70-poimen.yaml

@@ -27,15 +27,10 @@ metadata:
     app.kubernetes.io/part-of: poimen
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    # ── Image Updater ─────────────────────────────────────────────────────────
+    # ── Image promotion ──────────────────────────────────────────────────────
-    argocd-image-updater.argoproj.io/image-list: "workflows=forgejo.riotpiao.com/riotpiao-poimen/poimen-workflows,memory=forgejo.riotpiao.com/riotpiao-poimen/poimen-memory,frontend=forgejo.riotpiao.com/riotpiao-poimen/poimen-frontend"
+    # Forgejo package webhook → api-gw patches kustomize.images per source.
-    argocd-image-updater.argoproj.io/workflows.update-strategy: digest
+    # Source index mapping: 0=workflows, 1=memory, 3=frontend.
-    argocd-image-updater.argoproj.io/workflows.allow-tags: regexp:^latest$
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/memory.update-strategy: digest
-    argocd-image-updater.argoproj.io/memory.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/frontend.update-strategy: digest
-    argocd-image-updater.argoproj.io/frontend.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   # Multi-source: each repo owns its own k8s manifests.

k8s/argocd/apps/71-poimen-toolkit.yaml

@@ -5,11 +5,9 @@ metadata:
   namespace: argocd
   annotations:
     argocd.argoproj.io/sync-wave: "7"
-    argocd-image-updater.argoproj.io/image-list: |
+    # Image promotion: Forgejo package webhook → api-gw.
-      controller=ghcr.io/riotpiao-poimen/poimen-toolkit-controller
+    # Tracks: poimen-toolkit-controller, pi-worker.
-    argocd-image-updater.argoproj.io/controller.update-strategy: digest
+    # Replaced ArgoCD Image Updater (removed 2026-09-10).
-    argocd-image-updater.argoproj.io/controller.allow-tags: regexp:^latest$
-    argocd-image-updater.argoproj.io/write-back-method: argocd
 spec:
   project: homelab
   source:

k8s/infra/argocd-image-updater/values.yaml

@@ -1,70 +0,0 @@
-# ArgoCD Image Updater configuration
-# Watches Forgejo registry and updates ArgoCD Applications with new image tags
-
-config:
-  # Registry configuration - Forgejo allows anonymous pulls
-  registries:
-    - name: forgejo
-      api_url: https://forgejo.riotpiao.com
-      prefix: forgejo.riotpiao.com
-      default: true
-      insecure: false
-
-  # Log level
-  logLevel: debug
-
-  # ArgoCD API server
-  argocd:
-    grpcWeb: true
-    serverAddress: argocd-server.argocd.svc.cluster.local
-    insecure: true
-    plaintext: true
-
-  # Git write-back configuration (for multi-source Applications)
-  git:
-    # Commit author for image updates
-    user:
-      name: "ArgoCD Image Updater"
-      email: "[email protected]"
-    # Use SSH keys from ArgoCD's known hosts + credentials
-    # Image Updater inherits ArgoCD's git credentials (mounted via ArgoCD secret)
-
-# Mount ArgoCD's git credentials for write-back
-extraVolumes:
-  - name: argocd-ssh-known-hosts-cm
-    configMap:
-      name: argocd-ssh-known-hosts-cm
-      defaultMode: 0644
-  - name: argocd-gpg-keys-cm
-    configMap:
-      name: argocd-gpg-keys-cm
-      optional: true
-      defaultMode: 0644
-  - name: argocd-gpg-pubring
-    configMap:
-      name: argocd-gpg-pubring-cm
-      optional: true
-      defaultMode: 0644
-
-extraVolumeMounts:
-  - name: argocd-ssh-known-hosts-cm
-    mountPath: /etc/ssh/ssh_known_hosts.d/argocd-ssh-known-hosts
-    subPath: ssh_known_hosts
-  - name: argocd-gpg-keys-cm
-    mountPath: /etc/gpg/source
-  - name: argocd-gpg-pubring
-    mountPath: /etc/gpg/pubring
-
-# Extra environment variables
-extraEnv:
-  - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
-    value: "true"
-
-# Resources
-resources:
-  requests:
-    cpu: 50m
-    memory: 64Mi
-  limits:
-    cpu: 200m
-    memory: 128Mi

k8s/infra/longhorn/longhorn-add-disks-job.yaml

@@ -117,6 +117,22 @@ spec:
             echo "  ✓ talos-cp-3 already has $CP3_DISKS disks configured"
           fi
 
+          # worker-1: enable scheduling, reserve 400Gi for LLM weights,
+          # clear eviction flag that was set when scheduling was disabled.
+          echo "Configuring worker-1 disk..."
+          kubectl -n longhorn-system patch nodes.longhorn.io worker-1 --type merge -p '{
+            "spec": {
+              "disks": {
+                "default-disk-1030400000000": {
+                  "allowScheduling": true,
+                  "evictionRequested": false,
+                  "storageReserved": 429496729600
+                }
+              }
+            }
+          }'
+          echo "  ✓ worker-1 disk configured (400Gi reserved)"
+
           echo
           echo "Waiting for disks to be ready..."
           sleep 10

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,33 +1,25 @@
-# Longhorn Node CRDs for cp-2 and cp-3.
+# Longhorn Node CRDs — enable scheduling on nodes that need explicit opt-in.
-# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
-# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# Control-plane nodes (cp-2, cp-3) have taints so Longhorn doesn't auto-discover.
-# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# worker-1 was manually disabled; re-enabled 2026-09-28 to relieve degraded-volume
-# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# pressure (400Gi reserved for LLM weights via the add-disks-job, ~300Gi available
-# `default-disk` key here never matched the live one, so the Application sat
+# for general replica scheduling).
-# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
-# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# What these objects are actually for is `allowScheduling: true` on tainted
+# `spec.disks` deliberately absent — Longhorn owns disk identity and writes
-# control-plane nodes. That is all they need to declare.
+# storageReserved/diskType/evictionRequested itself. Disk-level config lives in
+# longhorn-add-disks-job.yaml. These objects only set `allowScheduling: true`.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
+metadata: { name: talos-cp-2, namespace: longhorn-system }
-  name: talos-cp-2
+spec: { name: talos-cp-2, allowScheduling: true, tags: [] }
-  namespace: longhorn-system
-spec:
-  name: talos-cp-2
-  allowScheduling: true
-  tags: []
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
-metadata:
+metadata: { name: talos-cp-3, namespace: longhorn-system }
-  name: talos-cp-3
+spec: { name: talos-cp-3, allowScheduling: true, tags: [] }
-  namespace: longhorn-system
+---
-spec:
+apiVersion: longhorn.io/v1beta2
-  name: talos-cp-3
+kind: Node
-  allowScheduling: true
+metadata: { name: worker-1, namespace: longhorn-system }
-  tags: []
+spec: { name: worker-1, allowScheduling: true, tags: [] }