From 346209631139a65929a9f9d14d9d20363cb4a951 Mon Sep 17 00:00:00 2001 From: rock Date: Fri, 11 Sep 2026 09:31:51 +0900 Subject: [PATCH] fix(longhorn): isolate GPU worker from general storage scheduling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove expand-replicas-job (blindly forced all volumes to 3 replicas, ignoring StorageClass settings) - Add diskSelector: 'storage' to longhorn and longhorn-cnpg StorageClasses so replicas only land on CP nodes (cp-1, cp-2, cp-3) - Tag all CP node disks with 'storage' via PostSync job (disk names are runtime-discovered, can't hardcode in Node CRs) - Disable scheduling on worker-1 Node CR — only longhorn-llm-local (diskSelector: 'llm') can use it - worker-1 is GPU-only: llm-models and comfyui use dedicated SCs --- k8s/infra/longhorn/expand-replicas-job.yaml | 96 ------------------- k8s/infra/longhorn/kustomization.yaml | 4 +- .../longhorn/longhorn-cnpg-storageclass.yaml | 1 + k8s/infra/longhorn/longhorn-nodes.yaml | 30 +++--- k8s/infra/longhorn/longhorn-storageclass.yaml | 6 +- .../longhorn/longhorn-tag-disks-job.yaml | 80 ++++++++++++++++ 6 files changed, 106 insertions(+), 111 deletions(-) delete mode 100644 k8s/infra/longhorn/expand-replicas-job.yaml create mode 100644 k8s/infra/longhorn/longhorn-tag-disks-job.yaml diff --git a/k8s/infra/longhorn/expand-replicas-job.yaml b/k8s/infra/longhorn/expand-replicas-job.yaml deleted file mode 100644 index f08d81f..0000000 --- a/k8s/infra/longhorn/expand-replicas-job.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# PostSync hook Job that expands all Longhorn volumes to 3 replicas. -apiVersion: batch/v1 -kind: Job -metadata: - name: longhorn-expand-replicas - namespace: longhorn-system - annotations: - argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: BeforeHookCreation -spec: - backoffLimit: 3 - template: - metadata: - name: longhorn-expand-replicas - spec: - restartPolicy: Never - serviceAccountName: longhorn-expand-replicas - containers: - - name: expand - image: bitnami/kubectl:latest - command: - - /bin/bash - - -c - - | - set -euo pipefail - - echo "Waiting for all 3 Longhorn nodes to be Ready..." - for i in {1..30}; do - READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \ - jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length') - - if [ "$READY_COUNT" -ge 3 ]; then - echo "✓ All 3 nodes Ready" - break - fi - - echo " $READY_COUNT/3 nodes ready, waiting..." - sleep 10 - - if [ $i -eq 30 ]; then - echo "✗ Timeout waiting for 3 nodes" - exit 1 - fi - done - - echo - echo "Expanding volumes with < 3 replicas..." - - VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \ - jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name') - - if [ -z "$VOLUMES" ]; then - echo " No volumes need expansion" - else - echo "$VOLUMES" | while read -r vol; do - CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}') - echo " $vol: $CURRENT → 3 replicas" - kubectl -n longhorn-system patch volume "$vol" --type merge \ - -p '{"spec":{"numberOfReplicas":3}}' - done - fi - - echo - echo "Done. Final replica counts:" - kubectl -n longhorn-system get volumes.longhorn.io \ - -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: longhorn-expand-replicas - namespace: longhorn-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: longhorn-expand-replicas - namespace: longhorn-system -rules: -- apiGroups: ["longhorn.io"] - resources: ["volumes", "nodes"] - verbs: ["get", "list", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: longhorn-expand-replicas - namespace: longhorn-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: longhorn-expand-replicas -subjects: -- kind: ServiceAccount - name: longhorn-expand-replicas - namespace: longhorn-system diff --git a/k8s/infra/longhorn/kustomization.yaml b/k8s/infra/longhorn/kustomization.yaml index bba17bf..fdc1dc1 100644 --- a/k8s/infra/longhorn/kustomization.yaml +++ b/k8s/infra/longhorn/kustomization.yaml @@ -8,11 +8,11 @@ resources: - longhorn-servicemonitor.yaml - longhorn-taint-toleration.yaml - longhorn-nodes.yaml - - expand-replicas-job.yaml + - longhorn-tag-disks-job.yaml - patch-csi-tolerations-job.yaml - longhorn-add-disks-job.yaml # Add extra disks to talos-cp-2 # Longhorn deployed via bootstrap script or Helm. # These manifests configure it: unified StorageClass (default, 3 replicas), # Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit # Node CRDs for cp-2/cp-3, CSI plugin tolerations, and a PostSync hook Job -# that ensures all existing volumes have 3 replicas. +# that configures storage for the cluster. diff --git a/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml b/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml index cca83f3..5c9108e 100644 --- a/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml +++ b/k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml @@ -27,6 +27,7 @@ provisioner: driver.longhorn.io allowVolumeExpansion: true parameters: numberOfReplicas: "3" + diskSelector: "storage" # Only schedule on CP node disks, not GPU worker staleReplicaTimeout: "30" fromBackup: "" dataLocality: "best-effort" diff --git a/k8s/infra/longhorn/longhorn-nodes.yaml b/k8s/infra/longhorn/longhorn-nodes.yaml index 52e4b3d..4eb4e61 100644 --- a/k8s/infra/longhorn/longhorn-nodes.yaml +++ b/k8s/infra/longhorn/longhorn-nodes.yaml @@ -1,16 +1,14 @@ -# Longhorn Node CRDs for cp-2 and cp-3. -# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them. -# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes. +# Longhorn Node CRDs for cp-2, cp-3, and worker-1. +# cp-2/cp-3 have the control-plane taint, so Longhorn doesn't auto-discover them. +# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 CP nodes. # -# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the -# entry itself (`default-disk-080400000000`, not `default-disk`) and writes -# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a -# `default-disk` key here never matched the live one, so the Application sat -# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the -# same /var/lib/longhorn path — which is worse than the drift it was fixing. +# `spec.disks` is deliberately absent for CP nodes. Longhorn owns disk identity: +# it names the entry itself and writes `storageReserved`, `diskType` and +# `evictionRequested` into it. Disk tags are applied via kubectl patch (see +# longhorn-tag-disks-job.yaml) since disk names are runtime-discovered. # -# What these objects are actually for is `allowScheduling: true` on tainted -# control-plane nodes. That is all they need to declare. +# worker-1 has scheduling disabled so only StorageClasses with explicit +# diskSelector (e.g. `llm` for longhorn-llm-local) can use it. --- apiVersion: longhorn.io/v1beta2 kind: Node @@ -31,3 +29,13 @@ spec: name: talos-cp-3 allowScheduling: true tags: [] +--- +apiVersion: longhorn.io/v1beta2 +kind: Node +metadata: + name: worker-1 + namespace: longhorn-system +spec: + name: worker-1 + allowScheduling: false + tags: [] diff --git a/k8s/infra/longhorn/longhorn-storageclass.yaml b/k8s/infra/longhorn/longhorn-storageclass.yaml index 908d506..9081f07 100644 --- a/k8s/infra/longhorn/longhorn-storageclass.yaml +++ b/k8s/infra/longhorn/longhorn-storageclass.yaml @@ -6,13 +6,15 @@ metadata: name: longhorn annotations: storageclass.kubernetes.io/is-default-class: "true" - description: "Longhorn distributed storage - 3 replicas, WaitForFirstConsumer" + description: "Longhorn distributed storage - 3 replicas, CP nodes only" + argocd.argoproj.io/sync-options: Replace=true,Force=true provisioner: driver.longhorn.io allowVolumeExpansion: true reclaimPolicy: Delete volumeBindingMode: Immediate # Immediate binding for StatefulSets (matches deployed config) parameters: - numberOfReplicas: "3" # HA across all 3 nodes + numberOfReplicas: "3" # HA across all 3 CP nodes + diskSelector: "storage" # Only schedule on CP node disks, not GPU worker staleReplicaTimeout: "30" fromBackup: "" dataLocality: "disabled" # Match deployed config (not best-effort) diff --git a/k8s/infra/longhorn/longhorn-tag-disks-job.yaml b/k8s/infra/longhorn/longhorn-tag-disks-job.yaml new file mode 100644 index 0000000..aef1897 --- /dev/null +++ b/k8s/infra/longhorn/longhorn-tag-disks-job.yaml @@ -0,0 +1,80 @@ +# PostSync hook: tag all CP node disks with "storage" so that +# diskSelector: "storage" in the default StorageClasses restricts +# replicas to CP nodes only, keeping worker-1 free for GPU workloads. +apiVersion: batch/v1 +kind: Job +metadata: + name: longhorn-tag-disks + namespace: longhorn-system + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation +spec: + backoffLimit: 3 + template: + metadata: + name: longhorn-tag-disks + spec: + restartPolicy: Never + serviceAccountName: longhorn-tag-disks + containers: + - name: tag + image: bitnami/kubectl:latest + command: + - /bin/bash + - -c + - | + set -euo pipefail + + CP_NODES="talos-cp-1 talos-cp-2 talos-cp-3" + + for node in $CP_NODES; do + echo "Processing $node..." + DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \ + jq -r '.spec.disks | to_entries[] | select(.key != "paperless-media") | .key') + + for disk in $DISKS; do + HAS_TAG=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \ + jq -r ".spec.disks[\"$disk\"].tags // [] | index(\"storage\") // empty") + + if [ -z "$HAS_TAG" ]; then + echo " Tagging $disk with 'storage'" + kubectl -n longhorn-system patch nodes.longhorn.io "$node" --type merge \ + -p "{\"spec\":{\"disks\":{\"$disk\":{\"tags\":[\"storage\"]}}}}" + else + echo " $disk already tagged" + fi + done + done + + echo "✓ All CP disks tagged with 'storage'" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: longhorn-tag-disks + namespace: longhorn-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhorn-tag-disks + namespace: longhorn-system +rules: +- apiGroups: ["longhorn.io"] + resources: ["nodes"] + verbs: ["get", "list", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhorn-tag-disks + namespace: longhorn-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhorn-tag-disks +subjects: +- kind: ServiceAccount + name: longhorn-tag-disks + namespace: longhorn-system