fix(longhorn): isolate GPU worker from general storage scheduling

- Remove expand-replicas-job (blindly forced all volumes to 3 replicas,
  ignoring StorageClass settings)
- Add diskSelector: 'storage' to longhorn and longhorn-cnpg StorageClasses
  so replicas only land on CP nodes (cp-1, cp-2, cp-3)
- Tag all CP node disks with 'storage' via PostSync job
  (disk names are runtime-discovered, can't hardcode in Node CRs)
- Disable scheduling on worker-1 Node CR — only longhorn-llm-local
  (diskSelector: 'llm') can use it
- worker-1 is GPU-only: llm-models and comfyui use dedicated SCs

k8s/apps/llm-serving/embeddings.yaml

@@ -45,14 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-embeddings
-        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-serving/ornith.yaml

@@ -83,14 +83,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-ornith
-        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.

k8s/apps/llm-serving/reasoning.yaml

@@ -104,14 +104,6 @@ spec:
         name: models
       - mountPath: /dev/shm
         name: shm
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reasoning
-        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     maxReplicas: 1

k8s/apps/llm-serving/reranker.yaml

@@ -45,14 +45,6 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
-    podMetadata:
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "8080"
-        prometheus.io/path: "/metrics"
-      labels:
-        app.kubernetes.io/name: llm-reranker
-        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/infra/argocd-image-updater/values.yaml

@@ -57,6 +57,8 @@ extraVolumeMounts:
 
 # Extra environment variables
 extraEnv:
+  - name: ARGOCD_GRPC_WEB
+    value: "true"
   - name: GIT_SSH_KNOWN_HOSTS_CONFIG_MAP_ENABLED
     value: "true"
 

k8s/infra/databases/memory-db.yaml

@@ -9,7 +9,7 @@ metadata:
   annotations:
     argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
 spec:
-  instances: 3
+  instances: 2
   imageName: ghcr.io/cloudnative-pg/postgresql:16.2
   bootstrap:
     initdb:

k8s/infra/longhorn/expand-replicas-job.yaml

@@ -1,96 +0,0 @@
-# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: longhorn-expand-replicas
-  namespace: longhorn-system
-  annotations:
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-spec:
-  backoffLimit: 3
-  template:
-    metadata:
-      name: longhorn-expand-replicas
-    spec:
-      restartPolicy: Never
-      serviceAccountName: longhorn-expand-replicas
-      containers:
-      - name: expand
-        image: bitnami/kubectl:latest
-        command:
-        - /bin/bash
-        - -c
-        - |
-          set -euo pipefail
-          
-          echo "Waiting for all 3 Longhorn nodes to be Ready..."
-          for i in {1..30}; do
-            READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \
-              jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length')
-            
-            if [ "$READY_COUNT" -ge 3 ]; then
-              echo "✓ All 3 nodes Ready"
-              break
-            fi
-            
-            echo "  $READY_COUNT/3 nodes ready, waiting..."
-            sleep 10
-            
-            if [ $i -eq 30 ]; then
-              echo "✗ Timeout waiting for 3 nodes"
-              exit 1
-            fi
-          done
-          
-          echo
-          echo "Expanding volumes with < 3 replicas..."
-          
-          VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \
-            jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name')
-          
-          if [ -z "$VOLUMES" ]; then
-            echo "  No volumes need expansion"
-          else
-            echo "$VOLUMES" | while read -r vol; do
-              CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
-              echo "  $vol: $CURRENT → 3 replicas"
-              kubectl -n longhorn-system patch volume "$vol" --type merge \
-                -p '{"spec":{"numberOfReplicas":3}}'
-            done
-          fi
-          
-          echo
-          echo "Done. Final replica counts:"
-          kubectl -n longhorn-system get volumes.longhorn.io \
-            -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: longhorn-expand-replicas
-  namespace: longhorn-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: longhorn-expand-replicas
-  namespace: longhorn-system
-rules:
-- apiGroups: ["longhorn.io"]
-  resources: ["volumes", "nodes"]
-  verbs: ["get", "list", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: longhorn-expand-replicas
-  namespace: longhorn-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: longhorn-expand-replicas
-subjects:
-- kind: ServiceAccount
-  name: longhorn-expand-replicas
-  namespace: longhorn-system

k8s/infra/longhorn/kustomization.yaml

@@ -8,11 +8,11 @@ resources:
   - longhorn-servicemonitor.yaml
   - longhorn-taint-toleration.yaml
   - longhorn-nodes.yaml
-  - expand-replicas-job.yaml
+  - longhorn-tag-disks-job.yaml
   - patch-csi-tolerations-job.yaml
   - longhorn-add-disks-job.yaml  # Add extra disks to talos-cp-2
 # Longhorn deployed via bootstrap script or Helm.
 # These manifests configure it: unified StorageClass (default, 3 replicas),
 # Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
 # Node CRDs for cp-2/cp-3, CSI plugin tolerations, and a PostSync hook Job
-# that ensures all existing volumes have 3 replicas.
+# that configures storage for the cluster.

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -27,6 +27,7 @@ provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 parameters:
   numberOfReplicas: "3"
+  diskSelector: "storage"           # Only schedule on CP node disks, not GPU worker
   staleReplicaTimeout: "30"
   fromBackup: ""
   dataLocality: "best-effort"

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,16 +1,14 @@
-# Longhorn Node CRDs for cp-2 and cp-3.
+# Longhorn Node CRDs for cp-2, cp-3, and worker-1.
-# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
+# cp-2/cp-3 have the control-plane taint, so Longhorn doesn't auto-discover them.
-# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 CP nodes.
 #
-# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
+# `spec.disks` is deliberately absent for CP nodes. Longhorn owns disk identity:
-# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
+# it names the entry itself and writes `storageReserved`, `diskType` and
-# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
+# `evictionRequested` into it. Disk tags are applied via kubectl patch (see
-# `default-disk` key here never matched the live one, so the Application sat
+# longhorn-tag-disks-job.yaml) since disk names are runtime-discovered.
-# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
-# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# What these objects are actually for is `allowScheduling: true` on tainted
+# worker-1 has scheduling disabled so only StorageClasses with explicit
-# control-plane nodes. That is all they need to declare.
+# diskSelector (e.g. `llm` for longhorn-llm-local) can use it.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
@@ -31,3 +29,13 @@ spec:
   name: talos-cp-3
   allowScheduling: true
   tags: []
+---
+apiVersion: longhorn.io/v1beta2
+kind: Node
+metadata:
+  name: worker-1
+  namespace: longhorn-system
+spec:
+  name: worker-1
+  allowScheduling: false
+  tags: []

k8s/infra/longhorn/longhorn-storageclass.yaml

@@ -6,13 +6,15 @@ metadata:
   name: longhorn
   annotations:
     storageclass.kubernetes.io/is-default-class: "true"
-    description: "Longhorn distributed storage - 3 replicas, WaitForFirstConsumer"
+    description: "Longhorn distributed storage - 3 replicas, CP nodes only"
+    argocd.argoproj.io/sync-options: Replace=true,Force=true
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 reclaimPolicy: Delete
 volumeBindingMode: Immediate  # Immediate binding for StatefulSets (matches deployed config)
 parameters:
-  numberOfReplicas: "3"           # HA across all 3 nodes
+  numberOfReplicas: "3"           # HA across all 3 CP nodes
+  diskSelector: "storage"           # Only schedule on CP node disks, not GPU worker
   staleReplicaTimeout: "30"
   fromBackup: ""
   dataLocality: "disabled"        # Match deployed config (not best-effort)

k8s/infra/longhorn/longhorn-tag-disks-job.yaml

@@ -0,0 +1,80 @@
+# PostSync hook: tag all CP node disks with "storage" so that
+# diskSelector: "storage" in the default StorageClasses restricts
+# replicas to CP nodes only, keeping worker-1 free for GPU workloads.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-tag-disks
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: longhorn-tag-disks
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-tag-disks
+      containers:
+      - name: tag
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+
+          CP_NODES="talos-cp-1 talos-cp-2 talos-cp-3"
+
+          for node in $CP_NODES; do
+            echo "Processing $node..."
+            DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \
+              jq -r '.spec.disks | to_entries[] | select(.key != "paperless-media") | .key')
+
+            for disk in $DISKS; do
+              HAS_TAG=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \
+                jq -r ".spec.disks[\"$disk\"].tags // [] | index(\"storage\") // empty")
+
+              if [ -z "$HAS_TAG" ]; then
+                echo "  Tagging $disk with 'storage'"
+                kubectl -n longhorn-system patch nodes.longhorn.io "$node" --type merge \
+                  -p "{\"spec\":{\"disks\":{\"$disk\":{\"tags\":[\"storage\"]}}}}"
+              else
+                echo "  $disk already tagged"
+              fi
+            done
+          done
+
+          echo "✓ All CP disks tagged with 'storage'"
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-tag-disks
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-tag-disks
+  namespace: longhorn-system
+rules:
+- apiGroups: ["longhorn.io"]
+  resources: ["nodes"]
+  verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-tag-disks
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-tag-disks
+subjects:
+- kind: ServiceAccount
+  name: longhorn-tag-disks
+  namespace: longhorn-system

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -1,30 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: vllm
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: vllm
-    app.kubernetes.io/part-of: llm-serving
-spec:
-  namespaceSelector:
-    matchNames:
-      - llm-serving
-  selector:
-    matchLabels:
-      app.kubernetes.io/part-of: llm-serving
-  endpoints:
-    - port: http
-      interval: 30s
-      scrapeTimeout: 10s
-      path: /metrics
-      scheme: http
-      relabelings:
-        - sourceLabels: [__meta_kubernetes_namespace]
-          targetLabel: namespace
-        - sourceLabels: [__meta_kubernetes_pod_name]
-          targetLabel: pod
-        - sourceLabels: [__meta_kubernetes_service_name]
-          targetLabel: service
-        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
-          targetLabel: app