feat: vLLM Prometheus metrics scraping (#37)

Add ServiceMonitor for llm-serving namespace. Wire prometheus.io annotations to all vLLM pods (reasoning, ornith, embeddings, reranker). Scrape /metrics@8080 every 30s with proper relabeling.

k8s/apps/llm-serving/embeddings.yaml

@@ -45,6 +45,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-embeddings
+        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/apps/llm-serving/ornith.yaml

@@ -83,6 +83,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-ornith
+        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     # 1 replica -- ornith:35b only. qwen2.5:3b moved to CPU on cp-2.

k8s/apps/llm-serving/reasoning.yaml

@@ -104,6 +104,14 @@ spec:
         name: models
       - mountPath: /dev/shm
         name: shm
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-reasoning
+        app.kubernetes.io/part-of: llm-serving
     deploymentStrategy:
       type: Recreate
     maxReplicas: 1

k8s/apps/llm-serving/reranker.yaml

@@ -45,6 +45,14 @@ spec:
       volumeMounts:
       - mountPath: /mnt/models
         name: models
+    podMetadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
+      labels:
+        app.kubernetes.io/name: llm-reranker
+        app.kubernetes.io/part-of: llm-serving
     maxReplicas: 1
     minReplicas: 1
     nodeSelector:

k8s/infra/longhorn/expand-replicas-job.yaml

@@ -0,0 +1,96 @@
+# PostSync hook Job that expands all Longhorn volumes to 3 replicas.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 3
+  template:
+    metadata:
+      name: longhorn-expand-replicas
+    spec:
+      restartPolicy: Never
+      serviceAccountName: longhorn-expand-replicas
+      containers:
+      - name: expand
+        image: bitnami/kubectl:latest
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+          
+          echo "Waiting for all 3 Longhorn nodes to be Ready..."
+          for i in {1..30}; do
+            READY_COUNT=$(kubectl -n longhorn-system get nodes.longhorn.io -o json | \
+              jq -r '[.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True"))] | length')
+            
+            if [ "$READY_COUNT" -ge 3 ]; then
+              echo "✓ All 3 nodes Ready"
+              break
+            fi
+            
+            echo "  $READY_COUNT/3 nodes ready, waiting..."
+            sleep 10
+            
+            if [ $i -eq 30 ]; then
+              echo "✗ Timeout waiting for 3 nodes"
+              exit 1
+            fi
+          done
+          
+          echo
+          echo "Expanding volumes with < 3 replicas..."
+          
+          VOLUMES=$(kubectl -n longhorn-system get volumes.longhorn.io -o json | \
+            jq -r '.items[] | select(.spec.numberOfReplicas < 3) | .metadata.name')
+          
+          if [ -z "$VOLUMES" ]; then
+            echo "  No volumes need expansion"
+          else
+            echo "$VOLUMES" | while read -r vol; do
+              CURRENT=$(kubectl -n longhorn-system get volume "$vol" -o jsonpath='{.spec.numberOfReplicas}')
+              echo "  $vol: $CURRENT → 3 replicas"
+              kubectl -n longhorn-system patch volume "$vol" --type merge \
+                -p '{"spec":{"numberOfReplicas":3}}'
+            done
+          fi
+          
+          echo
+          echo "Done. Final replica counts:"
+          kubectl -n longhorn-system get volumes.longhorn.io \
+            -o custom-columns='NAME:.metadata.name,REPLICAS:.spec.numberOfReplicas' | head -20
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+rules:
+- apiGroups: ["longhorn.io"]
+  resources: ["volumes", "nodes"]
+  verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: longhorn-expand-replicas
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: longhorn-expand-replicas
+subjects:
+- kind: ServiceAccount
+  name: longhorn-expand-replicas
+  namespace: longhorn-system

k8s/infra/longhorn/kustomization.yaml

@@ -8,11 +8,11 @@ resources:
   - longhorn-servicemonitor.yaml
   - longhorn-taint-toleration.yaml
   - longhorn-nodes.yaml
-  - longhorn-tag-disks-job.yaml
+  - expand-replicas-job.yaml
   - patch-csi-tolerations-job.yaml
   - longhorn-add-disks-job.yaml  # Add extra disks to talos-cp-2
 # Longhorn deployed via bootstrap script or Helm.
 # These manifests configure it: unified StorageClass (default, 3 replicas),
 # Prometheus ServiceMonitor, taint toleration for control-plane nodes, explicit
 # Node CRDs for cp-2/cp-3, CSI plugin tolerations, and a PostSync hook Job
-# that configures storage for the cluster.
+# that ensures all existing volumes have 3 replicas.

k8s/infra/longhorn/longhorn-cnpg-storageclass.yaml

@@ -27,7 +27,6 @@ provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 parameters:
   numberOfReplicas: "3"
-  diskSelector: "storage"           # Only schedule on CP node disks, not GPU worker
   staleReplicaTimeout: "30"
   fromBackup: ""
   dataLocality: "best-effort"

k8s/infra/longhorn/longhorn-nodes.yaml

@@ -1,14 +1,16 @@
-# Longhorn Node CRDs for cp-2, cp-3, and worker-1.
+# Longhorn Node CRDs for cp-2 and cp-3.
-# cp-2/cp-3 have the control-plane taint, so Longhorn doesn't auto-discover them.
+# These nodes have the control-plane taint, so Longhorn doesn't auto-discover them.
-# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 CP nodes.
+# Explicit Node CRDs + the taint-toleration setting enable storage across all 3 nodes.
 #
-# `spec.disks` is deliberately absent for CP nodes. Longhorn owns disk identity:
+# `spec.disks` is deliberately absent. Longhorn owns disk identity: it names the
-# it names the entry itself and writes `storageReserved`, `diskType` and
+# entry itself (`default-disk-080400000000`, not `default-disk`) and writes
-# `evictionRequested` into it. Disk tags are applied via kubectl patch (see
+# `storageReserved`, `diskType` and `evictionRequested` into it. Declaring a
-# longhorn-tag-disks-job.yaml) since disk names are runtime-discovered.
+# `default-disk` key here never matched the live one, so the Application sat
+# OutOfSync and selfHeal kept trying to add a SECOND disk record pointing at the
+# same /var/lib/longhorn path — which is worse than the drift it was fixing.
 #
-# worker-1 has scheduling disabled so only StorageClasses with explicit
+# What these objects are actually for is `allowScheduling: true` on tainted
-# diskSelector (e.g. `llm` for longhorn-llm-local) can use it.
+# control-plane nodes. That is all they need to declare.
 ---
 apiVersion: longhorn.io/v1beta2
 kind: Node
@@ -29,13 +31,3 @@ spec:
   name: talos-cp-3
   allowScheduling: true
   tags: []
----
-apiVersion: longhorn.io/v1beta2
-kind: Node
-metadata:
-  name: worker-1
-  namespace: longhorn-system
-spec:
-  name: worker-1
-  allowScheduling: false
-  tags: []

k8s/infra/longhorn/longhorn-storageclass.yaml

@@ -6,15 +6,13 @@ metadata:
   name: longhorn
   annotations:
     storageclass.kubernetes.io/is-default-class: "true"
-    description: "Longhorn distributed storage - 3 replicas, CP nodes only"
+    description: "Longhorn distributed storage - 3 replicas, WaitForFirstConsumer"
-    argocd.argoproj.io/sync-options: Replace=true,Force=true
 provisioner: driver.longhorn.io
 allowVolumeExpansion: true
 reclaimPolicy: Delete
 volumeBindingMode: Immediate  # Immediate binding for StatefulSets (matches deployed config)
 parameters:
-  numberOfReplicas: "3"           # HA across all 3 CP nodes
+  numberOfReplicas: "3"           # HA across all 3 nodes
-  diskSelector: "storage"           # Only schedule on CP node disks, not GPU worker
   staleReplicaTimeout: "30"
   fromBackup: ""
   dataLocality: "disabled"        # Match deployed config (not best-effort)

k8s/infra/longhorn/longhorn-tag-disks-job.yaml

@@ -1,80 +0,0 @@
-# PostSync hook: tag all CP node disks with "storage" so that
-# diskSelector: "storage" in the default StorageClasses restricts
-# replicas to CP nodes only, keeping worker-1 free for GPU workloads.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: longhorn-tag-disks
-  namespace: longhorn-system
-  annotations:
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-spec:
-  backoffLimit: 3
-  template:
-    metadata:
-      name: longhorn-tag-disks
-    spec:
-      restartPolicy: Never
-      serviceAccountName: longhorn-tag-disks
-      containers:
-      - name: tag
-        image: bitnami/kubectl:latest
-        command:
-        - /bin/bash
-        - -c
-        - |
-          set -euo pipefail
-
-          CP_NODES="talos-cp-1 talos-cp-2 talos-cp-3"
-
-          for node in $CP_NODES; do
-            echo "Processing $node..."
-            DISKS=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \
-              jq -r '.spec.disks | to_entries[] | select(.key != "paperless-media") | .key')
-
-            for disk in $DISKS; do
-              HAS_TAG=$(kubectl -n longhorn-system get nodes.longhorn.io "$node" -o json | \
-                jq -r ".spec.disks[\"$disk\"].tags // [] | index(\"storage\") // empty")
-
-              if [ -z "$HAS_TAG" ]; then
-                echo "  Tagging $disk with 'storage'"
-                kubectl -n longhorn-system patch nodes.longhorn.io "$node" --type merge \
-                  -p "{\"spec\":{\"disks\":{\"$disk\":{\"tags\":[\"storage\"]}}}}"
-              else
-                echo "  $disk already tagged"
-              fi
-            done
-          done
-
-          echo "✓ All CP disks tagged with 'storage'"
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: longhorn-tag-disks
-  namespace: longhorn-system
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: longhorn-tag-disks
-  namespace: longhorn-system
-rules:
-- apiGroups: ["longhorn.io"]
-  resources: ["nodes"]
-  verbs: ["get", "list", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: longhorn-tag-disks
-  namespace: longhorn-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: longhorn-tag-disks
-subjects:
-- kind: ServiceAccount
-  name: longhorn-tag-disks
-  namespace: longhorn-system

k8s/infra/monitoring/servicemonitors/vllm.yaml

@@ -0,0 +1,30 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vllm
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: vllm
+    app.kubernetes.io/part-of: llm-serving
+spec:
+  namespaceSelector:
+    matchNames:
+      - llm-serving
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: llm-serving
+  endpoints:
+    - port: http
+      interval: 30s
+      scrapeTimeout: 10s
+      path: /metrics
+      scheme: http
+      relabelings:
+        - sourceLabels: [__meta_kubernetes_namespace]
+          targetLabel: namespace
+        - sourceLabels: [__meta_kubernetes_pod_name]
+          targetLabel: pod
+        - sourceLabels: [__meta_kubernetes_service_name]
+          targetLabel: service
+        - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+          targetLabel: app