k8s/apps/llm-serving/coder.yaml

@@ -50,17 +50,28 @@ spec:
       - --quantization=gptq
       - --dtype=float16
       - --kv-cache-dtype=auto
-      # Native window is 262144. 64k keeps the KV cache comfortable on one card;
+      # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN.
-      # raise only after measuring.
+      # One card cannot hold it: measured KV pool on a single V100 was 132,672
-      - --max-model-len=65536
+      # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room
-      - --tensor-parallel-size=1
+      # for the larger pool. Measured after boot: see the KV line in the log.
+      - --max-model-len=262144
+      - --tensor-parallel-size=2
       - --pipeline-parallel-size=1
       - --gpu-memory-utilization=0.90
-      - --max-num-seqs=16
+      # A single 262,144-token request consumes the whole pool, so keep few slots.
+      - --max-num-seqs=4
       - --max-num-batched-tokens=8192
       - --long-prefill-token-threshold=2048
       - --enable-chunked-prefill
       - --enable-prefix-caching
+      # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph
+      # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only
+      # "No available shared memory broadcast block found in 60 seconds". The front-end
+      # never received the ready message. vLLM's custom all-reduce registers its own
+      # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback
+      # and is correct here, because TP=2 across two PCIe V100s gains nothing from the
+      # custom kernel anyway.
+      - --disable-custom-all-reduce
       - --reasoning-parser=qwen3
       - --enable-auto-tool-choice
       - --tool-call-parser=hermes
@@ -89,19 +100,22 @@ spec:
         httpGet:
           path: /health
           port: 8080
-        # The coder checkpoint is a 16.9 GB cold download on first start, then
+        # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile,
-        # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it.
+        # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom.
-        failureThreshold: 80
+        failureThreshold: 120
         periodSeconds: 15
       resources:
+        # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request
+        # never schedules. Two vLLM workers keep the weights on the GPU and need
+        # little host RAM, so the request stays small and the limit allows burst.
         requests:
-          cpu: "4"
-          memory: 12Gi
-          nvidia.com/gpu: "1"
-        limits:
           cpu: "8"
-          memory: 36Gi
+          memory: 12Gi
-          nvidia.com/gpu: "1"
+          nvidia.com/gpu: "2"
+        limits:
+          cpu: "16"
+          memory: 40Gi
+          nvidia.com/gpu: "2"
       volumeMounts:
       - mountPath: /mnt/models
         name: models
@@ -115,4 +129,6 @@ spec:
     - name: shm
       emptyDir:
         medium: Memory
-        sizeLimit: 2Gi
+        # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is
+        # too small and shows up as an init hang or a bus error.
+        sizeLimit: 8Gi