From d97fd6aba7f5f48432870d9bece48d5f3d4cb8ea Mon Sep 17 00:00:00 2001 From: rock Date: Sat, 31 Oct 2026 17:21:54 -0700 Subject: [PATCH] feat(coder): serve 256K context on TP=2, disable custom all-reduce Coder-30B-A3B is natively 262,144 tokens with rope_scaling null. One V100 could not hold it: the measured KV pool was 132,672 tokens. TP=2 halves the weights per card to 7.85 GiB each and frees room for the larger pool. Measured after boot, both cards worker-1: Available KV cache memory per GPU : 19.96 GiB GPU KV cache size : 436,112 tokens Max concurrency at 262,144 : 1.66x max_model_len (both served names) : 262144 POST /v1/chat/completions : 200 in 2.83s Weights, per worker : 7.8516 GiB torch.compile : 78 s Graph capture : 6 s One fatal detail, measured twice. With vLLM's default custom all-reduce the engine reaches "Graph capturing finished in 6 secs" and then hangs. The only output afterwards was "No available shared memory broadcast block found in 60 seconds", repeated for 14 minutes, and the front-end never received the ready message. The first attempt died on its startup probe at 20 min. `--disable-custom-all-reduce` fixes it. Custom all-reduce registers its own CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback and costs nothing here, because TP=2 across two PCIe V100s gains nothing from the custom kernel. Both workers now register cleanly and the API binds at 17:18. Two supporting changes, both required: - memory request 12Gi. worker-1 exposes only 63,495 MiB allocatable, so the earlier 24Gi request could never schedule. - /dev/shm 8Gi. NCCL all-reduce moves through it, and the 64Mi default shows up as an init hang. `reasoning` remains a served name at the same 262,144 so the client needs no model-name change. --- k8s/apps/llm-serving/coder.yaml | 46 ++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/k8s/apps/llm-serving/coder.yaml b/k8s/apps/llm-serving/coder.yaml index 8b1b05f..b4386e3 100644 --- a/k8s/apps/llm-serving/coder.yaml +++ b/k8s/apps/llm-serving/coder.yaml @@ -50,17 +50,28 @@ spec: - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto - # Native window is 262144. 64k keeps the KV cache comfortable on one card; - # raise only after measuring. - - --max-model-len=65536 - - --tensor-parallel-size=1 + # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN. + # One card cannot hold it: measured KV pool on a single V100 was 132,672 + # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room + # for the larger pool. Measured after boot: see the KV line in the log. + - --max-model-len=262144 + - --tensor-parallel-size=2 - --pipeline-parallel-size=1 - --gpu-memory-utilization=0.90 - - --max-num-seqs=16 + # A single 262,144-token request consumes the whole pool, so keep few slots. + - --max-num-seqs=4 - --max-num-batched-tokens=8192 - --long-prefill-token-threshold=2048 - --enable-chunked-prefill - --enable-prefix-caching + # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph + # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only + # "No available shared memory broadcast block found in 60 seconds". The front-end + # never received the ready message. vLLM's custom all-reduce registers its own + # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback + # and is correct here, because TP=2 across two PCIe V100s gains nothing from the + # custom kernel anyway. + - --disable-custom-all-reduce - --reasoning-parser=qwen3 - --enable-auto-tool-choice - --tool-call-parser=hermes @@ -89,19 +100,22 @@ spec: httpGet: path: /health port: 8080 - # The coder checkpoint is a 16.9 GB cold download on first start, then - # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it. - failureThreshold: 80 + # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile, + # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom. + failureThreshold: 120 periodSeconds: 15 resources: + # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request + # never schedules. Two vLLM workers keep the weights on the GPU and need + # little host RAM, so the request stays small and the limit allows burst. requests: - cpu: "4" - memory: 12Gi - nvidia.com/gpu: "1" - limits: cpu: "8" - memory: 36Gi - nvidia.com/gpu: "1" + memory: 12Gi + nvidia.com/gpu: "2" + limits: + cpu: "16" + memory: 40Gi + nvidia.com/gpu: "2" volumeMounts: - mountPath: /mnt/models name: models @@ -115,4 +129,6 @@ spec: - name: shm emptyDir: medium: Memory - sizeLimit: 2Gi + # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is + # too small and shows up as an init hang or a bus error. + sizeLimit: 8Gi