diff --git a/k8s/apps/llm-serving/coder.yaml b/k8s/apps/llm-serving/coder.yaml index 8b1b05f..b4386e3 100644 --- a/k8s/apps/llm-serving/coder.yaml +++ b/k8s/apps/llm-serving/coder.yaml @@ -50,17 +50,28 @@ spec: - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto - # Native window is 262144. 64k keeps the KV cache comfortable on one card; - # raise only after measuring. - - --max-model-len=65536 - - --tensor-parallel-size=1 + # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN. + # One card cannot hold it: measured KV pool on a single V100 was 132,672 + # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room + # for the larger pool. Measured after boot: see the KV line in the log. + - --max-model-len=262144 + - --tensor-parallel-size=2 - --pipeline-parallel-size=1 - --gpu-memory-utilization=0.90 - - --max-num-seqs=16 + # A single 262,144-token request consumes the whole pool, so keep few slots. + - --max-num-seqs=4 - --max-num-batched-tokens=8192 - --long-prefill-token-threshold=2048 - --enable-chunked-prefill - --enable-prefix-caching + # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph + # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only + # "No available shared memory broadcast block found in 60 seconds". The front-end + # never received the ready message. vLLM's custom all-reduce registers its own + # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback + # and is correct here, because TP=2 across two PCIe V100s gains nothing from the + # custom kernel anyway. + - --disable-custom-all-reduce - --reasoning-parser=qwen3 - --enable-auto-tool-choice - --tool-call-parser=hermes @@ -89,19 +100,22 @@ spec: httpGet: path: /health port: 8080 - # The coder checkpoint is a 16.9 GB cold download on first start, then - # ~20 s of weight load and ~80 s of torch.compile. 80 x 15 s covers it. - failureThreshold: 80 + # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile, + # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom. + failureThreshold: 120 periodSeconds: 15 resources: + # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request + # never schedules. Two vLLM workers keep the weights on the GPU and need + # little host RAM, so the request stays small and the limit allows burst. requests: - cpu: "4" - memory: 12Gi - nvidia.com/gpu: "1" - limits: cpu: "8" - memory: 36Gi - nvidia.com/gpu: "1" + memory: 12Gi + nvidia.com/gpu: "2" + limits: + cpu: "16" + memory: 40Gi + nvidia.com/gpu: "2" volumeMounts: - mountPath: /mnt/models name: models @@ -115,4 +129,6 @@ spec: - name: shm emptyDir: medium: Memory - sizeLimit: 2Gi + # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is + # too small and shows up as an init hang or a bus error. + sizeLimit: 8Gi