diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 5fde4ae..2ddad69 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -23,8 +23,20 @@ spec: # Requires vLLM >=0.17.0 -- v0.11.0 errors with "Model architectures # ['Qwen3_5MoeForConditionalGeneration'] are not supported for now." # moe_wna16 is the checkpoint's documented quantization kernel; - # compute-capability requirement on sm70 is UNVERIFIED going in -- - # this rollout is the real test. --kv-cache-dtype stays auto, not + # confirmed working on sm70 -- weights loaded fine (10.6GiB/rank at + # TP=2, no crash). What actually broke TP=2 was a shm_broadcast + # deadlock between the two worker processes post-load (matches a + # known vLLM bug class, e.g. vllm-ascend#9405 -- logic bug in the + # broadcast ring buffer, unaffected by shm size/timeouts/eager-mode, + # all of which were tried there too). Dropped to tensor-parallel-size=1 + # to sidestep the cross-process sync entirely -- full unsharded + # weights (~21GB, roughly 2x the per-rank figure above) fit on one + # 32GB V100 with room for KV cache. gpu-memory-utilization raised + # accordingly (0.5 was sized for the TP=2 split, too low for + # unsharded weights on a single GPU). Second V100 sits idle for now; + # pipeline-parallel-size=2 is the next thing to try if it's needed + # back, since PP unblocked the previous model instead of TP too. + # --kv-cache-dtype stays auto, not # fp8_e5m2: V100 has no FP8 tensor cores at all (Hopper/Ada only), # hardware-blocked regardless of vLLM version. tool-call-parser # changed hermes -> qwen3_coder per the checkpoint's own README @@ -39,9 +51,9 @@ spec: - --quantization=moe_wna16 - --dtype=float16 - --kv-cache-dtype=auto - - --tensor-parallel-size=2 + - --tensor-parallel-size=1 - --max-model-len=16384 - - --gpu-memory-utilization=0.5 + - --gpu-memory-utilization=0.85 - --max-num-seqs=4 - --enable-chunked-prefill - --enable-prefix-caching @@ -82,11 +94,11 @@ spec: limits: cpu: '16' memory: 36Gi - nvidia.com/gpu: '2' + nvidia.com/gpu: '1' requests: cpu: '8' memory: 12Gi - nvidia.com/gpu: '2' + nvidia.com/gpu: '1' startupProbe: failureThreshold: 240 httpGet: