fix: use pipeline parallelism instead of TP for reasoning-predictor (bnb prequant blocks TP)

2026-08-21 13:21:41 -07:00
parent 65f3f25932
commit 7a149bfb48
1 changed files with 2 additions and 1 deletions
@@ -28,7 +28,8 @@ spec:
      - --quantization=bitsandbytes
      - --dtype=float16
      - --kv-cache-dtype=auto
-      - --tensor-parallel-size=2
+      - --tensor-parallel-size=1
+      - --pipeline-parallel-size=2
      - --max-model-len=16384
      - --gpu-memory-utilization=0.90
      - --max-num-seqs=4