diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 0dd677c..19aae96 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -28,7 +28,8 @@ spec: - --quantization=bitsandbytes - --dtype=float16 - --kv-cache-dtype=auto - - --tensor-parallel-size=2 + - --tensor-parallel-size=1 + - --pipeline-parallel-size=2 - --max-model-len=16384 - --gpu-memory-utilization=0.90 - --max-num-seqs=4