diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 18ffdf1..19aae96 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -29,6 +29,7 @@ spec: - --dtype=float16 - --kv-cache-dtype=auto - --tensor-parallel-size=1 + - --pipeline-parallel-size=2 - --max-model-len=16384 - --gpu-memory-utilization=0.90 - --max-num-seqs=4 @@ -85,11 +86,11 @@ spec: limits: cpu: '16' memory: 36Gi - nvidia.com/gpu: '1' + nvidia.com/gpu: '2' requests: cpu: '8' memory: 12Gi - nvidia.com/gpu: '1' + nvidia.com/gpu: '2' startupProbe: failureThreshold: 80 httpGet: @@ -103,8 +104,8 @@ spec: name: shm deploymentStrategy: type: Recreate - maxReplicas: 2 - minReplicas: 2 + maxReplicas: 1 + minReplicas: 1 nodeSelector: kubernetes.io/hostname: worker-1 runtimeClassName: nvidia