diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 0ae0f8c..8920f7b 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -38,6 +38,20 @@ spec: # 400s, hermes returns a real tool_calls response. - --enable-auto-tool-choice - --tool-call-parser=hermes + # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks + # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4 + # + max-model-len=16384 means concurrent long sequences compete for + # the same GPU KV space). No extra dependency, built into vLLM core. + # num_cpu_blocks is a starting point sized against the +4Gi/replica + # headroom added below (worker-1 has ~18Gi of request headroom across + # both replicas as of 2026-08-19) -- watch actual host memory usage + # and adjust; block_size=128 tokens matches vLLM's own example. + # Note: 0.11.0 ships the original (fragmented, small-transfer-block) + # version of this connector -- 0.12.0 consolidates KV data into one + # contiguous block per request and is reported an order of magnitude + # faster for this specific feature, so this is a real but not yet + # optimal implementation until the image gets bumped. + - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":2000,"block_size":128}} - --host=0.0.0.0 - --port=8080 env: @@ -60,11 +74,11 @@ spec: resources: limits: cpu: '16' - memory: 16Gi + memory: 24Gi nvidia.com/gpu: '1' requests: cpu: '8' - memory: 8Gi + memory: 12Gi nvidia.com/gpu: '1' startupProbe: failureThreshold: 80