diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 34fe884..18ffdf1 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -45,21 +45,23 @@ spec: # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4 # + max-model-len=16384 means concurrent long sequences compete for # the same GPU KV space). No extra dependency, built into vLLM core. - # num_cpu_blocks=2000 hung the pod at startup ("Allocating 64 CPU - # tensors..." then nothing -- 64 is this model's layer count, one - # pinned host tensor per layer, each sized for every CPU block; 2000 - # was oversized enough to stall pinning that much host memory, likely - # blowing well past the pod's memory limit). Dropped to a small, - # known-safe starting point -- confirm it actually comes up healthy, - # then watch real host memory usage and raise it deliberately rather - # than guessing a round number again. block_size=128 tokens matches - # vLLM's own example. + # num_cpu_blocks=2000 hung the pod at startup on the old model (2000 x + # ~32MB/block blew well past the pod's memory limit). num_cpu_blocks=32 + # was the safe-recovery value after that -- only ~1GB of real DRAM + # (32 blocks x 128 tokens x 256KB/token-across-all-64-layers, fp16), + # basically a token-count safety valve, not real offload capacity. + # This model: 64 layers, 8 KV heads x 128 head_dim, fp16 -> ~256KB of + # KV per token across all layers -> ~32MB per 128-token block. + # num_cpu_blocks=256 -> ~8GB of actual DRAM offload (32,768 tokens), + # comfortably under the pod's 36Gi limit alongside the ~20GB bnb-4bit + # weights. Watch real host memory on boot before raising further -- + # block_size=128 tokens matches vLLM's own example. # Note: 0.11.0 ships the original (fragmented, small-transfer-block) # version of this connector -- 0.12.0 consolidates KV data into one # contiguous block per request and is reported an order of magnitude # faster for this specific feature, so this is a real but not yet # optimal implementation until the image gets bumped. - - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":32,"block_size":128}} + - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":256,"block_size":128}} - --host=0.0.0.0 - --port=8080 env: