From 7bec3a8c498e2ee0eee4baa068ccc75c810d4ca2 Mon Sep 17 00:00:00 2001
From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com>
Date: Wed, 19 Aug 2026 15:02:28 -0700
Subject: [PATCH] feat(llm-serving): offload reasoning's KV cache to CPU DRAM

vLLM 0.11.0's native OffloadingConnector -- spills KV blocks to CPU RAM on
preemption instead of discarding them, avoiding recompute. Built into vLLM
core, no extra dependency. Bumped memory request/limit (+4Gi/replica) to
give the CPU block pool real room; worker-1 had ~18Gi of request headroom
across both replicas.
---
 k8s/apps/llm-serving/reasoning.yaml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml
index 0ae0f8c..8920f7b 100644
--- a/k8s/apps/llm-serving/reasoning.yaml
+++ b/k8s/apps/llm-serving/reasoning.yaml
@@ -38,6 +38,20 @@ spec:
       # 400s, hermes returns a real tool_calls response.
       - --enable-auto-tool-choice
       - --tool-call-parser=hermes
+      # vLLM 0.11.0's native OffloadingConnector -- spills KV cache blocks
+      # to CPU DRAM instead of discarding them on preemption (max-num-seqs=4
+      # + max-model-len=16384 means concurrent long sequences compete for
+      # the same GPU KV space). No extra dependency, built into vLLM core.
+      # num_cpu_blocks is a starting point sized against the +4Gi/replica
+      # headroom added below (worker-1 has ~18Gi of request headroom across
+      # both replicas as of 2026-08-19) -- watch actual host memory usage
+      # and adjust; block_size=128 tokens matches vLLM's own example.
+      # Note: 0.11.0 ships the original (fragmented, small-transfer-block)
+      # version of this connector -- 0.12.0 consolidates KV data into one
+      # contiguous block per request and is reported an order of magnitude
+      # faster for this specific feature, so this is a real but not yet
+      # optimal implementation until the image gets bumped.
+      - --kv-transfer-config={"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"num_cpu_blocks":2000,"block_size":128}}
       - --host=0.0.0.0
       - --port=8080
       env:
@@ -60,11 +74,11 @@ spec:
       resources:
         limits:
           cpu: '16'
-          memory: 16Gi
+          memory: 24Gi
           nvidia.com/gpu: '1'
         requests:
           cpu: '8'
-          memory: 8Gi
+          memory: 12Gi
           nvidia.com/gpu: '1'
       startupProbe:
         failureThreshold: 80