From 67760720fc463d9d3a2d8e819b94c33be9b82292 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 4 Nov 2026 06:19:41 +0800 Subject: [PATCH 1/5] k8s(llm-serving): client-facing Service for candle-vllm coder Creates coder-predictor selecting app=candle-vllm on ports 80 and 8080, so clients keep one stable host name across the vLLM -> candle-vllm handoff and only this selector changes from now on. The retired KTML predictor requested two V100s and never reached Ready. --- k8s/apps/llm-serving/coder-service.yaml | 40 +++++++++++++++++++++++++ k8s/apps/llm-serving/kustomization.yaml | 1 + 2 files changed, 41 insertions(+) create mode 100644 k8s/apps/llm-serving/coder-service.yaml diff --git a/k8s/apps/llm-serving/coder-service.yaml b/k8s/apps/llm-serving/coder-service.yaml new file mode 100644 index 0000000..83aec11 --- /dev/null +++ b/k8s/apps/llm-serving/coder-service.yaml @@ -0,0 +1,40 @@ +# Stable client-facing name for the coder chat model, deliberately decoupled +# from the process that serves it. +# +# poimen-memory addresses a Service, not a pod and not an InferenceService. The +# vLLM -> candle-vllm handoff therefore changes this file's selector and nothing +# else, so the client config never changes again and there is no window in which +# the host name is wrong. +# +# The previous KTML predictor requested two V100s and never reached Ready, so it +# held half the GPU pool while serving nothing. candle-vllm replaces it. +# +# Two ports on purpose: +# - 80 matches every other predictor Service in this namespace, and matches +# a client base URL that carries no port number. +# - 8080 is the container port. Publishing it directly avoids a port +# translation that is easy to get wrong in a curl. +# +# Do not delete and recreate this Service: a delete drops the ClusterIP, and a +# client that cached the name during the gap fails until it re-resolves. +apiVersion: v1 +kind: Service +metadata: + name: coder-predictor + namespace: llm-serving + labels: + app.kubernetes.io/part-of: llm-serving + app: coder-predictor +spec: + type: ClusterIP + selector: + app: candle-vllm + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 + - name: http-8080 + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index 925774a..0aa63fc 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -16,6 +16,7 @@ resources: - clef-serve.yaml # Stable client-facing name for the chat model. poimen-memory points at this Service, # so the vLLM -> candle-vllm handoff is a selector edit here and no client change. + - coder-service.yaml - reasoning-long-service.yaml # ornith is retired: its InferenceService still reports Ready=True while running zero # replicas, because desired equals available at 0. Its manifest was never committed, -- 2.54.0 From db9715990540d46a14ec6ba49c8ea435af14ac1f Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 4 Nov 2026 18:47:54 +0800 Subject: [PATCH 2/5] feat(llm-serving): run clef-serve so /v1/systemone answers clef-serve was parked at 0 while candle-vllm took a card. worker-1 holds four V100s, so the coder takes one and clef takes one, leaving two free for a TP=2 candle run. The gateway already routes the clef-flash model to clef-serve:80 on /v1/systemone, so this single replica count is what turns that route on. --- k8s/apps/llm-serving/clef-serve.yaml | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/k8s/apps/llm-serving/clef-serve.yaml b/k8s/apps/llm-serving/clef-serve.yaml index 73c8614..f2dc8c4 100644 --- a/k8s/apps/llm-serving/clef-serve.yaml +++ b/k8s/apps/llm-serving/clef-serve.yaml @@ -33,19 +33,17 @@ metadata: app: clef-serve app.kubernetes.io/part-of: llm-serving spec: - # Parked at 0 to free the only V100 that was not serving anything, so candle-vllm can - # run its Phase 0 on one card. All four cards were allocated (coder 2, reasoning 1, - # this 1) and a new GPU workload had nowhere to land. + # One card. worker-1 holds four V100s: candle-vllm takes one for the coder, and + # this pod takes one, so two stay free for a TP=2 candle run. # - # This is the only GPU consumer in the namespace that is not vLLM, which is why it is - # the first card released: the standing instruction is to keep the vLLM pods healthy - # until candle-vllm answers, and this pod is not one of them. + # Enabled 2026-11-04 to serve /v1/systemone, which the gateway already routes to + # this Service. It was parked at 0 while candle-vllm was brought up on the same + # cards, and it served no inference request between 2026-10-30T06:23:40Z and then. # - # It was serving nothing. The last 30 log lines at park time were 30/30 kubelet - # /health probes and no inference request since start (2026-10-30T06:23:40Z). Nothing - # in git selects this service. Restore it once the rollout leaves a card free: - # 30B-A3B takes 1, Coder-Next TP=2 takes 2, so card 4 is this one. - replicas: 0 + # Cold start is about four minutes: the pod pip-installs on every boot because the + # torch base image cannot be pushed to the registry (Cloudflare answers 413 above + # 100 MB per layer). startupProbe allows for that. + replicas: 1 # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves # the new pod Pending forever waiting for a card the old pod still holds. strategy: -- 2.54.0 From f6fca4705b21d09dc2bc6e2895501a39cd7adc81 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 5 Nov 2026 09:11:46 +0800 Subject: [PATCH 3/5] feat(llm-serving): run coder as fp16 14B across two V100s Replace the GPTQ W4A16 30B-A3B coder with Qwen2.5-Coder-14B-Instruct at fp16 on TP=2. The V100 has FP16 tensor cores and no INT4 ones, so the 4-bit path paid a dequantisation pass on the SM. Profiling showed SM at 98% with the memory controller at 19%, and the batch-1 step cost matched a full-model read. Same harness, 16 streams: ITL 71.5 -> 39.7 ms, 215 -> 287 tok/s. The deployment keeps the candle-vllm name and the app: candle-vllm label, so coder-service.yaml and every client keep working unchanged. --- k8s/apps/llm-serving/coder-14b-fp16.yaml | 102 +++++++++++++++++++++++ k8s/apps/llm-serving/kustomization.yaml | 9 ++ 2 files changed, 111 insertions(+) create mode 100644 k8s/apps/llm-serving/coder-14b-fp16.yaml diff --git a/k8s/apps/llm-serving/coder-14b-fp16.yaml b/k8s/apps/llm-serving/coder-14b-fp16.yaml new file mode 100644 index 0000000..69be1c4 --- /dev/null +++ b/k8s/apps/llm-serving/coder-14b-fp16.yaml @@ -0,0 +1,102 @@ +# Qwen2.5-Coder-14B-Instruct, fp16, tensor-parallel across two V100s. +# +# Why this model and this precision. The previous workload was Qwen3-Coder-30B-A3B +# in GPTQ W4A16 on one V100. Profiling that deployment showed SM utilisation at +# 98% while the memory controller sat at 19%: the V100 has FP16 tensor cores and +# no INT4 ones, so every decode step ran a dequantisation pass on the SM. That is +# a hardware-generation mismatch, not a tunable. Sequence throughput matches the +# arithmetic: at batch 1 the step cost equalled a full-model read (17.6 ms against +# 15 GB / 900 GB/s = 16.7 ms) while the routed experts alone are ~1.5 GB. +# +# Measured on this hardware, same harness, same 16-stream load: +# +# GPTQ W4A16 30B-A3B, TP=1 : ITL 71.5 ms, 215 tok/s aggregate +# fp16 14B, TP=2 : ITL 39.7 ms, 287 tok/s aggregate +# +# The dense model degrades only 1.4x from 1 to 16 streams (27.9 -> 39.7 ms) while +# the MoE degrades 4.1x (17.6 -> 71.5 ms), because a dense model reads its weights +# once per step and amortises across the batch. The MoE pays expert weight traffic +# per sequence. Single-stream latency is the one case where the MoE still wins +# (17.6 ms against 27.9 ms), since it activates ~3B parameters per token. +# +# TP=2 notes that cost time to establish, so they are recorded here: +# - Sharding needs no code change. Every dimension divides by 2: attention heads +# 40 -> 20, KV heads 8 -> 4, intermediate_size 13824 -> 6912, vocab +# 152064 -> 76032. Verify this first for any replacement model. +# - TP=2 on a GPTQ checkpoint is what crashed (CUDA_ERROR_ILLEGAL_ADDRESS in +# forward()) on the previous model. The cause was quantised weight sharding +# and the g_idx guard in src/openai/models/linear.rs, not TP itself. A dense +# fp16 checkpoint has no g_idx, no packed weights and no scales, so it takes +# the plain column/row-parallel path and the crash does not reproduce. +# - These are V100-PCIE cards with no NVLink, so every layer's all-reduce +# crosses PCIe. It shows up in TTFT (275 ms at 1 stream, 1090 ms at 16) +# more than in ITL. +# - KV capacity is 11635 MB per rank, about 59K context tokens. The fp16 +# weights consume the room the KV pool would otherwise use. This is well +# short of the 256K the previous model targeted, and it is the price of +# leaving the dequantisation path. +# +# Weights must be in the PVC as /mnt/models/coder-14b-fp16 (28 GB, 6 shards). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: candle-vllm + namespace: llm-serving + labels: + app.kubernetes.io/part-of: llm-serving + app: candle-vllm +spec: + replicas: 1 + strategy: + # One replica holds both GPUs, so a rolling update would need a second pair. + # Recreate keeps the rollout to a single weights load. + type: Recreate + selector: + matchLabels: + app: candle-vllm + template: + metadata: + labels: + app.kubernetes.io/part-of: llm-serving + app: candle-vllm + spec: + containers: + - name: candle + image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim + command: ["/usr/local/bin/candle-vllm"] + # fp16, not GPTQ, and two ranks, not one. See the header for the + # measurements and the TP=2 findings. + args: + - --w + - /mnt/models/coder-14b-fp16 + - --d + - 0,1 + - --dtype + - f16 + - --p + - "8080" + - --max-num-seqs + - "32" + # 11.6 GB of KV per rank, about 59K context tokens. The fp16 weights + # leave no room for more. + - --kv-fraction + - "0.85" + ports: + - containerPort: 8080 + resources: + limits: + cpu: "8" + memory: 36Gi + nvidia.com/gpu: "2" + requests: + cpu: "4" + memory: 12Gi + nvidia.com/gpu: "2" + volumeMounts: + - name: models + mountPath: /mnt/models + readOnly: true + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index 0aa63fc..fa86d46 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -13,6 +13,15 @@ resources: # clef-flash runs through clef-serve.yaml, not through an SGLang runtime. SGLang needs # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100. + # coder-14b-fp16.yaml owns the `candle-vllm` Deployment name and the + # `app: candle-vllm` label that coder-service.yaml selects. It must be the only + # workload with that label, or the Service splits traffic across two models. + # + # fp16 on two V100s, not GPTQ on one: measured 39.7 ms ITL and 287 tok/s at 16 + # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16 + # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while + # the memory controller idles at 19%. + - coder-14b-fp16.yaml - clef-serve.yaml # Stable client-facing name for the chat model. poimen-memory points at this Service, # so the vLLM -> candle-vllm handoff is a selector edit here and no client change. -- 2.54.0 From 7bdc8b2de2b84040be69aadeb8c0bcb742d5e889 Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 5 Nov 2026 09:22:48 +0800 Subject: [PATCH 4/5] fix(llm-serving): set runtimeClassName nvidia on the candle deployment Without it the container runs under runc, libcuda.so is never injected and candle-vllm panics in cudarc at startup. The pod reached Running but crashed on every restart. Verified live: pod Ready, coder-predictor endpoints populated, /v1/models reports coder-14b-fp16 and a completion returns OK. --- k8s/apps/llm-serving/coder-14b-fp16.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/k8s/apps/llm-serving/coder-14b-fp16.yaml b/k8s/apps/llm-serving/coder-14b-fp16.yaml index 69be1c4..f8f1f4a 100644 --- a/k8s/apps/llm-serving/coder-14b-fp16.yaml +++ b/k8s/apps/llm-serving/coder-14b-fp16.yaml @@ -60,6 +60,10 @@ spec: app.kubernetes.io/part-of: llm-serving app: candle-vllm spec: + # Required. Without this the container runs under runc, libcuda.so is never + # injected, and candle-vllm panics in cudarc at startup with + # "Unable to dynamically load the cuda shared library". + runtimeClassName: nvidia containers: - name: candle image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim -- 2.54.0 From 3ab127e581995173ee24449b6bc61fc2e227078c Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 5 Nov 2026 09:35:47 +0800 Subject: [PATCH 5/5] feat(llm-serving): serve clef-flash on candle-vllm Adds clef-candle.yaml: clef-flash (hybrid Qwen3_5, 8 attention + 24 GDN layers) on one V100 via candle-vllm, and repoints the clef-serve Service selector at it. The Service name and ClusterIP do not move, so no client changes. Runs on one card because the GDN recurrent state is per-sequence and cannot be sharded the way attention KV can. --kv-fraction 0.30 gives about 66.8K KV tokens, which is ample for a single-shot classifier-style endpoint. The image is candle-vllm:sm70, built by .gitea/workflows/candle-vllm-sm70.yaml on every merge to main. It must be a build containing the GDN conv fix: older images zero the key channel band in causal_conv1d_fwd, starve the delta rule and make every linear-attention layer return zero. That failure is silent - the response is well formed, just wrong. Verified against the reference implementation: token ids 420/420 identical, beta 8647.021 vs 8646.982, delta-rule and gated-norm absmax equal to six decimals, all three head answers agree. --- k8s/apps/llm-serving/clef-candle.yaml | 80 +++++++++++++++++++++++++ k8s/apps/llm-serving/clef-serve.yaml | 5 +- k8s/apps/llm-serving/kustomization.yaml | 4 ++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 k8s/apps/llm-serving/clef-candle.yaml diff --git a/k8s/apps/llm-serving/clef-candle.yaml b/k8s/apps/llm-serving/clef-candle.yaml new file mode 100644 index 0000000..248bccb --- /dev/null +++ b/k8s/apps/llm-serving/clef-candle.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clef-candle + namespace: llm-serving + labels: + app.kubernetes.io/part-of: llm-serving + app: clef-candle +spec: + replicas: 1 + strategy: + # One replica holds one GPU, so Recreate keeps the rollout to a single + # weights load instead of needing a second card for a surge pod. + type: Recreate + selector: + matchLabels: + app: clef-candle + template: + metadata: + labels: + app.kubernetes.io/part-of: llm-serving + # The clef-serve Service now selects this label (see clef-serve.yaml), + # so clients keep addressing clef-serve:80 unchanged. The reference + # pytorch Deployment must stay at replicas 0 while this runs, or the + # Service splits traffic across two different implementations. + app: clef-candle + spec: + # Required. Without the nvidia runtime class libcuda.so is never injected + # and candle-vllm panics in cudarc at startup. + runtimeClassName: nvidia + containers: + - name: candle + # :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every + # merge to main. It MUST be a build that contains the GDN conv fix + # (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the + # key channel band in causal_conv1d_fwd, which starves the delta rule + # and makes every linear-attention layer return zero. The symptom is + # a well-formed /v1/systemone response with wrong answers, not an error. + image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70 + command: ["/usr/local/bin/candle-vllm"] + # clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24 + # GDN layers that carry per-sequence recurrent state. It runs on ONE + # card because the state is not shardable the way attention KV is. + args: + - --w + - /mnt/models/clef-flash + - --d + - "0" + - --dtype + - f16 + - --p + - "8080" + - --max-num-seqs + - "4" + # About 66.8K KV tokens. Clef is a single-shot classifier-style + # endpoint, so it does not need the coder's pool. + - --kv-fraction + - "0.30" + # Prefix caching stays on: clef requests repeat the schema block. + - --prefill-chunk-size + - "1024" + ports: + - containerPort: 8080 + resources: + limits: + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: models + mountPath: /mnt/models + readOnly: true + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/clef-serve.yaml b/k8s/apps/llm-serving/clef-serve.yaml index f2dc8c4..a8b820f 100644 --- a/k8s/apps/llm-serving/clef-serve.yaml +++ b/k8s/apps/llm-serving/clef-serve.yaml @@ -127,8 +127,11 @@ metadata: labels: app.kubernetes.io/part-of: llm-serving spec: + # Points at the candle-vllm implementation (clef-candle.yaml). Switching back + # to the reference pytorch server is this line plus that Deployment's replicas; + # no client changes, because the Service name and ClusterIP do not move. selector: - app: clef-serve + app: clef-candle ports: - name: http port: 80 diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index fa86d46..5eb7737 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -21,6 +21,10 @@ resources: # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16 # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while # the memory controller idles at 19%. + # clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU) + # and owns the clef-candle label that clef-serve.yaml's Service now selects. + # It must not sync against an image built before the GDN conv fix. + - clef-candle.yaml - coder-14b-fp16.yaml - clef-serve.yaml # Stable client-facing name for the chat model. poimen-memory points at this Service, -- 2.54.0