From 3ab127e581995173ee24449b6bc61fc2e227078c Mon Sep 17 00:00:00 2001 From: rock Date: Thu, 5 Nov 2026 09:35:47 +0800 Subject: [PATCH] feat(llm-serving): serve clef-flash on candle-vllm Adds clef-candle.yaml: clef-flash (hybrid Qwen3_5, 8 attention + 24 GDN layers) on one V100 via candle-vllm, and repoints the clef-serve Service selector at it. The Service name and ClusterIP do not move, so no client changes. Runs on one card because the GDN recurrent state is per-sequence and cannot be sharded the way attention KV can. --kv-fraction 0.30 gives about 66.8K KV tokens, which is ample for a single-shot classifier-style endpoint. The image is candle-vllm:sm70, built by .gitea/workflows/candle-vllm-sm70.yaml on every merge to main. It must be a build containing the GDN conv fix: older images zero the key channel band in causal_conv1d_fwd, starve the delta rule and make every linear-attention layer return zero. That failure is silent - the response is well formed, just wrong. Verified against the reference implementation: token ids 420/420 identical, beta 8647.021 vs 8646.982, delta-rule and gated-norm absmax equal to six decimals, all three head answers agree. --- k8s/apps/llm-serving/clef-candle.yaml | 80 +++++++++++++++++++++++++ k8s/apps/llm-serving/clef-serve.yaml | 5 +- k8s/apps/llm-serving/kustomization.yaml | 4 ++ 3 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 k8s/apps/llm-serving/clef-candle.yaml diff --git a/k8s/apps/llm-serving/clef-candle.yaml b/k8s/apps/llm-serving/clef-candle.yaml new file mode 100644 index 0000000..248bccb --- /dev/null +++ b/k8s/apps/llm-serving/clef-candle.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clef-candle + namespace: llm-serving + labels: + app.kubernetes.io/part-of: llm-serving + app: clef-candle +spec: + replicas: 1 + strategy: + # One replica holds one GPU, so Recreate keeps the rollout to a single + # weights load instead of needing a second card for a surge pod. + type: Recreate + selector: + matchLabels: + app: clef-candle + template: + metadata: + labels: + app.kubernetes.io/part-of: llm-serving + # The clef-serve Service now selects this label (see clef-serve.yaml), + # so clients keep addressing clef-serve:80 unchanged. The reference + # pytorch Deployment must stay at replicas 0 while this runs, or the + # Service splits traffic across two different implementations. + app: clef-candle + spec: + # Required. Without the nvidia runtime class libcuda.so is never injected + # and candle-vllm panics in cudarc at startup. + runtimeClassName: nvidia + containers: + - name: candle + # :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every + # merge to main. It MUST be a build that contains the GDN conv fix + # (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the + # key channel band in causal_conv1d_fwd, which starves the delta rule + # and makes every linear-attention layer return zero. The symptom is + # a well-formed /v1/systemone response with wrong answers, not an error. + image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70 + command: ["/usr/local/bin/candle-vllm"] + # clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24 + # GDN layers that carry per-sequence recurrent state. It runs on ONE + # card because the state is not shardable the way attention KV is. + args: + - --w + - /mnt/models/clef-flash + - --d + - "0" + - --dtype + - f16 + - --p + - "8080" + - --max-num-seqs + - "4" + # About 66.8K KV tokens. Clef is a single-shot classifier-style + # endpoint, so it does not need the coder's pool. + - --kv-fraction + - "0.30" + # Prefix caching stays on: clef requests repeat the schema block. + - --prefill-chunk-size + - "1024" + ports: + - containerPort: 8080 + resources: + limits: + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + requests: + cpu: "4" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: models + mountPath: /mnt/models + readOnly: true + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/clef-serve.yaml b/k8s/apps/llm-serving/clef-serve.yaml index f2dc8c4..a8b820f 100644 --- a/k8s/apps/llm-serving/clef-serve.yaml +++ b/k8s/apps/llm-serving/clef-serve.yaml @@ -127,8 +127,11 @@ metadata: labels: app.kubernetes.io/part-of: llm-serving spec: + # Points at the candle-vllm implementation (clef-candle.yaml). Switching back + # to the reference pytorch server is this line plus that Deployment's replicas; + # no client changes, because the Service name and ClusterIP do not move. selector: - app: clef-serve + app: clef-candle ports: - name: http port: 80 diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index fa86d46..5eb7737 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -21,6 +21,10 @@ resources: # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16 # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while # the memory controller idles at 19%. + # clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU) + # and owns the clef-candle label that clef-serve.yaml's Service now selects. + # It must not sync against an image built before the GDN conv fix. + - clef-candle.yaml - coder-14b-fp16.yaml - clef-serve.yaml # Stable client-facing name for the chat model. poimen-memory points at this Service,