From 10ce4f8109be132a524655896154bcac5cded7ee Mon Sep 17 00:00:00 2001
From: rock <locartrock@gmail.com>
Date: Thu, 5 Nov 2026 01:40:42 +0000
Subject: [PATCH] feat(llm-serving): serve clef-flash on candle-vllm (#109)

Co-authored-by: rock <locartrock@gmail.com>
---
 k8s/apps/llm-serving/clef-candle.yaml   | 80 +++++++++++++++++++++++++
 k8s/apps/llm-serving/clef-serve.yaml    |  5 +-
 k8s/apps/llm-serving/kustomization.yaml |  4 ++
 3 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 k8s/apps/llm-serving/clef-candle.yaml

diff --git a/k8s/apps/llm-serving/clef-candle.yaml b/k8s/apps/llm-serving/clef-candle.yaml
new file mode 100644
index 0000000..248bccb
--- /dev/null
+++ b/k8s/apps/llm-serving/clef-candle.yaml
@@ -0,0 +1,80 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: clef-candle
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: clef-candle
+spec:
+  replicas: 1
+  strategy:
+    # One replica holds one GPU, so Recreate keeps the rollout to a single
+    # weights load instead of needing a second card for a surge pod.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: clef-candle
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: llm-serving
+        # The clef-serve Service now selects this label (see clef-serve.yaml),
+        # so clients keep addressing clef-serve:80 unchanged. The reference
+        # pytorch Deployment must stay at replicas 0 while this runs, or the
+        # Service splits traffic across two different implementations.
+        app: clef-candle
+    spec:
+      # Required. Without the nvidia runtime class libcuda.so is never injected
+      # and candle-vllm panics in cudarc at startup.
+      runtimeClassName: nvidia
+      containers:
+        - name: candle
+          # :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every
+          # merge to main. It MUST be a build that contains the GDN conv fix
+          # (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the
+          # key channel band in causal_conv1d_fwd, which starves the delta rule
+          # and makes every linear-attention layer return zero. The symptom is
+          # a well-formed /v1/systemone response with wrong answers, not an error.
+          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
+          command: ["/usr/local/bin/candle-vllm"]
+          # clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
+          # GDN layers that carry per-sequence recurrent state. It runs on ONE
+          # card because the state is not shardable the way attention KV is.
+          args:
+            - --w
+            - /mnt/models/clef-flash
+            - --d
+            - "0"
+            - --dtype
+            - f16
+            - --p
+            - "8080"
+            - --max-num-seqs
+            - "4"
+            # About 66.8K KV tokens. Clef is a single-shot classifier-style
+            # endpoint, so it does not need the coder's pool.
+            - --kv-fraction
+            - "0.30"
+            # Prefix caching stays on: clef requests repeat the schema block.
+            - --prefill-chunk-size
+            - "1024"
+          ports:
+            - containerPort: 8080
+          resources:
+            limits:
+              cpu: "8"
+              memory: 32Gi
+              nvidia.com/gpu: "1"
+            requests:
+              cpu: "4"
+              memory: 16Gi
+              nvidia.com/gpu: "1"
+          volumeMounts:
+            - name: models
+              mountPath: /mnt/models
+              readOnly: true
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llm-models
diff --git a/k8s/apps/llm-serving/clef-serve.yaml b/k8s/apps/llm-serving/clef-serve.yaml
index f2dc8c4..a8b820f 100644
--- a/k8s/apps/llm-serving/clef-serve.yaml
+++ b/k8s/apps/llm-serving/clef-serve.yaml
@@ -127,8 +127,11 @@ metadata:
   labels:
     app.kubernetes.io/part-of: llm-serving
 spec:
+  # Points at the candle-vllm implementation (clef-candle.yaml). Switching back
+  # to the reference pytorch server is this line plus that Deployment's replicas;
+  # no client changes, because the Service name and ClusterIP do not move.
   selector:
-    app: clef-serve
+    app: clef-candle
   ports:
   - name: http
     port: 80
diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml
index fa86d46..5eb7737 100644
--- a/k8s/apps/llm-serving/kustomization.yaml
+++ b/k8s/apps/llm-serving/kustomization.yaml
@@ -21,6 +21,10 @@ resources:
   # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
   # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
   # the memory controller idles at 19%.
+  # clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU)
+  # and owns the clef-candle label that clef-serve.yaml's Service now selects.
+  # It must not sync against an image built before the GDN conv fix.
+  - clef-candle.yaml
   - coder-14b-fp16.yaml
   - clef-serve.yaml
   # Stable client-facing name for the chat model. poimen-memory points at this Service,