Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9617bbe814 | ||
|
|
64a3f131b5 | ||
|
|
32f1d50e5e | ||
|
|
3ab127e581 |
@@ -0,0 +1,85 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: clef-candle
|
||||||
|
namespace: llm-serving
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: llm-serving
|
||||||
|
app: clef-candle
|
||||||
|
spec:
|
||||||
|
# HOLD AT 0 until candle-vllm:sm70 contains the GDN conv fix (candle-vllm PR #5).
|
||||||
|
# An older :sm70 zeroes the key channel band, starves the delta rule and makes
|
||||||
|
# every linear-attention layer return zero. The response is still HTTP 200 and
|
||||||
|
# well formed, so the failure is silent. Raise to 1 only after CI publishes a
|
||||||
|
# :sm70 build from the merge that carries the fix.
|
||||||
|
replicas: 0
|
||||||
|
strategy:
|
||||||
|
# One replica holds one GPU, so Recreate keeps the rollout to a single
|
||||||
|
# weights load instead of needing a second card for a surge pod.
|
||||||
|
type: Recreate
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: clef-candle
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/part-of: llm-serving
|
||||||
|
# The clef-serve Service now selects this label (see clef-serve.yaml),
|
||||||
|
# so clients keep addressing clef-serve:80 unchanged. The reference
|
||||||
|
# pytorch Deployment must stay at replicas 0 while this runs, or the
|
||||||
|
# Service splits traffic across two different implementations.
|
||||||
|
app: clef-candle
|
||||||
|
spec:
|
||||||
|
# Required. Without the nvidia runtime class libcuda.so is never injected
|
||||||
|
# and candle-vllm panics in cudarc at startup.
|
||||||
|
runtimeClassName: nvidia
|
||||||
|
containers:
|
||||||
|
- name: candle
|
||||||
|
# :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every
|
||||||
|
# merge to main. It MUST be a build that contains the GDN conv fix
|
||||||
|
# (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the
|
||||||
|
# key channel band in causal_conv1d_fwd, which starves the delta rule
|
||||||
|
# and makes every linear-attention layer return zero. The symptom is
|
||||||
|
# a well-formed /v1/systemone response with wrong answers, not an error.
|
||||||
|
image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
|
||||||
|
command: ["/usr/local/bin/candle-vllm"]
|
||||||
|
# clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
|
||||||
|
# GDN layers that carry per-sequence recurrent state. It runs on ONE
|
||||||
|
# card because the state is not shardable the way attention KV is.
|
||||||
|
args:
|
||||||
|
- --w
|
||||||
|
- /mnt/models/clef-flash
|
||||||
|
- --d
|
||||||
|
- "0"
|
||||||
|
- --dtype
|
||||||
|
- f16
|
||||||
|
- --p
|
||||||
|
- "8080"
|
||||||
|
- --max-num-seqs
|
||||||
|
- "4"
|
||||||
|
# About 66.8K KV tokens. Clef is a single-shot classifier-style
|
||||||
|
# endpoint, so it does not need the coder's pool.
|
||||||
|
- --kv-fraction
|
||||||
|
- "0.30"
|
||||||
|
# Prefix caching stays on: clef requests repeat the schema block.
|
||||||
|
- --prefill-chunk-size
|
||||||
|
- "1024"
|
||||||
|
ports:
|
||||||
|
- containerPort: 8080
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "8"
|
||||||
|
memory: 32Gi
|
||||||
|
nvidia.com/gpu: "1"
|
||||||
|
requests:
|
||||||
|
cpu: "4"
|
||||||
|
memory: 16Gi
|
||||||
|
nvidia.com/gpu: "1"
|
||||||
|
volumeMounts:
|
||||||
|
- name: models
|
||||||
|
mountPath: /mnt/models
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: models
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: llm-models
|
||||||
@@ -127,8 +127,11 @@ metadata:
|
|||||||
labels:
|
labels:
|
||||||
app.kubernetes.io/part-of: llm-serving
|
app.kubernetes.io/part-of: llm-serving
|
||||||
spec:
|
spec:
|
||||||
|
# Points at the candle-vllm implementation (clef-candle.yaml). Switching back
|
||||||
|
# to the reference pytorch server is this line plus that Deployment's replicas;
|
||||||
|
# no client changes, because the Service name and ClusterIP do not move.
|
||||||
selector:
|
selector:
|
||||||
app: clef-serve
|
app: clef-candle
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
port: 80
|
port: 80
|
||||||
|
|||||||
@@ -21,6 +21,10 @@ resources:
|
|||||||
# streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
|
# streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
|
||||||
# tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
|
# tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
|
||||||
# the memory controller idles at 19%.
|
# the memory controller idles at 19%.
|
||||||
|
# clef-candle.yaml serves clef-flash on candle-vllm (hybrid Qwen3_5, one GPU)
|
||||||
|
# and owns the clef-candle label that clef-serve.yaml's Service now selects.
|
||||||
|
# It must not sync against an image built before the GDN conv fix.
|
||||||
|
- clef-candle.yaml
|
||||||
- coder-14b-fp16.yaml
|
- coder-14b-fp16.yaml
|
||||||
- clef-serve.yaml
|
- clef-serve.yaml
|
||||||
# Stable client-facing name for the chat model. poimen-memory points at this Service,
|
# Stable client-facing name for the chat model. poimen-memory points at this Service,
|
||||||
|
|||||||
Reference in New Issue
Block a user