93 lines
3.7 KiB
YAML
93 lines
3.7 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: clef-candle
|
|
namespace: llm-serving
|
|
labels:
|
|
app.kubernetes.io/part-of: llm-serving
|
|
app: clef-candle
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
# One replica holds one GPU, so Recreate keeps the rollout to a single
|
|
# weights load instead of needing a second card for a surge pod.
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: clef-candle
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: llm-serving
|
|
# The clef-serve Service now selects this label (see clef-serve.yaml),
|
|
# so clients keep addressing clef-serve:80 unchanged. The reference
|
|
# pytorch Deployment must stay at replicas 0 while this runs, or the
|
|
# Service splits traffic across two different implementations.
|
|
app: clef-candle
|
|
spec:
|
|
# Required. Without the nvidia runtime class libcuda.so is never injected
|
|
# and candle-vllm panics in cudarc at startup.
|
|
runtimeClassName: nvidia
|
|
containers:
|
|
- name: candle
|
|
# :sm70 is built by .gitea/workflows/candle-vllm-sm70.yaml on every
|
|
# merge to main. It MUST be a build that contains the GDN conv fix
|
|
# (candle-vllm PR: feat/clef-systemone). Older :sm70 images zero the
|
|
# key channel band in causal_conv1d_fwd, which starves the delta rule
|
|
# and makes every linear-attention layer return zero. The symptom is
|
|
# a well-formed /v1/systemone response with wrong answers, not an error.
|
|
image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
|
|
# :sm70 is a moving tag. Default IfNotPresent keeps a cached copy and a
|
|
# rebuilt image never reaches the pod, so a merged fix looks like it did
|
|
# not land. Always forces the pull on every pod start.
|
|
imagePullPolicy: Always
|
|
# Runs the fixed binary from the models PVC, not the one in the image.
|
|
# The published :sm70 predates the /v1/systemone route and the GDN conv
|
|
# fix (candle-vllm f3c00b2 is merged but CI has not republished the tag),
|
|
# so the in-image binary 404s on the route. The PVC copy at
|
|
# /mnt/models/bin/candle-vllm contains both fixes.
|
|
#
|
|
# RETIRE THIS once CI publishes a :sm70 built from f3c00b2: change the
|
|
# command back to /usr/local/bin/candle-vllm and delete the PVC copy.
|
|
command: ["/mnt/models/bin/candle-vllm"]
|
|
# clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
|
|
# GDN layers that carry per-sequence recurrent state. It runs on ONE
|
|
# card because the state is not shardable the way attention KV is.
|
|
args:
|
|
- --w
|
|
- /mnt/models/clef-flash
|
|
- --d
|
|
- "0"
|
|
- --dtype
|
|
- f16
|
|
- --p
|
|
- "8080"
|
|
- --max-num-seqs
|
|
- "4"
|
|
# About 66.8K KV tokens. Clef is a single-shot classifier-style
|
|
# endpoint, so it does not need the coder's pool.
|
|
- --kv-fraction
|
|
- "0.30"
|
|
# Prefix caching stays on: clef requests repeat the schema block.
|
|
- --prefill-chunk-size
|
|
- "1024"
|
|
ports:
|
|
- containerPort: 8080
|
|
resources:
|
|
limits:
|
|
cpu: "8"
|
|
memory: 32Gi
|
|
nvidia.com/gpu: "1"
|
|
requests:
|
|
cpu: "4"
|
|
memory: 16Gi
|
|
nvidia.com/gpu: "1"
|
|
volumeMounts:
|
|
- name: models
|
|
mountPath: /mnt/models
|
|
readOnly: true
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|