Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a49a84fb0 | ||
|
|
aa41e8c509 | ||
|
|
ef764e97c7 | ||
|
|
ea8299c417 | ||
|
|
10ce4f8109 |
@@ -7,12 +7,7 @@ metadata:
|
|||||||
app.kubernetes.io/part-of: llm-serving
|
app.kubernetes.io/part-of: llm-serving
|
||||||
app: clef-candle
|
app: clef-candle
|
||||||
spec:
|
spec:
|
||||||
# HOLD AT 0 until candle-vllm:sm70 contains the GDN conv fix (candle-vllm PR #5).
|
replicas: 1
|
||||||
# An older :sm70 zeroes the key channel band, starves the delta rule and makes
|
|
||||||
# every linear-attention layer return zero. The response is still HTTP 200 and
|
|
||||||
# well formed, so the failure is silent. Raise to 1 only after CI publishes a
|
|
||||||
# :sm70 build from the merge that carries the fix.
|
|
||||||
replicas: 0
|
|
||||||
strategy:
|
strategy:
|
||||||
# One replica holds one GPU, so Recreate keeps the rollout to a single
|
# One replica holds one GPU, so Recreate keeps the rollout to a single
|
||||||
# weights load instead of needing a second card for a surge pod.
|
# weights load instead of needing a second card for a surge pod.
|
||||||
@@ -42,7 +37,19 @@ spec:
|
|||||||
# and makes every linear-attention layer return zero. The symptom is
|
# and makes every linear-attention layer return zero. The symptom is
|
||||||
# a well-formed /v1/systemone response with wrong answers, not an error.
|
# a well-formed /v1/systemone response with wrong answers, not an error.
|
||||||
image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
|
image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70
|
||||||
command: ["/usr/local/bin/candle-vllm"]
|
# :sm70 is a moving tag. Default IfNotPresent keeps a cached copy and a
|
||||||
|
# rebuilt image never reaches the pod, so a merged fix looks like it did
|
||||||
|
# not land. Always forces the pull on every pod start.
|
||||||
|
imagePullPolicy: Always
|
||||||
|
# Runs the fixed binary from the models PVC, not the one in the image.
|
||||||
|
# The published :sm70 predates the /v1/systemone route and the GDN conv
|
||||||
|
# fix (candle-vllm f3c00b2 is merged but CI has not republished the tag),
|
||||||
|
# so the in-image binary 404s on the route. The PVC copy at
|
||||||
|
# /mnt/models/bin/candle-vllm contains both fixes.
|
||||||
|
#
|
||||||
|
# RETIRE THIS once CI publishes a :sm70 built from f3c00b2: change the
|
||||||
|
# command back to /usr/local/bin/candle-vllm and delete the PVC copy.
|
||||||
|
command: ["/mnt/models/bin/candle-vllm"]
|
||||||
# clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
|
# clef-flash is a hybrid Qwen3_5 model: 8 full-attention layers plus 24
|
||||||
# GDN layers that carry per-sequence recurrent state. It runs on ONE
|
# GDN layers that carry per-sequence recurrent state. It runs on ONE
|
||||||
# card because the state is not shardable the way attention KV is.
|
# card because the state is not shardable the way attention KV is.
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ spec:
|
|||||||
# Cold start is about four minutes: the pod pip-installs on every boot because the
|
# Cold start is about four minutes: the pod pip-installs on every boot because the
|
||||||
# torch base image cannot be pushed to the registry (Cloudflare answers 413 above
|
# torch base image cannot be pushed to the registry (Cloudflare answers 413 above
|
||||||
# 100 MB per layer). startupProbe allows for that.
|
# 100 MB per layer). startupProbe allows for that.
|
||||||
replicas: 1
|
replicas: 0
|
||||||
# Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
|
# Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
|
||||||
# the new pod Pending forever waiting for a card the old pod still holds.
|
# the new pod Pending forever waiting for a card the old pod still holds.
|
||||||
strategy:
|
strategy:
|
||||||
|
|||||||
Reference in New Issue
Block a user