rock
41dcda819e
The Clef runtime was running from hand-applied objects that existed only in the cluster.
A rebuild would have lost it.
Three things move into git:
- clef-serve.yaml: the Deployment and the Service the gateway routes to
- configMapGenerator for clef-serve/app.py, so an edit to the code rolls the pod
instead of leaving a stale copy mounted
- clef-flash.yaml is deleted: it described an SGLang predictor that cannot run on a
V100, and its slot is now clef-serve.yaml
Recorded in the manifest, because both cost hours: SGLang needs CUDA 13 for Clef and
CUDA 13 has no sm_70 kernels, and device_map over two cards breaks the forward pass with
'indices should be either on cpu or on the same device as the indexed tensor'.
Still open and written down in the file: the image cannot be pushed, because Cloudflare
answers 413 for layers above 100 MB. So each start pip-installs and needs PyPI.
126 lines
4.3 KiB
YAML
126 lines
4.3 KiB
YAML
# clef-flash: a decision model, not a chat model.
|
|
#
|
|
# It reads a `state` plus a schema of typed `questions` and returns one probability
|
|
# per allowed option in a single prefill pass. There is no decoding, no sampling and
|
|
# no free-form text. The public route is /v1/systemone, and its request and response
|
|
# shapes are Clef's own: no `messages`, no `choices`.
|
|
#
|
|
# Why not SGLang: Clef support landed in SGLang v0.5.22, which upstream publishes only
|
|
# on CUDA 13. CUDA 13 dropped Volta kernels and every card here is a Tesla V100
|
|
# (compute capability 7.0), so SGLang aborts with "no kernel image is available for
|
|
# execution on the device". SGLang also states sm75 as its floor regardless.
|
|
#
|
|
# So this runs the implementation the model ships: joint_schema_model.py, which has
|
|
# load_release_model, encode_record, collate_records and systemone. The weights are
|
|
# 17.75 GiB of BF16, and fp16 costs the same bytes per parameter. On one 32 GiB card
|
|
# that leaves room for a 32k state, which is why this is one GPU and not two: with
|
|
# device_map over two cards the embedding table and the tensors indexing into it land
|
|
# on different devices, and the forward pass dies with
|
|
# "indices should be either on cpu or on the same device as the indexed tensor".
|
|
#
|
|
# Known gap: the image cannot be pushed to the registry. Cloudflare fronts
|
|
# forgejo.riotpiao.com and answers 413 Payload Too Large for any layer above 100 MB,
|
|
# and the torch base alone is 7.58 GB. So this mounts app.py from a ConfigMap and pip
|
|
# installs on every start. A cold start therefore needs PyPI reachable, and takes
|
|
# about four minutes. Build the image in-cluster, or unproxy the registry host, before
|
|
# treating this as production.
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: clef-serve
|
|
namespace: llm-serving
|
|
labels:
|
|
app: clef-serve
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
replicas: 1
|
|
# Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
|
|
# the new pod Pending forever waiting for a card the old pod still holds.
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: clef-serve
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: clef-serve
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
nodeSelector:
|
|
kubernetes.io/hostname: worker-1
|
|
containers:
|
|
- name: clef
|
|
image: pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime
|
|
command: ["/bin/bash","-lc"]
|
|
args:
|
|
- |
|
|
set -e
|
|
pip install --no-cache-dir transformers==5.10.2 accelerate fastapi 'uvicorn[standard]' pillow safetensors sentencepiece protobuf
|
|
exec uvicorn app:create_app --factory --host 0.0.0.0 --port 8080
|
|
workingDir: /app
|
|
env:
|
|
- name: PYTHONUNBUFFERED
|
|
value: "1"
|
|
- name: MODEL_PATH
|
|
value: /mnt/models/clef-flash
|
|
# V100 has no bf16, and the release weights are bf16. fp16 is the same width.
|
|
- name: DTYPE
|
|
value: float16
|
|
# The state is truncated at this length without warning, so raise it only with
|
|
# spare VRAM: 17.75 GiB of weights plus 128 KiB of KV per token.
|
|
- name: MAX_LEN
|
|
value: "32768"
|
|
ports:
|
|
- containerPort: 8080
|
|
# A cold start pip-installs and loads 17.75 GiB, measured at about four minutes.
|
|
startupProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
failureThreshold: 120
|
|
periodSeconds: 10
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
periodSeconds: 10
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 12Gi
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "8"
|
|
memory: 40Gi
|
|
nvidia.com/gpu: "1"
|
|
volumeMounts:
|
|
- name: app
|
|
mountPath: /app
|
|
- name: models
|
|
mountPath: /mnt/models
|
|
readOnly: true
|
|
volumes:
|
|
- name: app
|
|
configMap:
|
|
name: clef-serve-app
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: clef-serve
|
|
namespace: llm-serving
|
|
labels:
|
|
app.kubernetes.io/part-of: llm-serving
|
|
spec:
|
|
selector:
|
|
app: clef-serve
|
|
ports:
|
|
- name: http
|
|
port: 80
|
|
targetPort: 8080
|