diff --git a/k8s/apps/llm-serving/clef-flash.yaml b/k8s/apps/llm-serving/clef-flash.yaml new file mode 100644 index 0000000..4464506 --- /dev/null +++ b/k8s/apps/llm-serving/clef-flash.yaml @@ -0,0 +1,147 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + # KServe injects the prometheus.io/{scrape,port,path} annotations into the + # generated pod only when this flag is on. SGLang exposes engine metrics on + # the same port as its API, so this is what makes them visible at all. + serving.kserve.io/enable-prometheus-scraping: "true" + labels: + app.kubernetes.io/name: llm-clef-flash + app.kubernetes.io/part-of: llm-serving + name: clef-flash + namespace: llm-serving +spec: + predictor: + containers: + - args: + # Clef-flash: Cloudflare's 9.4B decision model, Apache-2.0, self-hosted. + # + # This is NOT a chat model and it does not replace `reasoning`. It reads + # a state plus a schema of typed questions and returns a probability per + # allowed option in a single prefill pass. There is no decode step and no + # free-form text output. Consumers call POST /v1/systemone, not + # /v1/chat/completions. + # + # Why SGLang and not vLLM: Clef needs the separate joint_schema_head + # weights (joint_head.safetensors) loaded on top of the Qwen3.5-9B + # backbone, and it needs the /v1/systemone route. Only SGLang implements + # these (upstream PR #42721, released in v0.5.22). vLLM has a qwen3_5 + # model class but would ignore the joint head and expose no systemone + # endpoint, so it cannot serve this model correctly. + # + # Image: the `dev-clef` tag is a pinned nightly because Clef support + # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by + # digest when it ships; do not leave a nightly tag in git long term. + # + # Context: 24576. This is deliberate and it is NOT raised to 128K. + # - 24576 is exactly what Cloudflare serves for clef-flash on Workers + # AI, so self-hosted and hosted answers match. + # - The SGLang cookbook defaults to 16384 and its published accuracy + # runs stop near 16K. Nothing public validates quality above that. + # - Cloudflare cut their hosted window from 64K to 24K on purpose and + # points long-context users at the 27B `clef` instead. + # - Latency, not memory, is the real cap: Clef has no decode, so cost + # scales with input tokens. On a V100 without flash attention, 128K + # would push a single decision into the tens of seconds, which + # defeats the point of a classifier. + # + # KV cache math, from config.json: 32 layers, but only 8 use full + # attention (every 4th). 8 * 2 * 4 kv_heads * 256 head_dim * 2 B + # = 32 KB/token. At 24576 tokens that is only 0.75 GiB. The other 24 + # layers use linear attention and hold a fixed 48 MiB state regardless + # of length. Memory is therefore dominated by the BF16 weights (18.8 GB), + # not the cache, which is why one V100-32GB is enough. + - serve + - --model-path=Cloudflare/clef-flash + - --host=0.0.0.0 + - --port=8080 + - --context-length=24576 + - --tp-size=1 + # Leave headroom for the vision encoder and the joint head. The model + # reports 41 GB at a 64K window on H200 including engine overhead; at + # 24K on a 32 GB card this keeps the static pool bounded. + - --mem-fraction-static=0.80 + # Clef runs a single prefill and disables radix cache, chunked prefill + # and CUDA graphs itself. Set them explicitly so a future default change + # cannot silently re-enable them. + - --disable-radix-cache + # Triton is the only attention backend with a chance on sm70. FlashInfer + # requires sm75+ and FA3 requires sm90. SGLang's install guide states the + # pairing explicitly: on backends below sm75 you must set + # --attention-backend triton --sampling-backend pytorch + # Both flags are required. Triton alone leaves the sampler on a FlashInfer + # kernel, which fails on Volta. + # + # This is still the main untested assumption here: Cloudflare validated + # the dev-clef image on H200, B200 and B300 only. SGLang upstream also + # warns that Triton >=3.3.0 has a Dot() bug for compute capability < 80. + # If the pod crashes, check the Triton version in the image first, then + # fall back to the sglang-V100 fork, which ships purpose-built SM70 + # kernels. + - --attention-backend=triton + - --sampling-backend=pytorch + env: + - name: HF_HOME + value: /mnt/models + - name: HF_HUB_CACHE + value: /mnt/models/hub + image: lmsysorg/sglang:dev-clef + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 32Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 16Gi + nvidia.com/gpu: '1' + startupProbe: + # The image is large and the checkpoint is 18.8 GB on first pull. + # 120 * 15s = 30 min ceiling for the initial download. + failureThreshold: 120 + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + # podMetadata is NOT valid on this KServe version; annotations and labels + # sit directly under predictor. See the note in qwen3-moe.yaml. + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + prometheus.io/path: "/metrics" + labels: + app.kubernetes.io/name: llm-clef-flash + app.kubernetes.io/part-of: llm-serving + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - emptyDir: + medium: Memory + # SGLang needs a large shared-memory segment for the tokenizer and + # message-queue processes. The upstream run command uses --shm-size 32g. + sizeLimit: 8Gi + name: shm diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index e7da0a8..d6f4952 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -10,6 +10,7 @@ kind: Kustomization # way after any edit here: a GPU predictor restart is a weights reload measured # in tens of seconds, not a rolling update. resources: + - clef-flash.yaml - embeddings.yaml - ornith.yaml - reasoning.yaml