apiVersion: serving.kserve.io/v1beta1 kind: InferenceService metadata: annotations: serving.kserve.io/deploymentMode: RawDeployment # KServe injects the prometheus.io/{scrape,port,path} annotations into the # generated pod only when this flag is on. SGLang exposes engine metrics on # the same port as its API, so this is what makes them visible at all. serving.kserve.io/enable-prometheus-scraping: "true" labels: app.kubernetes.io/name: llm-clef-flash app.kubernetes.io/part-of: llm-serving name: clef-flash namespace: llm-serving spec: predictor: containers: - args: # Clef-flash: Cloudflare's 9.4B decision model, Apache-2.0, self-hosted. # # This is NOT a chat model and it does not replace `reasoning`. It reads # a state plus a schema of typed questions and returns a probability per # allowed option in a single prefill pass. There is no decode step and no # free-form text output. Consumers call POST /v1/systemone, not # /v1/chat/completions. # # Why SGLang and not vLLM: Clef needs the separate joint_schema_head # weights (joint_head.safetensors) loaded on top of the Qwen3.5-9B # backbone, and it needs the /v1/systemone route. Only SGLang implements # these (upstream PR #42721, released in v0.5.22). vLLM has a qwen3_5 # model class but would ignore the joint head and expose no systemone # endpoint, so it cannot serve this model correctly. # # Image: the `dev-clef` tag is a pinned nightly because Clef support # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by # digest when it ships; do not leave a nightly tag in git long term. # # Context: 24576. This is deliberate and it is NOT raised to 128K. # - 24576 is exactly what Cloudflare serves for clef-flash on Workers # AI, so self-hosted and hosted answers match. # - The SGLang cookbook defaults to 16384 and its published accuracy # runs stop near 16K. Nothing public validates quality above that. # - Cloudflare cut their hosted window from 64K to 24K on purpose and # points long-context users at the 27B `clef` instead. # - Latency, not memory, is the real cap: Clef has no decode, so cost # scales with input tokens. On a V100 without flash attention, 128K # would push a single decision into the tens of seconds, which # defeats the point of a classifier. # # KV cache math, from config.json: 32 layers, but only 8 use full # attention (every 4th). 8 * 2 * 4 kv_heads * 256 head_dim * 2 B # = 32 KB/token. At 24576 tokens that is only 0.75 GiB. The other 24 # layers use linear attention and hold a fixed 48 MiB state regardless # of length. Memory is therefore dominated by the BF16 weights (18.8 GB), # not the cache, which is why one V100-32GB is enough. - serve - --model-path=Cloudflare/clef-flash - --host=0.0.0.0 - --port=8080 - --context-length=24576 - --tp-size=1 # Leave headroom for the vision encoder and the joint head. The model # reports 41 GB at a 64K window on H200 including engine overhead; at # 24K on a 32 GB card this keeps the static pool bounded. - --mem-fraction-static=0.80 # Clef runs a single prefill and disables radix cache, chunked prefill # and CUDA graphs itself. Set them explicitly so a future default change # cannot silently re-enable them. - --disable-radix-cache # Triton is the only attention backend with a chance on sm70. FlashInfer # requires sm75+ and FA3 requires sm90. SGLang's install guide states the # pairing explicitly: on backends below sm75 you must set # --attention-backend triton --sampling-backend pytorch # Both flags are required. Triton alone leaves the sampler on a FlashInfer # kernel, which fails on Volta. # # This is still the main untested assumption here: Cloudflare validated # the dev-clef image on H200, B200 and B300 only. SGLang upstream also # warns that Triton >=3.3.0 has a Dot() bug for compute capability < 80. # If the pod crashes, check the Triton version in the image first, then # fall back to the sglang-V100 fork, which ships purpose-built SM70 # kernels. - --attention-backend=triton - --sampling-backend=pytorch env: - name: HF_HOME value: /mnt/models - name: HF_HUB_CACHE value: /mnt/models/hub image: lmsysorg/sglang:dev-clef name: kserve-container ports: - containerPort: 8080 protocol: TCP readinessProbe: httpGet: path: /health port: 8080 periodSeconds: 10 resources: limits: cpu: '16' memory: 32Gi nvidia.com/gpu: '1' requests: cpu: '8' # 8Gi, not 16Gi. worker-1 holds 64Gi and the other predictors already # request about 55Gi, so a 16Gi request left the pod Pending forever: # "1 Insufficient memory". The weights live on the GPU, so host memory is # for SGLang's own bookkeeping. The limit stays at 32Gi, so a burst that # needs more is allowed; only the scheduler reservation is smaller. memory: 8Gi nvidia.com/gpu: '1' startupProbe: # The image is large and the checkpoint is 18.8 GB on first pull. # 120 * 15s = 30 min ceiling for the initial download. failureThreshold: 120 httpGet: path: /health port: 8080 periodSeconds: 15 volumeMounts: - mountPath: /mnt/models name: models - mountPath: /dev/shm name: shm # podMetadata is NOT valid on this KServe version; annotations and labels # sit directly under predictor. See the note in qwen3-moe.yaml. annotations: prometheus.io/scrape: "true" prometheus.io/port: "8080" prometheus.io/path: "/metrics" labels: app.kubernetes.io/name: llm-clef-flash app.kubernetes.io/part-of: llm-serving deploymentStrategy: type: Recreate maxReplicas: 1 minReplicas: 1 nodeSelector: kubernetes.io/hostname: worker-1 runtimeClassName: nvidia volumes: - name: models persistentVolumeClaim: claimName: llm-models - emptyDir: medium: Memory # SGLang needs a large shared-memory segment for the tokenizer and # message-queue processes. The upstream run command uses --shm-size 32g. sizeLimit: 8Gi name: shm