homelab/k8s/apps/llm-serving/clef-flash.yaml

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  annotations:
    serving.kserve.io/deploymentMode: RawDeployment
    # KServe injects the prometheus.io/{scrape,port,path} annotations into the
    # generated pod only when this flag is on. SGLang exposes engine metrics on
    # the same port as its API, so this is what makes them visible at all.
    serving.kserve.io/enable-prometheus-scraping: "true"
  labels:
    app.kubernetes.io/name: llm-clef-flash
    app.kubernetes.io/part-of: llm-serving
  name: clef-flash
  namespace: llm-serving
spec:
  predictor:
    containers:
    - args:
      # Clef-flash: Cloudflare's 9.4B decision model, Apache-2.0, self-hosted.
      #
      # This is NOT a chat model and it does not replace `reasoning`. It reads
      # a state plus a schema of typed questions and returns a probability per
      # allowed option in a single prefill pass. There is no decode step and no
      # free-form text output. Consumers call POST /v1/systemone, not
      # /v1/chat/completions.
      #
      # Why SGLang and not vLLM: Clef needs the separate joint_schema_head
      # weights (joint_head.safetensors) loaded on top of the Qwen3.5-9B
      # backbone, and it needs the /v1/systemone route. Only SGLang implements
      # these (upstream PR #42721, released in v0.5.22). vLLM has a qwen3_5
      # model class but would ignore the joint head and expose no systemone
      # endpoint, so it cannot serve this model correctly.
      #
      # BLOCKED ON HARDWARE. worker-1 holds Tesla V100-PCIE-32GB cards, compute
      # capability 7.0. The pinned image ships torch 2.14.1+cu130, whose kernels cover
      # CC 8.0 and above only:
      #
      #   Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0.
      #   Your installed torch==2.14.1+cu130 does not include kernels for this GPU.
      #
      # So the pod starts and then dies in the model loader. minReplicas is 0 until
      # either a build with sm_70 kernels exists or the model moves to newer hardware;
      # serving it on CPU is not viable for a 9.4B backbone.
      #
      # Image: the `dev-clef` tag is a pinned nightly because Clef support
      # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by
      # digest when it ships; do not leave a nightly tag in git long term.
      #
      # Context: 24576. This is deliberate and it is NOT raised to 128K.
      #   - 24576 is exactly what Cloudflare serves for clef-flash on Workers
      #     AI, so self-hosted and hosted answers match.
      #   - The SGLang cookbook defaults to 16384 and its published accuracy
      #     runs stop near 16K. Nothing public validates quality above that.
      #   - Cloudflare cut their hosted window from 64K to 24K on purpose and
      #     points long-context users at the 27B `clef` instead.
      #   - Latency, not memory, is the real cap: Clef has no decode, so cost
      #     scales with input tokens. On a V100 without flash attention, 128K
      #     would push a single decision into the tens of seconds, which
      #     defeats the point of a classifier.
      #
      # KV cache math, from config.json: 32 layers, but only 8 use full
      # attention (every 4th). 8 * 2 * 4 kv_heads * 256 head_dim * 2 B
      # = 32 KB/token. At 24576 tokens that is only 0.75 GiB. The other 24
      # layers use linear attention and hold a fixed 48 MiB state regardless
      # of length. Memory is therefore dominated by the BF16 weights (18.8 GB),
      # not the cache, which is why one V100-32GB is enough.
      # The image ships its CLI at /opt/sglang/bin/sglang and puts no `serve` on
      # PATH, so a bare `serve` failed with "exec: serve: not found" on every start
      # (19 restarts). Name the binary by its full path.
      - /opt/sglang/bin/sglang
      - serve
      - --model-path=Cloudflare/clef-flash
      - --host=0.0.0.0
      - --port=8080
      - --context-length=24576
      - --tp-size=1
      # Leave headroom for the vision encoder and the joint head. The model
      # reports 41 GB at a 64K window on H200 including engine overhead; at
      # 24K on a 32 GB card this keeps the static pool bounded.
      - --mem-fraction-static=0.80
      # Clef runs a single prefill and disables radix cache, chunked prefill
      # and CUDA graphs itself. Set them explicitly so a future default change
      # cannot silently re-enable them.
      - --disable-radix-cache
      # Triton is the only attention backend with a chance on sm70. FlashInfer
      # requires sm75+ and FA3 requires sm90. SGLang's install guide states the
      # pairing explicitly: on backends below sm75 you must set
      #     --attention-backend triton --sampling-backend pytorch
      # Both flags are required. Triton alone leaves the sampler on a FlashInfer
      # kernel, which fails on Volta.
      #
      # This is still the main untested assumption here: Cloudflare validated
      # the dev-clef image on H200, B200 and B300 only. SGLang upstream also
      # warns that Triton >=3.3.0 has a Dot() bug for compute capability < 80.
      # If the pod crashes, check the Triton version in the image first, then
      # fall back to the sglang-V100 fork, which ships purpose-built SM70
      # kernels.
      - --attention-backend=triton
      - --sampling-backend=pytorch
      env:
      - name: HF_HOME
        value: /mnt/models
      - name: HF_HUB_CACHE
        value: /mnt/models/hub
      image: lmsysorg/sglang:dev-clef
      name: kserve-container
      ports:
      - containerPort: 8080
        protocol: TCP
      readinessProbe:
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 10
      resources:
        limits:
          cpu: '16'
          memory: 32Gi
          nvidia.com/gpu: '1'
        requests:
          cpu: '8'
          # 8Gi, not 16Gi. worker-1 holds 64Gi and the other predictors already
          # request about 55Gi, so a 16Gi request left the pod Pending forever:
          # "1 Insufficient memory". The weights live on the GPU, so host memory is
          # for SGLang's own bookkeeping. The limit stays at 32Gi, so a burst that
          # needs more is allowed; only the scheduler reservation is smaller.
          memory: 8Gi
          nvidia.com/gpu: '1'
      startupProbe:
        # The image is large and the checkpoint is 18.8 GB on first pull.
        # 120 * 15s = 30 min ceiling for the initial download.
        failureThreshold: 120
        httpGet:
          path: /health
          port: 8080
        periodSeconds: 15
      volumeMounts:
      - mountPath: /mnt/models
        name: models
      - mountPath: /dev/shm
        name: shm
    # podMetadata is NOT valid on this KServe version; annotations and labels
    # sit directly under predictor. See the note in qwen3-moe.yaml.
    annotations:
      prometheus.io/scrape: "true"
      prometheus.io/port: "8080"
      prometheus.io/path: "/metrics"
    labels:
      app.kubernetes.io/name: llm-clef-flash
      app.kubernetes.io/part-of: llm-serving
    deploymentStrategy:
      type: Recreate
    maxReplicas: 1
    # 0 until the hardware or the build changes: see the hardware note above.
    minReplicas: 0
    nodeSelector:
      kubernetes.io/hostname: worker-1
    runtimeClassName: nvidia
    volumes:
    - name: models
      persistentVolumeClaim:
        claimName: llm-models
    - emptyDir:
        medium: Memory
        # SGLang needs a large shared-memory segment for the tokenizer and
        # message-queue processes. The upstream run command uses --shm-size 32g.
        sizeLimit: 8Gi
      name: shm