homelab/k8s/apps/llm-serving/clef-serve.yaml

# clef-flash: a decision model, not a chat model.
#
# It reads a `state` plus a schema of typed `questions` and returns one probability
# per allowed option in a single prefill pass. There is no decoding, no sampling and
# no free-form text. The public route is /v1/systemone, and its request and response
# shapes are Clef's own: no `messages`, no `choices`.
#
# Why not SGLang: Clef support landed in SGLang v0.5.22, which upstream publishes only
# on CUDA 13. CUDA 13 dropped Volta kernels and every card here is a Tesla V100
# (compute capability 7.0), so SGLang aborts with "no kernel image is available for
# execution on the device". SGLang also states sm75 as its floor regardless.
#
# So this runs the implementation the model ships: joint_schema_model.py, which has
# load_release_model, encode_record, collate_records and systemone. The weights are
# 17.75 GiB of BF16, and fp16 costs the same bytes per parameter. On one 32 GiB card
# that leaves room for a 32k state, which is why this is one GPU and not two: with
# device_map over two cards the embedding table and the tensors indexing into it land
# on different devices, and the forward pass dies with
# "indices should be either on cpu or on the same device as the indexed tensor".
#
# Known gap: the image cannot be pushed to the registry. Cloudflare fronts
# forgejo.riotpiao.com and answers 413 Payload Too Large for any layer above 100 MB,
# and the torch base alone is 7.58 GB. So this mounts app.py from a ConfigMap and pip
# installs on every start. A cold start therefore needs PyPI reachable, and takes
# about four minutes. Build the image in-cluster, or unproxy the registry host, before
# treating this as production.
apiVersion: apps/v1
kind: Deployment
metadata:
  name: clef-serve
  namespace: llm-serving
  labels:
    app: clef-serve
    app.kubernetes.io/part-of: llm-serving
spec:
  # One card. worker-1 holds four V100s: candle-vllm takes one for the coder, and
  # this pod takes one, so two stay free for a TP=2 candle run.
  #
  # Enabled 2026-11-04 to serve /v1/systemone, which the gateway already routes to
  # this Service. It was parked at 0 while candle-vllm was brought up on the same
  # cards, and it served no inference request between 2026-10-30T06:23:40Z and then.
  #
  # Cold start is about four minutes: the pod pip-installs on every boot because the
  # torch base image cannot be pushed to the registry (Cloudflare answers 413 above
  # 100 MB per layer). startupProbe allows for that.
  replicas: 1
  # Recreate, not RollingUpdate. Every GPU card is allocated, so a rolling update leaves
  # the new pod Pending forever waiting for a card the old pod still holds.
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: clef-serve
  template:
    metadata:
      labels:
        app: clef-serve
        app.kubernetes.io/part-of: llm-serving
    spec:
      runtimeClassName: nvidia
      nodeSelector:
        kubernetes.io/hostname: worker-1
      containers:
      - name: clef
        image: pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime
        command: ["/bin/bash","-lc"]
        args:
        - |
          set -e
          pip install --no-cache-dir transformers==5.10.2 accelerate fastapi 'uvicorn[standard]' pillow safetensors sentencepiece protobuf
          exec uvicorn app:create_app --factory --host 0.0.0.0 --port 8080
        workingDir: /app
        env:
        - name: PYTHONUNBUFFERED
          value: "1"
        - name: MODEL_PATH
          value: /mnt/models/clef-flash
        # V100 has no bf16, and the release weights are bf16. fp16 is the same width.
        - name: DTYPE
          value: float16
        # The state is truncated at this length without warning, so raise it only with
        # spare VRAM: 17.75 GiB of weights plus 128 KiB of KV per token.
        - name: MAX_LEN
          value: "32768"
        ports:
        - containerPort: 8080
        # A cold start pip-installs and loads 17.75 GiB, measured at about four minutes.
        startupProbe:
          httpGet:
            path: /health
            port: 8080
          failureThreshold: 120
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 10
        resources:
          requests:
            cpu: "4"
            memory: 12Gi
            nvidia.com/gpu: "1"
          limits:
            cpu: "8"
            memory: 40Gi
            nvidia.com/gpu: "1"
        volumeMounts:
        - name: app
          mountPath: /app
        - name: models
          mountPath: /mnt/models
          readOnly: true
      volumes:
      - name: app
        configMap:
          name: clef-serve-app
      - name: models
        persistentVolumeClaim:
          claimName: llm-models
---
apiVersion: v1
kind: Service
metadata:
  name: clef-serve
  namespace: llm-serving
  labels:
    app.kubernetes.io/part-of: llm-serving
spec:
  # Points at the candle-vllm implementation (clef-candle.yaml). Switching back
  # to the reference pytorch server is this line plus that Deployment's replicas;
  # no client changes, because the Service name and ClusterIP do not move.
  selector:
    app: clef-candle
  ports:
  - name: http
    port: 80
    targetPort: 8080