Story Crater Bot
5a8fc1885a
deepseek_v3 400s on this checkpoint: "could not locate tool call start/end tokens in the tokenizer". unsloth/DeepSeek-R1-Distill-Qwen-32B is a Qwen2.5 base distilled on R1 reasoning traces -- it kept R1's <think> format but never got DeepSeek-V3's own special tool-call tokens registered in its tokenizer. hermes parses from text patterns instead of special tokens, so it works against the underlying Qwen tokenizer.
96 lines
2.9 KiB
YAML
96 lines
2.9 KiB
YAML
apiVersion: serving.kserve.io/v1beta1
|
|
kind: InferenceService
|
|
metadata:
|
|
annotations:
|
|
serving.kserve.io/deploymentMode: RawDeployment
|
|
labels:
|
|
app.kubernetes.io/name: llm-reasoning
|
|
app.kubernetes.io/part-of: llm-serving
|
|
name: reasoning
|
|
namespace: llm-serving
|
|
spec:
|
|
predictor:
|
|
containers:
|
|
- args:
|
|
- --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
|
|
- --served-model-name=reasoning
|
|
- --quantization=bitsandbytes
|
|
- --dtype=float16
|
|
- --kv-cache-dtype=auto
|
|
- --tensor-parallel-size=1
|
|
- --max-model-len=16384
|
|
- --gpu-memory-utilization=0.90
|
|
- --max-num-seqs=4
|
|
- --enable-chunked-prefill
|
|
- --enable-prefix-caching
|
|
- --reasoning-parser=deepseek_r1
|
|
# Without these, any client sending tool_choice="auto" (pi does, for
|
|
# Read/Bash/etc.) gets a 400: "auto" tool choice requires
|
|
# --enable-auto-tool-choice and --tool-call-parser to be set.
|
|
# deepseek_v3 (matching --reasoning-parser above) 400s here --
|
|
# "DeepSeek-V3 Tool parser could not locate tool call start/end
|
|
# tokens in the tokenizer" -- this checkpoint is a Qwen2.5-32B base
|
|
# distilled on R1 traces, so its tokenizer never got DeepSeek-V3's
|
|
# own special tool-call tokens registered even though it kept R1's
|
|
# <think> reasoning format. hermes parses tool calls from plain text
|
|
# patterns instead of special tokens, so it works against the
|
|
# underlying Qwen tokenizer regardless. Verified live: deepseek_v3
|
|
# 400s, hermes returns a real tool_calls response.
|
|
- --enable-auto-tool-choice
|
|
- --tool-call-parser=hermes
|
|
- --host=0.0.0.0
|
|
- --port=8080
|
|
env:
|
|
- name: VLLM_USE_FLASHINFER_SAMPLER
|
|
value: '0'
|
|
- name: VLLM_ATTENTION_BACKEND
|
|
value: TRITON_ATTN
|
|
- name: HF_HOME
|
|
value: /mnt/models
|
|
image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
|
|
name: kserve-container
|
|
ports:
|
|
- containerPort: 8080
|
|
protocol: TCP
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
periodSeconds: 10
|
|
resources:
|
|
limits:
|
|
cpu: '16'
|
|
memory: 16Gi
|
|
nvidia.com/gpu: '1'
|
|
requests:
|
|
cpu: '8'
|
|
memory: 8Gi
|
|
nvidia.com/gpu: '1'
|
|
startupProbe:
|
|
failureThreshold: 80
|
|
httpGet:
|
|
path: /health
|
|
port: 8080
|
|
periodSeconds: 15
|
|
volumeMounts:
|
|
- mountPath: /mnt/models
|
|
name: models
|
|
- mountPath: /dev/shm
|
|
name: shm
|
|
deploymentStrategy:
|
|
type: Recreate
|
|
maxReplicas: 2
|
|
minReplicas: 2
|
|
nodeSelector:
|
|
kubernetes.io/hostname: worker-1
|
|
runtimeClassName: nvidia
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim:
|
|
claimName: llm-models
|
|
- emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 2Gi
|
|
name: shm
|
|
|