diff --git a/k8s/apps/llm-serving/coder-14b-fp16.yaml b/k8s/apps/llm-serving/coder-14b-fp16.yaml index 69be1c4..f8f1f4a 100644 --- a/k8s/apps/llm-serving/coder-14b-fp16.yaml +++ b/k8s/apps/llm-serving/coder-14b-fp16.yaml @@ -60,6 +60,10 @@ spec: app.kubernetes.io/part-of: llm-serving app: candle-vllm spec: + # Required. Without this the container runs under runc, libcuda.so is never + # injected, and candle-vllm panics in cudarc at startup with + # "Unable to dynamically load the cuda shared library". + runtimeClassName: nvidia containers: - name: candle image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim