From 15a22abb1f076c69860ec627834e0942c275da54 Mon Sep 17 00:00:00 2001 From: rock Date: Wed, 28 Oct 2026 02:41:47 -0700 Subject: [PATCH] fix(llm-serving): clef-flash cannot run on a V100 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI path was wrong and is now correct, so the pod reaches the model loader — and dies there: Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0. Your installed torch==2.14.1+cu130 does not include kernels for this GPU. Every GPU on worker-1 is a V100 (CC 7.0) and the pinned dev-clef image ships kernels for CC 8.0 and above. Scale to zero so the pod stops crash-looping, and record the reason in the manifest. Serving a 9.4B backbone on CPU is not a substitute. --- k8s/apps/llm-serving/clef-flash.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/k8s/apps/llm-serving/clef-flash.yaml b/k8s/apps/llm-serving/clef-flash.yaml index 8a71e23..7caf344 100644 --- a/k8s/apps/llm-serving/clef-flash.yaml +++ b/k8s/apps/llm-serving/clef-flash.yaml @@ -31,6 +31,17 @@ spec: # model class but would ignore the joint head and expose no systemone # endpoint, so it cannot serve this model correctly. # + # BLOCKED ON HARDWARE. worker-1 holds Tesla V100-PCIE-32GB cards, compute + # capability 7.0. The pinned image ships torch 2.14.1+cu130, whose kernels cover + # CC 8.0 and above only: + # + # Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0. + # Your installed torch==2.14.1+cu130 does not include kernels for this GPU. + # + # So the pod starts and then dies in the model loader. minReplicas is 0 until + # either a build with sm_70 kernels exists or the model moves to newer hardware; + # serving it on CPU is not viable for a 9.4B backbone. + # # Image: the `dev-clef` tag is a pinned nightly because Clef support # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by # digest when it ships; do not leave a nightly tag in git long term. @@ -140,7 +151,8 @@ spec: deploymentStrategy: type: Recreate maxReplicas: 1 - minReplicas: 1 + # 0 until the hardware or the build changes: see the hardware note above. + minReplicas: 0 nodeSelector: kubernetes.io/hostname: worker-1 runtimeClassName: nvidia