diff --git a/k8s/apps/llm-serving/clef-flash.yaml b/k8s/apps/llm-serving/clef-flash.yaml index 8a71e23..7caf344 100644 --- a/k8s/apps/llm-serving/clef-flash.yaml +++ b/k8s/apps/llm-serving/clef-flash.yaml @@ -31,6 +31,17 @@ spec: # model class but would ignore the joint head and expose no systemone # endpoint, so it cannot serve this model correctly. # + # BLOCKED ON HARDWARE. worker-1 holds Tesla V100-PCIE-32GB cards, compute + # capability 7.0. The pinned image ships torch 2.14.1+cu130, whose kernels cover + # CC 8.0 and above only: + # + # Found GPU0 Tesla V100-PCIE-32GB which is of cuda capability 7.0. + # Your installed torch==2.14.1+cu130 does not include kernels for this GPU. + # + # So the pod starts and then dies in the model loader. minReplicas is 0 until + # either a build with sm_70 kernels exists or the model moves to newer hardware; + # serving it on CPU is not viable for a 9.4B backbone. + # # Image: the `dev-clef` tag is a pinned nightly because Clef support # landed after v0.5.21. Move to the official v0.5.22+ tag and re-pin by # digest when it ships; do not leave a nightly tag in git long term. @@ -140,7 +151,8 @@ spec: deploymentStrategy: type: Recreate maxReplicas: 1 - minReplicas: 1 + # 0 until the hardware or the build changes: see the hardware note above. + minReplicas: 0 nodeSelector: kubernetes.io/hostname: worker-1 runtimeClassName: nvidia