diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index d33a568..5fde4ae 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -52,6 +52,13 @@ spec: # for this specific GPTQ-Int4 release -- not hermes. - --enable-auto-tool-choice - --tool-call-parser=qwen3_coder + # skip CUDA graph capture / torch.compile -- this arch's custom ops + # (mamba_mixer2, gdn_attention_core) are compiling for the first time + # ever on this hardware with no cache, and startupProbe kept killing + # the pod mid-compile every ~20min before it could finish. Trade some + # runtime throughput for a startup that actually completes; revisit + # once this is confirmed working end to end. + - --enforce-eager - --host=0.0.0.0 - --port=8080 env: @@ -81,7 +88,7 @@ spec: memory: 12Gi nvidia.com/gpu: '2' startupProbe: - failureThreshold: 80 + failureThreshold: 240 httpGet: path: /health port: 8080