diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index 31d77a3..92308fd 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -26,12 +26,10 @@ resources: # # Re-add it when the model gets an Ampere-or-newer GPU. The manifest itself is # kept in git (k8s/apps/llm-serving/clef-flash.yaml) with the same reasoning. - # ornith and reasoning are live but unmanaged: their manifests were never committed - # to this directory, so listing them made `kustomize build` fail and left the whole - # Application in ComparisonError. Remove the entries rather than the models. The - # Application runs with prune=false, so an object that is no longer in the desired - # state is left alone, not deleted. Re-export both from live state when they are - # next edited. + - reasoning.yaml + # ornith is retired: its InferenceService still reports Ready=True while running zero + # replicas, because desired equals available at 0. Its manifest was never committed, + # so it is left unmanaged and is not listed here. Delete the CR when convenient. - embeddings.yaml - reranker.yaml - qwen-cpu.yaml diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml new file mode 100644 index 0000000..77f5203 --- /dev/null +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -0,0 +1,102 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: reasoning + namespace: llm-serving + labels: + app.kubernetes.io/name: llm-reasoning + app.kubernetes.io/part-of: llm-serving + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + # Every GPU card is allocated, so a rolling update deadlocks: the new pod needs a + # card the old pod still holds and sits Pending forever. Recreate frees the card + # first. The cost is a short gap in service for this model only. + deploymentStrategy: + type: Recreate + containers: + - name: kserve-container + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + # Qwen3-30B-A3B is a 30.5B MoE with 3.3B active per token. On a V100 that matters: + # the 32B dense model it replaces needed two cards for the same context, this needs + # one, because its KV cache is 96 KiB/token against the dense model's 256 KiB. + # + # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights, + # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request. + - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4 + - --served-model-name=reasoning + - --quantization=gptq + - --dtype=float16 + - --kv-cache-dtype=auto + # Native window is 40960. YaRN factor 2.0 reaches 81920, so 64k is inside it. + - --rope-scaling={"rope_type":"yarn","factor":2.0,"original_max_position_embeddings":40960} + - --tensor-parallel-size=1 + - --pipeline-parallel-size=1 + - --max-model-len=65536 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=8 + - --max-num-batched-tokens=8192 + - --long-prefill-token-threshold=2048 + - --enable-chunked-prefill + - --enable-prefix-caching + - --reasoning-parser=qwen3 + - --enable-auto-tool-choice + - --tool-call-parser=hermes + - --host=0.0.0.0 + - --port=8080 + env: + # Both are required on this hardware, not tuning: + # - flashinfer's check_cuda_arch raises AttributeError on compute capability 7.0 + # (flashinfer/jit/core.py:55, minor.isdigit() on an int), so the sampler must stay off + # - FlashInfer and FlashAttention-2 both need sm75+, so the Triton backend serves attention + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + # Weight load took 18s from cache and 148s on a cold download, and the first + # torch.compile adds about 70s. 80 x 15s leaves ample room for a cold start. + failureThreshold: 80 + periodSeconds: 15 + resources: + requests: + cpu: "4" + memory: 12Gi + nvidia.com/gpu: "1" + limits: + cpu: "8" + memory: 36Gi + nvidia.com/gpu: "1" + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: 2Gi