diff --git a/k8s/apps/llm-serving/coder.yaml b/k8s/apps/llm-serving/coder.yaml new file mode 100644 index 0000000..b4386e3 --- /dev/null +++ b/k8s/apps/llm-serving/coder.yaml @@ -0,0 +1,134 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: coder + namespace: llm-serving + labels: + app.kubernetes.io/name: llm-coder + app.kubernetes.io/part-of: llm-serving + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + # Two GPUs are free, so this rolls out beside `reasoning` instead of over it. + # Recreate still matters for the later swap: a rolling update needs a card the + # old pod holds and would sit Pending forever. + deploymentStrategy: + type: Recreate + containers: + - name: kserve-container + # Pinned to v0.11.0 on purpose. This is the last image measured to work on this + # node: torch 2.8.0+cu128, arch list sm_70..sm_120, driver 580.159.04. + # v0.15.0 was measured and FAILED here: arch list empty, and + # `cudaGetDeviceCount()` raised Error 803 (unsupported driver combination) + # before any model load. Do not raise this tag without re-running that probe. + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + # Qwen3-Coder-30B-A3B-Instruct: 30.5B total, 3.3B active. Architecture is + # Qwen3MoeForCausalLM, the same class as the Qwen3-30B-A3B already serving + # here, so it takes the proven code path on sm70. + # + # Qwen ships this model only as BF16 (61.1 GB) and FP8 (no FP8 on Volta). + # 4-bit GPTQ is the only form that fits one 32 GB card. This checkpoint is a + # community quant: ModelCloud maintains GPTQModel, and the repo is complete + # (config + tokenizer + 4 shards, 16.9 GB). Verified by loading, not by trust. + # + # Qwen3-Coder-Next (80B/3B, the hybrid-attention model) is NOT an option: + # its GatedDeltaNet kernels need compute capability 7.5+ and the `fla` module, + # and sm70 has neither. Measured, not assumed. + - --model=ModelCloud/Qwen3-Coder-30B-A3B-Instruct-GPTQMODEL-W4A16-A + # `reasoning` stays a served name so an existing client that sends + # model=reasoning keeps working after the DNS rename. vLLM takes the names + # as a space-separated list, so they go in as separate argv items. + - --served-model-name + - coder + - reasoning + - --quantization=gptq + - --dtype=float16 + - --kv-cache-dtype=auto + # Native window is 262144 and rope_scaling is null, so 256K needs no YaRN. + # One card cannot hold it: measured KV pool on a single V100 was 132,672 + # tokens. TP=2 halves the weights per card (7.8 GiB each), which frees room + # for the larger pool. Measured after boot: see the KV line in the log. + - --max-model-len=262144 + - --tensor-parallel-size=2 + - --pipeline-parallel-size=1 + - --gpu-memory-utilization=0.90 + # A single 262,144-token request consumes the whole pool, so keep few slots. + - --max-num-seqs=4 + - --max-num-batched-tokens=8192 + - --long-prefill-token-threshold=2048 + - --enable-chunked-prefill + - --enable-prefix-caching + # Measured: with the default (custom all-reduce ON) the engine finished CUDA graph + # capture in 6s at 17:01:56 and then went silent for 14+ minutes, emitting only + # "No available shared memory broadcast block found in 60 seconds". The front-end + # never received the ready message. vLLM's custom all-reduce registers its own + # CUDA graph addresses and deadlocks on this Volta/PCIe pair. NCCL is the fallback + # and is correct here, because TP=2 across two PCIe V100s gains nothing from the + # custom kernel anyway. + - --disable-custom-all-reduce + - --reasoning-parser=qwen3 + - --enable-auto-tool-choice + - --tool-call-parser=hermes + - --host=0.0.0.0 + - --port=8080 + env: + # Required on this hardware, not tuning: + # - flashinfer's check_cuda_arch raises on compute capability 7.0, + # so the sampler must stay off + # - FlashInfer and FlashAttention-2 need sm75+, so Triton serves attention + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + startupProbe: + httpGet: + path: /health + port: 8080 + # TP=2 at 256K needs ~10 min before the API binds: 22s weights, 80s torch.compile, + # 85s KV profiling, then graph capture. 120 x 15s = 30 min of headroom. + failureThreshold: 120 + periodSeconds: 15 + resources: + # worker-1 exposes only 63,495 MiB of allocatable memory, so a 24Gi request + # never schedules. Two vLLM workers keep the weights on the GPU and need + # little host RAM, so the request stays small and the limit allows burst. + requests: + cpu: "8" + memory: 12Gi + nvidia.com/gpu: "2" + limits: + cpu: "16" + memory: 40Gi + nvidia.com/gpu: "2" + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + # NCCL all-reduce over TP=2 moves through /dev/shm. The 64Mi default is + # too small and shows up as an init hang or a bus error. + sizeLimit: 8Gi diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index 8f0fb81..801ae29 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -14,7 +14,14 @@ resources: # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100. - clef-serve.yaml + # coder = Qwen3-Coder-30B-A3B-Instruct, 4-bit GPTQ, TP=2 at 262,144 tokens. It serves + # both `coder` and `reasoning`, so poimen-memory needs a host change only, never a + # model-name change. reasoning.yaml below can be dropped once that host change lands. + - coder.yaml - reasoning.yaml + # Stable client-facing name for the chat model. poimen-memory points at this Service, + # so the vLLM -> candle-vllm handoff is a selector edit here and no client change. + - reasoning-long-service.yaml # ornith is retired: its InferenceService still reports Ready=True while running zero # replicas, because desired equals available at 0. Its manifest was never committed, # so it is left unmanaged and is not listed here. Delete the CR when convenient. diff --git a/k8s/apps/llm-serving/networkpolicy.yaml b/k8s/apps/llm-serving/networkpolicy.yaml index d403e60..17a1ecd 100644 --- a/k8s/apps/llm-serving/networkpolicy.yaml +++ b/k8s/apps/llm-serving/networkpolicy.yaml @@ -66,6 +66,21 @@ spec: port: 80 - protocol: TCP port: 8000 + # Allow from poimen namespace (poimen-memory ingest + query). + # The client-side egress rule is the other half and must stay in step: + # NetworkPolicy poimen/poimen-memory-llm-egress permits port 8080 to pods + # labelled part-of=llm-serving, which is the same selector used here. This + # policy had no matching ingress rule at all, so the packet was allowed out + # and dropped here. The client then saw a bare timeout and the predictor log + # stayed empty, which is why this failure looked like a hung inference engine + # for 34 days. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: poimen + ports: + - protocol: TCP + port: 8080 # Allow intra-namespace (pod-to-pod within llm-serving) - from: - podSelector: diff --git a/k8s/apps/llm-serving/reasoning-long-service.yaml b/k8s/apps/llm-serving/reasoning-long-service.yaml new file mode 100644 index 0000000..2b05330 --- /dev/null +++ b/k8s/apps/llm-serving/reasoning-long-service.yaml @@ -0,0 +1,43 @@ +# Stable client-facing name for the chat model, deliberately decoupled from the +# process that currently serves it. +# +# poimen-memory addresses this Service, not a pod and not an InferenceService. That +# means the vLLM -> candle-vllm handoff changes only this file's selector. The client +# config never changes again, so the ingest cutover has no window in which the host +# name is wrong. +# +# Two ports on purpose: +# - 80 matches every other predictor Service in this namespace, and matches +# poimen-memory's LLM_CHAT_BASE, which carries no port number. +# - 8080 is the container port. It is also the only port llm-serving-ingress and +# poimen-memory-llm-egress permit, and the port the handoff readiness check +# uses. Publishing it directly avoids a port translation that is easy to get +# wrong in a curl. +# +# The selector names the vLLM reasoning pod today. To hand off, replace the three +# selector labels with candle-vllm's pod labels and nothing else. Do not delete and +# recreate this Service: a delete drops the ClusterIP, and a client that cached the +# name during the gap will fail until it re-resolves. +apiVersion: v1 +kind: Service +metadata: + name: reasoning-long-predictor + namespace: llm-serving + labels: + app.kubernetes.io/part-of: llm-serving + app: reasoning-long-predictor +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: llm-reasoning + component: predictor + serving.kserve.io/inferenceservice: reasoning + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 8080 + - name: http-8080 + protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml index 77f5203..de87548 100644 --- a/k8s/apps/llm-serving/reasoning.yaml +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -30,7 +30,28 @@ spec: # Measured on this cluster, one V100-32GB, 64k context: 15.55 GiB of weights, # 137k tokens of KV cache, 2.09x concurrency at 65536 tokens per request. - --model=Qwen/Qwen3-30B-A3B-GPTQ-Int4 - - --served-model-name=reasoning + # Two served names, one pod, on purpose. `reasoning-long` is the name the goal map + # uses for this model, and the name qwen-finetune wiring targets. It is NOT a + # separate InferenceService: this model already occupies the only free card, and a + # second pod on the same weights cannot schedule (4 of 4 GPUs allocated: coder 2, + # this 1, clef-serve 1). Serving both names from one process costs nothing and + # keeps the two callers working without a second weights copy. + # + # `reasoning` is deliberately still served here. poimen-memory pins + # LLM_CHAT_BASE=reasoning-predictor, so dropping the name here breaks ingest + # immediately. coder-predictor also serves `reasoning`; that ambiguity is known + # and documented in the PR. Resolve it by moving the client host, not by + # removing the name here. + # + # The two names are separate list entries, NOT one comma-joined value. This + # image's parser returns ['reasoning,reasoning-long'] for the comma form, i.e. a + # single model whose name contains a comma, and api_server.py then iterates + # args.served_model_name to build the model list. Verified against vLLM 0.11.0: + # FlexibleArgumentParser().parse_args([... "--served-model-name", "a", "b"]) + # -> ['a', 'b'] + - --served-model-name + - reasoning + - reasoning-long - --quantization=gptq - --dtype=float16 - --kv-cache-dtype=auto