2 changed files with 111 additions and 0 deletions
@@ -0,0 +1,102 @@
+# Qwen2.5-Coder-14B-Instruct, fp16, tensor-parallel across two V100s.
+#
+# Why this model and this precision. The previous workload was Qwen3-Coder-30B-A3B
+# in GPTQ W4A16 on one V100. Profiling that deployment showed SM utilisation at
+# 98% while the memory controller sat at 19%: the V100 has FP16 tensor cores and
+# no INT4 ones, so every decode step ran a dequantisation pass on the SM. That is
+# a hardware-generation mismatch, not a tunable. Sequence throughput matches the
+# arithmetic: at batch 1 the step cost equalled a full-model read (17.6 ms against
+# 15 GB / 900 GB/s = 16.7 ms) while the routed experts alone are ~1.5 GB.
+#
+# Measured on this hardware, same harness, same 16-stream load:
+#
+#   GPTQ W4A16 30B-A3B, TP=1  : ITL 71.5 ms, 215 tok/s aggregate
+#   fp16 14B, TP=2            : ITL 39.7 ms, 287 tok/s aggregate
+#
+# The dense model degrades only 1.4x from 1 to 16 streams (27.9 -> 39.7 ms) while
+# the MoE degrades 4.1x (17.6 -> 71.5 ms), because a dense model reads its weights
+# once per step and amortises across the batch. The MoE pays expert weight traffic
+# per sequence. Single-stream latency is the one case where the MoE still wins
+# (17.6 ms against 27.9 ms), since it activates ~3B parameters per token.
+#
+# TP=2 notes that cost time to establish, so they are recorded here:
+#   - Sharding needs no code change. Every dimension divides by 2: attention heads
+#     40 -> 20, KV heads 8 -> 4, intermediate_size 13824 -> 6912, vocab
+#     152064 -> 76032. Verify this first for any replacement model.
+#   - TP=2 on a GPTQ checkpoint is what crashed (CUDA_ERROR_ILLEGAL_ADDRESS in
+#     forward()) on the previous model. The cause was quantised weight sharding
+#     and the g_idx guard in src/openai/models/linear.rs, not TP itself. A dense
+#     fp16 checkpoint has no g_idx, no packed weights and no scales, so it takes
+#     the plain column/row-parallel path and the crash does not reproduce.
+#   - These are V100-PCIE cards with no NVLink, so every layer's all-reduce
+#     crosses PCIe. It shows up in TTFT (275 ms at 1 stream, 1090 ms at 16)
+#     more than in ITL.
+#   - KV capacity is 11635 MB per rank, about 59K context tokens. The fp16
+#     weights consume the room the KV pool would otherwise use. This is well
+#     short of the 256K the previous model targeted, and it is the price of
+#     leaving the dequantisation path.
+#
+# Weights must be in the PVC as /mnt/models/coder-14b-fp16 (28 GB, 6 shards).
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: candle-vllm
+  namespace: llm-serving
+  labels:
+    app.kubernetes.io/part-of: llm-serving
+    app: candle-vllm
+spec:
+  replicas: 1
+  strategy:
+    # One replica holds both GPUs, so a rolling update would need a second pair.
+    # Recreate keeps the rollout to a single weights load.
+    type: Recreate
+  selector:
+    matchLabels:
+      app: candle-vllm
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/part-of: llm-serving
+        app: candle-vllm
+    spec:
+      containers:
+        - name: candle
+          image: forgejo.riotpiao.com/riotpiao-poimen/candle-vllm:sm70-gptq-slim
+          command: ["/usr/local/bin/candle-vllm"]
+          # fp16, not GPTQ, and two ranks, not one. See the header for the
+          # measurements and the TP=2 findings.
+          args:
+            - --w
+            - /mnt/models/coder-14b-fp16
+            - --d
+            - 0,1
+            - --dtype
+            - f16
+            - --p
+            - "8080"
+            - --max-num-seqs
+            - "32"
+            # 11.6 GB of KV per rank, about 59K context tokens. The fp16 weights
+            # leave no room for more.
+            - --kv-fraction
+            - "0.85"
+          ports:
+            - containerPort: 8080
+          resources:
+            limits:
+              cpu: "8"
+              memory: 36Gi
+              nvidia.com/gpu: "2"
+            requests:
+              cpu: "4"
+              memory: 12Gi
+              nvidia.com/gpu: "2"
+          volumeMounts:
+            - name: models
+              mountPath: /mnt/models
+              readOnly: true
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llm-models
@@ -13,6 +13,15 @@ resources:
  # clef-flash runs through clef-serve.yaml, not through an SGLang runtime. SGLang needs
  # v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. The model
  # instead runs the implementation it ships (joint_schema_model.py) in fp16 on one V100.
+  # coder-14b-fp16.yaml owns the `candle-vllm` Deployment name and the
+  # `app: candle-vllm` label that coder-service.yaml selects. It must be the only
+  # workload with that label, or the Service splits traffic across two models.
+  #
+  # fp16 on two V100s, not GPTQ on one: measured 39.7 ms ITL and 287 tok/s at 16
+  # streams against 71.5 ms and 215 tok/s for the 4-bit MoE. The V100 has FP16
+  # tensor cores and no INT4 ones, so 4-bit pays dequantisation on the SM while
+  # the memory controller idles at 19%.
+  - coder-14b-fp16.yaml
  - clef-serve.yaml
  # Stable client-facing name for the chat model. poimen-memory points at this Service,
  # so the vLLM -> candle-vllm handoff is a selector edit here and no client change.