From 4fb1c6feeb45d76320e1ad3f95121c836a4f34f3 Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:02:53 -0700 Subject: [PATCH] =?UTF-8?q?feat(gpu):=20serve=206=20models=20on=20worker-1?= =?UTF-8?q?=20via=20KServe=20=E2=80=94=20vLLM=20v0.11.0=20(bitsandbytes)?= =?UTF-8?q?=20+=20Ollama=20+=20TEI,=20plus=20RuntimeClass/privileged-PSA?= =?UTF-8?q?=20prereqs=20and=20a=20local-NVMe=20StorageClass,=20working=20a?= =?UTF-8?q?round=20Volta=20sm=5F70=20limits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layout on 4x Tesla V100 32GB (PCIe, no NVLink), all TP=1: GPU0+1 vLLM DeepSeek-R1-Distill-Qwen-32B bnb-nf4 (2 replicas) GPU2 Ollama ornith:35b + qwen2.5:3b-instruct (co-resident) GPU3 vLLM Qwen2.5-Math-PRM-7B (reward model) CPU TEI nomic-embed-text-v2-moe, bge-reranker-base Volta constraints, each verified against live output rather than config: - vLLM pinned v0.11.0: sm_70 dropped from CUDA_SUPPORTED_ARCHS at v0.11.1. - AWQ hard-rejected (needs sm_75). GPTQ passes vLLM's min_capability=60 gate but is NUMERICALLY WRONG on sm_70 — emits garbage logits. Proven by an fp16 control run producing correct text on an identical backend. bitsandbytes nf4 verified correct by output. - flashinfer's check_cuda_arch() crashes on any sm_7x (calls .isdigit() on an int) -> VLLM_USE_FLASHINFER_SAMPLER=0. - xformers has no sm_70 kernel for V1's paged-attention bias, and V0 was removed in v0.11.0 -> TRITON_ATTN. - Ornith is Qwen3.5-MoE hybrid-attention; vLLM added that arch after dropping Volta, so no build has both -> Ollama, which also multiplexes a second model on the same card for free. Cluster prereqs that were absent: - RuntimeClass nvidia: the Talos toolkit extension registers the containerd handler but not the k8s object; without it every pod is rejected at admission. - gpu-system pinned to privileged PSA: a device plugin cannot satisfy the cluster-default baseline, it must mount hostPath. - device-plugin affinity=null: the chart requires NFD labels that do not exist here, so it matched zero nodes and reported desiredNumberScheduled=0 silently. - Recreate strategy on GPU services: with GPUs allocated exactly 4/4, a RollingUpdate surge pod has no card and deadlocks the rollout. - longhorn-llm-local SC (1 replica, strict-local, disk tag llm): the default 3-replica class could not place the volume at all (every control-plane disk was at its over-provisioning ceiling), and this keeps ~60GB of weights on worker-1's own NVMe instead of reading them over the network. deploy-gpu-serving.sh sequences ArgoCD syncs (or helm/kubectl in --manual mode) and never applies a manifest absent from git; doctor/unstick/teardown stages exist so this is diagnosable without ad-hoc kubectl archaeology. --- k8s/apps/gpu-runtimeclass/kustomization.yaml | 6 + k8s/apps/gpu-runtimeclass/namespace.yaml | 23 ++ k8s/apps/gpu-runtimeclass/runtimeclass.yaml | 18 + .../inferenceservice-embeddings.yaml | 74 ++++ .../llm-serving/inferenceservice-ornith.yaml | 107 +++++ .../inferenceservice-reasoning.yaml | 133 ++++++ .../inferenceservice-reranker.yaml | 71 ++++ .../inferenceservice-verifier.yaml | 101 +++++ k8s/apps/llm-serving/kustomization.yaml | 13 + k8s/apps/llm-serving/namespace.yaml | 4 + k8s/apps/llm-serving/networkpolicy.yaml | 43 ++ k8s/apps/llm-serving/pvc-models.yaml | 32 ++ k8s/apps/llm-serving/storageclass.yaml | 38 ++ k8s/argocd/apps/70-gpu-serving.yaml | 156 +++++++ scripts/deploy-gpu-serving.sh | 381 ++++++++++++++++++ 15 files changed, 1200 insertions(+) create mode 100644 k8s/apps/gpu-runtimeclass/kustomization.yaml create mode 100644 k8s/apps/gpu-runtimeclass/namespace.yaml create mode 100644 k8s/apps/gpu-runtimeclass/runtimeclass.yaml create mode 100644 k8s/apps/llm-serving/inferenceservice-embeddings.yaml create mode 100644 k8s/apps/llm-serving/inferenceservice-ornith.yaml create mode 100644 k8s/apps/llm-serving/inferenceservice-reasoning.yaml create mode 100644 k8s/apps/llm-serving/inferenceservice-reranker.yaml create mode 100644 k8s/apps/llm-serving/inferenceservice-verifier.yaml create mode 100644 k8s/apps/llm-serving/kustomization.yaml create mode 100644 k8s/apps/llm-serving/namespace.yaml create mode 100644 k8s/apps/llm-serving/networkpolicy.yaml create mode 100644 k8s/apps/llm-serving/pvc-models.yaml create mode 100644 k8s/apps/llm-serving/storageclass.yaml create mode 100644 k8s/argocd/apps/70-gpu-serving.yaml create mode 100755 scripts/deploy-gpu-serving.sh diff --git a/k8s/apps/gpu-runtimeclass/kustomization.yaml b/k8s/apps/gpu-runtimeclass/kustomization.yaml new file mode 100644 index 0000000..3ede467 --- /dev/null +++ b/k8s/apps/gpu-runtimeclass/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# No namespace: RuntimeClass is cluster-scoped. +resources: + - namespace.yaml + - runtimeclass.yaml diff --git a/k8s/apps/gpu-runtimeclass/namespace.yaml b/k8s/apps/gpu-runtimeclass/namespace.yaml new file mode 100644 index 0000000..4279e23 --- /dev/null +++ b/k8s/apps/gpu-runtimeclass/namespace.yaml @@ -0,0 +1,23 @@ +# Namespace for GPU node-level plumbing (device plugin, and later DCGM). +# +# PodSecurity must be `privileged` here. The cluster default from the Talos +# controlplane config is `enforce: baseline` with exemptions only for +# kube-system, and a device plugin cannot satisfy baseline: it has to mount the +# kubelet device-plugin socket and the CDI/driver directories as hostPath +# volumes, which baseline forbids outright: +# +# Error creating: pods "nvidia-device-plugin-xxxxx" is forbidden: +# violates PodSecurity "baseline:latest": hostPath volumes +# (volumes "kubelet-device-plugins-dir", "mps-root", "mps-shm", "cdi-root") +# +# This is inherent to how device plugins work, not a workaround. Scope is +# limited to this namespace; the engine namespace (llm-serving) stays on the +# cluster default. +apiVersion: v1 +kind: Namespace +metadata: + name: gpu-system + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/k8s/apps/gpu-runtimeclass/runtimeclass.yaml b/k8s/apps/gpu-runtimeclass/runtimeclass.yaml new file mode 100644 index 0000000..f49968a --- /dev/null +++ b/k8s/apps/gpu-runtimeclass/runtimeclass.yaml @@ -0,0 +1,18 @@ +# Cluster-scoped prerequisite for every GPU workload on worker-1. +# +# The Talos nvidia-container-toolkit extension already registers the containerd +# runtime handler (/etc/cri/conf.d/10-nvidia-container-runtime.part -> +# plugins."io.containerd.cri.v1.runtime".containerd.runtimes.nvidia), but the +# Kubernetes RuntimeClass object is separate and is NOT created by the +# extension. Without it every pod carrying runtimeClassName: nvidia is rejected +# at admission with: +# pods "..." is forbidden: pod rejected: RuntimeClass "nvidia" not found +# +# Deliberately NOT setting nvidia as containerd's default_runtime_name (the +# 20-customization.part patch in the Talos guide): that would route every pod on +# the node through the NVIDIA runtime. Opting in per-pod is narrower. +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia diff --git a/k8s/apps/llm-serving/inferenceservice-embeddings.yaml b/k8s/apps/llm-serving/inferenceservice-embeddings.yaml new file mode 100644 index 0000000..f9e822b --- /dev/null +++ b/k8s/apps/llm-serving/inferenceservice-embeddings.yaml @@ -0,0 +1,74 @@ +# Embeddings — Nomic Embed Text v2 (MoE), on CPU via HuggingFace TEI. +# +# CPU, not GPU, deliberately. All 4 V100s are claimed by the generation models, +# and the device plugin hands out WHOLE GPUs — a 5th GPU-requesting pod is +# unschedulable no matter how much VRAM is free. Sharing would need global +# time-slicing, which on a single node cannot be scoped to one card and would let +# the scheduler co-locate two ~20GB models and OOM both. +# +# worker-1 has 96 cores with ~250m requested, and this is a 475M-param encoder +# (305M active). Retrieval runs once per agent-loop iteration, not per token, so +# CPU latency here is immaterial. This is also what Plan 1 originally specified. +# +# TEI (not vLLM) because it is purpose-built for encoders and explicitly lists +# nomic-embed-text-v2-moe as supported. +# +# NOTE: Nomic v2 requires task prefixes on the CLIENT side — +# documents: "search_document: " +# queries: "search_query: " +# Embedding without the prefix silently degrades retrieval quality. +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: embeddings + labels: + app.kubernetes.io/name: llm-embeddings + app.kubernetes.io/part-of: llm-serving +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + # Pinned to worker-1 only so it can share the RWO models PVC with the GPU + # pods (RWO = single node, any number of pods on it). + nodeSelector: + kubernetes.io/hostname: worker-1 + containers: + - name: kserve-container + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + args: + - --model-id=nomic-ai/nomic-embed-text-v2-moe + - --port=8080 + - --hostname=0.0.0.0 + # Truncate rather than 413 on over-long input. + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "8" + memory: 4Gi + limits: + cpu: "16" + memory: 8Gi + volumeMounts: + - name: models + mountPath: /mnt/models + startupProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/inferenceservice-ornith.yaml b/k8s/apps/llm-serving/inferenceservice-ornith.yaml new file mode 100644 index 0000000..8ed2216 --- /dev/null +++ b/k8s/apps/llm-serving/inferenceservice-ornith.yaml @@ -0,0 +1,107 @@ +# Action engine — Ornith-1.0-35B on Ollama. +# +# Why not vLLM like the other two: Ornith is +# Qwen3_5MoeForConditionalGeneration (Qwen3.5 MoE, 256 experts / 8 active, +# hybrid attention — 30 linear_attention + 10 full_attention layers). vLLM's +# Qwen3.5 support landed 2026-07-29, AFTER vLLM dropped Volta (sm_70) at +# v0.11.1. No vLLM build has both, so Ornith cannot run on vLLM on a V100. +# +# Ollama ships `ornith:35b` in its library and runs a llama-server runner +# underneath, which keeps Volta support. q4 is ~21GB — fits one 32GB V100 with +# room for KV. +# +# OLLAMA_KEEP_ALIVE=-1 is load-bearing: the harness calls this every loop +# iteration, and Ollama's default is to evict an idle model after 5m, which +# would add a ~21GB reload to a random future request. +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: ornith + labels: + app.kubernetes.io/name: llm-ornith + app.kubernetes.io/part-of: llm-serving +spec: + predictor: + minReplicas: 1 + # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4, + # so a surge pod has no card to claim and sits Pending while the old pod is + # never torn down — a deadlock. Recreate tears down first, accepting a brief + # gap during updates. + deploymentStrategy: + type: Recreate + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + containers: + - name: kserve-container + image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f + # `ollama serve` does not pull models, and `ollama pull` needs a running + # server — so background the server, wait for it, pull, then hand the + # foreground back to serve. + command: + - /bin/sh + - -c + - | + set -e + ollama serve & + SERVE_PID=$! + until ollama list >/dev/null 2>&1; do sleep 2; done + ollama pull ornith:35b + ollama pull qwen2.5:3b-instruct + wait $SERVE_PID + env: + # Match the port the other two engines use. + - name: OLLAMA_HOST + value: "0.0.0.0:8080" + - name: OLLAMA_MODELS + value: /mnt/models/ollama + # Ollama defaults to a 4096 context, far too small for an agentic + # coding model. Ornith's hybrid attention means only 10 of its 40 + # layers hold a conventional KV cache, so 32K is affordable in the + # ~11GiB left after its 21GB of weights. + - name: OLLAMA_CONTEXT_LENGTH + value: "32768" + # Never evict — this model is on the harness's hot path. + - name: OLLAMA_KEEP_ALIVE + value: "-1" + # Serial agent loop; no benefit from parallel slots. + - name: OLLAMA_NUM_PARALLEL + value: "1" + # 2, so ornith and the small utility model stay co-resident on GPU2 + # instead of evicting one another on every alternating request. + - name: OLLAMA_MAX_LOADED_MODELS + value: "2" + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "8" + memory: 8Gi + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: models + mountPath: /mnt/models + # Probes must confirm the MODEL is present, not just that the server + # answers. Ollama's `GET /` returns 200 ("Ollama is running") the moment + # `ollama serve` binds — which is before the ~21GB pull finishes. An + # httpGet probe would therefore mark this pod Ready with no model + # loaded, and KServe would route traffic to it. + startupProbe: + exec: + command: ["/bin/sh", "-c", "ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | grep -q qwen2.5"] + periodSeconds: 15 + failureThreshold: 120 + readinessProbe: + exec: + command: ["/bin/sh", "-c", "ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | grep -q qwen2.5"] + periodSeconds: 10 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/inferenceservice-reasoning.yaml b/k8s/apps/llm-serving/inferenceservice-reasoning.yaml new file mode 100644 index 0000000..7e8e0bb --- /dev/null +++ b/k8s/apps/llm-serving/inferenceservice-reasoning.yaml @@ -0,0 +1,133 @@ +# Reasoning engine — DeepSeek-R1-Distill-Qwen-32B, GPTQ INT4, vLLM. +# +# TP=1 with 2 data-parallel replicas (GPU0 + GPU1) rather than one TP=2 engine: +# worker-1 has NO NVLink, so tensor-parallel's per-token all-reduce would cross +# PCIe on every decode step. Two independent replicas need zero inter-GPU +# communication and KServe load-balances them behind one Service. +# +# vLLM is pinned to v0.11.0 — the LAST release that compiles sm_70 (Volta) +# kernels. v0.11.1 dropped 7.0 from CUDA_SUPPORTED_ARCHS. Do not bump this +# without re-checking CMakeLists.txt, or every pod dies with "no kernel image". +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: reasoning + labels: + app.kubernetes.io/name: llm-reasoning + app.kubernetes.io/part-of: llm-serving +spec: + predictor: + minReplicas: 2 + # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4, + # so a surge pod has no card to claim and sits Pending while the old pod is + # never torn down — a deadlock. Recreate tears down first, accepting a brief + # gap during updates. + deploymentStrategy: + type: Recreate + maxReplicas: 2 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + containers: + - name: kserve-container + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + # GPTQ, NOT AWQ. vLLM hard-refuses AWQ below compute capability 75: + # "The quantization method awq is not supported for the current GPU. + # Minimum capability: 75. Current capability: 70." + # V100 is sm_70. GPTQ's min capability is 60, so it runs. (gptq_marlin + # needs 80 and fp8 needs 80 — both also out.) Same 19.3GB footprint. + # desc_act=False in this build: no activation reordering, faster. + - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit + - --served-model-name=reasoning + # bitsandbytes nf4. GPTQ passed vLLM's min_capability=60 check but was + # numerically WRONG on sm_70 (garbage logits) — proven by the fp16 + # control run producing correct text with the identical backend. bnb + # declares min_capability=70, but treat that as unverified until the + # output itself is checked. + # NOTE the repo sets bnb_4bit_compute_dtype=bfloat16, which Volta does + # not have; --dtype=float16 must override it. + - --quantization=bitsandbytes + # Volta has no bf16 — must be explicit, the repo's weights are bf16. + - --dtype=float16 + # No FP8 KV on Volta; stays fp16. + - --kv-cache-dtype=auto + - --tensor-parallel-size=1 + - --max-model-len=16384 + # VRAM budget on a 32GiB V100: 0.92 => ~29.4GiB, minus ~18GiB of GPTQ + # weights leaves ~11GiB for KV + activations. One full 32K sequence + # costs 32768 x 256KB = 8GiB of KV, so 8 concurrent full-length + # sequences is not physically possible here — 4 is honest, and a + # serial single-user harness never needs more. + - --gpu-memory-utilization=0.90 + - --max-num-seqs=4 + # Smooths Volta's slow prefill (no FlashAttention2 on sm_70). + - --enable-chunked-prefill + - --enable-prefix-caching + # Splits … into its own channel. + - --reasoning-parser=deepseek_r1 + - --host=0.0.0.0 + - --port=8080 + env: + # FlashAttention2 requires sm_80; Volta must fall back to xformers. + # flashinfer's check_cuda_arch() has an upstream bug that crashes on + # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit() + # on an int, so instead of reporting "unsupported" it raises + # AttributeError: 'int' object has no attribute 'isdigit' + # and engine init dies. Default is None (auto-detect), which walks + # straight into that path. 0 disables the flashinfer sampler outright. + # Only affects the generate runner — the verifier (pooling) never hits + # the sampler, which is why it started fine and this did not. + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + # TRITON_ATTN, not XFORMERS. On sm_70 every xformers kernel is + # rejected for V1's paged-attention bias type: + # fa2F / triton_splitKF -> require sm_80 + # cutlassF -> supports sm_70 but not + # PagedBlockDiagonalCausalWithOffsetPaddedKeysMask + # -> NotImplementedError kills EngineCore on the FIRST request, which + # takes the whole pod down (vLLM treats engine death as fatal). + # V0, whose hand-written paged kernels did support sm_70, was REMOVED + # in v0.11.0, so VLLM_USE_V1=0 has nothing to fall back to. + # Triton JIT-compiles for the local arch, so it is the last option. + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "8" + memory: 8Gi + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: models + mountPath: /mnt/models + - name: shm + mountPath: /dev/shm + startupProbe: + httpGet: + path: /health + port: 8080 + # Cold start pulls ~18Gi of weights over Longhorn, then loads to VRAM. + periodSeconds: 15 + failureThreshold: 80 + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: 2Gi diff --git a/k8s/apps/llm-serving/inferenceservice-reranker.yaml b/k8s/apps/llm-serving/inferenceservice-reranker.yaml new file mode 100644 index 0000000..7e23deb --- /dev/null +++ b/k8s/apps/llm-serving/inferenceservice-reranker.yaml @@ -0,0 +1,71 @@ +# Reranker — BAAI/bge-reranker-base, on CPU via HuggingFace TEI. +# +# Second stage of retrieval: the embedding model fetches a coarse top-k by +# vector similarity, this cross-encoder re-scores those candidates against the +# query directly. That is what fixes the "semantic dilution" problem in Plan 1 — +# a single embedding vector cannot represent a large chunk faithfully, so +# ranking by cosine alone surfaces near-misses. +# +# CPU for the same reason as the embedding service: all 4 GPUs are claimed and +# the device plugin allocates whole cards. A 568M cross-encoder scoring ~20-50 +# candidates per query is well within CPU budget. +# +# Arch is XLMRobertaForSequenceClassification, which TEI serves as /rerank. +# 278M params — smaller than v2-m3 (568M) and English/Chinese rather than +# multilingual, which suits code+docs retrieval and is faster on CPU. +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: reranker + labels: + app.kubernetes.io/name: llm-reranker + app.kubernetes.io/part-of: llm-serving +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + # Same worker-1 pin as the embedding service, to share the RWO models PVC. + nodeSelector: + kubernetes.io/hostname: worker-1 + containers: + - name: kserve-container + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + args: + # bge-reranker-base, NOT v2-m3. TEI's CPU image starts the ONNX + # Runtime backend and v2-m3 ships no ONNX files, so it dies with + # "Model ONNX files not found in the repository". This build does. + - --model-id=BAAI/bge-reranker-base + - --port=8080 + - --hostname=0.0.0.0 + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "8" + memory: 4Gi + limits: + cpu: "16" + memory: 8Gi + volumeMounts: + - name: models + mountPath: /mnt/models + startupProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/inferenceservice-verifier.yaml b/k8s/apps/llm-serving/inferenceservice-verifier.yaml new file mode 100644 index 0000000..c33b05f --- /dev/null +++ b/k8s/apps/llm-serving/inferenceservice-verifier.yaml @@ -0,0 +1,101 @@ +# Verifier — Qwen2.5-Math-PRM-7B, step-level process reward model, vLLM. +# +# Model choice was constrained by vLLM v0.11.0's registry: its arch +# (Qwen2ForProcessRewardModel) is natively registered, whereas the smaller +# community PRMs are Qwen2ForTokenClassification / Qwen2ForPrmModel, neither of +# which v0.11.0 can load (ForTokenClassification is absent from +# _SUFFIX_TO_DEFAULTS, so it won't even auto-convert). +# +# --runner pooling, NOT --task reward: --task is [DEPRECATED] in v0.11.0. +# Scoring goes to /pooling, not /v1/completions — this is a reward model, it +# returns scores, not tokens. +# +# Gets a whole dedicated GPU despite only needing ~15Gi: it sits on the +# harness's critical path (every reasoning->action->verify iteration waits on +# it), so isolation from the generation engines' decode loops matters more than +# the idle VRAM. +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: verifier + labels: + app.kubernetes.io/name: llm-verifier + app.kubernetes.io/part-of: llm-serving +spec: + predictor: + minReplicas: 1 + # Recreate, not the default RollingUpdate: GPUs are allocated exactly 4/4, + # so a surge pod has no card to claim and sits Pending while the old pod is + # never torn down — a deadlock. Recreate tears down first, accepting a brief + # gap during updates. + deploymentStrategy: + type: Recreate + maxReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + containers: + - name: kserve-container + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + args: + - --model=Qwen/Qwen2.5-Math-PRM-7B + - --served-model-name=verifier + # Pooling runner => reward scoring. Weights are bf16; Volta needs fp16. + - --runner=pooling + - --dtype=float16 + - --tensor-parallel-size=1 + - --max-model-len=4096 + - --max-num-seqs=8 + - --host=0.0.0.0 + - --port=8080 + env: + # flashinfer's check_cuda_arch() has an upstream bug that crashes on + # ANY sm_7x GPU: `elif major == 7 and minor.isdigit()` calls .isdigit() + # on an int, so instead of reporting "unsupported" it raises + # AttributeError: 'int' object has no attribute 'isdigit' + # and engine init dies. Default is None (auto-detect), which walks + # straight into that path. 0 disables the flashinfer sampler outright. + # Only affects the generate runner — the verifier (pooling) never hits + # the sampler, which is why it started fine and this did not. + - name: VLLM_USE_FLASHINFER_SAMPLER + value: "0" + - name: VLLM_ATTENTION_BACKEND + value: XFORMERS + - name: HF_HOME + value: /mnt/models + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: "4" + memory: 8Gi + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: 16Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: models + mountPath: /mnt/models + - name: shm + mountPath: /dev/shm + startupProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - name: shm + emptyDir: + medium: Memory + sizeLimit: 1Gi diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml new file mode 100644 index 0000000..6bc50c1 --- /dev/null +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: llm-serving +resources: + - namespace.yaml + - storageclass.yaml + - pvc-models.yaml + - inferenceservice-reasoning.yaml + - inferenceservice-ornith.yaml + - inferenceservice-verifier.yaml + - inferenceservice-embeddings.yaml + - inferenceservice-reranker.yaml + - networkpolicy.yaml diff --git a/k8s/apps/llm-serving/namespace.yaml b/k8s/apps/llm-serving/namespace.yaml new file mode 100644 index 0000000..600226b --- /dev/null +++ b/k8s/apps/llm-serving/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: llm-serving diff --git a/k8s/apps/llm-serving/networkpolicy.yaml b/k8s/apps/llm-serving/networkpolicy.yaml new file mode 100644 index 0000000..58bc9cf --- /dev/null +++ b/k8s/apps/llm-serving/networkpolicy.yaml @@ -0,0 +1,43 @@ +# Default-deny ingress for the serving pods. +# +# This is a real compensating control, not hygiene: vLLM is pinned to v0.11.0 +# (forced — last release with Volta kernels), which sits below the patch line on +# several advisories that will never be backported to that branch, incl. +# CVE-2026-54234 (remote DoS) and GHSA-7m6h-x95x-82q5 (cross-user data leak). +# Those are all remote/unauthenticated attack surface, so keeping the engines +# reachable only from opted-in in-cluster clients is what keeps exposure low. +# +# Consumers opt in with label `llm-client: "true"`. Do NOT expose these via +# Ingress. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: llm-serving-default-deny +spec: + podSelector: + matchLabels: + app.kubernetes.io/part-of: llm-serving + policyTypes: + - Ingress + ingress: + - from: + # Any pod, any namespace, that explicitly opts in as an LLM client. + - namespaceSelector: {} + podSelector: + matchLabels: + llm-client: "true" + # Sibling engines (harness may chain calls between them). + - podSelector: + matchLabels: + app.kubernetes.io/part-of: llm-serving + ports: + - protocol: TCP + port: 8080 + - from: + # Prometheus scraping /metrics. + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ports: + - protocol: TCP + port: 8080 diff --git a/k8s/apps/llm-serving/pvc-models.yaml b/k8s/apps/llm-serving/pvc-models.yaml new file mode 100644 index 0000000..8d417da --- /dev/null +++ b/k8s/apps/llm-serving/pvc-models.yaml @@ -0,0 +1,32 @@ +# Shared HuggingFace cache for all three engines. +# +# ReadWriteOnce is correct here despite six pods mounting it: RWO means "one +# NODE", and every pod in this app is pinned to worker-1 via nodeSelector, so +# they share the volume legally. If a pod is ever allowed onto another node, +# this must become RWX first. +# +# StorageClass is longhorn-llm-local (1 replica, strict-local, disk tag `llm`) +# — NOT the default 3-replica class, which could not place this volume at all: +# every control-plane disk was already at its over-provisioning ceiling. +# +# Sizing (measured, not estimated): +# reasoning GPTQ INT4 19.3 GB +# ornith:35b q4 (ollama) 21.0 GB +# verifier Qwen2.5-Math-PRM-7B fp16 15.3 GB +# nomic-embed-text-v2-moe (CPU) 1.9 GB +# bge-reranker-base (CPU) 1.1 GB +# ------------------------------------------ +# total ~58.6 GB (+ HF temp during pulls) +# The two reasoning replicas share ONE on-disk copy; they differ only in which +# GPU they load it onto. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: llm-models +spec: + accessModes: + - ReadWriteOnce + storageClassName: longhorn-llm-local + resources: + requests: + storage: 120Gi diff --git a/k8s/apps/llm-serving/storageclass.yaml b/k8s/apps/llm-serving/storageclass.yaml new file mode 100644 index 0000000..07b5c11 --- /dev/null +++ b/k8s/apps/llm-serving/storageclass.yaml @@ -0,0 +1,38 @@ +# Dedicated StorageClass for model weights on worker-1's local NVMe. +# +# Why not the default `longhorn` class (3 replicas, network-attached): +# +# 1. numberOfReplicas: 1 — model weights are re-downloadable from HuggingFace. +# Replicating them 3x buys nothing; losing a replica costs a re-pull, not +# data. The repo's "never delete a PVC without replicas/backups" rule exists +# for irreplaceable data, which this is not. +# +# 2. dataLocality: strict-local — keeps the single replica on the SAME node as +# the pod. All engines are pinned to worker-1, so weights are read from its +# local 751GB NVMe instead of over the network from a control-plane node. +# Removes ~60GB of network reads on every cold start. +# +# 3. diskSelector: llm — restricts this class to disks tagged `llm`, i.e. only +# worker-1's disk. Equally important, worker-1's disk carries that tag so +# UNTAGGED volumes (any ordinary cluster PVC) will not land on it. Before +# tagging, worker-1 had been silently hosting a replica of cicd/runner-dind, +# consuming GPU-node storage for general cluster workloads. +# +# The default 3-replica class also physically could not place this volume: all +# three control-plane disks were already at their over-provisioning ceiling +# (storage-over-provisioning-percentage=100, 30% reserved), so a 120Gi x3 +# request failed with ReplicaSchedulingFailure on every node. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-llm-local +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Delete +volumeBindingMode: Immediate +parameters: + numberOfReplicas: "1" + dataLocality: "strict-local" + diskSelector: "llm" + staleReplicaTimeout: "30" + fsType: "ext4" diff --git a/k8s/argocd/apps/70-gpu-serving.yaml b/k8s/argocd/apps/70-gpu-serving.yaml new file mode 100644 index 0000000..5ab7545 --- /dev/null +++ b/k8s/argocd/apps/70-gpu-serving.yaml @@ -0,0 +1,156 @@ +# Wave 9-11 — GPU serving stack on worker-1 (4x Tesla V100 32GB). +# +# Ordering matters: device plugin must expose nvidia.com/gpu and the KServe CRDs +# must exist before any InferenceService is applied, hence three waves. +# +# NOTE the chart versions below are v-PREFIXED (v0.15.2, not 0.15.2) — that is +# how the KServe OCI tags are published; the unprefixed form 404s. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: gpu-runtimeclass + namespace: argocd + annotations: + # Wave 8: must precede the device plugin, whose DaemonSet sets + # runtimeClassName: nvidia and is rejected at admission if the + # RuntimeClass does not exist yet. + argocd.argoproj.io/sync-wave: "8" +spec: + project: homelab + source: + repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/gpu-runtimeclass + destination: + server: https://kubernetes.default.svc + # Cluster-scoped resource; namespace is only the app's default context. + namespace: gpu-system + syncPolicy: + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: nvidia-device-plugin + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "9" +spec: + project: homelab + source: + repoURL: https://nvidia.github.io/k8s-device-plugin + chart: nvidia-device-plugin + targetRevision: "0.19.3" + helm: + values: | + # Driver + container toolkit are supplied by Talos system extensions + # baked into the installer image (nonfree-kmod-nvidia-lts / + # nvidia-container-toolkit-lts). This chart ONLY advertises the GPUs to + # the kubelet — it does not and must not install drivers. + runtimeClassName: nvidia + nodeSelector: + nvidia.com/gpu: "true" + # Drop the chart's default nodeAffinity. It requires one of three + # Node-Feature-Discovery labels (feature.node.kubernetes.io/pci-10de.present, + # .../cpu-model.vendor_id=NVIDIA, or nvidia.com/gpu.present). NFD is not + # installed and Talos sets nvidia.com/gpu (no ".present" suffix), so the + # affinity matches zero nodes and the DaemonSet silently reports + # desiredNumberScheduled=0 with no events. nodeSelector is the constraint. + affinity: null + destination: + server: https://kubernetes.default.svc + namespace: gpu-system + syncPolicy: + # Manual sync for first bring-up: watch device-plugin -> KServe -> models + # come up in order, and avoid auto-deploying while worker-1 is cordoned. + # Switch to `automated: {prune: true, selfHeal: true}` once proven. + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kserve-crd + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "9" +spec: + project: homelab + source: + repoURL: oci://ghcr.io/kserve/charts + chart: kserve-crd + targetRevision: v0.15.2 + destination: + server: https://kubernetes.default.svc + namespace: kserve + syncPolicy: + # Manual sync for first bring-up: watch device-plugin -> KServe -> models + # come up in order, and avoid auto-deploying while worker-1 is cordoned. + # Switch to `automated: {prune: true, selfHeal: true}` once proven. + syncOptions: + - CreateNamespace=true + # InferenceService CRD exceeds the annotation size limit for + # client-side apply. + - ServerSideApply=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kserve + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + project: homelab + source: + repoURL: oci://ghcr.io/kserve/charts + chart: kserve + targetRevision: v0.15.2 + helm: + values: | + kserve: + controller: + # RawDeployment => plain Deployments/Services, no Knative, no Istio. + # v0.18 renames this mode "Standard"; do not bump without checking. + deploymentMode: RawDeployment + gateway: + ingressGateway: + # Route through the existing ingress-nginx, not Gateway API. + # NOTE the nesting: it is gateway.ingressGateway.enableGatewayApi, + # not gateway.enableGatewayApi — Helm silently ignores the wrong + # key rather than erroring. + enableGatewayApi: false + destination: + server: https://kubernetes.default.svc + namespace: kserve + syncPolicy: + # Manual sync for first bring-up: watch device-plugin -> KServe -> models + # come up in order, and avoid auto-deploying while worker-1 is cordoned. + # Switch to `automated: {prune: true, selfHeal: true}` once proven. + syncOptions: + - CreateNamespace=true + - ServerSideApply=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: llm-serving + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "11" +spec: + project: homelab + source: + repoURL: git@github.com:Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/llm-serving + destination: + server: https://kubernetes.default.svc + namespace: llm-serving + syncPolicy: + # Manual sync for first bring-up: watch device-plugin -> KServe -> models + # come up in order, and avoid auto-deploying while worker-1 is cordoned. + # Switch to `automated: {prune: true, selfHeal: true}` once proven. + syncOptions: + - CreateNamespace=true diff --git a/scripts/deploy-gpu-serving.sh b/scripts/deploy-gpu-serving.sh new file mode 100755 index 0000000..77ab6cb --- /dev/null +++ b/scripts/deploy-gpu-serving.sh @@ -0,0 +1,381 @@ +#!/usr/bin/env bash +# +# Deploy the GPU serving stack on worker-1: +# NVIDIA device plugin -> KServe (RawDeployment) -> 3 engines +# GPU0,1 vLLM v0.11.0 deepseek-r1-distill-qwen-32b-awq (2 replicas) +# GPU2 Ollama ornith:35b +# GPU3 vLLM v0.11.0 Qwen2.5-Math-PRM-7B (reward model) +# +# Two modes: +# argocd (default) -- syncs the ArgoCD Applications from +# k8s/argocd/apps/70-gpu-serving.yaml. Git is the source +# of truth; this only sequences the syncs. +# manual (--manual) -- bootstraps directly with helm + kubectl, for first +# bring-up before the Applications are committed. Applies +# the SAME manifests from k8s/apps/llm-serving/, so ArgoCD +# adopts them cleanly later (matches the repo's existing +# k8s/bootstrap/phaseN-* pattern). +# +# Usage: +# ./scripts/deploy-gpu-serving.sh --manual # full manual bootstrap +# ./scripts/deploy-gpu-serving.sh --manual gpu-plugin # one stage +# ./scripts/deploy-gpu-serving.sh --manual --dry-run # show, don't run +# ./scripts/deploy-gpu-serving.sh # via ArgoCD +# +set -euo pipefail + +NODE=worker-1 +NS=llm-serving +KSERVE_NS=kserve +GPU_NS=gpu-system +KSERVE_VER=v0.15.2 # v-PREFIXED; the unprefixed tag 404s +NVDP_VER=0.19.3 +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +MODE=argocd +DRY=false + +log() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } +info() { printf ' %s\n' "$*"; } +die() { printf '\033[31mERROR: %s\033[0m\n' "$*" >&2; exit 1; } +run() { if $DRY; then info "[dry-run] $*"; else "$@"; fi; } +need() { command -v "$1" >/dev/null 2>&1 || die "missing required tool: $1"; } + +sync_app() { + local app=$1 + log "argocd sync: $app" + run argocd app sync "$app" --timeout 600 + run argocd app wait "$app" --health --timeout 600 +} + +# ---------------------------------------------------------------- stages + +preflight() { + log "preflight" + need kubectl + [[ $MODE == argocd ]] && need argocd + [[ $MODE == manual ]] && need helm + + kubectl get node "$NODE" >/dev/null 2>&1 || die "node $NODE not found" + local ready + ready=$(kubectl get node "$NODE" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') + [[ $ready == True ]] || die "node $NODE is not Ready" + info "node $NODE: Ready" + + # Driver ships in the Talos installer image as a system extension, not from + # the cluster. Missing label => node was built from the wrong schematic and + # no amount of device-plugin will help. + kubectl get node "$NODE" -o jsonpath='{.metadata.labels}' | grep -q 'nonfree-kmod-nvidia' \ + || die "NVIDIA driver extension label absent — worker-1 not on the GPU schematic" + info "NVIDIA driver extension: present" + + kubectl get node "$NODE" -o jsonpath='{.metadata.labels.nvidia\.com/gpu}' 2>/dev/null | grep -q true \ + || die "node label nvidia.com/gpu=true missing — device plugin nodeSelector will not match" + info "node label nvidia.com/gpu=true: present" + + [[ $(kubectl get node "$NODE" -o jsonpath='{.spec.unschedulable}') == true ]] \ + && info "node is cordoned (expected; uncordoned before engines deploy)" + info "mode: $MODE" +} + +runtimeclass() { + # Must exist before ANY pod with runtimeClassName: nvidia is admitted — + # including the device plugin itself. The Talos extension registers the + # containerd handler but not this object. + if [[ $MODE == manual ]]; then + log "apply RuntimeClass nvidia" + run kubectl apply -k "$REPO_ROOT/k8s/apps/gpu-runtimeclass" + else + sync_app gpu-runtimeclass + fi + $DRY || kubectl get runtimeclass nvidia >/dev/null 2>&1 \ + && info "RuntimeClass nvidia: present" +} + +gpu_plugin() { + runtimeclass + if [[ $MODE == manual ]]; then + log "helm install: nvidia-device-plugin $NVDP_VER" + # Driver + container toolkit come from Talos system extensions. This chart + # only advertises the GPUs to the kubelet — it must not install drivers. + run helm upgrade --install nvidia-device-plugin nvidia-device-plugin \ + --repo https://nvidia.github.io/k8s-device-plugin \ + --version "$NVDP_VER" \ + -n "$GPU_NS" --create-namespace \ + --set runtimeClassName=nvidia \ + `# --set-string, NOT --set: nodeSelector is map[string]string in the` \ + `# PodSpec schema, and plain --set coerces "true" to a YAML boolean,` \ + `# which the API server rejects as a type violation.` \ + --set-string nodeSelector."nvidia\.com/gpu"=true \ + `# Drop the chart's default nodeAffinity. It requires one of three` \ + `# Node-Feature-Discovery labels (feature.node.kubernetes.io/pci-10de.present,` \ + `# .../cpu-model.vendor_id=NVIDIA, or nvidia.com/gpu.present). NFD is not` \ + `# installed here and Talos sets nvidia.com/gpu (no ".present" suffix), so` \ + `# the affinity matches zero nodes and the DaemonSet silently reports` \ + `# desiredNumberScheduled=0. The nodeSelector above is our constraint.` \ + `# Deliberately NO --wait: it blocks on DaemonSet readiness, and any` \ + `# admission failure then times helm out and wedges the release in` \ + `# pending-upgrade, blocking all later upgrades. The allocatable-GPU` \ + `# poll below is the real readiness signal.` \ + --set affinity=null + else + sync_app nvidia-device-plugin + fi + + log "waiting for nvidia.com/gpu to register on $NODE" + $DRY && { info "[dry-run] would verify allocatable == 4"; return 0; } + local n="" + for _ in $(seq 1 60); do + n=$(kubectl get node "$NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || true) + [[ -n $n && $n != 0 ]] && break + sleep 5 + done + [[ $n == 4 ]] || die "expected 4 allocatable GPUs, got '${n:-none}'" + info "allocatable nvidia.com/gpu: $n" +} + +kserve() { + if [[ $MODE == manual ]]; then + log "helm install: kserve-crd $KSERVE_VER" + # No --wait on helm (a timeout leaves the release wedged in pending-upgrade, + # blocking every later upgrade). kubectl wait below is the readiness signal + # and does not touch helm release state. + run helm upgrade --install kserve-crd "oci://ghcr.io/kserve/charts/kserve-crd" \ + --version "$KSERVE_VER" -n "$KSERVE_NS" --create-namespace + + run kubectl wait --for=condition=established --timeout=120s \ + crd/inferenceservices.serving.kserve.io + + log "helm install: kserve $KSERVE_VER (RawDeployment)" + # NOTE the nesting on enableGatewayApi: gateway.ingressGateway.enableGatewayApi. + # Helm silently ignores a wrong key rather than erroring. + run helm upgrade --install kserve "oci://ghcr.io/kserve/charts/kserve" \ + --version "$KSERVE_VER" -n "$KSERVE_NS" \ + --set kserve.controller.deploymentMode=RawDeployment \ + --set kserve.controller.gateway.ingressGateway.enableGatewayApi=false + # Controller readiness via kubectl, not helm --wait, for the same reason. + run kubectl rollout status deploy -n "$KSERVE_NS" --timeout=600s + else + sync_app kserve-crd + run kubectl wait --for=condition=established --timeout=120s \ + crd/inferenceservices.serving.kserve.io + sync_app kserve + fi + info "KServe ready (RawDeployment mode)" +} + +uncordon() { + log "uncordon $NODE" + # Must precede the engines. The device-plugin DaemonSet tolerates the + # unschedulable taint automatically; the engine Deployments do not and would + # sit Pending forever. + run kubectl uncordon "$NODE" +} + +engines() { + if [[ $MODE == manual ]]; then + log "kubectl apply -k k8s/apps/llm-serving" + run kubectl apply -k "$REPO_ROOT/k8s/apps/llm-serving" + else + sync_app llm-serving + fi + + log "waiting for InferenceServices (first start pulls ~60GB of weights)" + run kubectl wait --for=condition=Ready --timeout=2400s \ + inferenceservice --all -n "$NS" \ + || info "not all Ready yet — check: kubectl get pods -n $NS" +} + +smoke() { + log "smoke test" + $DRY && { info "[dry-run] would curl the three engines"; return 0; } + + # Engines are ClusterIP behind a default-deny NetworkPolicy, so probe from + # inside the cluster with the llm-client label that the policy allows. + # + # KServe names the Service "-predictor" (constants.PredictorServiceName) + # on port 80 -> containerPort 8080 — NOT "" on 8080. + probe() { + local isvc=$1 path=$2 body=$3 + printf ' %-24s ' "$isvc" + if kubectl run "smoke-$RANDOM" -n "$NS" --rm -i --restart=Never -q \ + --image=curlimages/curl:8.11.1 \ + --labels="llm-client=true" \ + --command -- curl -sf -m 300 -X POST \ + "http://${isvc}-predictor.${NS}.svc.cluster.local/${path}" \ + -H 'Content-Type: application/json' -d "$body" >/dev/null 2>&1; then + printf '\033[32mOK\033[0m\n' + else + printf '\033[31mFAIL\033[0m\n' + fi + } + + # Thinking models (reasoning, ornith) spend their first tokens inside a + # block, so a small max_tokens returns EMPTY content and looks like a + # failure. Give them room. + probe reasoning v1/completions \ + '{"model":"reasoning","prompt":"2+2=","max_tokens":200}' + probe ornith v1/chat/completions \ + '{"model":"ornith:35b","messages":[{"role":"user","content":"hi"}],"max_tokens":400}' + # Same Ollama endpoint, second co-resident model on GPU2. + probe ornith v1/chat/completions \ + '{"model":"qwen2.5:3b-instruct","messages":[{"role":"user","content":"hi"}],"max_tokens":64}' + # Reward model: scores via /pooling with an `input` field, NOT /v1/completions. + probe verifier pooling \ + '{"model":"verifier","input":"Step 1: 2+2=4."}' + # TEI encoders (CPU): /embed and /rerank, not an OpenAI-shaped API. + # Nomic v2 needs the search_document:/search_query: prefix from the client. + probe embeddings embed \ + '{"inputs":"search_query: hello"}' + probe reranker rerank \ + '{"query":"how to sort a list","texts":["use sorted()","unrelated text"]}' +} + +status() { + log "status" + kubectl get node "$NODE" -o wide + echo + kubectl get inferenceservice -n "$NS" 2>/dev/null || info "no InferenceServices yet" + echo + kubectl get pods -n "$NS" -o wide 2>/dev/null || true + echo + info "GPU allocatable: $(kubectl get node "$NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || echo none)" +} + +# Every check that this bring-up has actually needed, in one command, so +# troubleshooting never requires ad-hoc kubectl archaeology again. Read-only. +doctor() { + log "doctor — full diagnostic" + set +e # advisory only: never abort on an absent resource + local fail=0 + chk() { # chk