Story Crater Bot
4fb1c6feeb
Layout on 4x Tesla V100 32GB (PCIe, no NVLink), all TP=1: GPU0+1 vLLM DeepSeek-R1-Distill-Qwen-32B bnb-nf4 (2 replicas) GPU2 Ollama ornith:35b + qwen2.5:3b-instruct (co-resident) GPU3 vLLM Qwen2.5-Math-PRM-7B (reward model) CPU TEI nomic-embed-text-v2-moe, bge-reranker-base Volta constraints, each verified against live output rather than config: - vLLM pinned v0.11.0: sm_70 dropped from CUDA_SUPPORTED_ARCHS at v0.11.1. - AWQ hard-rejected (needs sm_75). GPTQ passes vLLM's min_capability=60 gate but is NUMERICALLY WRONG on sm_70 — emits garbage logits. Proven by an fp16 control run producing correct text on an identical backend. bitsandbytes nf4 verified correct by output. - flashinfer's check_cuda_arch() crashes on any sm_7x (calls .isdigit() on an int) -> VLLM_USE_FLASHINFER_SAMPLER=0. - xformers has no sm_70 kernel for V1's paged-attention bias, and V0 was removed in v0.11.0 -> TRITON_ATTN. - Ornith is Qwen3.5-MoE hybrid-attention; vLLM added that arch after dropping Volta, so no build has both -> Ollama, which also multiplexes a second model on the same card for free. Cluster prereqs that were absent: - RuntimeClass nvidia: the Talos toolkit extension registers the containerd handler but not the k8s object; without it every pod is rejected at admission. - gpu-system pinned to privileged PSA: a device plugin cannot satisfy the cluster-default baseline, it must mount hostPath. - device-plugin affinity=null: the chart requires NFD labels that do not exist here, so it matched zero nodes and reported desiredNumberScheduled=0 silently. - Recreate strategy on GPU services: with GPUs allocated exactly 4/4, a RollingUpdate surge pod has no card and deadlocks the rollout. - longhorn-llm-local SC (1 replica, strict-local, disk tag llm): the default 3-replica class could not place the volume at all (every control-plane disk was at its over-provisioning ceiling), and this keeps ~60GB of weights on worker-1's own NVMe instead of reading them over the network. deploy-gpu-serving.sh sequences ArgoCD syncs (or helm/kubectl in --manual mode) and never applies a manifest absent from git; doctor/unstick/teardown stages exist so this is diagnosable without ad-hoc kubectl archaeology.
157 lines
5.3 KiB
YAML
157 lines
5.3 KiB
YAML
# Wave 9-11 — GPU serving stack on worker-1 (4x Tesla V100 32GB).
|
|
#
|
|
# Ordering matters: device plugin must expose nvidia.com/gpu and the KServe CRDs
|
|
# must exist before any InferenceService is applied, hence three waves.
|
|
#
|
|
# NOTE the chart versions below are v-PREFIXED (v0.15.2, not 0.15.2) — that is
|
|
# how the KServe OCI tags are published; the unprefixed form 404s.
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: gpu-runtimeclass
|
|
namespace: argocd
|
|
annotations:
|
|
# Wave 8: must precede the device plugin, whose DaemonSet sets
|
|
# runtimeClassName: nvidia and is rejected at admission if the
|
|
# RuntimeClass does not exist yet.
|
|
argocd.argoproj.io/sync-wave: "8"
|
|
spec:
|
|
project: homelab
|
|
source:
|
|
repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
|
|
targetRevision: main
|
|
path: k8s/apps/gpu-runtimeclass
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
# Cluster-scoped resource; namespace is only the app's default context.
|
|
namespace: gpu-system
|
|
syncPolicy:
|
|
syncOptions:
|
|
- CreateNamespace=true
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: nvidia-device-plugin
|
|
namespace: argocd
|
|
annotations:
|
|
argocd.argoproj.io/sync-wave: "9"
|
|
spec:
|
|
project: homelab
|
|
source:
|
|
repoURL: https://nvidia.github.io/k8s-device-plugin
|
|
chart: nvidia-device-plugin
|
|
targetRevision: "0.19.3"
|
|
helm:
|
|
values: |
|
|
# Driver + container toolkit are supplied by Talos system extensions
|
|
# baked into the installer image (nonfree-kmod-nvidia-lts /
|
|
# nvidia-container-toolkit-lts). This chart ONLY advertises the GPUs to
|
|
# the kubelet — it does not and must not install drivers.
|
|
runtimeClassName: nvidia
|
|
nodeSelector:
|
|
nvidia.com/gpu: "true"
|
|
# Drop the chart's default nodeAffinity. It requires one of three
|
|
# Node-Feature-Discovery labels (feature.node.kubernetes.io/pci-10de.present,
|
|
# .../cpu-model.vendor_id=NVIDIA, or nvidia.com/gpu.present). NFD is not
|
|
# installed and Talos sets nvidia.com/gpu (no ".present" suffix), so the
|
|
# affinity matches zero nodes and the DaemonSet silently reports
|
|
# desiredNumberScheduled=0 with no events. nodeSelector is the constraint.
|
|
affinity: null
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
namespace: gpu-system
|
|
syncPolicy:
|
|
# Manual sync for first bring-up: watch device-plugin -> KServe -> models
|
|
# come up in order, and avoid auto-deploying while worker-1 is cordoned.
|
|
# Switch to `automated: {prune: true, selfHeal: true}` once proven.
|
|
syncOptions:
|
|
- CreateNamespace=true
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: kserve-crd
|
|
namespace: argocd
|
|
annotations:
|
|
argocd.argoproj.io/sync-wave: "9"
|
|
spec:
|
|
project: homelab
|
|
source:
|
|
repoURL: oci://ghcr.io/kserve/charts
|
|
chart: kserve-crd
|
|
targetRevision: v0.15.2
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
namespace: kserve
|
|
syncPolicy:
|
|
# Manual sync for first bring-up: watch device-plugin -> KServe -> models
|
|
# come up in order, and avoid auto-deploying while worker-1 is cordoned.
|
|
# Switch to `automated: {prune: true, selfHeal: true}` once proven.
|
|
syncOptions:
|
|
- CreateNamespace=true
|
|
# InferenceService CRD exceeds the annotation size limit for
|
|
# client-side apply.
|
|
- ServerSideApply=true
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: kserve
|
|
namespace: argocd
|
|
annotations:
|
|
argocd.argoproj.io/sync-wave: "10"
|
|
spec:
|
|
project: homelab
|
|
source:
|
|
repoURL: oci://ghcr.io/kserve/charts
|
|
chart: kserve
|
|
targetRevision: v0.15.2
|
|
helm:
|
|
values: |
|
|
kserve:
|
|
controller:
|
|
# RawDeployment => plain Deployments/Services, no Knative, no Istio.
|
|
# v0.18 renames this mode "Standard"; do not bump without checking.
|
|
deploymentMode: RawDeployment
|
|
gateway:
|
|
ingressGateway:
|
|
# Route through the existing ingress-nginx, not Gateway API.
|
|
# NOTE the nesting: it is gateway.ingressGateway.enableGatewayApi,
|
|
# not gateway.enableGatewayApi — Helm silently ignores the wrong
|
|
# key rather than erroring.
|
|
enableGatewayApi: false
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
namespace: kserve
|
|
syncPolicy:
|
|
# Manual sync for first bring-up: watch device-plugin -> KServe -> models
|
|
# come up in order, and avoid auto-deploying while worker-1 is cordoned.
|
|
# Switch to `automated: {prune: true, selfHeal: true}` once proven.
|
|
syncOptions:
|
|
- CreateNamespace=true
|
|
- ServerSideApply=true
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Application
|
|
metadata:
|
|
name: llm-serving
|
|
namespace: argocd
|
|
annotations:
|
|
argocd.argoproj.io/sync-wave: "11"
|
|
spec:
|
|
project: homelab
|
|
source:
|
|
repoURL: [email protected]:Riotpiaole/riotpiao.homelab.com.git
|
|
targetRevision: main
|
|
path: k8s/apps/llm-serving
|
|
destination:
|
|
server: https://kubernetes.default.svc
|
|
namespace: llm-serving
|
|
syncPolicy:
|
|
# Manual sync for first bring-up: watch device-plugin -> KServe -> models
|
|
# come up in order, and avoid auto-deploying while worker-1 is cordoned.
|
|
# Switch to `automated: {prune: true, selfHeal: true}` once proven.
|
|
syncOptions:
|
|
- CreateNamespace=true
|