From e87780b22f2d17ec91da730580b89839a049705c Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Sat, 15 Aug 2026 22:45:38 -0700 Subject: [PATCH] feat(llm-serving): adopt InferenceServices into ArgoCD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five model servers were applied by hand and tracked nowhere. Exported live, kubectl diff empty on all five, so the first sync adopts without restarting. prune: false — KServe copies isvc labels to its child Deployment, so ArgoCD would prune children it does not own and KServe would recreate them. --- k8s/apps/llm-serving/embeddings.yaml | 56 +++++++++++++++ k8s/apps/llm-serving/kustomization.yaml | 19 ++++++ k8s/apps/llm-serving/ornith.yaml | 90 +++++++++++++++++++++++++ k8s/apps/llm-serving/reasoning.yaml | 81 ++++++++++++++++++++++ k8s/apps/llm-serving/reranker.yaml | 56 +++++++++++++++ k8s/apps/llm-serving/verifier.yaml | 76 +++++++++++++++++++++ k8s/argocd/apps/56-llm-serving.yaml | 58 ++++++++++++++++ 7 files changed, 436 insertions(+) create mode 100644 k8s/apps/llm-serving/embeddings.yaml create mode 100644 k8s/apps/llm-serving/kustomization.yaml create mode 100644 k8s/apps/llm-serving/ornith.yaml create mode 100644 k8s/apps/llm-serving/reasoning.yaml create mode 100644 k8s/apps/llm-serving/reranker.yaml create mode 100644 k8s/apps/llm-serving/verifier.yaml create mode 100644 k8s/argocd/apps/56-llm-serving.yaml diff --git a/k8s/apps/llm-serving/embeddings.yaml b/k8s/apps/llm-serving/embeddings.yaml new file mode 100644 index 0000000..5ceb6b3 --- /dev/null +++ b/k8s/apps/llm-serving/embeddings.yaml @@ -0,0 +1,56 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-embeddings + app.kubernetes.io/part-of: llm-serving + name: embeddings + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model-id=nomic-ai/nomic-embed-text-v2-moe + - --port=8080 + - --hostname=0.0.0.0 + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 8Gi + requests: + cpu: '8' + memory: 4Gi + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumeMounts: + - mountPath: /mnt/models + name: models + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml new file mode 100644 index 0000000..ea788ac --- /dev/null +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Explicit allowlist, matching k8s/apps/api. Anything added to this directory +# and not listed here is silently dropped — no error, no drift shown. +# +# These five were adopted from live state on 2026-08-15; they had been applied +# by hand and carried no ArgoCD ownership. Each was exported and verified with +# `kubectl diff -f ` returning empty before the Application below was +# created, so the first sync was a no-op rather than a redeploy. Re-verify that +# way after any edit here: a GPU predictor restart is a weights reload measured +# in tens of seconds, not a rolling update. +resources: + - embeddings.yaml + - ornith.yaml + - reasoning.yaml + - reranker.yaml + - verifier.yaml +# No namespace transformer: every file sets its own, and the transformer would +# rewrite metadata.namespace on anything cross-namespace added later. diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml new file mode 100644 index 0000000..ec8754e --- /dev/null +++ b/k8s/apps/llm-serving/ornith.yaml @@ -0,0 +1,90 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-ornith + app.kubernetes.io/part-of: llm-serving + name: ornith + namespace: llm-serving +spec: + predictor: + containers: + - command: + - /bin/sh + - -c + - 'set -e + + ollama serve & + + SERVE_PID=$! + + until ollama list >/dev/null 2>&1; do sleep 2; done + + ollama pull ornith:35b + + ollama pull qwen2.5:3b-instruct + + wait $SERVE_PID + + ' + env: + - name: OLLAMA_HOST + value: 0.0.0.0:8080 + - name: OLLAMA_MODELS + value: /mnt/models/ollama + - name: OLLAMA_CONTEXT_LENGTH + value: '32768' + - name: OLLAMA_KEEP_ALIVE + value: '-1' + - name: OLLAMA_NUM_PARALLEL + value: '1' + - name: OLLAMA_MAX_LOADED_MODELS + value: '2' + image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -c + - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | + grep -q qwen2.5 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + exec: + command: + - /bin/sh + - -c + - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null | + grep -q qwen2.5 + failureThreshold: 120 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/reasoning.yaml b/k8s/apps/llm-serving/reasoning.yaml new file mode 100644 index 0000000..44f091c --- /dev/null +++ b/k8s/apps/llm-serving/reasoning.yaml @@ -0,0 +1,81 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-reasoning + app.kubernetes.io/part-of: llm-serving + name: reasoning + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit + - --served-model-name=reasoning + - --quantization=bitsandbytes + - --dtype=float16 + - --kv-cache-dtype=auto + - --tensor-parallel-size=1 + - --max-model-len=16384 + - --gpu-memory-utilization=0.90 + - --max-num-seqs=4 + - --enable-chunked-prefill + - --enable-prefix-caching + - --reasoning-parser=deepseek_r1 + - --host=0.0.0.0 + - --port=8080 + env: + - name: VLLM_USE_FLASHINFER_SAMPLER + value: '0' + - name: VLLM_ATTENTION_BACKEND + value: TRITON_ATTN + - name: HF_HOME + value: /mnt/models + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 80 + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + deploymentStrategy: + type: Recreate + maxReplicas: 2 + minReplicas: 2 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - emptyDir: + medium: Memory + sizeLimit: 2Gi + name: shm + diff --git a/k8s/apps/llm-serving/reranker.yaml b/k8s/apps/llm-serving/reranker.yaml new file mode 100644 index 0000000..eded1cd --- /dev/null +++ b/k8s/apps/llm-serving/reranker.yaml @@ -0,0 +1,56 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-reranker + app.kubernetes.io/part-of: llm-serving + name: reranker + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model-id=BAAI/bge-reranker-base + - --port=8080 + - --hostname=0.0.0.0 + - --auto-truncate + env: + - name: HUGGINGFACE_HUB_CACHE + value: /mnt/models + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 8Gi + requests: + cpu: '8' + memory: 4Gi + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + volumeMounts: + - mountPath: /mnt/models + name: models + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + diff --git a/k8s/apps/llm-serving/verifier.yaml b/k8s/apps/llm-serving/verifier.yaml new file mode 100644 index 0000000..8e20847 --- /dev/null +++ b/k8s/apps/llm-serving/verifier.yaml @@ -0,0 +1,76 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + labels: + app.kubernetes.io/name: llm-verifier + app.kubernetes.io/part-of: llm-serving + name: verifier + namespace: llm-serving +spec: + predictor: + containers: + - args: + - --model=Qwen/Qwen2.5-Math-PRM-7B + - --served-model-name=verifier + - --runner=pooling + - --dtype=float16 + - --tensor-parallel-size=1 + - --max-model-len=4096 + - --max-num-seqs=8 + - --host=0.0.0.0 + - --port=8080 + env: + - name: VLLM_USE_FLASHINFER_SAMPLER + value: '0' + - name: VLLM_ATTENTION_BACKEND + value: XFORMERS + - name: HF_HOME + value: /mnt/models + image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + httpGet: + path: /health + port: 8080 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '4' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + failureThreshold: 60 + httpGet: + path: /health + port: 8080 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + - mountPath: /dev/shm + name: shm + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models + - emptyDir: + medium: Memory + sizeLimit: 1Gi + name: shm + diff --git a/k8s/argocd/apps/56-llm-serving.yaml b/k8s/argocd/apps/56-llm-serving.yaml new file mode 100644 index 0000000..5544104 --- /dev/null +++ b/k8s/argocd/apps/56-llm-serving.yaml @@ -0,0 +1,58 @@ +# Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`). +# +# Syncs before wave 7 (Kong), so the predictor Services exist before the routes +# that point at them. KServe itself is part of the substrate; this Application +# owns only the InferenceServices. +# +# Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed +# by hand — no ArgoCD ownership, present in no repo — so every change to them +# was drift by definition. Each manifest was exported from the cluster and +# verified with `kubectl diff` returning empty before this file existed; the +# first sync therefore adopted them without restarting anything. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: llm-serving + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "6" +spec: + project: homelab + revisionHistoryLimit: 3 + source: + repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git + targetRevision: main + path: k8s/apps/llm-serving + destination: + server: https://kubernetes.default.svc + namespace: llm-serving + syncPolicy: + automated: + # `prune: false` here, unlike every other Application in this repo, and it + # is not an oversight. + # + # ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label + # (argocd-cm `application.instanceLabelKey`). KServe copies an + # InferenceService's labels onto the Deployment and Service it generates — + # visible today as `app.kubernetes.io/name` and `part-of` on + # `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe + # propagates that tracking label to children that are not in git, ArgoCD + # reads them as extraneous, prunes them, and KServe recreates them. That + # loop churns GPU pods. + # + # Deleting an InferenceService therefore means deleting the file AND + # removing the object, rather than relying on prune. + prune: false + selfHeal: true + syncOptions: + - CreateNamespace=true + # KServe CRDs are large; server-side apply avoids the + # "metadata.annotations: Too long" failure client-side apply hits, and is + # the correct mode for adopting objects an operator also writes to. + - ServerSideApply=true + retry: + limit: 3 + backoff: + duration: 10s + factor: 2 + maxDuration: 3m