feat(llm-serving): adopt InferenceServices into ArgoCD

Five model servers were applied by hand and tracked nowhere. Exported live, kubectl diff empty on all five, so the first sync adopts without restarting. prune: false — KServe copies isvc labels to its child Deployment, so ArgoCD would prune children it does not own and KServe would recreate them.
2026-08-15 22:45:38 -07:00
parent 826a2acda3
commit e87780b22f
7 changed files with 436 additions and 0 deletions
@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-embeddings
+    app.kubernetes.io/part-of: llm-serving
+  name: embeddings
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=nomic-ai/nomic-embed-text-v2-moe
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Explicit allowlist, matching k8s/apps/api. Anything added to this directory
+# and not listed here is silently dropped — no error, no drift shown.
+#
+# These five were adopted from live state on 2026-08-15; they had been applied
+# by hand and carried no ArgoCD ownership. Each was exported and verified with
+# `kubectl diff -f <file>` returning empty before the Application below was
+# created, so the first sync was a no-op rather than a redeploy. Re-verify that
+# way after any edit here: a GPU predictor restart is a weights reload measured
+# in tens of seconds, not a rolling update.
+resources:
+  - embeddings.yaml
+  - ornith.yaml
+  - reasoning.yaml
+  - reranker.yaml
+  - verifier.yaml
+# No namespace transformer: every file sets its own, and the transformer would
+# rewrite metadata.namespace on anything cross-namespace added later.
@@ -0,0 +1,90 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-ornith
+    app.kubernetes.io/part-of: llm-serving
+  name: ornith
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - command:
+      - /bin/sh
+      - -c
+      - 'set -e
+
+        ollama serve &
+
+        SERVE_PID=$!
+
+        until ollama list >/dev/null 2>&1; do sleep 2; done
+
+        ollama pull ornith:35b
+
+        ollama pull qwen2.5:3b-instruct
+
+        wait $SERVE_PID
+
+        '
+      env:
+      - name: OLLAMA_HOST
+        value: 0.0.0.0:8080
+      - name: OLLAMA_MODELS
+        value: /mnt/models/ollama
+      - name: OLLAMA_CONTEXT_LENGTH
+        value: '32768'
+      - name: OLLAMA_KEEP_ALIVE
+        value: '-1'
+      - name: OLLAMA_NUM_PARALLEL
+        value: '1'
+      - name: OLLAMA_MAX_LOADED_MODELS
+        value: '2'
+      image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null |
+            grep -q qwen2.5
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        exec:
+          command:
+          - /bin/sh
+          - -c
+          - ollama list 2>/dev/null | grep -q ornith && ollama list 2>/dev/null |
+            grep -q qwen2.5
+        failureThreshold: 120
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+
@@ -0,0 +1,81 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reasoning
+    app.kubernetes.io/part-of: llm-serving
+  name: reasoning
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model=unsloth/DeepSeek-R1-Distill-Qwen-32B-bnb-4bit
+      - --served-model-name=reasoning
+      - --quantization=bitsandbytes
+      - --dtype=float16
+      - --kv-cache-dtype=auto
+      - --tensor-parallel-size=1
+      - --max-model-len=16384
+      - --gpu-memory-utilization=0.90
+      - --max-num-seqs=4
+      - --enable-chunked-prefill
+      - --enable-prefix-caching
+      - --reasoning-parser=deepseek_r1
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: '0'
+      - name: VLLM_ATTENTION_BACKEND
+        value: TRITON_ATTN
+      - name: HF_HOME
+        value: /mnt/models
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '8'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        failureThreshold: 80
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 2
+    minReplicas: 2
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        sizeLimit: 2Gi
+      name: shm
+
@@ -0,0 +1,56 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-reranker
+    app.kubernetes.io/part-of: llm-serving
+  name: reranker
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model-id=BAAI/bge-reranker-base
+      - --port=8080
+      - --hostname=0.0.0.0
+      - --auto-truncate
+      env:
+      - name: HUGGINGFACE_HUB_CACHE
+        value: /mnt/models
+      image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.8.2@sha256:4d632b76bd14cb57044a1ffb0ad48ab0ba4939e705a9a615ccc740658575c26e
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 8Gi
+        requests:
+          cpu: '8'
+          memory: 4Gi
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+
@@ -0,0 +1,76 @@
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+  labels:
+    app.kubernetes.io/name: llm-verifier
+    app.kubernetes.io/part-of: llm-serving
+  name: verifier
+  namespace: llm-serving
+spec:
+  predictor:
+    containers:
+    - args:
+      - --model=Qwen/Qwen2.5-Math-PRM-7B
+      - --served-model-name=verifier
+      - --runner=pooling
+      - --dtype=float16
+      - --tensor-parallel-size=1
+      - --max-model-len=4096
+      - --max-num-seqs=8
+      - --host=0.0.0.0
+      - --port=8080
+      env:
+      - name: VLLM_USE_FLASHINFER_SAMPLER
+        value: '0'
+      - name: VLLM_ATTENTION_BACKEND
+        value: XFORMERS
+      - name: HF_HOME
+        value: /mnt/models
+      image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c
+      name: kserve-container
+      ports:
+      - containerPort: 8080
+        protocol: TCP
+      readinessProbe:
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 10
+      resources:
+        limits:
+          cpu: '16'
+          memory: 16Gi
+          nvidia.com/gpu: '1'
+        requests:
+          cpu: '4'
+          memory: 8Gi
+          nvidia.com/gpu: '1'
+      startupProbe:
+        failureThreshold: 60
+        httpGet:
+          path: /health
+          port: 8080
+        periodSeconds: 15
+      volumeMounts:
+      - mountPath: /mnt/models
+        name: models
+      - mountPath: /dev/shm
+        name: shm
+    deploymentStrategy:
+      type: Recreate
+    maxReplicas: 1
+    minReplicas: 1
+    nodeSelector:
+      kubernetes.io/hostname: worker-1
+    runtimeClassName: nvidia
+    volumes:
+    - name: models
+      persistentVolumeClaim:
+        claimName: llm-models
+    - emptyDir:
+        medium: Memory
+        sizeLimit: 1Gi
+      name: shm
+
@@ -0,0 +1,58 @@
+# Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`).
+#
+# Syncs before wave 7 (Kong), so the predictor Services exist before the routes
+# that point at them. KServe itself is part of the substrate; this Application
+# owns only the InferenceServices.
+#
+# Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed
+# by hand — no ArgoCD ownership, present in no repo — so every change to them
+# was drift by definition. Each manifest was exported from the cluster and
+# verified with `kubectl diff` returning empty before this file existed; the
+# first sync therefore adopted them without restarting anything.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: llm-serving
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "6"
+spec:
+  project: homelab
+  revisionHistoryLimit: 3
+  source:
+    repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+    targetRevision: main
+    path: k8s/apps/llm-serving
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: llm-serving
+  syncPolicy:
+    automated:
+      # `prune: false` here, unlike every other Application in this repo, and it
+      # is not an oversight.
+      #
+      # ArgoCD tracks ownership with the `argocd.argoproj.io/instance` label
+      # (argocd-cm `application.instanceLabelKey`). KServe copies an
+      # InferenceService's labels onto the Deployment and Service it generates —
+      # visible today as `app.kubernetes.io/name` and `part-of` on
+      # `ornith-predictor`. So once ArgoCD labels an InferenceService, KServe
+      # propagates that tracking label to children that are not in git, ArgoCD
+      # reads them as extraneous, prunes them, and KServe recreates them. That
+      # loop churns GPU pods.
+      #
+      # Deleting an InferenceService therefore means deleting the file AND
+      # removing the object, rather than relying on prune.
+      prune: false
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      # KServe CRDs are large; server-side apply avoids the
+      # "metadata.annotations: Too long" failure client-side apply hits, and is
+      # the correct mode for adopting objects an operator also writes to.
+      - ServerSideApply=true
+    retry:
+      limit: 3
+      backoff:
+        duration: 10s
+        factor: 2
+        maxDuration: 3m