diff --git a/k8s/apps/api/llm-routes.yaml b/k8s/apps/api/llm-routes.yaml index 4652b88..29c6f0f 100644 --- a/k8s/apps/api/llm-routes.yaml +++ b/k8s/apps/api/llm-routes.yaml @@ -9,10 +9,14 @@ # ── Model -> upstream map (verified live) ─────────────────────────────────── # reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B # ornith:35b -> ornith-predictor Ollama -# qwen2.5:3b-instruct -> ornith-predictor Ollama (same pod!) +# qwen2.5:3b-instruct -> grm-predictor Ollama (dedicated GPU -- +# retired verifier-predictor's +# vLLM PRM slot; verification/ +# judge traffic no longer +# contends with ornith:35b's +# planner/implementer traffic) # nomic-embed-text-v2 -> embeddings-predictor TEI # bge-reranker-base -> reranker-predictor TEI -# Qwen2.5-Math-PRM-7B -> verifier-predictor vLLM pooling # # ── Why path-per-model, and why the body is rewritten ─────────────────────── # Kong matches routes on host, path, method and headers — never on the request @@ -20,12 +24,10 @@ # `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does # multi-target model routing, is Enterprise-only). # -# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct` -# share ONE Ollama pod, and Ollama still reads which model to load from the -# body's `model` field. If only the path selected the route, a client calling -# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the -# 35B model. So each chat route force-overwrites `model` in the body, making the -# path the single source of truth. Callers may omit `model` entirely. +# Hence the model is in the path, and each chat route force-overwrites `model` +# in the body regardless, so a client calling /v1/qwen/... with some other +# `model` value in the body can't silently get routed to the wrong weights. +# Callers may omit `model` entirely. # # ── Timeouts ─────────────────────────────────────────────────────────────── # Kong's upstream timeouts default to 60000ms. A 32B model generating a long @@ -56,8 +58,7 @@ config: {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0}, {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0}, {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0}, - {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}, - {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0} + {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0} ]} --- apiVersion: networking.k8s.io/v1 @@ -213,7 +214,7 @@ spec: pathType: Prefix backend: service: - name: ornith-predictor + name: grm-predictor port: number: 80 --- @@ -286,32 +287,3 @@ spec: name: reranker-predictor port: number: 80 ---- -# ── POST /v1/score ────────────────────────────────────────────────────────── -# The process reward model. Returns scores, not tokens, so it is deliberately -# not under /chat/completions. vLLM serves /v1/score natively (verified), so no -# rewrite is needed. -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-score - namespace: llm-serving - annotations: - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "600000" - konghq.com/write-timeout: "600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/score - pathType: Prefix - backend: - service: - name: verifier-predictor - port: - number: 80 diff --git a/k8s/apps/llm-serving/grm.yaml b/k8s/apps/llm-serving/grm.yaml new file mode 100644 index 0000000..1e8a904 --- /dev/null +++ b/k8s/apps/llm-serving/grm.yaml @@ -0,0 +1,96 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + annotations: + serving.kserve.io/deploymentMode: RawDeployment + # Same Kong-timeout rationale as ornith.yaml: these configure the Service + # KServe generates, not the Ingress, and matter once OLLAMA_KEEP_ALIVE=-1 + # stops covering a cold load after a pod restart. + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" + labels: + app.kubernetes.io/name: llm-grm + app.kubernetes.io/part-of: llm-serving + name: grm + namespace: llm-serving +spec: + predictor: + containers: + - command: + - /bin/sh + - -c + - 'set -e + + ollama serve & + + SERVE_PID=$! + + until ollama list >/dev/null 2>&1; do sleep 2; done + + ollama pull qwen2.5:3b-instruct + + ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true + + wait $SERVE_PID + + ' + env: + - name: OLLAMA_HOST + value: 0.0.0.0:8080 + - name: OLLAMA_MODELS + value: /mnt/models/ollama + - name: OLLAMA_CONTEXT_LENGTH + value: '32768' + - name: OLLAMA_KEEP_ALIVE + value: '-1' + - name: OLLAMA_NUM_PARALLEL + value: '1' + # Room for a second verification/reward model alongside qwen2.5:3b + # without a redeploy -- matches ornith.yaml's pattern, one dedicated + # GPU now free for it instead of contending with ornith:35b's. + - name: OLLAMA_MAX_LOADED_MODELS + value: '2' + image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f + name: kserve-container + ports: + - containerPort: 8080 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/sh + - -c + - ollama ps 2>/dev/null | grep -q qwen2.5 + periodSeconds: 10 + resources: + limits: + cpu: '16' + memory: 16Gi + nvidia.com/gpu: '1' + requests: + cpu: '8' + memory: 8Gi + nvidia.com/gpu: '1' + startupProbe: + exec: + command: + - /bin/sh + - -c + - ollama ps 2>/dev/null | grep -q qwen2.5 + failureThreshold: 120 + periodSeconds: 15 + volumeMounts: + - mountPath: /mnt/models + name: models + deploymentStrategy: + type: Recreate + maxReplicas: 1 + minReplicas: 1 + nodeSelector: + kubernetes.io/hostname: worker-1 + runtimeClassName: nvidia + volumes: + - name: models + persistentVolumeClaim: + claimName: llm-models diff --git a/k8s/apps/llm-serving/kustomization.yaml b/k8s/apps/llm-serving/kustomization.yaml index ea788ac..338cfd7 100644 --- a/k8s/apps/llm-serving/kustomization.yaml +++ b/k8s/apps/llm-serving/kustomization.yaml @@ -11,9 +11,9 @@ kind: Kustomization # in tens of seconds, not a rolling update. resources: - embeddings.yaml + - grm.yaml - ornith.yaml - reasoning.yaml - reranker.yaml - - verifier.yaml # No namespace transformer: every file sets its own, and the transformer would # rewrite metadata.namespace on anything cross-namespace added later. diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml index fb0c4bc..0293f96 100644 --- a/k8s/apps/llm-serving/ornith.yaml +++ b/k8s/apps/llm-serving/ornith.yaml @@ -38,12 +38,8 @@ spec: ollama pull ornith:35b - ollama pull qwen2.5:3b-instruct - ollama run ornith:35b "ok" >/dev/null 2>&1 || true - ollama run qwen2.5:3b-instruct "ok" >/dev/null 2>&1 || true - wait $SERVE_PID ' @@ -58,8 +54,11 @@ spec: value: '-1' - name: OLLAMA_NUM_PARALLEL value: '1' + # qwen2.5:3b-instruct moved to its own dedicated GPU (llm-serving/grm.yaml) + # so verification/judge traffic no longer contends with ornith:35b's + # planner/implementer traffic on this one -- single model here now. - name: OLLAMA_MAX_LOADED_MODELS - value: '2' + value: '1' image: ollama/ollama:0.32.9@sha256:1685741456770df6e3cceb2a945a5f75e020f658d1701509668d6f4688f1dd3f name: kserve-container ports: @@ -70,8 +69,7 @@ spec: command: - /bin/sh - -c - - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | - grep -q qwen2.5 + - ollama ps 2>/dev/null | grep -q ornith periodSeconds: 10 resources: limits: @@ -87,8 +85,7 @@ spec: command: - /bin/sh - -c - - ollama ps 2>/dev/null | grep -q ornith && ollama ps 2>/dev/null | - grep -q qwen2.5 + - ollama ps 2>/dev/null | grep -q ornith failureThreshold: 120 periodSeconds: 15 volumeMounts: diff --git a/k8s/apps/llm-serving/verifier.yaml b/k8s/apps/llm-serving/verifier.yaml deleted file mode 100644 index 8e20847..0000000 --- a/k8s/apps/llm-serving/verifier.yaml +++ /dev/null @@ -1,76 +0,0 @@ -apiVersion: serving.kserve.io/v1beta1 -kind: InferenceService -metadata: - annotations: - serving.kserve.io/deploymentMode: RawDeployment - labels: - app.kubernetes.io/name: llm-verifier - app.kubernetes.io/part-of: llm-serving - name: verifier - namespace: llm-serving -spec: - predictor: - containers: - - args: - - --model=Qwen/Qwen2.5-Math-PRM-7B - - --served-model-name=verifier - - --runner=pooling - - --dtype=float16 - - --tensor-parallel-size=1 - - --max-model-len=4096 - - --max-num-seqs=8 - - --host=0.0.0.0 - - --port=8080 - env: - - name: VLLM_USE_FLASHINFER_SAMPLER - value: '0' - - name: VLLM_ATTENTION_BACKEND - value: XFORMERS - - name: HF_HOME - value: /mnt/models - image: vllm/vllm-openai:v0.11.0@sha256:014a95f21c9edf6abe0aea6b07353f96baa4ec291c427bb1176dc7c93a85845c - name: kserve-container - ports: - - containerPort: 8080 - protocol: TCP - readinessProbe: - httpGet: - path: /health - port: 8080 - periodSeconds: 10 - resources: - limits: - cpu: '16' - memory: 16Gi - nvidia.com/gpu: '1' - requests: - cpu: '4' - memory: 8Gi - nvidia.com/gpu: '1' - startupProbe: - failureThreshold: 60 - httpGet: - path: /health - port: 8080 - periodSeconds: 15 - volumeMounts: - - mountPath: /mnt/models - name: models - - mountPath: /dev/shm - name: shm - deploymentStrategy: - type: Recreate - maxReplicas: 1 - minReplicas: 1 - nodeSelector: - kubernetes.io/hostname: worker-1 - runtimeClassName: nvidia - volumes: - - name: models - persistentVolumeClaim: - claimName: llm-models - - emptyDir: - medium: Memory - sizeLimit: 1Gi - name: shm -