Story Crater Bot
50d00ae350
Frees verifier's GPU from an underused vLLM PRM deployment. qwen2.5:3b-instruct moves off ornith-predictor's shared pod onto its own dedicated GPU (grm.yaml), so verification/judge traffic stops contending with ornith:35b's agent traffic. /v1/qwen/chat/completions now points at grm-predictor; path unchanged.
290 lines
10 KiB
YAML
290 lines
10 KiB
YAML
# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped.
|
|
#
|
|
# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress
|
|
# can only reference a Service in its own namespace and the predictor Services
|
|
# are there. The Kong ingress controller watches all namespaces, so the routes
|
|
# still land on the gateway. They are synced by the `kong` Application (which
|
|
# has a `path: k8s/apps/api` source) so all gateway config stays in one place.
|
|
#
|
|
# ── Model -> upstream map (verified live) ───────────────────────────────────
|
|
# reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B
|
|
# ornith:35b -> ornith-predictor Ollama
|
|
# qwen2.5:3b-instruct -> grm-predictor Ollama (dedicated GPU --
|
|
# retired verifier-predictor's
|
|
# vLLM PRM slot; verification/
|
|
# judge traffic no longer
|
|
# contends with ornith:35b's
|
|
# planner/implementer traffic)
|
|
# nomic-embed-text-v2 -> embeddings-predictor TEI
|
|
# bge-reranker-base -> reranker-predictor TEI
|
|
#
|
|
# ── Why path-per-model, and why the body is rewritten ───────────────────────
|
|
# Kong matches routes on host, path, method and headers — never on the request
|
|
# body. So a single /v1/chat/completions endpoint that dispatches on the body's
|
|
# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
|
|
# multi-target model routing, is Enterprise-only).
|
|
#
|
|
# Hence the model is in the path, and each chat route force-overwrites `model`
|
|
# in the body regardless, so a client calling /v1/qwen/... with some other
|
|
# `model` value in the body can't silently get routed to the wrong weights.
|
|
# Callers may omit `model` entirely.
|
|
#
|
|
# ── Timeouts ───────────────────────────────────────────────────────────────
|
|
# Kong's upstream timeouts default to 60000ms. A 32B model generating a long
|
|
# answer on a Volta GPU routinely exceeds that, and the client would see a
|
|
# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds.
|
|
|
|
# ── GET /v1/models ──────────────────────────────────────────────────────────
|
|
# Served entirely by Kong via request-termination: the plugin short-circuits in
|
|
# the access phase, so the backend below is never contacted. It only exists
|
|
# because an Ingress rule requires a backend.
|
|
#
|
|
# The list is static, which means it can drift from what the engines actually
|
|
# serve — notably if the Ollama pull list in the ornith InferenceService
|
|
# changes. Verify with:
|
|
# curl -s $SVC/v1/models (against each *-predictor)
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-models-list
|
|
namespace: llm-serving
|
|
plugin: request-termination
|
|
config:
|
|
status_code: 200
|
|
content_type: application/json
|
|
body: |
|
|
{"object":"list","data":[
|
|
{"id":"reasoning","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}
|
|
]}
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-models
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "GET"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/models
|
|
pathType: Exact
|
|
backend:
|
|
# Never actually called — request-termination answers first.
|
|
service:
|
|
name: reasoning-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/reasoning/chat/completions ─────────────────────────────────────
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-reasoning
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
# `add` only applies when the field is absent, `replace` only when present.
|
|
# Both are needed to force the value in either case.
|
|
add:
|
|
body:
|
|
- "model:reasoning"
|
|
replace:
|
|
body:
|
|
- "model:reasoning"
|
|
# The model lives in the path for routing; the upstream still expects the
|
|
# canonical OpenAI path.
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-reasoning
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/reasoning/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: reasoning-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/ornith/chat/completions ────────────────────────────────────────
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-ornith
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
add:
|
|
body:
|
|
- "model:ornith:35b"
|
|
replace:
|
|
body:
|
|
- "model:ornith:35b"
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-ornith
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/ornith/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: ornith-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/qwen/chat/completions ──────────────────────────────────────────
|
|
# Same upstream pod as ornith — only the forced body `model` differs. Both stay
|
|
# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and
|
|
# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request.
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-qwen
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
add:
|
|
body:
|
|
- "model:qwen2.5:3b-instruct"
|
|
replace:
|
|
body:
|
|
- "model:qwen2.5:3b-instruct"
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-qwen
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/qwen/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: grm-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/embeddings ─────────────────────────────────────────────────────
|
|
# No path-per-model and no rewrite: there is exactly one embeddings backend, so
|
|
# there is nothing to disambiguate, and TEI already serves the canonical
|
|
# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists).
|
|
# That makes an OpenAI SDK a drop-in here.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-embeddings
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "600000"
|
|
konghq.com/write-timeout: "600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/embeddings
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: embeddings-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/rerank ─────────────────────────────────────────────────────────
|
|
# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing
|
|
# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely
|
|
# needs the rewrite that embeddings does not.
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-rerank
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
replace:
|
|
uri: /rerank
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-rerank
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "600000"
|
|
konghq.com/write-timeout: "600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/rerank
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: reranker-predictor
|
|
port:
|
|
number: 80
|