Story Crater Bot
e6ada95b39
Kong key-auth rejected the Authorization: Bearer header every OpenAI-SDK-compatible client sends (verified: raw apikey header works, Bearer doesn't), so it's commented out and stripped from every llm-routes.yaml annotation until there's a Bearer-compatible fix. agent-pod now clones and builds the agent-manager fork from source at container start (no prebuilt binary shipped -- wrong arch and over ConfigMap's size cap) and ships coordinator.js alongside hub.js, so multiple repos can run the pipeline concurrently in one pod via kubectl exec. hub.js keeps its existing role as the container's foreground process, unchanged.
318 lines
11 KiB
YAML
318 lines
11 KiB
YAML
# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped.
|
|
#
|
|
# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress
|
|
# can only reference a Service in its own namespace and the predictor Services
|
|
# are there. The Kong ingress controller watches all namespaces, so the routes
|
|
# still land on the gateway. They are synced by the `kong` Application (which
|
|
# has a `path: k8s/apps/api` source) so all gateway config stays in one place.
|
|
#
|
|
# ── Model -> upstream map (verified live) ───────────────────────────────────
|
|
# reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B
|
|
# ornith:35b -> ornith-predictor Ollama
|
|
# qwen2.5:3b-instruct -> ornith-predictor Ollama (same pod!)
|
|
# nomic-embed-text-v2 -> embeddings-predictor TEI
|
|
# bge-reranker-base -> reranker-predictor TEI
|
|
# Qwen2.5-Math-PRM-7B -> verifier-predictor vLLM pooling
|
|
#
|
|
# ── Why path-per-model, and why the body is rewritten ───────────────────────
|
|
# Kong matches routes on host, path, method and headers — never on the request
|
|
# body. So a single /v1/chat/completions endpoint that dispatches on the body's
|
|
# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
|
|
# multi-target model routing, is Enterprise-only).
|
|
#
|
|
# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct`
|
|
# share ONE Ollama pod, and Ollama still reads which model to load from the
|
|
# body's `model` field. If only the path selected the route, a client calling
|
|
# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the
|
|
# 35B model. So each chat route force-overwrites `model` in the body, making the
|
|
# path the single source of truth. Callers may omit `model` entirely.
|
|
#
|
|
# ── Timeouts ───────────────────────────────────────────────────────────────
|
|
# Kong's upstream timeouts default to 60000ms. A 32B model generating a long
|
|
# answer on a Volta GPU routinely exceeds that, and the client would see a
|
|
# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds.
|
|
|
|
# ── GET /v1/models ──────────────────────────────────────────────────────────
|
|
# Served entirely by Kong via request-termination: the plugin short-circuits in
|
|
# the access phase, so the backend below is never contacted. It only exists
|
|
# because an Ingress rule requires a backend.
|
|
#
|
|
# The list is static, which means it can drift from what the engines actually
|
|
# serve — notably if the Ollama pull list in the ornith InferenceService
|
|
# changes. Verify with:
|
|
# curl -s $SVC/v1/models (against each *-predictor)
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-models-list
|
|
namespace: llm-serving
|
|
plugin: request-termination
|
|
config:
|
|
status_code: 200
|
|
content_type: application/json
|
|
body: |
|
|
{"object":"list","data":[
|
|
{"id":"reasoning","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0},
|
|
{"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0}
|
|
]}
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-models
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "GET"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/models
|
|
pathType: Exact
|
|
backend:
|
|
# Never actually called — request-termination answers first.
|
|
service:
|
|
name: reasoning-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/reasoning/chat/completions ─────────────────────────────────────
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-reasoning
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
# `add` only applies when the field is absent, `replace` only when present.
|
|
# Both are needed to force the value in either case.
|
|
add:
|
|
body:
|
|
- "model:reasoning"
|
|
replace:
|
|
body:
|
|
- "model:reasoning"
|
|
# The model lives in the path for routing; the upstream still expects the
|
|
# canonical OpenAI path.
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-reasoning
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/reasoning/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: reasoning-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/ornith/chat/completions ────────────────────────────────────────
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-ornith
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
add:
|
|
body:
|
|
- "model:ornith:35b"
|
|
replace:
|
|
body:
|
|
- "model:ornith:35b"
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-ornith
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/ornith/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: ornith-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/qwen/chat/completions ──────────────────────────────────────────
|
|
# Same upstream pod as ornith — only the forced body `model` differs. Both stay
|
|
# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and
|
|
# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request.
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-qwen
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
add:
|
|
body:
|
|
- "model:qwen2.5:3b-instruct"
|
|
replace:
|
|
body:
|
|
- "model:qwen2.5:3b-instruct"
|
|
uri: /v1/chat/completions
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-chat-qwen
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "3600000"
|
|
konghq.com/write-timeout: "3600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/qwen/chat/completions
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: ornith-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/embeddings ─────────────────────────────────────────────────────
|
|
# No path-per-model and no rewrite: there is exactly one embeddings backend, so
|
|
# there is nothing to disambiguate, and TEI already serves the canonical
|
|
# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists).
|
|
# That makes an OpenAI SDK a drop-in here.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-embeddings
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "600000"
|
|
konghq.com/write-timeout: "600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/embeddings
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: embeddings-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/rerank ─────────────────────────────────────────────────────────
|
|
# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing
|
|
# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely
|
|
# needs the rewrite that embeddings does not.
|
|
apiVersion: configuration.konghq.com/v1
|
|
kind: KongPlugin
|
|
metadata:
|
|
name: llm-rewrite-rerank
|
|
namespace: llm-serving
|
|
plugin: request-transformer
|
|
config:
|
|
replace:
|
|
uri: /rerank
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-rerank
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "600000"
|
|
konghq.com/write-timeout: "600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/rerank
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: reranker-predictor
|
|
port:
|
|
number: 80
|
|
---
|
|
# ── POST /v1/score ──────────────────────────────────────────────────────────
|
|
# The process reward model. Returns scores, not tokens, so it is deliberately
|
|
# not under /chat/completions. vLLM serves /v1/score natively (verified), so no
|
|
# rewrite is needed.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: llm-score
|
|
namespace: llm-serving
|
|
annotations:
|
|
konghq.com/strip-path: "false"
|
|
konghq.com/methods: "POST"
|
|
konghq.com/connect-timeout: "10000"
|
|
konghq.com/read-timeout: "600000"
|
|
konghq.com/write-timeout: "600000"
|
|
spec:
|
|
ingressClassName: kong
|
|
rules:
|
|
- host: api.riotpiao.com
|
|
http:
|
|
paths:
|
|
- path: /v1/score
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: verifier-predictor
|
|
port:
|
|
number: 80
|