From 245a03e951636dd2c0f573ce9d39c108975cfabe Mon Sep 17 00:00:00 2001 From: Story Crater Bot <19826264+Riotpiaole@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:47:45 -0700 Subject: [PATCH] =?UTF-8?q?feat(api):=20add=20DeepSeek-shaped=20LLM=20API?= =?UTF-8?q?=20on=20Kong=20=E2=80=94=20/v1/models,=20per-model=20chat=20com?= =?UTF-8?q?pletions,=20embeddings,=20rerank,=20score;=20disable=20Kong=20r?= =?UTF-8?q?esponse=20buffering=20so=20stream:true=20actually=20streams?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kong matches routes on host/path/method/header, never on the request body, so a single /v1/chat/completions dispatching on body.model is not expressible in Kong OSS (ai-proxy-advanced, which does multi-target model routing, is Enterprise). Model therefore goes in the path: GET /v1/models static list (request-termination) POST /v1/reasoning/chat/completions reasoning-predictor (vLLM) POST /v1/ornith/chat/completions ornith-predictor (Ollama) POST /v1/qwen/chat/completions ornith-predictor (Ollama, same pod) POST /v1/embeddings embeddings-predictor (TEI) POST /v1/rerank reranker-predictor (TEI) POST /v1/score verifier-predictor (vLLM pooling) - each chat route force-overwrites body.model via request-transformer add+replace: ornith:35b and qwen2.5:3b-instruct share one Ollama pod, so without this a client hitting /v1/qwen with "model":"ornith:35b" would silently get the 35B - routes live in ns llm-serving, not api: an Ingress can only reference a Service in its own namespace, and KIC watches all namespaces - embeddings and score need no rewrite (TEI/vLLM already serve the canonical paths); rerank does, since /v1/rerank 404s and only /rerank exists - read/write timeouts 1h: Kong defaults to 60s, which a 32B model on Volta exceeds mid-generation and returns 504 - nginx_proxy_proxy_buffering=off: buffered responses lump or stall SSE, and both hops (nginx Ingress and Kong) must be unbuffered or the buffered one wins - no auth for now, per decision; api.riotpiao.com is reachable through nginx, so GPU time is currently unauthenticated --- k8s/apps/api/kong-values.yaml | 10 + k8s/apps/api/kustomization.yaml | 1 + k8s/apps/api/llm-routes.yaml | 317 ++++++++++++++++++++++++++++++++ 3 files changed, 328 insertions(+) create mode 100644 k8s/apps/api/llm-routes.yaml diff --git a/k8s/apps/api/kong-values.yaml b/k8s/apps/api/kong-values.yaml index 48ac0e9..117285e 100644 --- a/k8s/apps/api/kong-values.yaml +++ b/k8s/apps/api/kong-values.yaml @@ -35,6 +35,16 @@ env: # config in a database mutated through the Admin API — state outside git, plus # migration Jobs on every upgrade. database: "off" + # `nginx_proxy_` injects a directive into the proxy location block; + # this renders `proxy_buffering off;`. + # + # Required for LLM streaming. With buffering on (the default) nginx accumulates + # the upstream response before forwarding, so an SSE stream from + # `"stream": true` arrives in lumps or stalls until the generation finishes — + # which defeats the point of streaming. The matching setting is already on the + # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the + # buffered one dominates. + nginx_proxy_proxy_buffering: "off" ingressController: enabled: true diff --git a/k8s/apps/api/kustomization.yaml b/k8s/apps/api/kustomization.yaml index 95b8f1b..14753b8 100644 --- a/k8s/apps/api/kustomization.yaml +++ b/k8s/apps/api/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization # or it is silently dropped with no error and no drift shown. resources: - ingress.yaml + - llm-routes.yaml # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own # namespace, and the transformer rewrites metadata.namespace on every resource # it builds, which is a trap for anything cross-namespace added later. diff --git a/k8s/apps/api/llm-routes.yaml b/k8s/apps/api/llm-routes.yaml new file mode 100644 index 0000000..5fe1602 --- /dev/null +++ b/k8s/apps/api/llm-routes.yaml @@ -0,0 +1,317 @@ +# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped. +# +# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress +# can only reference a Service in its own namespace and the predictor Services +# are there. The Kong ingress controller watches all namespaces, so the routes +# still land on the gateway. They are synced by the `kong` Application (which +# has a `path: k8s/apps/api` source) so all gateway config stays in one place. +# +# ── Model -> upstream map (verified live) ─────────────────────────────────── +# reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B +# ornith:35b -> ornith-predictor Ollama +# qwen2.5:3b-instruct -> ornith-predictor Ollama (same pod!) +# nomic-embed-text-v2 -> embeddings-predictor TEI +# bge-reranker-base -> reranker-predictor TEI +# Qwen2.5-Math-PRM-7B -> verifier-predictor vLLM pooling +# +# ── Why path-per-model, and why the body is rewritten ─────────────────────── +# Kong matches routes on host, path, method and headers — never on the request +# body. So a single /v1/chat/completions endpoint that dispatches on the body's +# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does +# multi-target model routing, is Enterprise-only). +# +# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct` +# share ONE Ollama pod, and Ollama still reads which model to load from the +# body's `model` field. If only the path selected the route, a client calling +# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the +# 35B model. So each chat route force-overwrites `model` in the body, making the +# path the single source of truth. Callers may omit `model` entirely. +# +# ── Timeouts ─────────────────────────────────────────────────────────────── +# Kong's upstream timeouts default to 60000ms. A 32B model generating a long +# answer on a Volta GPU routinely exceeds that, and the client would see a +# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds. + +# ── GET /v1/models ────────────────────────────────────────────────────────── +# Served entirely by Kong via request-termination: the plugin short-circuits in +# the access phase, so the backend below is never contacted. It only exists +# because an Ingress rule requires a backend. +# +# The list is static, which means it can drift from what the engines actually +# serve — notably if the Ollama pull list in the ornith InferenceService +# changes. Verify with: +# curl -s $SVC/v1/models (against each *-predictor) +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-models-list + namespace: llm-serving +plugin: request-termination +config: + status_code: 200 + content_type: application/json + body: | + {"object":"list","data":[ + {"id":"reasoning","object":"model","owned_by":"homelab","created":0}, + {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0}, + {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0}, + {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0}, + {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}, + {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0} + ]} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-models + namespace: llm-serving + annotations: + konghq.com/plugins: llm-models-list + konghq.com/strip-path: "false" + konghq.com/methods: "GET" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/models + pathType: Exact + backend: + # Never actually called — request-termination answers first. + service: + name: reasoning-predictor + port: + number: 80 +--- +# ── POST /v1/reasoning/chat/completions ───────────────────────────────────── +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-reasoning + namespace: llm-serving +plugin: request-transformer +config: + # `add` only applies when the field is absent, `replace` only when present. + # Both are needed to force the value in either case. + add: + body: + - "model:reasoning" + replace: + body: + - "model:reasoning" + # The model lives in the path for routing; the upstream still expects the + # canonical OpenAI path. + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-reasoning + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-reasoning + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/reasoning/chat/completions + pathType: Prefix + backend: + service: + name: reasoning-predictor + port: + number: 80 +--- +# ── POST /v1/ornith/chat/completions ──────────────────────────────────────── +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-ornith + namespace: llm-serving +plugin: request-transformer +config: + add: + body: + - "model:ornith:35b" + replace: + body: + - "model:ornith:35b" + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-ornith + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-ornith + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/ornith/chat/completions + pathType: Prefix + backend: + service: + name: ornith-predictor + port: + number: 80 +--- +# ── POST /v1/qwen/chat/completions ────────────────────────────────────────── +# Same upstream pod as ornith — only the forced body `model` differs. Both stay +# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and +# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request. +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-qwen + namespace: llm-serving +plugin: request-transformer +config: + add: + body: + - "model:qwen2.5:3b-instruct" + replace: + body: + - "model:qwen2.5:3b-instruct" + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-qwen + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-qwen + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/qwen/chat/completions + pathType: Prefix + backend: + service: + name: ornith-predictor + port: + number: 80 +--- +# ── POST /v1/embeddings ───────────────────────────────────────────────────── +# No path-per-model and no rewrite: there is exactly one embeddings backend, so +# there is nothing to disambiguate, and TEI already serves the canonical +# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists). +# That makes an OpenAI SDK a drop-in here. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-embeddings + namespace: llm-serving + annotations: + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/embeddings + pathType: Prefix + backend: + service: + name: embeddings-predictor + port: + number: 80 +--- +# ── POST /v1/rerank ───────────────────────────────────────────────────────── +# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing +# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely +# needs the rewrite that embeddings does not. +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-rerank + namespace: llm-serving +plugin: request-transformer +config: + replace: + uri: /rerank +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-rerank + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-rerank + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/rerank + pathType: Prefix + backend: + service: + name: reranker-predictor + port: + number: 80 +--- +# ── POST /v1/score ────────────────────────────────────────────────────────── +# The process reward model. Returns scores, not tokens, so it is deliberately +# not under /chat/completions. vLLM serves /v1/score natively (verified), so no +# rewrite is needed. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-score + namespace: llm-serving + annotations: + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/score + pathType: Prefix + backend: + service: + name: verifier-predictor + port: + number: 80