diff --git a/k8s/apps/api/kong-values.yaml b/k8s/apps/api/kong-values.yaml index 48ac0e9..117285e 100644 --- a/k8s/apps/api/kong-values.yaml +++ b/k8s/apps/api/kong-values.yaml @@ -35,6 +35,16 @@ env: # config in a database mutated through the Admin API — state outside git, plus # migration Jobs on every upgrade. database: "off" + # `nginx_proxy_` injects a directive into the proxy location block; + # this renders `proxy_buffering off;`. + # + # Required for LLM streaming. With buffering on (the default) nginx accumulates + # the upstream response before forwarding, so an SSE stream from + # `"stream": true` arrives in lumps or stalls until the generation finishes — + # which defeats the point of streaming. The matching setting is already on the + # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the + # buffered one dominates. + nginx_proxy_proxy_buffering: "off" ingressController: enabled: true diff --git a/k8s/apps/api/kustomization.yaml b/k8s/apps/api/kustomization.yaml index 95b8f1b..14753b8 100644 --- a/k8s/apps/api/kustomization.yaml +++ b/k8s/apps/api/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization # or it is silently dropped with no error and no drift shown. resources: - ingress.yaml + - llm-routes.yaml # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own # namespace, and the transformer rewrites metadata.namespace on every resource # it builds, which is a trap for anything cross-namespace added later. diff --git a/k8s/apps/api/llm-routes.yaml b/k8s/apps/api/llm-routes.yaml new file mode 100644 index 0000000..5fe1602 --- /dev/null +++ b/k8s/apps/api/llm-routes.yaml @@ -0,0 +1,317 @@ +# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped. +# +# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress +# can only reference a Service in its own namespace and the predictor Services +# are there. The Kong ingress controller watches all namespaces, so the routes +# still land on the gateway. They are synced by the `kong` Application (which +# has a `path: k8s/apps/api` source) so all gateway config stays in one place. +# +# ── Model -> upstream map (verified live) ─────────────────────────────────── +# reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B +# ornith:35b -> ornith-predictor Ollama +# qwen2.5:3b-instruct -> ornith-predictor Ollama (same pod!) +# nomic-embed-text-v2 -> embeddings-predictor TEI +# bge-reranker-base -> reranker-predictor TEI +# Qwen2.5-Math-PRM-7B -> verifier-predictor vLLM pooling +# +# ── Why path-per-model, and why the body is rewritten ─────────────────────── +# Kong matches routes on host, path, method and headers — never on the request +# body. So a single /v1/chat/completions endpoint that dispatches on the body's +# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does +# multi-target model routing, is Enterprise-only). +# +# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct` +# share ONE Ollama pod, and Ollama still reads which model to load from the +# body's `model` field. If only the path selected the route, a client calling +# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the +# 35B model. So each chat route force-overwrites `model` in the body, making the +# path the single source of truth. Callers may omit `model` entirely. +# +# ── Timeouts ─────────────────────────────────────────────────────────────── +# Kong's upstream timeouts default to 60000ms. A 32B model generating a long +# answer on a Volta GPU routinely exceeds that, and the client would see a +# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds. + +# ── GET /v1/models ────────────────────────────────────────────────────────── +# Served entirely by Kong via request-termination: the plugin short-circuits in +# the access phase, so the backend below is never contacted. It only exists +# because an Ingress rule requires a backend. +# +# The list is static, which means it can drift from what the engines actually +# serve — notably if the Ollama pull list in the ornith InferenceService +# changes. Verify with: +# curl -s $SVC/v1/models (against each *-predictor) +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-models-list + namespace: llm-serving +plugin: request-termination +config: + status_code: 200 + content_type: application/json + body: | + {"object":"list","data":[ + {"id":"reasoning","object":"model","owned_by":"homelab","created":0}, + {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0}, + {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0}, + {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0}, + {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}, + {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0} + ]} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-models + namespace: llm-serving + annotations: + konghq.com/plugins: llm-models-list + konghq.com/strip-path: "false" + konghq.com/methods: "GET" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/models + pathType: Exact + backend: + # Never actually called — request-termination answers first. + service: + name: reasoning-predictor + port: + number: 80 +--- +# ── POST /v1/reasoning/chat/completions ───────────────────────────────────── +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-reasoning + namespace: llm-serving +plugin: request-transformer +config: + # `add` only applies when the field is absent, `replace` only when present. + # Both are needed to force the value in either case. + add: + body: + - "model:reasoning" + replace: + body: + - "model:reasoning" + # The model lives in the path for routing; the upstream still expects the + # canonical OpenAI path. + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-reasoning + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-reasoning + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/reasoning/chat/completions + pathType: Prefix + backend: + service: + name: reasoning-predictor + port: + number: 80 +--- +# ── POST /v1/ornith/chat/completions ──────────────────────────────────────── +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-ornith + namespace: llm-serving +plugin: request-transformer +config: + add: + body: + - "model:ornith:35b" + replace: + body: + - "model:ornith:35b" + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-ornith + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-ornith + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/ornith/chat/completions + pathType: Prefix + backend: + service: + name: ornith-predictor + port: + number: 80 +--- +# ── POST /v1/qwen/chat/completions ────────────────────────────────────────── +# Same upstream pod as ornith — only the forced body `model` differs. Both stay +# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and +# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request. +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-qwen + namespace: llm-serving +plugin: request-transformer +config: + add: + body: + - "model:qwen2.5:3b-instruct" + replace: + body: + - "model:qwen2.5:3b-instruct" + uri: /v1/chat/completions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-chat-qwen + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-qwen + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "3600000" + konghq.com/write-timeout: "3600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/qwen/chat/completions + pathType: Prefix + backend: + service: + name: ornith-predictor + port: + number: 80 +--- +# ── POST /v1/embeddings ───────────────────────────────────────────────────── +# No path-per-model and no rewrite: there is exactly one embeddings backend, so +# there is nothing to disambiguate, and TEI already serves the canonical +# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists). +# That makes an OpenAI SDK a drop-in here. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-embeddings + namespace: llm-serving + annotations: + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/embeddings + pathType: Prefix + backend: + service: + name: embeddings-predictor + port: + number: 80 +--- +# ── POST /v1/rerank ───────────────────────────────────────────────────────── +# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing +# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely +# needs the rewrite that embeddings does not. +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: llm-rewrite-rerank + namespace: llm-serving +plugin: request-transformer +config: + replace: + uri: /rerank +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-rerank + namespace: llm-serving + annotations: + konghq.com/plugins: llm-rewrite-rerank + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/rerank + pathType: Prefix + backend: + service: + name: reranker-predictor + port: + number: 80 +--- +# ── POST /v1/score ────────────────────────────────────────────────────────── +# The process reward model. Returns scores, not tokens, so it is deliberately +# not under /chat/completions. vLLM serves /v1/score natively (verified), so no +# rewrite is needed. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: llm-score + namespace: llm-serving + annotations: + konghq.com/strip-path: "false" + konghq.com/methods: "POST" + konghq.com/connect-timeout: "10000" + konghq.com/read-timeout: "600000" + konghq.com/write-timeout: "600000" +spec: + ingressClassName: kong + rules: + - host: api.riotpiao.com + http: + paths: + - path: /v1/score + pathType: Prefix + backend: + service: + name: verifier-predictor + port: + number: 80