homelab/k8s/apps/api/llm-routes.yaml

# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped.
#
# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress
# can only reference a Service in its own namespace and the predictor Services
# are there. The Kong ingress controller watches all namespaces, so the routes
# still land on the gateway. They are synced by the `kong` Application (which
# has a `path: k8s/apps/api` source) so all gateway config stays in one place.
#
# ── Model -> upstream map (verified live) ───────────────────────────────────
#   reasoning              -> reasoning-predictor   vLLM, DeepSeek-R1-Distill-32B
#   ornith:35b             -> ornith-predictor      Ollama
#   qwen2.5:3b-instruct    -> ornith-predictor      Ollama  (same pod!)
#   nomic-embed-text-v2    -> embeddings-predictor  TEI
#   bge-reranker-base      -> reranker-predictor    TEI
#   Qwen2.5-Math-PRM-7B    -> verifier-predictor    vLLM pooling
#
# ── Why path-per-model, and why the body is rewritten ───────────────────────
# Kong matches routes on host, path, method and headers — never on the request
# body. So a single /v1/chat/completions endpoint that dispatches on the body's
# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
# multi-target model routing, is Enterprise-only).
#
# Hence the model is in the path. But `ornith:35b` and `qwen2.5:3b-instruct`
# share ONE Ollama pod, and Ollama still reads which model to load from the
# body's `model` field. If only the path selected the route, a client calling
# /v1/qwen/... with `"model": "ornith:35b"` in the body would silently get the
# 35B model. So each chat route force-overwrites `model` in the body, making the
# path the single source of truth. Callers may omit `model` entirely.
#
# ── Timeouts ───────────────────────────────────────────────────────────────
# Kong's upstream timeouts default to 60000ms. A 32B model generating a long
# answer on a Volta GPU routinely exceeds that, and the client would see a
# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds.

# ── GET /v1/models ──────────────────────────────────────────────────────────
# Served entirely by Kong via request-termination: the plugin short-circuits in
# the access phase, so the backend below is never contacted. It only exists
# because an Ingress rule requires a backend.
#
# The list is static, which means it can drift from what the engines actually
# serve — notably if the Ollama pull list in the ornith InferenceService
# changes. Verify with:
#   curl -s $SVC/v1/models   (against each *-predictor)
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: llm-models-list
  namespace: llm-serving
plugin: request-termination
config:
  status_code: 200
  content_type: application/json
  body: |
    {"object":"list","data":[
      {"id":"reasoning","object":"model","owned_by":"homelab","created":0},
      {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
      {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
      {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
      {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0},
      {"id":"Qwen/Qwen2.5-Math-PRM-7B","object":"model","owned_by":"homelab","created":0}
    ]}
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-models
  namespace: llm-serving
  annotations:
    konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml
    konghq.com/strip-path: "false"
    konghq.com/methods: "GET"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/models
            pathType: Exact
            backend:
              # Never actually called — request-termination answers first.
              service:
                name: reasoning-predictor
                port:
                  number: 80
---
# ── POST /v1/reasoning/chat/completions ─────────────────────────────────────
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: llm-rewrite-reasoning
  namespace: llm-serving
plugin: request-transformer
config:
  # `add` only applies when the field is absent, `replace` only when present.
  # Both are needed to force the value in either case.
  add:
    body:
      - "model:reasoning"
  replace:
    body:
      - "model:reasoning"
    # The model lives in the path for routing; the upstream still expects the
    # canonical OpenAI path.
    uri: /v1/chat/completions
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-chat-reasoning
  namespace: llm-serving
  annotations:
    konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "3600000"
    konghq.com/write-timeout: "3600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/reasoning/chat/completions
            pathType: Prefix
            backend:
              service:
                name: reasoning-predictor
                port:
                  number: 80
---
# ── POST /v1/ornith/chat/completions ────────────────────────────────────────
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: llm-rewrite-ornith
  namespace: llm-serving
plugin: request-transformer
config:
  add:
    body:
      - "model:ornith:35b"
  replace:
    body:
      - "model:ornith:35b"
    uri: /v1/chat/completions
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-chat-ornith
  namespace: llm-serving
  annotations:
    konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "3600000"
    konghq.com/write-timeout: "3600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/ornith/chat/completions
            pathType: Prefix
            backend:
              service:
                name: ornith-predictor
                port:
                  number: 80
---
# ── POST /v1/qwen/chat/completions ──────────────────────────────────────────
# Same upstream pod as ornith — only the forced body `model` differs. Both stay
# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and
# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request.
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: llm-rewrite-qwen
  namespace: llm-serving
plugin: request-transformer
config:
  add:
    body:
      - "model:qwen2.5:3b-instruct"
  replace:
    body:
      - "model:qwen2.5:3b-instruct"
    uri: /v1/chat/completions
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-chat-qwen
  namespace: llm-serving
  annotations:
    konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "3600000"
    konghq.com/write-timeout: "3600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/qwen/chat/completions
            pathType: Prefix
            backend:
              service:
                name: ornith-predictor
                port:
                  number: 80
---
# ── POST /v1/embeddings ─────────────────────────────────────────────────────
# No path-per-model and no rewrite: there is exactly one embeddings backend, so
# there is nothing to disambiguate, and TEI already serves the canonical
# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists).
# That makes an OpenAI SDK a drop-in here.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-embeddings
  namespace: llm-serving
  annotations:
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "600000"
    konghq.com/write-timeout: "600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/embeddings
            pathType: Prefix
            backend:
              service:
                name: embeddings-predictor
                port:
                  number: 80
---
# ── POST /v1/rerank ─────────────────────────────────────────────────────────
# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing
# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely
# needs the rewrite that embeddings does not.
apiVersion: configuration.konghq.com/v1
kind: KongPlugin
metadata:
  name: llm-rewrite-rerank
  namespace: llm-serving
plugin: request-transformer
config:
  replace:
    uri: /rerank
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-rerank
  namespace: llm-serving
  annotations:
    konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "600000"
    konghq.com/write-timeout: "600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/rerank
            pathType: Prefix
            backend:
              service:
                name: reranker-predictor
                port:
                  number: 80
---
# ── POST /v1/score ──────────────────────────────────────────────────────────
# The process reward model. Returns scores, not tokens, so it is deliberately
# not under /chat/completions. vLLM serves /v1/score natively (verified), so no
# rewrite is needed.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: llm-score
  namespace: llm-serving
  annotations:
    konghq.com/strip-path: "false"
    konghq.com/methods: "POST"
    konghq.com/connect-timeout: "10000"
    konghq.com/read-timeout: "600000"
    konghq.com/write-timeout: "600000"
spec:
  ingressClassName: kong
  rules:
    - host: api.riotpiao.com
      http:
        paths:
          - path: /v1/score
            pathType: Prefix
            backend:
              service:
                name: verifier-predictor
                port:
                  number: 80