When the upstream LLM server expects a different model name than what
clients send, the gateway now rewrites the 'model' field in the request
body before forwarding.
Config example:
models:
- name: "ornith:35b" # client-facing name
address: "ornith-predictor:80"
upstreamModel: "qwen2.5:72b" # what upstream expects
Fixes 400 'model is required' errors when upstream model names differ.
This commit is contained in:
@@ -122,6 +122,20 @@ func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// If upstream expects a different model name, rewrite the body
|
||||
if modelUpstream.UpstreamModel != "" && modelUpstream.UpstreamModel != modelName {
|
||||
payload["model"] = modelUpstream.UpstreamModel
|
||||
newBody, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, &modelValidationError{
|
||||
Kind: "invalid_request",
|
||||
Message: fmt.Sprintf("failed to rewrite model name: %v", err),
|
||||
}
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(newBody))
|
||||
r.ContentLength = int64(len(newBody))
|
||||
}
|
||||
|
||||
// Determine the upstream path based on the request path
|
||||
upstreamPath := path
|
||||
if path == "/v1/rerank" {
|
||||
|
||||
Reference in New Issue
Block a user