feat(phase2): complete openai api surfaces 2.3-2.7
- 2.3: unknown model errors (400 + RFC 9457 problem+json with valid_models) - 2.5: GET /v1/models endpoint (derived from config, not hardcoded) - 2.6: POST /v1/embeddings passthrough (body-based dispatch, no rewrite) - 2.7: POST /v1/rerank with path rewrite (/v1/rerank → /rerank) - wire proxy.Handler in main.go (was using dummy handler) - 140+ tests passing, race detector clean - all requests: client → nginx → gateway → upstreams - ready for config deployment to go live
This commit is contained in:
+63
-13
@@ -13,12 +13,12 @@ import (
|
||||
)
|
||||
|
||||
// RouteRequest determines which upstream should handle the request.
|
||||
// For /v1/chat/completions, it uses body-based dispatch (reads JSON to find "model" field).
|
||||
// For other routes, it looks up by path prefix.
|
||||
// For /v1/chat/completions, /v1/embeddings, and /v1/rerank, it uses body-based dispatch.
|
||||
// For other routes, it looks up by path in the configured routes.
|
||||
func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
|
||||
// For /v1/chat/completions, use body-based dispatch
|
||||
if r.URL.Path == "/v1/chat/completions" && r.Method == "POST" {
|
||||
return h.routeByModel(r)
|
||||
// For /v1/chat/completions, /v1/embeddings, /v1/rerank use body-based dispatch
|
||||
if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank") {
|
||||
return h.routeByModel(r, r.URL.Path)
|
||||
}
|
||||
|
||||
// For other paths, try to find a matching route by path
|
||||
@@ -45,17 +45,25 @@ func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
|
||||
|
||||
// routeByModel reads the request body to find the "model" field and routes accordingly.
|
||||
// The body is preserved for forwarding to the upstream.
|
||||
func (h *Handler) routeByModel(r *http.Request) (*Route, error) {
|
||||
// Returns a modelValidationError for client errors (invalid JSON, missing/unknown model).
|
||||
// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank)
|
||||
func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) {
|
||||
// If there's no body, we can't determine the model
|
||||
if r.Body == nil {
|
||||
return nil, fmt.Errorf("request body required")
|
||||
return nil, &modelValidationError{
|
||||
Kind: "missing_model",
|
||||
Message: "request body required",
|
||||
}
|
||||
}
|
||||
|
||||
// Read the body to extract the model name
|
||||
// We need to be careful to preserve the body for the upstream
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read request body: %w", err)
|
||||
return nil, &modelValidationError{
|
||||
Kind: "invalid_request",
|
||||
Message: fmt.Sprintf("failed to read request body: %v", err),
|
||||
}
|
||||
}
|
||||
|
||||
// Restore the body so it can be read again by the upstream
|
||||
@@ -64,26 +72,68 @@ func (h *Handler) routeByModel(r *http.Request) (*Route, error) {
|
||||
// Parse the JSON to find the model field
|
||||
var payload map[string]interface{}
|
||||
if err := json.Unmarshal(bodyBytes, &payload); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON in request body: %w", err)
|
||||
return nil, &modelValidationError{
|
||||
Kind: "invalid_json",
|
||||
Message: "request body is not valid JSON",
|
||||
}
|
||||
}
|
||||
|
||||
// Extract the model name
|
||||
modelName, ok := payload["model"].(string)
|
||||
modelVal, hasModel := payload["model"]
|
||||
if !hasModel {
|
||||
return nil, &modelValidationError{
|
||||
Kind: "missing_model",
|
||||
Message: "'model' field is missing",
|
||||
}
|
||||
}
|
||||
|
||||
// Handle null model
|
||||
if modelVal == nil {
|
||||
return nil, &modelValidationError{
|
||||
Kind: "null_model",
|
||||
Message: "'model' field is null",
|
||||
}
|
||||
}
|
||||
|
||||
// Extract as string
|
||||
modelName, ok := modelVal.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("model field missing or not a string")
|
||||
return nil, &modelValidationError{
|
||||
Kind: "missing_model",
|
||||
Message: "'model' field must be a string",
|
||||
}
|
||||
}
|
||||
|
||||
// Handle empty string
|
||||
if modelName == "" {
|
||||
return nil, &modelValidationError{
|
||||
Kind: "empty_model",
|
||||
Message: "'model' field cannot be empty",
|
||||
}
|
||||
}
|
||||
|
||||
// Look up the model in the registry
|
||||
modelUpstream := h.config.LookupModel(modelName)
|
||||
if modelUpstream == nil {
|
||||
return nil, fmt.Errorf("unknown model: %q", modelName)
|
||||
return nil, &modelValidationError{
|
||||
Kind: "unknown_model",
|
||||
Message: fmt.Sprintf("unknown model: %q", modelName),
|
||||
Model: modelName,
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the upstream path based on the request path
|
||||
upstreamPath := path
|
||||
if path == "/v1/rerank" {
|
||||
// Rerank endpoint uses /rerank path on upstream
|
||||
upstreamPath = "/rerank"
|
||||
}
|
||||
|
||||
// Create a route for this model with appropriate timeouts
|
||||
// These are sensible defaults for LLM models
|
||||
upstreamCfg := &config.Upstream{
|
||||
Address: modelUpstream.Address,
|
||||
PathRewrite: "/v1/chat/completions",
|
||||
PathRewrite: upstreamPath,
|
||||
ConnectTimeout: h.defaultConnectTimeout,
|
||||
ReadTimeout: h.defaultReadTimeout,
|
||||
WriteTimeout: h.defaultWriteTimeout,
|
||||
|
||||
Reference in New Issue
Block a user