feat(gateway): dispatch /v1/systemone by model

Clef-flash is a decision model, not a chat model. It serves Clef's own
System One / Jev API at /v1/systemone: a state plus a schema of typed
questions in, one probability per allowed option out, single prefill
pass, no decoding and no free-form text.

Router previously dispatched by body `model` only for
/v1/chat/completions, /v1/embeddings and /v1/rerank. A request to
/v1/systemone therefore fell through to the configured routes, found
none, and answered "no routes configured". Adding the path to the
dispatch predicate is the whole fix: routeByModel already sets
upstreamPath := path, so the upstream receives /v1/systemone unchanged.
There is deliberately no /rerank-style rewrite.

Config: `ornith:35b` is removed with the Ollama predictor, and its GPU
now serves clef-flash. `qwen2.5:3b-instruct` already pointed at qwen-cpu
and is unchanged. Callers of ornith:35b must move to `reasoning`, which
keeps its serving name while running Qwen3-30B-A3B.

Tests cover body dispatch, unknown_model parity with the other three
paths, and that the upstream path is not rewritten.
This commit is contained in:
Admin Bot
2026-10-27 23:32:24 -07:00
parent 67fe20d7a3
commit d521d7a48b
3 changed files with 177 additions and 5 deletions
+7 -4
View File
@@ -13,11 +13,14 @@ import (
)
// RouteRequest determines which upstream should handle the request.
// For /v1/chat/completions, /v1/embeddings, and /v1/rerank, it uses body-based dispatch.
// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone, it
// uses body-based dispatch.
// For other routes, it looks up by path in the configured routes.
func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
// For /v1/chat/completions, /v1/embeddings, /v1/rerank use body-based dispatch
if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank") {
// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone use
// body-based dispatch. /v1/systemone is Clef's decision API: same
// body-based model lookup, different request/response schema.
if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank" || r.URL.Path == "/v1/systemone") {
return h.routeByModel(r, r.URL.Path)
}
@@ -46,7 +49,7 @@ func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
// routeByModel reads the request body to find the "model" field and routes accordingly.
// The body is preserved for forwarding to the upstream.
// Returns a modelValidationError for client errors (invalid JSON, missing/unknown model).
// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank)
// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank, /v1/systemone)
func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) {
// If there's no body, we can't determine the model
if r.Body == nil {
+169
View File
@@ -0,0 +1,169 @@
package proxy
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
)
// TestSystemoneRoutesByModel verifies POST /v1/systemone dispatches on the body
// "model" field instead of falling through to path-based route lookup. Before
// that path joined body dispatch, the request reached the route table and
// failed with "no routes configured" when no routes were defined.
func TestSystemoneRoutesByModel(t *testing.T) {
clefCalled := false
clefServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
clefCalled = true
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, `{"decision":"allow"}`)
}))
defer clefServer.Close()
clefAddr := strings.TrimPrefix(clefServer.URL, "http://")
cfg := &config.Config{
Models: map[string]*config.ModelUpstream{
"clef-flash": {
Name: "clef-flash",
Address: clefAddr,
},
},
Routes: make(map[string]*config.Route),
}
handler := New(cfg)
defer handler.Close()
server := httptest.NewServer(handler)
defer server.Close()
resp, err := http.Post(
server.URL+"/v1/systemone",
"application/json",
bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)),
)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if !clefCalled {
t.Errorf("upstream clef-flash service was not called")
}
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200 for known systemone model, got %d", resp.StatusCode)
}
body, readErr := io.ReadAll(resp.Body)
if readErr != nil {
t.Fatalf("failed to read response body: %v", readErr)
}
if strings.Contains(string(body), "no routes configured") {
t.Errorf("systemone request fell through to route lookup: %s", string(body))
}
if !strings.Contains(string(body), `"decision":"allow"`) {
t.Errorf("expected upstream response body, got %s", string(body))
}
}
// TestSystemoneUnknownModelValidationError verifies an unknown model on
// /v1/systemone returns the same modelValidationError kind as the other three
// body-dispatched paths.
func TestSystemoneUnknownModelValidationError(t *testing.T) {
cfg := &config.Config{
Models: map[string]*config.ModelUpstream{
"clef-flash": {
Name: "clef-flash",
Address: "localhost:9000",
},
},
Routes: make(map[string]*config.Route),
}
handler := New(cfg)
defer handler.Close()
paths := []string{
"/v1/chat/completions",
"/v1/embeddings",
"/v1/rerank",
"/v1/systemone",
}
for _, path := range paths {
req := httptest.NewRequest(
"POST",
path,
strings.NewReader(`{"model":"unknown-model"}`),
)
route, err := handler.RouteRequest(req)
if route != nil {
t.Errorf("%s: expected nil route for unknown model", path)
}
var validationErr *modelValidationError
if !errors.As(err, &validationErr) {
t.Fatalf("%s: expected modelValidationError, got %v", path, err)
}
if validationErr.Kind != "unknown_model" {
t.Errorf("%s: expected kind unknown_model, got %s", path, validationErr.Kind)
}
}
}
// TestSystemoneUpstreamPathNotRewritten verifies routeByModel keeps the request
// path for /v1/systemone: upstream must see /v1/systemone, not
// /v1/chat/completions. /v1/rerank is the only endpoint with a path rewrite.
func TestSystemoneUpstreamPathNotRewritten(t *testing.T) {
upstreamPath := ""
upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
upstreamPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, `{"decision":"deny"}`)
}))
defer upstreamServer.Close()
upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
cfg := &config.Config{
Models: map[string]*config.ModelUpstream{
"clef-flash": {
Name: "clef-flash",
Address: upstreamAddr,
},
},
Routes: make(map[string]*config.Route),
}
handler := New(cfg)
defer handler.Close()
server := httptest.NewServer(handler)
defer server.Close()
resp, err := http.Post(
server.URL+"/v1/systemone",
"application/json",
bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)),
)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if upstreamPath != "/v1/systemone" {
t.Errorf("expected upstream path /v1/systemone, got %s", upstreamPath)
}
if upstreamPath == "/v1/chat/completions" {
t.Errorf("upstream path was rewritten to /v1/chat/completions")
}
}
File diff suppressed because one or more lines are too long