From d521d7a48b53e3461920965d9cb953bf04468e4c Mon Sep 17 00:00:00 2001 From: Admin Bot Date: Tue, 27 Oct 2026 23:32:24 -0700 Subject: [PATCH] feat(gateway): dispatch /v1/systemone by model Clef-flash is a decision model, not a chat model. It serves Clef's own System One / Jev API at /v1/systemone: a state plus a schema of typed questions in, one probability per allowed option out, single prefill pass, no decoding and no free-form text. Router previously dispatched by body `model` only for /v1/chat/completions, /v1/embeddings and /v1/rerank. A request to /v1/systemone therefore fell through to the configured routes, found none, and answered "no routes configured". Adding the path to the dispatch predicate is the whole fix: routeByModel already sets upstreamPath := path, so the upstream receives /v1/systemone unchanged. There is deliberately no /rerank-style rewrite. Config: `ornith:35b` is removed with the Ollama predictor, and its GPU now serves clef-flash. `qwen2.5:3b-instruct` already pointed at qwen-cpu and is unchanged. Callers of ornith:35b must move to `reasoning`, which keeps its serving name while running Qwen3-30B-A3B. Tests cover body dispatch, unknown_model parity with the other three paths, and that the upstream path is not rewritten. --- internal/proxy/router.go | 11 +- internal/proxy/systemone_test.go | 169 +++++++++++++++++++++++++++++++ k8s/gateway-config-secret.yaml | 2 +- 3 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 internal/proxy/systemone_test.go diff --git a/internal/proxy/router.go b/internal/proxy/router.go index 493c7d4..9159806 100644 --- a/internal/proxy/router.go +++ b/internal/proxy/router.go @@ -13,11 +13,14 @@ import ( ) // RouteRequest determines which upstream should handle the request. -// For /v1/chat/completions, /v1/embeddings, and /v1/rerank, it uses body-based dispatch. +// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone, it +// uses body-based dispatch. // For other routes, it looks up by path in the configured routes. func (h *Handler) RouteRequest(r *http.Request) (*Route, error) { - // For /v1/chat/completions, /v1/embeddings, /v1/rerank use body-based dispatch - if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank") { + // For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone use + // body-based dispatch. /v1/systemone is Clef's decision API: same + // body-based model lookup, different request/response schema. + if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank" || r.URL.Path == "/v1/systemone") { return h.routeByModel(r, r.URL.Path) } @@ -46,7 +49,7 @@ func (h *Handler) RouteRequest(r *http.Request) (*Route, error) { // routeByModel reads the request body to find the "model" field and routes accordingly. // The body is preserved for forwarding to the upstream. // Returns a modelValidationError for client errors (invalid JSON, missing/unknown model). -// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank) +// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank, /v1/systemone) func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) { // If there's no body, we can't determine the model if r.Body == nil { diff --git a/internal/proxy/systemone_test.go b/internal/proxy/systemone_test.go new file mode 100644 index 0000000..b4655e9 --- /dev/null +++ b/internal/proxy/systemone_test.go @@ -0,0 +1,169 @@ +package proxy + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "forgejo.riotpiao.com/rock/homelab-frontend/internal/config" +) + +// TestSystemoneRoutesByModel verifies POST /v1/systemone dispatches on the body +// "model" field instead of falling through to path-based route lookup. Before +// that path joined body dispatch, the request reached the route table and +// failed with "no routes configured" when no routes were defined. +func TestSystemoneRoutesByModel(t *testing.T) { + clefCalled := false + clefServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + clefCalled = true + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"decision":"allow"}`) + })) + defer clefServer.Close() + + clefAddr := strings.TrimPrefix(clefServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: clefAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Post( + server.URL+"/v1/systemone", + "application/json", + bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)), + ) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if !clefCalled { + t.Errorf("upstream clef-flash service was not called") + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200 for known systemone model, got %d", resp.StatusCode) + } + + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + t.Fatalf("failed to read response body: %v", readErr) + } + if strings.Contains(string(body), "no routes configured") { + t.Errorf("systemone request fell through to route lookup: %s", string(body)) + } + if !strings.Contains(string(body), `"decision":"allow"`) { + t.Errorf("expected upstream response body, got %s", string(body)) + } +} + +// TestSystemoneUnknownModelValidationError verifies an unknown model on +// /v1/systemone returns the same modelValidationError kind as the other three +// body-dispatched paths. +func TestSystemoneUnknownModelValidationError(t *testing.T) { + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: "localhost:9000", + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + paths := []string{ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/rerank", + "/v1/systemone", + } + for _, path := range paths { + req := httptest.NewRequest( + "POST", + path, + strings.NewReader(`{"model":"unknown-model"}`), + ) + route, err := handler.RouteRequest(req) + if route != nil { + t.Errorf("%s: expected nil route for unknown model", path) + } + var validationErr *modelValidationError + if !errors.As(err, &validationErr) { + t.Fatalf("%s: expected modelValidationError, got %v", path, err) + } + if validationErr.Kind != "unknown_model" { + t.Errorf("%s: expected kind unknown_model, got %s", path, validationErr.Kind) + } + } +} + +// TestSystemoneUpstreamPathNotRewritten verifies routeByModel keeps the request +// path for /v1/systemone: upstream must see /v1/systemone, not +// /v1/chat/completions. /v1/rerank is the only endpoint with a path rewrite. +func TestSystemoneUpstreamPathNotRewritten(t *testing.T) { + upstreamPath := "" + + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"decision":"deny"}`) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Post( + server.URL+"/v1/systemone", + "application/json", + bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)), + ) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if upstreamPath != "/v1/systemone" { + t.Errorf("expected upstream path /v1/systemone, got %s", upstreamPath) + } + if upstreamPath == "/v1/chat/completions" { + t.Errorf("upstream path was rewritten to /v1/chat/completions") + } +} diff --git a/k8s/gateway-config-secret.yaml b/k8s/gateway-config-secret.yaml index fdeb293..f79b768 100644 --- a/k8s/gateway-config-secret.yaml +++ b/k8s/gateway-config-secret.yaml @@ -7,4 +7,4 @@ metadata: app: api-gateway type: Opaque stringData: - config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"ornith:35b\"\n address: \"ornith-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n" + config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n# clef-flash is a DECISION model, not a chat model. It takes a `state` plus a\n# schema of typed `questions` and returns one probability per allowed option in\n# a single prefill pass. There is no decoding and no free-form text.\n#\n# Its request and response shapes are Clef's own (System One / Jev), NOT\n# OpenAI's: no `messages`, no `choices`. A standard OpenAI client cannot call\n# it, and the public path is /v1/systemone rather than /v1/chat/completions.\n#\n# The dispatcher resolves this model by the body's `model` field, then forwards\n# to the REQUEST path. The `path` value below records intent and must stay in\n# sync with the dispatch list in internal/proxy/router.go.\n- name: \"clef-flash\"\n address: \"clef-flash-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/systemone\"\n# `ornith:35b` retired: the Ollama Llama-35B predictor is gone and its GPU now\n# serves clef-flash. Callers must move to `reasoning`, which keeps the same\n# serving name while running Qwen3-30B-A3B on the same hardware.\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n"