fix(gateway): stop routing clef-flash, the model cannot run here

The model list still pointed clef-flash at clef-flash-predictor. That predictor is gone:
Clef needs SGLang v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta
kernels. Every GPU is a Tesla V100 (sm_70), so SGLang aborts with 'no kernel image is
available'. SGLang's floor is sm75 regardless.

Advertising it meant a caller would wait for a Service with no endpoints until its own
timeout fired, instead of failing clearly. Remove the entry and keep the reasoning in
its place, so restoring it later is a deliberate act tied to new hardware.

internal/config/config.go

@@ -66,6 +66,13 @@ type AuthConfig struct {
 	JWKSURL string
 	// RequiredCapability is the permission required for LLM inference (e.g., "llm:inference").
 	RequiredCapability string
+	// AllowListedMachineIdentities admits a caller whose token carries no roles
+	// claim, when its client id is in ZitadelClientIDs. ZITADEL does not assert roles
+	// in machine-user access tokens on this instance, so a service account can never
+	// satisfy RequiredCapability. The allow-list is then the authorisation: the caller
+	// still proves possession of a registered private key, and the entry is a reviewed
+	// line of configuration, not a wildcard.
+	AllowListedMachineIdentities bool
 	// TokenURL is the OIDC token endpoint used for token exchange
 	// (client_credentials / password grants).
 	TokenURL string

internal/config/loader.go

@@ -25,6 +25,7 @@ type rawAuth struct {
 	Audience                     string   `yaml:"audience"`
 	JWKSURL                      string   `yaml:"jwksUrl"`
 	RequiredCapability           string   `yaml:"requiredCapability"`
+	AllowListedMachineIdentities bool     `yaml:"allowListedMachineIdentities"`
 	TokenURL                     string   `yaml:"tokenUrl"`
 	ClientID                     string   `yaml:"clientId"`
 	ZitadelDomain                string   `yaml:"zitadelDomain"`
@@ -192,6 +193,7 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
 		Audience:                     raw.Auth.Audience,
 		JWKSURL:                      raw.Auth.JWKSURL,
 		RequiredCapability:           raw.Auth.RequiredCapability,
+		AllowListedMachineIdentities: raw.Auth.AllowListedMachineIdentities,
 		TokenURL:                     raw.Auth.TokenURL,
 		ClientID:                     raw.Auth.ClientID,
 		ClientSecret:                 os.Getenv("AUTH_CLIENT_SECRET"),

internal/proxy/machine_identity_test.go

@@ -0,0 +1,65 @@
+package proxy
+
+import (
+	"testing"
+
+	"github.com/golang-jwt/jwt/v5"
+
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
+)
+
+func handlerWith(allow bool, ids ...string) *Handler {
+	return &Handler{config: &config.Config{Auth: config.AuthConfig{
+		AllowListedMachineIdentities: allow,
+		ZitadelClientIDs:             ids,
+	}}}
+}
+
+func TestAllowListedMachineIdentityAdmitsANamedClientWithoutRoles(t *testing.T) {
+	h := handlerWith(true, "poimen-llm")
+	claims := jwt.MapClaims{"client_id": "poimen-llm", "sub": "397071051344839054"}
+	if !h.allowListedMachineIdentity(claims) {
+		t.Fatalf("an allow-listed machine client must be admitted when the token has no roles")
+	}
+}
+
+func TestAllowListedMachineIdentityRefusesAnUnknownClient(t *testing.T) {
+	h := handlerWith(true, "poimen-llm")
+	claims := jwt.MapClaims{"client_id": "someone-else"}
+	if h.allowListedMachineIdentity(claims) {
+		t.Fatalf("a client that is not on the list must not be admitted")
+	}
+}
+
+func TestAllowListedMachineIdentityRefusesWhenTheTokenStatesRoles(t *testing.T) {
+	h := handlerWith(true, "poimen-llm")
+	claims := jwt.MapClaims{
+		"client_id": "poimen-llm",
+		"urn:zitadel:iam:org:project:roles": map[string]interface{}{"memory:read": nil},
+	}
+	if h.allowListedMachineIdentity(claims) {
+		t.Fatalf("a token with roles must be judged by the role check, not by the allow-list")
+	}
+}
+
+func TestAllowListedMachineIdentityIsOffByDefault(t *testing.T) {
+	h := handlerWith(false, "poimen-llm")
+	claims := jwt.MapClaims{"client_id": "poimen-llm"}
+	if h.allowListedMachineIdentity(claims) {
+		t.Fatalf("without the setting the gateway must fall back to the role check only")
+	}
+}
+
+func TestRolesClaimReadsBothShapes(t *testing.T) {
+	asMap := jwt.MapClaims{"urn:zitadel:iam:org:project:roles": map[string]interface{}{"llm:inference": nil}}
+	if got := rolesClaim(asMap); len(got) != 1 || got[0] != "llm:inference" {
+		t.Fatalf("map shape must yield the role names, got %v", got)
+	}
+	asList := jwt.MapClaims{"roles": []interface{}{"llm:inference", "memory:read"}}
+	if got := rolesClaim(asList); len(got) != 2 {
+		t.Fatalf("list shape must yield every role, got %v", got)
+	}
+	if got := rolesClaim(jwt.MapClaims{}); len(got) != 0 {
+		t.Fatalf("a token with no roles must yield nothing, got %v", got)
+	}
+}

internal/proxy/proxy.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/golang-jwt/jwt/v5"
 	"io"
 	"net"
 	"net/http"
@@ -460,8 +461,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	proxy.ServeHTTP(respWriter, r)
 }
 
-
-
 // applyWarmup strips the X-Poimen-Warmup header, delays the request while
 // honouring context cancellation, and records the metric.
 // Returns false if the context was cancelled during the delay (caller must abort).
@@ -745,7 +744,7 @@ func (h *Handler) authenticateV1(w http.ResponseWriter, r *http.Request) bool {
 	identity.Inject(r, claims)
 	if caps := requiredCapabilities(h.config.Auth.RequiredCapability); len(caps) > 0 {
 		args := append(caps, "*")
-		if !h.jwtValidator.CheckPermissions(claims, args...) {
+		if !h.jwtValidator.CheckPermissions(claims, args...) && !h.allowListedMachineIdentity(claims) {
 			writeProblemDetail(w, http.StatusForbidden,
 				"https://api.example.com/problems/insufficient-permissions",
 				"Insufficient Permissions",
@@ -761,6 +760,59 @@ func (h *Handler) authenticateV1(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+// allowListedMachineIdentity admits a service account whose token carries no roles
+// claim, when its client id appears in the explicit allow-list.
+//
+// ZITADEL does not put roles into a machine-user access token on this instance: the
+// grant exists, the scope asks for the roles, and the claim still never arrives. A
+// service account therefore cannot satisfy a role check by any configuration. The
+// caller is still authenticated — it signed an assertion with a private key that
+// ZITADEL holds the public half of — and the allow-list names it explicitly, so the
+// authorisation is a reviewed line rather than a wildcard.
+func (h *Handler) allowListedMachineIdentity(claims jwt.MapClaims) bool {
+	if !h.config.Auth.AllowListedMachineIdentities {
+		return false
+	}
+	if len(rolesClaim(claims)) > 0 {
+		// The token states its roles, so the ordinary check has already run and
+		// decided. Falling through here would be a second, weaker rule.
+		return false
+	}
+	clientID, _ := claims["client_id"].(string)
+	if clientID == "" {
+		return false
+	}
+	for _, allowed := range h.config.Auth.ZitadelClientIDs {
+		if allowed == clientID {
+			logging.Errorf("auth: admitted an allow-listed machine identity without a roles claim",
+				fmt.Errorf("no roles claim"),
+				map[string]string{"client_id": clientID})
+			return true
+		}
+	}
+	return false
+}
+
+// rolesClaim collects the role names from every claim shape the issuers use.
+func rolesClaim(claims jwt.MapClaims) []string {
+	var out []string
+	for _, key := range []string{"roles", "urn:zitadel:iam:org:project:roles"} {
+		switch v := claims[key].(type) {
+		case []interface{}:
+			for _, item := range v {
+				if name, ok := item.(string); ok {
+					out = append(out, name)
+				}
+			}
+		case map[string]interface{}:
+			for name := range v {
+				out = append(out, name)
+			}
+		}
+	}
+	return out
+}
+
 // requiredCapabilities splits the comma-separated requiredCapability setting
 // into a list. Several spellings are accepted because the same capability is
 // named differently by each identity provider: Authentik used "llm:inference"

internal/proxy/router.go

@@ -13,11 +13,14 @@ import (
 )
 
 // RouteRequest determines which upstream should handle the request.
-// For /v1/chat/completions, /v1/embeddings, and /v1/rerank, it uses body-based dispatch.
+// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone, it
+// uses body-based dispatch.
 // For other routes, it looks up by path in the configured routes.
 func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
-	// For /v1/chat/completions, /v1/embeddings, /v1/rerank use body-based dispatch
-	if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank") {
+	// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone use
+	// body-based dispatch. /v1/systemone is Clef's decision API: same
+	// body-based model lookup, different request/response schema.
+	if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank" || r.URL.Path == "/v1/systemone") {
 		return h.routeByModel(r, r.URL.Path)
 	}
 
@@ -46,7 +49,7 @@ func (h *Handler) RouteRequest(r *http.Request) (*Route, error) {
 // routeByModel reads the request body to find the "model" field and routes accordingly.
 // The body is preserved for forwarding to the upstream.
 // Returns a modelValidationError for client errors (invalid JSON, missing/unknown model).
-// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank)
+// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank, /v1/systemone)
 func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) {
 	// If there's no body, we can't determine the model
 	if r.Body == nil {

internal/proxy/systemone_test.go

@@ -0,0 +1,169 @@
+package proxy
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
+)
+
+// TestSystemoneRoutesByModel verifies POST /v1/systemone dispatches on the body
+// "model" field instead of falling through to path-based route lookup. Before
+// that path joined body dispatch, the request reached the route table and
+// failed with "no routes configured" when no routes were defined.
+func TestSystemoneRoutesByModel(t *testing.T) {
+	clefCalled := false
+	clefServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		clefCalled = true
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, `{"decision":"allow"}`)
+	}))
+	defer clefServer.Close()
+
+	clefAddr := strings.TrimPrefix(clefServer.URL, "http://")
+
+	cfg := &config.Config{
+		Models: map[string]*config.ModelUpstream{
+			"clef-flash": {
+				Name:    "clef-flash",
+				Address: clefAddr,
+			},
+		},
+		Routes: make(map[string]*config.Route),
+	}
+
+	handler := New(cfg)
+	defer handler.Close()
+
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	resp, err := http.Post(
+		server.URL+"/v1/systemone",
+		"application/json",
+		bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)),
+	)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if !clefCalled {
+		t.Errorf("upstream clef-flash service was not called")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("expected 200 for known systemone model, got %d", resp.StatusCode)
+	}
+
+	body, readErr := io.ReadAll(resp.Body)
+	if readErr != nil {
+		t.Fatalf("failed to read response body: %v", readErr)
+	}
+	if strings.Contains(string(body), "no routes configured") {
+		t.Errorf("systemone request fell through to route lookup: %s", string(body))
+	}
+	if !strings.Contains(string(body), `"decision":"allow"`) {
+		t.Errorf("expected upstream response body, got %s", string(body))
+	}
+}
+
+// TestSystemoneUnknownModelValidationError verifies an unknown model on
+// /v1/systemone returns the same modelValidationError kind as the other three
+// body-dispatched paths.
+func TestSystemoneUnknownModelValidationError(t *testing.T) {
+	cfg := &config.Config{
+		Models: map[string]*config.ModelUpstream{
+			"clef-flash": {
+				Name:    "clef-flash",
+				Address: "localhost:9000",
+			},
+		},
+		Routes: make(map[string]*config.Route),
+	}
+
+	handler := New(cfg)
+	defer handler.Close()
+
+	paths := []string{
+		"/v1/chat/completions",
+		"/v1/embeddings",
+		"/v1/rerank",
+		"/v1/systemone",
+	}
+	for _, path := range paths {
+		req := httptest.NewRequest(
+			"POST",
+			path,
+			strings.NewReader(`{"model":"unknown-model"}`),
+		)
+		route, err := handler.RouteRequest(req)
+		if route != nil {
+			t.Errorf("%s: expected nil route for unknown model", path)
+		}
+		var validationErr *modelValidationError
+		if !errors.As(err, &validationErr) {
+			t.Fatalf("%s: expected modelValidationError, got %v", path, err)
+		}
+		if validationErr.Kind != "unknown_model" {
+			t.Errorf("%s: expected kind unknown_model, got %s", path, validationErr.Kind)
+		}
+	}
+}
+
+// TestSystemoneUpstreamPathNotRewritten verifies routeByModel keeps the request
+// path for /v1/systemone: upstream must see /v1/systemone, not
+// /v1/chat/completions. /v1/rerank is the only endpoint with a path rewrite.
+func TestSystemoneUpstreamPathNotRewritten(t *testing.T) {
+	upstreamPath := ""
+
+	upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		upstreamPath = r.URL.Path
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, `{"decision":"deny"}`)
+	}))
+	defer upstreamServer.Close()
+
+	upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://")
+
+	cfg := &config.Config{
+		Models: map[string]*config.ModelUpstream{
+			"clef-flash": {
+				Name:    "clef-flash",
+				Address: upstreamAddr,
+			},
+		},
+		Routes: make(map[string]*config.Route),
+	}
+
+	handler := New(cfg)
+	defer handler.Close()
+
+	server := httptest.NewServer(handler)
+	defer server.Close()
+
+	resp, err := http.Post(
+		server.URL+"/v1/systemone",
+		"application/json",
+		bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)),
+	)
+	if err != nil {
+		t.Fatalf("request failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if upstreamPath != "/v1/systemone" {
+		t.Errorf("expected upstream path /v1/systemone, got %s", upstreamPath)
+	}
+	if upstreamPath == "/v1/chat/completions" {
+		t.Errorf("upstream path was rewritten to /v1/chat/completions")
+	}
+}

k8s/network-policy.yaml

@@ -172,6 +172,18 @@ spec:
     ports:
     - protocol: TCP
       port: 3000
+  # Allow HTTPS to ingress-nginx. The gateway fetches JWKS and the OIDC
+  # discovery document from https://zitadel.riotpiao.com, which CoreDNS rewrites
+  # to this service. The service CIDR is 10.106.0.0/16, so the 10.96.0.0/12
+  # ipBlock below never covered it: the fetch hung until the context deadline and
+  # the pod failed its liveness probe.
+  - to:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: ingress-nginx
+    ports:
+    - protocol: TCP
+      port: 443
   # Allow to kubernetes API server (ArgoCD Application patching via in-cluster SA).
   # kube-apiserver runs on host network (192.168.1.162/166:6443) — must use ipBlock.
   # ClusterIP 10.96.0.1:443 also routes there, covered by the 10.96.0.0/12 CIDR.