diff --git a/internal/config/config.go b/internal/config/config.go index 5c1d437..726cda4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,6 +66,13 @@ type AuthConfig struct { JWKSURL string // RequiredCapability is the permission required for LLM inference (e.g., "llm:inference"). RequiredCapability string + // AllowListedMachineIdentities admits a caller whose token carries no roles + // claim, when its client id is in ZitadelClientIDs. ZITADEL does not assert roles + // in machine-user access tokens on this instance, so a service account can never + // satisfy RequiredCapability. The allow-list is then the authorisation: the caller + // still proves possession of a registered private key, and the entry is a reviewed + // line of configuration, not a wildcard. + AllowListedMachineIdentities bool // TokenURL is the OIDC token endpoint used for token exchange // (client_credentials / password grants). TokenURL string diff --git a/internal/config/loader.go b/internal/config/loader.go index 37a38ed..e0b2c85 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -20,16 +20,17 @@ type rawConfig struct { // rawAuth represents auth configuration in YAML. type rawAuth struct { - Enabled bool `yaml:"enabled"` - Issuer string `yaml:"issuer"` - Audience string `yaml:"audience"` - JWKSURL string `yaml:"jwksUrl"` - RequiredCapability string `yaml:"requiredCapability"` - TokenURL string `yaml:"tokenUrl"` - ClientID string `yaml:"clientId"` - ZitadelDomain string `yaml:"zitadelDomain"` - ZitadelInsecurePort string `yaml:"zitadelInsecurePort"` - ZitadelClientIDs []string `yaml:"zitadelClientIds"` + Enabled bool `yaml:"enabled"` + Issuer string `yaml:"issuer"` + Audience string `yaml:"audience"` + JWKSURL string `yaml:"jwksUrl"` + RequiredCapability string `yaml:"requiredCapability"` + AllowListedMachineIdentities bool `yaml:"allowListedMachineIdentities"` + TokenURL string `yaml:"tokenUrl"` + ClientID string `yaml:"clientId"` + ZitadelDomain string `yaml:"zitadelDomain"` + ZitadelInsecurePort string `yaml:"zitadelInsecurePort"` + ZitadelClientIDs []string `yaml:"zitadelClientIds"` } // rawRoute represents a single route in the YAML configuration. @@ -44,8 +45,8 @@ type rawModel struct { Address string `yaml:"address"` Addresses []string `yaml:"addresses"` Path string `yaml:"path"` - UpstreamModel string `yaml:"upstreamModel"` - AuthRequired *bool `yaml:"authRequired"` + UpstreamModel string `yaml:"upstreamModel"` + AuthRequired *bool `yaml:"authRequired"` } // rawAdapter represents a service adapter in the YAML configuration. @@ -187,17 +188,18 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo // Parse auth config authConfig := AuthConfig{ - Enabled: raw.Auth.Enabled, - Issuer: raw.Auth.Issuer, - Audience: raw.Auth.Audience, - JWKSURL: raw.Auth.JWKSURL, - RequiredCapability: raw.Auth.RequiredCapability, - TokenURL: raw.Auth.TokenURL, - ClientID: raw.Auth.ClientID, - ClientSecret: os.Getenv("AUTH_CLIENT_SECRET"), - ZitadelDomain: raw.Auth.ZitadelDomain, - ZitadelInsecurePort: raw.Auth.ZitadelInsecurePort, - ZitadelClientIDs: raw.Auth.ZitadelClientIDs, + Enabled: raw.Auth.Enabled, + Issuer: raw.Auth.Issuer, + Audience: raw.Auth.Audience, + JWKSURL: raw.Auth.JWKSURL, + RequiredCapability: raw.Auth.RequiredCapability, + AllowListedMachineIdentities: raw.Auth.AllowListedMachineIdentities, + TokenURL: raw.Auth.TokenURL, + ClientID: raw.Auth.ClientID, + ClientSecret: os.Getenv("AUTH_CLIENT_SECRET"), + ZitadelDomain: raw.Auth.ZitadelDomain, + ZitadelInsecurePort: raw.Auth.ZitadelInsecurePort, + ZitadelClientIDs: raw.Auth.ZitadelClientIDs, } return routes, models, adapters, authConfig, nil diff --git a/internal/proxy/machine_identity_test.go b/internal/proxy/machine_identity_test.go new file mode 100644 index 0000000..01b07e3 --- /dev/null +++ b/internal/proxy/machine_identity_test.go @@ -0,0 +1,65 @@ +package proxy + +import ( + "testing" + + "github.com/golang-jwt/jwt/v5" + + "forgejo.riotpiao.com/rock/homelab-frontend/internal/config" +) + +func handlerWith(allow bool, ids ...string) *Handler { + return &Handler{config: &config.Config{Auth: config.AuthConfig{ + AllowListedMachineIdentities: allow, + ZitadelClientIDs: ids, + }}} +} + +func TestAllowListedMachineIdentityAdmitsANamedClientWithoutRoles(t *testing.T) { + h := handlerWith(true, "poimen-llm") + claims := jwt.MapClaims{"client_id": "poimen-llm", "sub": "397071051344839054"} + if !h.allowListedMachineIdentity(claims) { + t.Fatalf("an allow-listed machine client must be admitted when the token has no roles") + } +} + +func TestAllowListedMachineIdentityRefusesAnUnknownClient(t *testing.T) { + h := handlerWith(true, "poimen-llm") + claims := jwt.MapClaims{"client_id": "someone-else"} + if h.allowListedMachineIdentity(claims) { + t.Fatalf("a client that is not on the list must not be admitted") + } +} + +func TestAllowListedMachineIdentityRefusesWhenTheTokenStatesRoles(t *testing.T) { + h := handlerWith(true, "poimen-llm") + claims := jwt.MapClaims{ + "client_id": "poimen-llm", + "urn:zitadel:iam:org:project:roles": map[string]interface{}{"memory:read": nil}, + } + if h.allowListedMachineIdentity(claims) { + t.Fatalf("a token with roles must be judged by the role check, not by the allow-list") + } +} + +func TestAllowListedMachineIdentityIsOffByDefault(t *testing.T) { + h := handlerWith(false, "poimen-llm") + claims := jwt.MapClaims{"client_id": "poimen-llm"} + if h.allowListedMachineIdentity(claims) { + t.Fatalf("without the setting the gateway must fall back to the role check only") + } +} + +func TestRolesClaimReadsBothShapes(t *testing.T) { + asMap := jwt.MapClaims{"urn:zitadel:iam:org:project:roles": map[string]interface{}{"llm:inference": nil}} + if got := rolesClaim(asMap); len(got) != 1 || got[0] != "llm:inference" { + t.Fatalf("map shape must yield the role names, got %v", got) + } + asList := jwt.MapClaims{"roles": []interface{}{"llm:inference", "memory:read"}} + if got := rolesClaim(asList); len(got) != 2 { + t.Fatalf("list shape must yield every role, got %v", got) + } + if got := rolesClaim(jwt.MapClaims{}); len(got) != 0 { + t.Fatalf("a token with no roles must yield nothing, got %v", got) + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 5c48558..ae4ce57 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/golang-jwt/jwt/v5" "io" "net" "net/http" @@ -45,7 +46,7 @@ type Handler struct { defaultWriteTimeout time.Duration defaultMaxBodySize int64 // CAG: per-model backend schedulers for cache-aware routing (#45/#46) - schedulers map[string]*Scheduler + schedulers map[string]*Scheduler // CAG: delay applied to warmup requests before queue entry (#45) warmupDelay time.Duration // engineObservers polls each model backend's /metrics for admission (#64) @@ -78,18 +79,18 @@ func (e *modelValidationError) Error() string { // RFC 9457 Problem Details type problemDetail struct { - Type string `json:"type"` - Title string `json:"title"` - Status int `json:"status"` - Detail string `json:"detail"` - ValidModels []string `json:"valid_models,omitempty"` + Type string `json:"type"` + Title string `json:"title"` + Status int `json:"status"` + Detail string `json:"detail"` + ValidModels []string `json:"valid_models,omitempty"` } // New creates a new reverse proxy handler from configuration. // It sets up connection pooling and rewriting rules for each route. func New(cfg *config.Config) *Handler { h := &Handler{ - routes: make(map[string]*Route), + routes: make(map[string]*Route), transports: make(map[string]*http.Transport), config: cfg, defaultConnectTimeout: 10 * time.Second, @@ -238,9 +239,9 @@ func New(cfg *config.Config) *Handler { for name, route := range cfg.Routes { // Create a transport per unique upstream address for connection reuse transport := h.getOrCreateTransport(route.Upstream.Address, &route.Upstream) - + upstreamURL := &url.URL{Scheme: "http", Host: route.Upstream.Address} - + r := &Route{ Name: name, Upstream: &route.Upstream, @@ -330,10 +331,10 @@ func directorFunc(req *http.Request, target *url.URL, upstream *config.Upstream) // Set the scheme and host req.URL.Scheme = target.Scheme req.URL.Host = target.Host - + // Update the Host header to the upstream address req.Host = target.Host - + // Strip hop-by-hop headers as defined in RFC 7230 Section 6.1 // These must not be forwarded to upstream hopByHopHeaders := map[string]bool{ @@ -346,20 +347,20 @@ func directorFunc(req *http.Request, target *url.URL, upstream *config.Upstream) "transfer-encoding": true, "upgrade": true, } - + // Also strip any headers listed in the Connection header if conn := req.Header.Get("Connection"); conn != "" { for _, h := range strings.Split(conn, ",") { hopByHopHeaders[strings.ToLower(strings.TrimSpace(h))] = true } } - + // Remove all hop-by-hop headers // The http.Header.Del method is case-insensitive, so we can delete using lowercase keys for header := range hopByHopHeaders { req.Header.Del(header) } - + // Handle X-Forwarded-For: append the immediate peer // Get the peer IP from the request RemoteAddr peerIP := getPeerIP(req.RemoteAddr) @@ -460,8 +461,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { proxy.ServeHTTP(respWriter, r) } - - // applyWarmup strips the X-Poimen-Warmup header, delays the request while // honouring context cancellation, and records the metric. // Returns false if the context was cancelled during the delay (caller must abort). @@ -745,7 +744,7 @@ func (h *Handler) authenticateV1(w http.ResponseWriter, r *http.Request) bool { identity.Inject(r, claims) if caps := requiredCapabilities(h.config.Auth.RequiredCapability); len(caps) > 0 { args := append(caps, "*") - if !h.jwtValidator.CheckPermissions(claims, args...) { + if !h.jwtValidator.CheckPermissions(claims, args...) && !h.allowListedMachineIdentity(claims) { writeProblemDetail(w, http.StatusForbidden, "https://api.example.com/problems/insufficient-permissions", "Insufficient Permissions", @@ -761,6 +760,59 @@ func (h *Handler) authenticateV1(w http.ResponseWriter, r *http.Request) bool { return true } +// allowListedMachineIdentity admits a service account whose token carries no roles +// claim, when its client id appears in the explicit allow-list. +// +// ZITADEL does not put roles into a machine-user access token on this instance: the +// grant exists, the scope asks for the roles, and the claim still never arrives. A +// service account therefore cannot satisfy a role check by any configuration. The +// caller is still authenticated — it signed an assertion with a private key that +// ZITADEL holds the public half of — and the allow-list names it explicitly, so the +// authorisation is a reviewed line rather than a wildcard. +func (h *Handler) allowListedMachineIdentity(claims jwt.MapClaims) bool { + if !h.config.Auth.AllowListedMachineIdentities { + return false + } + if len(rolesClaim(claims)) > 0 { + // The token states its roles, so the ordinary check has already run and + // decided. Falling through here would be a second, weaker rule. + return false + } + clientID, _ := claims["client_id"].(string) + if clientID == "" { + return false + } + for _, allowed := range h.config.Auth.ZitadelClientIDs { + if allowed == clientID { + logging.Errorf("auth: admitted an allow-listed machine identity without a roles claim", + fmt.Errorf("no roles claim"), + map[string]string{"client_id": clientID}) + return true + } + } + return false +} + +// rolesClaim collects the role names from every claim shape the issuers use. +func rolesClaim(claims jwt.MapClaims) []string { + var out []string + for _, key := range []string{"roles", "urn:zitadel:iam:org:project:roles"} { + switch v := claims[key].(type) { + case []interface{}: + for _, item := range v { + if name, ok := item.(string); ok { + out = append(out, name) + } + } + case map[string]interface{}: + for name := range v { + out = append(out, name) + } + } + } + return out +} + // requiredCapabilities splits the comma-separated requiredCapability setting // into a list. Several spellings are accepted because the same capability is // named differently by each identity provider: Authentik used "llm:inference" @@ -919,8 +971,8 @@ func (h *Handler) getValidModels() []string { // modelsListResponse represents the response for GET /v1/models type modelsListResponse struct { - Object string `json:"object"` - Data []modelsListEntry `json:"data"` + Object string `json:"object"` + Data []modelsListEntry `json:"data"` } // modelsListEntry represents a single model in the list diff --git a/internal/proxy/router.go b/internal/proxy/router.go index 493c7d4..9159806 100644 --- a/internal/proxy/router.go +++ b/internal/proxy/router.go @@ -13,11 +13,14 @@ import ( ) // RouteRequest determines which upstream should handle the request. -// For /v1/chat/completions, /v1/embeddings, and /v1/rerank, it uses body-based dispatch. +// For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone, it +// uses body-based dispatch. // For other routes, it looks up by path in the configured routes. func (h *Handler) RouteRequest(r *http.Request) (*Route, error) { - // For /v1/chat/completions, /v1/embeddings, /v1/rerank use body-based dispatch - if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank") { + // For /v1/chat/completions, /v1/embeddings, /v1/rerank and /v1/systemone use + // body-based dispatch. /v1/systemone is Clef's decision API: same + // body-based model lookup, different request/response schema. + if r.Method == "POST" && (r.URL.Path == "/v1/chat/completions" || r.URL.Path == "/v1/embeddings" || r.URL.Path == "/v1/rerank" || r.URL.Path == "/v1/systemone") { return h.routeByModel(r, r.URL.Path) } @@ -46,7 +49,7 @@ func (h *Handler) RouteRequest(r *http.Request) (*Route, error) { // routeByModel reads the request body to find the "model" field and routes accordingly. // The body is preserved for forwarding to the upstream. // Returns a modelValidationError for client errors (invalid JSON, missing/unknown model). -// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank) +// The path parameter indicates which endpoint is being called (/v1/chat/completions, /v1/embeddings, /v1/rerank, /v1/systemone) func (h *Handler) routeByModel(r *http.Request, path string) (*Route, error) { // If there's no body, we can't determine the model if r.Body == nil { diff --git a/internal/proxy/systemone_test.go b/internal/proxy/systemone_test.go new file mode 100644 index 0000000..b4655e9 --- /dev/null +++ b/internal/proxy/systemone_test.go @@ -0,0 +1,169 @@ +package proxy + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "forgejo.riotpiao.com/rock/homelab-frontend/internal/config" +) + +// TestSystemoneRoutesByModel verifies POST /v1/systemone dispatches on the body +// "model" field instead of falling through to path-based route lookup. Before +// that path joined body dispatch, the request reached the route table and +// failed with "no routes configured" when no routes were defined. +func TestSystemoneRoutesByModel(t *testing.T) { + clefCalled := false + clefServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + clefCalled = true + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"decision":"allow"}`) + })) + defer clefServer.Close() + + clefAddr := strings.TrimPrefix(clefServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: clefAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Post( + server.URL+"/v1/systemone", + "application/json", + bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)), + ) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if !clefCalled { + t.Errorf("upstream clef-flash service was not called") + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("expected 200 for known systemone model, got %d", resp.StatusCode) + } + + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + t.Fatalf("failed to read response body: %v", readErr) + } + if strings.Contains(string(body), "no routes configured") { + t.Errorf("systemone request fell through to route lookup: %s", string(body)) + } + if !strings.Contains(string(body), `"decision":"allow"`) { + t.Errorf("expected upstream response body, got %s", string(body)) + } +} + +// TestSystemoneUnknownModelValidationError verifies an unknown model on +// /v1/systemone returns the same modelValidationError kind as the other three +// body-dispatched paths. +func TestSystemoneUnknownModelValidationError(t *testing.T) { + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: "localhost:9000", + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + paths := []string{ + "/v1/chat/completions", + "/v1/embeddings", + "/v1/rerank", + "/v1/systemone", + } + for _, path := range paths { + req := httptest.NewRequest( + "POST", + path, + strings.NewReader(`{"model":"unknown-model"}`), + ) + route, err := handler.RouteRequest(req) + if route != nil { + t.Errorf("%s: expected nil route for unknown model", path) + } + var validationErr *modelValidationError + if !errors.As(err, &validationErr) { + t.Fatalf("%s: expected modelValidationError, got %v", path, err) + } + if validationErr.Kind != "unknown_model" { + t.Errorf("%s: expected kind unknown_model, got %s", path, validationErr.Kind) + } + } +} + +// TestSystemoneUpstreamPathNotRewritten verifies routeByModel keeps the request +// path for /v1/systemone: upstream must see /v1/systemone, not +// /v1/chat/completions. /v1/rerank is the only endpoint with a path rewrite. +func TestSystemoneUpstreamPathNotRewritten(t *testing.T) { + upstreamPath := "" + + upstreamServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + upstreamPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + fmt.Fprint(w, `{"decision":"deny"}`) + })) + defer upstreamServer.Close() + + upstreamAddr := strings.TrimPrefix(upstreamServer.URL, "http://") + + cfg := &config.Config{ + Models: map[string]*config.ModelUpstream{ + "clef-flash": { + Name: "clef-flash", + Address: upstreamAddr, + }, + }, + Routes: make(map[string]*config.Route), + } + + handler := New(cfg) + defer handler.Close() + + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Post( + server.URL+"/v1/systemone", + "application/json", + bytes.NewReader([]byte(`{"model":"clef-flash","state":"x","questions":{}}`)), + ) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if upstreamPath != "/v1/systemone" { + t.Errorf("expected upstream path /v1/systemone, got %s", upstreamPath) + } + if upstreamPath == "/v1/chat/completions" { + t.Errorf("upstream path was rewritten to /v1/chat/completions") + } +} diff --git a/k8s/gateway-config-secret.yaml b/k8s/gateway-config-secret.yaml index fdeb293..f79b768 100644 --- a/k8s/gateway-config-secret.yaml +++ b/k8s/gateway-config-secret.yaml @@ -7,4 +7,4 @@ metadata: app: api-gateway type: Opaque stringData: - config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"ornith:35b\"\n address: \"ornith-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n" + config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n# clef-flash is a DECISION model, not a chat model. It takes a `state` plus a\n# schema of typed `questions` and returns one probability per allowed option in\n# a single prefill pass. There is no decoding and no free-form text.\n#\n# Its request and response shapes are Clef's own (System One / Jev), NOT\n# OpenAI's: no `messages`, no `choices`. A standard OpenAI client cannot call\n# it, and the public path is /v1/systemone rather than /v1/chat/completions.\n#\n# The dispatcher resolves this model by the body's `model` field, then forwards\n# to the REQUEST path. The `path` value below records intent and must stay in\n# sync with the dispatch list in internal/proxy/router.go.\n- name: \"clef-flash\"\n address: \"clef-flash-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/systemone\"\n# `ornith:35b` retired: the Ollama Llama-35B predictor is gone and its GPU now\n# serves clef-flash. Callers must move to `reasoning`, which keeps the same\n# serving name while running Qwen3-30B-A3B on the same hardware.\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n"