From 9ff41e31312282f1d6993dae0df3d02d3b664f66 Mon Sep 17 00:00:00 2001 From: Admin Bot Date: Wed, 28 Oct 2026 10:30:17 -0700 Subject: [PATCH] fix(gateway): stop routing clef-flash, the model cannot run here The model list still pointed clef-flash at clef-flash-predictor. That predictor is gone: Clef needs SGLang v0.5.22+, published only on CUDA 13, and CUDA 13 dropped Volta kernels. Every GPU is a Tesla V100 (sm_70), so SGLang aborts with 'no kernel image is available'. SGLang's floor is sm75 regardless. Advertising it meant a caller would wait for a Service with no endpoints until its own timeout fired, instead of failing clearly. Remove the entry and keep the reasoning in its place, so restoring it later is a deliberate act tied to new hardware. --- k8s/gateway-config-secret.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/gateway-config-secret.yaml b/k8s/gateway-config-secret.yaml index 788df13..09f96a8 100644 --- a/k8s/gateway-config-secret.yaml +++ b/k8s/gateway-config-secret.yaml @@ -7,4 +7,4 @@ metadata: app: api-gateway type: Opaque stringData: - config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Machine users, one per cross-service caller. A machine-user token names its\n # caller in client_id, so the entry is the account name, not a number.\n - \"poimen-llm\" # poimen -> LLM inference\n - \"paperless-llm\" # paperless-ai -> LLM inference (sidecar mints)\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # ZITADEL puts no roles in a machine-user token on this instance, so a service\n # account can never satisfy requiredCapability. An explicit allow-list entry\n # authorises it instead; a token that does state its roles is still judged by\n # the role check.\n allowListedMachineIdentities: true\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n# clef-flash is a DECISION model, not a chat model. It takes a `state` plus a\n# schema of typed `questions` and returns one probability per allowed option in\n# a single prefill pass. There is no decoding and no free-form text.\n#\n# Its request and response shapes are Clef's own (System One / Jev), NOT\n# OpenAI's: no `messages`, no `choices`. A standard OpenAI client cannot call\n# it, and the public path is /v1/systemone rather than /v1/chat/completions.\n#\n# The dispatcher resolves this model by the body's `model` field, then forwards\n# to the REQUEST path. The `path` value below records intent and must stay in\n# sync with the dispatch list in internal/proxy/router.go.\n- name: \"clef-flash\"\n address: \"clef-flash-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/systemone\"\n# `ornith:35b` retired: the Ollama Llama-35B predictor is gone and its GPU now\n# serves clef-flash. Callers must move to `reasoning`, which keeps the same\n# serving name while running Qwen3-30B-A3B on the same hardware.\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n" + config.yaml: "# PRODUCTION GATEWAY CONFIGURATION\n# ==========================================\n# All upstream services MUST use Kubernetes internal service DNS names\n# Format: ..svc.cluster.local\n# \n# This ensures:\n# - Communication within cluster network only (no external IP exposure)\n# - Pod-to-pod service discovery via internal DNS\n# - Security policy enforcement at network level\n# - Service-level load balancing via kube-proxy\n#\n# Routing Pattern:\n# PREFERRED: X-Service header routing (e.g., X-Service: workflow)\n# Legacy: Path-based routing (e.g., /workflow) - being deprecated\n#\nauth:\n enabled: true\n # ZITADEL is the only identity provider. CoreDNS rewrites\n # zitadel.riotpiao.com to ingress-nginx, so discovery and JWKS stay inside\n # the cluster and TLS is the wildcard certificate.\n issuer: \"https://zitadel.riotpiao.com\"\n # zitadelDomain enables the official ZITADEL Go SDK path: OIDC discovery,\n # remote JWKS cache, and op.AccessTokenVerifier.\n zitadelDomain: \"zitadel.riotpiao.com\"\n # Audience allow-list: a caller token must carry one of these client IDs.\n # Add a consumer here before it may call the LLM models.\n zitadelClientIds:\n - \"396757800874672511\" # api-gw\n - \"396757803559027071\" # paperless (paperless-ai-agent)\n - \"396757803877794175\" # poimen\n - \"396757801210282367\" # argocd\n - \"396757804213403649\" # queue-sqs\n # Machine users, one per cross-service caller. A machine-user token names its\n # caller in client_id, so the entry is the account name, not a number.\n - \"poimen-llm\" # poimen -> LLM inference\n - \"paperless-llm\" # paperless-ai -> LLM inference (sidecar mints)\n # Two spellings accepted. ZITADEL does store role keys with a colon (this\n # project holds iam:read, memory:write and llm:inference), so the colon form\n # is primary and the hyphen form is the legacy alias.\n requiredCapability: \"llm-inference,llm:inference\"\n # ZITADEL puts no roles in a machine-user token on this instance, so a service\n # account can never satisfy requiredCapability. An explicit allow-list entry\n # authorises it instead; a token that does state its roles is still judged by\n # the role check.\n allowListedMachineIdentities: true\n # token exchange (client_credentials) against the same instance\n tokenUrl: \"https://zitadel.riotpiao.com/oauth/v2/token\"\n clientId: \"396757800874672511\"\n # jwksUrl MUST be set. cmd/gateway/main.go:144 builds the dispatcher\n # validator only when cfg.Auth.Enabled and cfg.Auth.JWKSURL are both set, so\n # an unset value leaves jwtValidator nil and every adapter request passes with\n # only a warning in the log. The SDK discovers the keys on its own path, but\n # the adapter dispatcher does not use that path.\n # In-cluster endpoint, not the public host. The public host resolves through\n # the CoreDNS rewrite to ingress-nginx and the fetch timed out there, so the\n # validator answered 403 with a context-deadline error on every caller. The\n # ZITADEL service answers JWKS on 8080 and the NetworkPolicy already allows\n # egress from this namespace to iam:8080.\n # The public host, not the service. The ZITADEL service port 8080 answers 404\n # for /oauth/v2/keys, while https://zitadel.riotpiao.com/oauth/v2/keys answers\n # 200 from a pod in this namespace.\n jwksUrl: \"https://zitadel.riotpiao.com/oauth/v2/keys\"\nroutes: []\nmodels:\n# All model services use internal Kubernetes DNS (llm-serving namespace)\n- name: \"reasoning\"\n address: \"reasoning-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n# clef-flash is NOT exposed: it cannot run on this hardware. It needs SGLang\n# v0.5.22+, which upstream publishes only on CUDA 13, and CUDA 13 dropped Volta\n# kernels. Every GPU here is a Tesla V100 (sm_70), so SGLang aborts with\n# \"no kernel image is available for execution on the device\". SGLang states sm75\n# as its floor anyway (README hardware list omits V100; sm75 removed in\n# sgl-project/sglang#9207; FlashInfer needs sm75+).\n#\n# Exposing it here would route callers to a Service with no endpoints, so the\n# request would hang until the client times out instead of failing clearly.\n# Restore the entry together with k8s/apps/llm-serving/clef-flash.yaml when an\n# Ampere-or-newer GPU is available.\n# `ornith:35b` retired: the Ollama Llama-35B predictor is gone and its GPU now\n# serves nothing. Callers must move to `reasoning`, which keeps the same\n# serving name while running Qwen3-30B-A3B on the same hardware.\n- name: \"qwen2.5:3b-instruct\"\n address: \"qwen-cpu.llm-serving.svc.cluster.local:80\"\n path: \"/v1/chat/completions\"\n- name: \"nomic-ai/nomic-embed-text-v2-moe\"\n address: \"embeddings-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/embeddings\"\n- name: \"BAAI/bge-reranker-base\"\n address: \"reranker-predictor.llm-serving.svc.cluster.local:80\"\n path: \"/v1/rerank\"\nadapters:\n- serviceName: sqs\n upstream:\n url: http://management-service.sqs.svc.cluster.local:9090\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: send-message\n methods:\n - verb: POST\n upstreamPath: /sqs/send\n - name: receive-message\n methods:\n - verb: POST\n upstreamPath: /sqs/receive\n - name: list-queues\n methods:\n - verb: GET\n upstreamPath: /sqs/queues\n- serviceName: workflow\n upstream:\n url: grpc://temporal-frontend.temporal.svc.cluster.local:7233\n timeoutSeconds: 60\n auth:\n required: true\n resources:\n - name: execute\n methods:\n - verb: POST\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ExecuteWorkflow\n - name: describe\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/DescribeWorkflowExecution\n - name: list\n methods:\n - verb: GET\n upstreamPath: /temporal.api.workflowservice.v1.WorkflowService/ListWorkflowExecutions\n- serviceName: memory\n upstream:\n url: http://poimen-memory.poimen.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: query\n methods:\n - verb: POST\n upstreamPath: /memory/query\n - name: ingest\n methods:\n - verb: POST\n upstreamPath: /memory/ingest\n - name: skills\n methods:\n - verb: GET\n upstreamPath: /memory/skills\n- serviceName: s3\n upstream:\n url: http://minio.storage.svc.cluster.local:80\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-objects\n methods:\n - verb: GET\n upstreamPath: /\n - name: get-object\n methods:\n - verb: GET\n upstreamPath: /\n - name: put-object\n methods:\n - verb: PUT\n upstreamPath: /\n- serviceName: iam\n upstream:\n url: http://zitadel.iam.svc.cluster.local:8080\n timeoutSeconds: 30\n auth:\n required: true\n resources:\n - name: list-roles\n methods:\n - verb: GET\n upstreamPath: /api/v3/roles\n - name: list-users\n methods:\n - verb: GET\n upstreamPath: /api/v3/users\n - name: create-role\n methods:\n - verb: POST\n upstreamPath: /api/v3/roles\n"