fix(api): retire Kong key-auth on model routes; agent-pod builds agent-manager fork + ships coordinator.js

Kong key-auth rejected the Authorization: Bearer header every OpenAI-SDK-compatible client sends (verified: raw apikey header works, Bearer doesn't), so it's commented out and stripped from every llm-routes.yaml annotation until there's a Bearer-compatible fix. agent-pod now clones and builds the agent-manager fork from source at container start (no prebuilt binary shipped -- wrong arch and over ConfigMap's size cap) and ships coordinator.js alongside hub.js, so multiple repos can run the pipeline concurrently in one pod via kubectl exec. hub.js keeps its existing role as the container's foreground process, unchanged.

k8s/apps/api/llm-routes.yaml

@@ -66,7 +66,7 @@ metadata:
   name: llm-models
   namespace: llm-serving
   annotations:
-    konghq.com/plugins: llm-models-list,model-key-auth
+    konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml
     konghq.com/strip-path: "false"
     konghq.com/methods: "GET"
 spec:
@@ -110,7 +110,7 @@ metadata:
   name: llm-chat-reasoning
   namespace: llm-serving
   annotations:
-    konghq.com/plugins: llm-rewrite-reasoning,model-key-auth
+    konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml
     konghq.com/strip-path: "false"
     konghq.com/methods: "POST"
     konghq.com/connect-timeout: "10000"
@@ -152,7 +152,7 @@ metadata:
   name: llm-chat-ornith
   namespace: llm-serving
   annotations:
-    konghq.com/plugins: llm-rewrite-ornith,model-key-auth
+    konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml
     konghq.com/strip-path: "false"
     konghq.com/methods: "POST"
     konghq.com/connect-timeout: "10000"
@@ -197,7 +197,7 @@ metadata:
   name: llm-chat-qwen
   namespace: llm-serving
   annotations:
-    konghq.com/plugins: llm-rewrite-qwen,model-key-auth
+    konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml
     konghq.com/strip-path: "false"
     konghq.com/methods: "POST"
     konghq.com/connect-timeout: "10000"
@@ -267,7 +267,7 @@ metadata:
   name: llm-rerank
   namespace: llm-serving
   annotations:
-    konghq.com/plugins: llm-rewrite-rerank,model-key-auth
+    konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml
     konghq.com/strip-path: "false"
     konghq.com/methods: "POST"
     konghq.com/connect-timeout: "10000"

k8s/apps/api/model-auth.yaml

@@ -1,13 +1,22 @@
 # API auth layer — Kong key-auth on the model routes.
 #
-# The model API (api.riotpiao.com/v1/...) requires a static API key, presented
-# OpenAI-style as `Authorization: Bearer <key>` (or `apikey: <key>`). The key
-# lives in the ksops-managed Secret model-invoke-apikey (labelled
-# konghq.com/credential: key-auth) and is bound to the KongConsumer below.
+# TEMPORARILY RETIRED: verified live that Kong's key-auth here does not accept
+# `Authorization: Bearer <key>` the way the comment below used to claim — a
+# raw `apikey: <key>` header succeeds (200), the same request with only
+# `Authorization: Bearer <key>` fails (401). No OpenAI-SDK-compatible client
+# (pi included) sends a raw apikey header or lets you customize the header
+# name, so every such client was hard-blocked. The KongPlugin below is
+# commented out and every route's `konghq.com/plugins` annotation in
+# llm-routes.yaml has `model-key-auth` stripped, so the model routes are
+# unauthenticated for now. Re-enable once there's a Bearer-compatible fix
+# (e.g. a request-transformer that copies the Bearer token into an `apikey`
+# header before key-auth runs) — do not just uncomment this as-is, that
+# reintroduces the exact block every real client hits.
 #
-# Issue the key to rock; use it as the OpenAI SDK api_key. Rotate by updating the
-# ksops secret. This is self-contained in Kong — the invoke path does not depend
-# on an Authentik token (Authentik still fronts every *human* dashboard SSO).
+# The key itself lives in the ksops-managed Secret model-invoke-apikey
+# (labelled konghq.com/credential: key-auth) and is bound to the KongConsumer
+# below, which stays defined (harmless without the plugin) so re-enabling
+# later is a two-line uncomment instead of a rebuild.
 ---
 apiVersion: configuration.konghq.com/v1
 kind: KongConsumer
@@ -19,31 +28,18 @@ metadata:
 username: model-invoker
 credentials:
   - model-invoke-apikey
----
-# key-auth: require the API key on the model routes. key_in_header accepts the
-# `apikey` header; key_in_bearer accepts `Authorization: Bearer <key>` so any
-# OpenAI-compatible SDK (api_key=..., base_url=https://api.riotpiao.com/v1) works
-# unchanged.
-#
-# Namespace `llm-serving`, not `api`: the ingress controller resolves a
-# `konghq.com/plugins` annotation against the annotated object's OWN namespace,
-# and all five model routes in llm-routes.yaml live in llm-serving. While this
-# sat in `api` the reference dangled, the plugin never bound, and every model
-# route served traffic with no key at all — verified: an unauthenticated
-# /v1/models and /v1/ornith/chat/completions both returned 200. A dangling
-# plugin reference is silent; it fails open, so re-test without a key after any
-# move rather than trusting that the object exists.
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: model-key-auth
-  namespace: llm-serving
-plugin: key-auth
-config:
-  key_names:
-    - apikey
-    - authorization
-  key_in_header: true
-  key_in_query: false
-  key_in_body: false
-  hide_credentials: true
+# ---
+# apiVersion: configuration.konghq.com/v1
+# kind: KongPlugin
+# metadata:
+#   name: model-key-auth
+#   namespace: llm-serving
+# plugin: key-auth
+# config:
+#   key_names:
+#     - apikey
+#     - authorization
+#   key_in_header: true
+#   key_in_query: false
+#   key_in_body: false
+#   hide_credentials: true