diff --git a/k8s/apps/agent-pod/console-ingress.yaml b/k8s/apps/agent-pod/console-ingress.yaml index afc8229..a73fcc8 100644 --- a/k8s/apps/agent-pod/console-ingress.yaml +++ b/k8s/apps/agent-pod/console-ingress.yaml @@ -1,14 +1,27 @@ # Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run # (trigger a new session) -- both are routes on the same hub.js service. +# +# Was ingressClassName: kong until Kong was retired on 2026-08-19. Pointed +# straight at nginx rather than through the replacement Go gateway because that +# gateway has no WebSocket upgrade support yet -- routing /console through it +# would break the console outright. nginx handles the upgrade natively. +# +# Path precedence: the nginx Ingress api/api catch-alls `/` on this same host +# to the gateway. nginx matches longest prefix first, so these three paths win +# over `/` and the rest of the host still reaches the gateway. apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: console namespace: agent-pod annotations: - konghq.com/strip-path: "false" + # A console WebSocket stays open across a whole agent session; nginx's 60s + # default read timeout would drop it mid-run. + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-buffering: "off" spec: - ingressClassName: kong + ingressClassName: nginx rules: - host: api.riotpiao.com http: diff --git a/k8s/apps/api/AUTH-PLAN.md b/k8s/apps/api/AUTH-PLAN.md deleted file mode 100644 index d3feb1c..0000000 --- a/k8s/apps/api/AUTH-PLAN.md +++ /dev/null @@ -1,83 +0,0 @@ -# API Auth Layer — Authentik service account + Kong JWT (model invoke) - -Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik -service account holding a valid **client_credentials** JWT can invoke the KServe -models. "Invoke role" = **possession of a JWT from the dedicated model-invoke -OAuth2 provider** (only the service account can obtain one). - -## Flow - -``` -service account ── client_credentials ──▶ Authentik token endpoint - (client_id + secret) https://authentik.riotpiao.com/application/o/token/ - │ - ▼ RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/) -client ── Authorization: Bearer ──▶ Kong (api.riotpiao.com/*) - jwt plugin: verify RS256 sig via Authentik JWKS, - check iss/exp → map to KongConsumer → allow - ▼ - KServe model (reasoning / ornith / ...) -``` - -Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in -**`jwt`** plugin: it validates an RS256 signature against a public key we pin on -a KongConsumer, keyed by the token's `iss`. - -## Changes - -### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py) -- New **service account** user `model-invoker` (type `service_account`, no - password; Authentik issues an app-password/token for M2M). -- New **OAuth2 provider + application** `model-invoke`: - - `client_type: confidential`, `grant_types: ["client_credentials"]` - - signing key = existing RS256 keypair (same as other providers) - - mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes - needed for M2M. -- Client secret written to k8s Secret `api/model-invoke-oidc` - (keys `client-id`, `client-secret`), labelled for whoever consumes it. -- Bind the service account so it (and only it) can use the provider. - -### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`) -- **KongConsumer** `model-invoker` (ns api). -- **`jwt` credential** on that consumer (a Secret of type - `konghq.com/v1/credential`): - - `algorithm: RS256` - - `key` = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/` - - `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing - cert (fetched from Authentik JWKS / cert, stored in git or ksops). -- **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`). - -### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml) -- Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation - (currently e.g. `llm-rewrite-reasoning`) → becomes - `llm-rewrite-reasoning,jwt-auth`. -- Leave `/models` list route open OR protect too (decision). - -## Client usage (after build) -```bash -TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \ - -d grant_type=client_credentials \ - -d client_id=model-invoke \ - -d client_secret= \ - -d scope=openid | jq -r .access_token) - -curl https://api.riotpiao.com/v1/chat/completions \ - -H "Authorization: Bearer $TOKEN" -d '{...}' -``` - -## Test plan -1. No token → Kong returns 401. -2. Valid client_credentials token → 200, model responds. -3. Expired/garbage token → 401. -4. Confirm the `/models` route behaviour matches the decision. - -## Open items / risks -- Authentik `client_credentials` for a *service account* may require an - **app-password / JWT-assertion** flow rather than plain client_secret POST — - verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs - the SA token). Adjust step 1 accordingly before wiring Kong. -- Pinning `rsa_public_key`: Authentik key rotation would break it — document a - rotation runbook, or have the provision script re-export the cert PEM into the - Kong credential on each run (keeps them in sync, same idea as ksops secrets). -- Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's - issuer is stable. diff --git a/k8s/apps/api/ingress.yaml b/k8s/apps/api/ingress.yaml index 2897bf1..1e76a19 100644 --- a/k8s/apps/api/ingress.yaml +++ b/k8s/apps/api/ingress.yaml @@ -5,14 +5,17 @@ # # nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its # default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards -# plain HTTP to kong-proxy. Kong then does the real routing, from Ingresses -# carrying `ingressClassName: kong`. +# plain HTTP to the gateway. # -# Catch-all `/` on purpose: everything under this host belongs to Kong. Listing -# per-API paths here would duplicate Kong's routing table inside nginx, and the -# two copies would drift. +# Backend was kong-proxy:80 until Kong was retired on 2026-08-19; it is now the +# Go gateway's Service, api-gateway:8080, deployed from rock/homelab-frontend. +# Reverting the cutover is a change to these two lines and nothing else. # -# In-cluster callers should prefer http://kong-proxy.api.svc.cluster.local +# Catch-all `/` on purpose: everything under this host belongs to the gateway. +# Listing per-API paths here would duplicate the gateway's routing table inside +# nginx, and the two copies would drift. +# +# In-cluster callers should prefer http://api-gateway.api.svc.cluster.local:8080 # directly. Resolving api.riotpiao.com sends them out to nginx and back in, # which is a pointless hairpin unless they need TLS or the public hostname. apiVersion: networking.k8s.io/v1 @@ -38,6 +41,6 @@ spec: pathType: Prefix backend: service: - name: kong-proxy + name: api-gateway port: - number: 80 + number: 8080 diff --git a/k8s/apps/api/kong-metrics.yaml b/k8s/apps/api/kong-metrics.yaml deleted file mode 100644 index d490e06..0000000 --- a/k8s/apps/api/kong-metrics.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the -# ingress controller apply it to every route on this Kong instance, so all -# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics -# without touching llm-routes.yaml. Scraped via kong-values.yaml's -# serviceMonitor (status listener, already on by chart default at :8100). -apiVersion: configuration.konghq.com/v1 -kind: KongClusterPlugin -metadata: - name: prometheus - annotations: - kubernetes.io/ingress.class: kong - labels: - global: "true" -plugin: prometheus -config: - status_code_metrics: true - latency_metrics: true - bandwidth_metrics: true - upstream_health_metrics: true diff --git a/k8s/apps/api/kong-values.yaml b/k8s/apps/api/kong-values.yaml deleted file mode 100644 index 8f284e5..0000000 --- a/k8s/apps/api/kong-values.yaml +++ /dev/null @@ -1,140 +0,0 @@ -# Kong Gateway — cluster-internal API gateway (namespace `api`). -# -# Chart: kong/kong 3.4.1 (appVersion 3.9). Only overrides are listed; every key -# here was checked against `helm show values kong/kong --version 3.4.1`, because -# Helm silently ignores unknown keys — a typo is a no-op, not an error. -# -# ── Topology ──────────────────────────────────────────────────────────────── -# external: client -> nginx (TLS, wildcard *.riotpiao.com) -> kong-proxy:80 -# internal: pod -> kong-proxy.api.svc.cluster.local:80 -# -# nginx stays the single edge and the only LoadBalancer (192.168.1.160). Kong is -# the policy/routing layer behind it, so it needs no LB IP and no TLS of its own -# — hence ClusterIP and proxy.tls disabled. Giving Kong its own IP from -# homelab-pool would mean duplicating cert-manager wiring and diverging from the -# CoreDNS convention that sends every *.riotpiao.com host to nginx. -# -# ── Routing model ─────────────────────────────────────────────────────────── -# Consumers publish an Ingress with `ingressClassName: kong`; the controller -# turns it into a Kong route. `nginx` remains the default IngressClass, so this -# is strictly opt-in and no existing Ingress changes behaviour. - -# Without this the release name is prefixed onto everything (`kong-kong-proxy`). -# Pinning it keeps the Service name stable and independent of the release name, -# which matters because the nginx Ingress in k8s/bootstrap/ingress/ingress.yaml -# references it by name. -fullnameOverride: kong - -# Two replicas so a node drain or rollout doesn't take the gateway down. Kong is -# stateless in DB-less mode, so replicas are pure redundancy. -replicaCount: 2 - -# Opt in to the `llm-serving-default-deny` NetworkPolicy, which admits port 8080 -# only from pods carrying this label. That policy is a compensating control, not -# hygiene: vLLM v0.11.0 is frozen on Volta and will never receive patches for -# several remote/unauthenticated advisories, so it must not be broadly reachable. -# -# Without this label Cilium DROPS the packets rather than refusing them, so the -# symptom is a request that hangs until the client's timeout — not a connection -# error. /v1/models still worked while this was missing, because -# request-termination answers inside Kong and never touches an upstream. -podLabels: - llm-client: "true" - -env: - # DB-less. Config comes from Kubernetes objects via the ingress controller, so - # git stays the source of truth. A Postgres-backed Kong would put live routing - # config in a database mutated through the Admin API — state outside git, plus - # migration Jobs on every upgrade. - database: "off" - # `nginx_proxy_` injects a directive into the proxy location block; - # this renders `proxy_buffering off;`. - # - # Required for LLM streaming. With buffering on (the default) nginx accumulates - # the upstream response before forwarding, so an SSE stream from - # `"stream": true` arrives in lumps or stalls until the generation finishes — - # which defeats the point of streaming. The matching setting is already on the - # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the - # buffered one dominates. - nginx_proxy_proxy_buffering: "off" - # Any plugin that rewrites the request body — request-transformer on the - # llm-chat-* routes — reads it through `kong.request.get_body()`, and that - # returns nothing once nginx has spilled the body past - # client_body_buffer_size into a temp file. The plugin then re-serializes a - # body with no `messages`, and the upstream answers - # HTTP 400 {"error":{"message":"[] is too short - 'messages'"}} - # Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400. - # An agent request carrying tool schemas clears that in one turn, so the - # buffer has to hold a whole conversation, not a chat message. - nginx_http_client_body_buffer_size: "16m" - nginx_http_client_max_body_size: "16m" - -ingressController: - enabled: true - ingressClass: kong - # The chart's ingress-class template is gated on - # `.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass"`, so a - # bare `helm template` renders nothing. ArgoCD passes --api-versions from the - # live cluster, so it does render there — verify `kubectl get ingressclass - # kong` after the first sync rather than assuming it. - createIngressClass: true - # Deliberately empty: setting is-default-class here would hijack every Ingress - # in the cluster that omits ingressClassName. nginx keeps that role. - ingressClassAnnotations: {} - -proxy: - enabled: true - # Chart default is LoadBalancer, which would claim an IP from homelab-pool. - type: ClusterIP - http: - enabled: true - servicePort: 80 - containerPort: 8000 - # nginx already terminated TLS; a second handshake to the same cluster buys - # nothing and would need Kong to hold its own certificate. - tls: - enabled: false - -# No Service for the Admin API. The controller reaches it over localhost inside -# the pod, so exposing it would only create an unauthenticated write path to the -# gateway's entire configuration. -admin: - enabled: false - -# Kong Manager UI — chart default is `enabled: true` with type NodePort, which -# would open a port on every node. Not wanted. -manager: - enabled: false - -resources: - requests: - cpu: 200m - memory: 256Mi - limits: - cpu: "2" - memory: 1Gi - -podDisruptionBudget: - enabled: true - minAvailable: 1 - -# Status listener (metrics/health) is on by default at :8100 (chart default, -# verified via `helm show values`). This just wires the ServiceMonitor the -# chart already knows how to generate for it, so kong_http_requests_total / -# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the -# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml. -serviceMonitor: - enabled: true - labels: - release: kube-prometheus-stack - -# Spread the two replicas across nodes; `ScheduleAnyway` so a single-node -# situation degrades to co-location instead of leaving a pod Pending. -topologySpreadConstraints: - - maxSkew: 1 - topologyKey: kubernetes.io/hostname - whenUnsatisfiable: ScheduleAnyway - labelSelector: - matchLabels: - app.kubernetes.io/name: kong - app.kubernetes.io/instance: kong diff --git a/k8s/apps/api/kustomization.yaml b/k8s/apps/api/kustomization.yaml index 5a9ea06..34c6f09 100644 --- a/k8s/apps/api/kustomization.yaml +++ b/k8s/apps/api/kustomization.yaml @@ -1,14 +1,14 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Explicit allowlist so kong-values.yaml in this directory is NOT treated as a -# manifest — it is Helm input consumed by the chart source of the `kong` -# Application, not a Kubernetes object. Anything new added here must be listed -# or it is silently dropped with no error and no drift shown. +# Explicit allowlist. Anything added to this directory and not listed here is +# silently dropped — no error, no drift shown. +# +# Down to a single Ingress since Kong was retired (2026-08-19). The Kong Helm +# values, the KongClusterPlugin for Prometheus, the six KongPlugin CRs behind +# the path-per-model LLM surface, the KongConsumer and the key-auth plan all +# went with it. resources: - ingress.yaml - - kong-metrics.yaml - - llm-routes.yaml - - model-auth.yaml # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own # namespace, and the transformer rewrites metadata.namespace on every resource # it builds, which is a trap for anything cross-namespace added later. diff --git a/k8s/apps/api/llm-routes.yaml b/k8s/apps/api/llm-routes.yaml deleted file mode 100644 index e33d3fe..0000000 --- a/k8s/apps/api/llm-routes.yaml +++ /dev/null @@ -1,288 +0,0 @@ -# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped. -# -# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress -# can only reference a Service in its own namespace and the predictor Services -# are there. The Kong ingress controller watches all namespaces, so the routes -# still land on the gateway. They are synced by the `kong` Application (which -# has a `path: k8s/apps/api` source) so all gateway config stays in one place. -# -# ── Model -> upstream map (verified live) ─────────────────────────────────── -# reasoning -> reasoning-predictor vLLM, DeepSeek-R1-Distill-32B, 2 replicas -# ornith:35b -> ornith-predictor Ollama, 2 replicas (retired verifier- -# qwen2.5:3b-instruct -> ornith-predictor Ollama predictor's vLLM PRM slot to get -# the 2nd GPU) -- k8s Service load-balances -# across both, each replica loads both -# models, so 2 concurrent implementer-style -# calls each land on an independent instance -# nomic-embed-text-v2 -> embeddings-predictor TEI -# bge-reranker-base -> reranker-predictor TEI -# -# ── Why path-per-model, and why the body is rewritten ─────────────────────── -# Kong matches routes on host, path, method and headers — never on the request -# body. So a single /v1/chat/completions endpoint that dispatches on the body's -# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does -# multi-target model routing, is Enterprise-only). -# -# Hence the model is in the path, and each chat route force-overwrites `model` -# in the body regardless, so a client calling /v1/qwen/... with some other -# `model` value in the body can't silently get routed to the wrong weights. -# Callers may omit `model` entirely. -# -# ── Timeouts ─────────────────────────────────────────────────────────────── -# Kong's upstream timeouts default to 60000ms. A 32B model generating a long -# answer on a Volta GPU routinely exceeds that, and the client would see a -# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds. - -# ── GET /v1/models ────────────────────────────────────────────────────────── -# Served entirely by Kong via request-termination: the plugin short-circuits in -# the access phase, so the backend below is never contacted. It only exists -# because an Ingress rule requires a backend. -# -# The list is static, which means it can drift from what the engines actually -# serve — notably if the Ollama pull list in the ornith InferenceService -# changes. Verify with: -# curl -s $SVC/v1/models (against each *-predictor) -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: llm-models-list - namespace: llm-serving -plugin: request-termination -config: - status_code: 200 - content_type: application/json - body: | - {"object":"list","data":[ - {"id":"reasoning","object":"model","owned_by":"homelab","created":0}, - {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0}, - {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0}, - {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0}, - {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0} - ]} ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-models - namespace: llm-serving - annotations: - konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml - konghq.com/strip-path: "false" - konghq.com/methods: "GET" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/models - pathType: Exact - backend: - # Never actually called — request-termination answers first. - service: - name: reasoning-predictor - port: - number: 80 ---- -# ── POST /v1/reasoning/chat/completions ───────────────────────────────────── -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: llm-rewrite-reasoning - namespace: llm-serving -plugin: request-transformer -config: - # `add` only applies when the field is absent, `replace` only when present. - # Both are needed to force the value in either case. - add: - body: - - "model:reasoning" - replace: - body: - - "model:reasoning" - # The model lives in the path for routing; the upstream still expects the - # canonical OpenAI path. - uri: /v1/chat/completions ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-chat-reasoning - namespace: llm-serving - annotations: - konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "3600000" - konghq.com/write-timeout: "3600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/reasoning/chat/completions - pathType: Prefix - backend: - service: - name: reasoning-predictor - port: - number: 80 ---- -# ── POST /v1/ornith/chat/completions ──────────────────────────────────────── -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: llm-rewrite-ornith - namespace: llm-serving -plugin: request-transformer -config: - add: - body: - - "model:ornith:35b" - replace: - body: - - "model:ornith:35b" - uri: /v1/chat/completions ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-chat-ornith - namespace: llm-serving - annotations: - konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "3600000" - konghq.com/write-timeout: "3600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/ornith/chat/completions - pathType: Prefix - backend: - service: - name: ornith-predictor - port: - number: 80 ---- -# ── POST /v1/qwen/chat/completions ────────────────────────────────────────── -# Same upstream pod as ornith — only the forced body `model` differs. Both stay -# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and -# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request. -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: llm-rewrite-qwen - namespace: llm-serving -plugin: request-transformer -config: - add: - body: - - "model:qwen2.5:3b-instruct" - replace: - body: - - "model:qwen2.5:3b-instruct" - uri: /v1/chat/completions ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-chat-qwen - namespace: llm-serving - annotations: - konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "3600000" - konghq.com/write-timeout: "3600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/qwen/chat/completions - pathType: Prefix - backend: - service: - name: ornith-predictor - port: - number: 80 ---- -# ── POST /v1/embeddings ───────────────────────────────────────────────────── -# No path-per-model and no rewrite: there is exactly one embeddings backend, so -# there is nothing to disambiguate, and TEI already serves the canonical -# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists). -# That makes an OpenAI SDK a drop-in here. -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-embeddings - namespace: llm-serving - annotations: - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "600000" - konghq.com/write-timeout: "600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/embeddings - pathType: Prefix - backend: - service: - name: embeddings-predictor - port: - number: 80 ---- -# ── POST /v1/rerank ───────────────────────────────────────────────────────── -# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing -# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely -# needs the rewrite that embeddings does not. -apiVersion: configuration.konghq.com/v1 -kind: KongPlugin -metadata: - name: llm-rewrite-rerank - namespace: llm-serving -plugin: request-transformer -config: - replace: - uri: /rerank ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: llm-rerank - namespace: llm-serving - annotations: - konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml - konghq.com/strip-path: "false" - konghq.com/methods: "POST" - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "600000" - konghq.com/write-timeout: "600000" -spec: - ingressClassName: kong - rules: - - host: api.riotpiao.com - http: - paths: - - path: /v1/rerank - pathType: Prefix - backend: - service: - name: reranker-predictor - port: - number: 80 diff --git a/k8s/apps/api/model-auth.yaml b/k8s/apps/api/model-auth.yaml deleted file mode 100644 index 10d325d..0000000 --- a/k8s/apps/api/model-auth.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# API auth layer — Kong key-auth on the model routes. -# -# TEMPORARILY RETIRED: verified live that Kong's key-auth here does not accept -# `Authorization: Bearer ` the way the comment below used to claim — a -# raw `apikey: ` header succeeds (200), the same request with only -# `Authorization: Bearer ` fails (401). No OpenAI-SDK-compatible client -# (pi included) sends a raw apikey header or lets you customize the header -# name, so every such client was hard-blocked. The KongPlugin below is -# commented out and every route's `konghq.com/plugins` annotation in -# llm-routes.yaml has `model-key-auth` stripped, so the model routes are -# unauthenticated for now. Re-enable once there's a Bearer-compatible fix -# (e.g. a request-transformer that copies the Bearer token into an `apikey` -# header before key-auth runs) — do not just uncomment this as-is, that -# reintroduces the exact block every real client hits. -# -# The key itself lives in the ksops-managed Secret model-invoke-apikey -# (labelled konghq.com/credential: key-auth) and is bound to the KongConsumer -# below, which stays defined (harmless without the plugin) so re-enabling -# later is a two-line uncomment instead of a rebuild. ---- -apiVersion: configuration.konghq.com/v1 -kind: KongConsumer -metadata: - name: model-invoker - namespace: api - annotations: - kubernetes.io/ingress.class: kong -username: model-invoker -credentials: - - model-invoke-apikey -# --- -# apiVersion: configuration.konghq.com/v1 -# kind: KongPlugin -# metadata: -# name: model-key-auth -# namespace: llm-serving -# plugin: key-auth -# config: -# key_names: -# - apikey -# - authorization -# key_in_header: true -# key_in_query: false -# key_in_body: false -# hide_credentials: true diff --git a/k8s/apps/llm-serving/ornith.yaml b/k8s/apps/llm-serving/ornith.yaml index a4ab0d2..97189f3 100644 --- a/k8s/apps/llm-serving/ornith.yaml +++ b/k8s/apps/llm-serving/ornith.yaml @@ -3,20 +3,15 @@ kind: InferenceService metadata: annotations: serving.kserve.io/deploymentMode: RawDeployment - # Kong reads its timeouts from the Kubernetes Service, not the Ingress — - # Ingress annotations configure Route entities (strip-path, methods, - # plugins), these configure the Service entity. They were on - # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s - # default in force. KServe propagates InferenceService annotations to the - # Service it generates, which is how they reach Kong from here. + # The konghq.com/{connect,read,write}-timeout annotations that used to live + # here went with Kong (retired 2026-08-19). They existed because Kong read + # its upstream timeouts off the Kubernetes Service, and its 60s default cut + # off the first request after any pod restart — a restart flushes VRAM and + # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the + # problem in steady state. # - # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no - # request ever waited on a cold load. A pod restart flushes VRAM, and - # loading ornith:35b takes longer than 60s, so the first request after any - # restart returned 504. - konghq.com/connect-timeout: "10000" - konghq.com/read-timeout: "3600000" - konghq.com/write-timeout: "3600000" + # The equivalent budget now belongs to the Go gateway's per-route timeout + # config in rock/homelab-frontend, not to an annotation on this object. labels: app.kubernetes.io/name: llm-ornith app.kubernetes.io/part-of: llm-serving diff --git a/k8s/argocd/apps/55-api-gateway.yaml b/k8s/argocd/apps/55-api-gateway.yaml index 8da65c8..406a602 100644 --- a/k8s/argocd/apps/55-api-gateway.yaml +++ b/k8s/argocd/apps/55-api-gateway.yaml @@ -1,39 +1,44 @@ -# Wave 7 — Kong, the cluster's internal API gateway (namespace `api`). +# Wave 7 — api-gw, the cluster's API gateway (namespace `api`). # -# Sits between nginx and the backend services: nginx owns the edge and TLS, -# Kong owns routing policy, auth and rate limiting. Wave 7 puts it after the -# data/messaging tiers it fronts and before the wave-8 applications that -# publish routes into it. +# Replaces Kong OSS 3.4.1, removed 2026-08-19. Kong existed to route +# `api.riotpiao.com`, but Kong OSS cannot dispatch on a request body, so the +# LLM surface had to be expressed as one path per model +# (`/v1/reasoning/chat/completions`, `/v1/ornith/...`, `/v1/qwen/...`) with a +# `request-transformer` plugin forcing the body's `model` field on each. The Go +# gateway reads the body and picks the upstream, so a single canonical +# `POST /v1/chat/completions` covers every model. See +# docs/adr/ADR-0001-retire-kong-for-go-gateway.md in the frontend repo. # -# DB-less: routing config comes from Kubernetes objects (Ingress with -# `ingressClassName: kong`, plus KongPlugin/KongConsumer CRDs), so git remains -# the source of truth and there are no migration Jobs on upgrade. +# Two sources: +# 1. rock/homelab-frontend on the in-cluster Forgejo — the gateway's own +# kustomization (Deployment, Service, ConfigMap, RBAC, NetworkPolicy). It +# sets `namespace: api` itself, so no transformer is needed here. The +# Forgejo host must stay listed in the `homelab` AppProject sourceRepos or +# this Application is rejected with "is not permitted in project". +# 2. k8s/apps/api in this repo — the nginx edge Ingress for +# api.riotpiao.com, inherited from the retired `kong` Application. It +# cannot move to k8s/bootstrap/ingress/ingress.yaml because that syncs in +# wave 1, before namespace `api` exists. # -# CRDs ship in the chart's crds/ directory; ArgoCD applies those by default -# (helm.skipCrds is left false). +# No resources-finalizer: deleting this Application leaves the workload running +# rather than cascading the delete. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: kong + name: api-gw namespace: argocd + labels: + app.kubernetes.io/name: api-gateway + app.kubernetes.io/component: gateway annotations: argocd.argoproj.io/sync-wave: "7" spec: project: homelab revisionHistoryLimit: 3 sources: - - repoURL: https://charts.konghq.com - chart: kong - targetRevision: "3.4.1" - helm: - valueFiles: - - $values/k8s/apps/api/kong-values.yaml - - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git + - repoURL: https://forgejo.riotpiao.com/rock/homelab-frontend.git targetRevision: main - ref: values - # The nginx Ingress for api.riotpiao.com. Kept in this Application rather - # than the central k8s/bootstrap/ingress/ingress.yaml because that one syncs - # in wave 1, before namespace `api` exists. + path: k8s - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git targetRevision: main path: k8s/apps/api @@ -46,13 +51,9 @@ spec: selfHeal: true syncOptions: - CreateNamespace=true - # The chart's CRDs exceed the annotation size limit that client-side - # apply relies on; server-side apply avoids the - # "metadata.annotations: Too long" failure CRDs commonly hit. - - ServerSideApply=true retry: - limit: 3 + limit: 5 backoff: - duration: 10s + duration: 5s factor: 2 maxDuration: 3m diff --git a/k8s/argocd/apps/56-llm-serving.yaml b/k8s/argocd/apps/56-llm-serving.yaml index 5544104..e712e7f 100644 --- a/k8s/argocd/apps/56-llm-serving.yaml +++ b/k8s/argocd/apps/56-llm-serving.yaml @@ -1,7 +1,7 @@ # Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`). # -# Syncs before wave 7 (Kong), so the predictor Services exist before the routes -# that point at them. KServe itself is part of the substrate; this Application +# Syncs before wave 7 (api-gw), so the predictor Services exist before the +# gateway that routes to them. KServe itself is part of the substrate; this Application # owns only the InferenceServices. # # Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed diff --git a/k8s/bootstrap/ingress/ingress.yaml b/k8s/bootstrap/ingress/ingress.yaml index 5ad7e2d..0999300 100644 --- a/k8s/bootstrap/ingress/ingress.yaml +++ b/k8s/bootstrap/ingress/ingress.yaml @@ -300,10 +300,11 @@ spec: port: number: 8080 --- -# NOTE: api.riotpiao.com (Kong) is deliberately NOT here. Its namespace `api` is +# NOTE: api.riotpiao.com is deliberately NOT here. Its namespace `api` is # created in wave 7, and this Application syncs in wave 1 — an Ingress into a # namespace that doesn't exist yet would fail and mark this whole app -# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced with Kong itself. +# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced by the api-gw +# Application. apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/k8s/infra/monitoring/dashboards/llm-frontend.yaml b/k8s/infra/monitoring/dashboards/llm-frontend.yaml index 8537562..ed5b4e4 100644 --- a/k8s/infra/monitoring/dashboards/llm-frontend.yaml +++ b/k8s/infra/monitoring/dashboards/llm-frontend.yaml @@ -7,15 +7,22 @@ metadata: grafana_dashboard: "1" annotations: grafana_folder: "LLM" - # Request rate/error/latency/bandwidth now come from Kong's prometheus - # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the - # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) -- - # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/ - # embeddings/rerank uniformly without per-backend instrumentation. - # Token-count metrics are still not available: that needs response-body - # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only). - # Predictor-level metrics (native Ollama/vLLM stats) also still need a - # dedicated exporter -- not added here. + # No request-level panels. The rate/error/latency/bandwidth row used to run + # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go + # gateway that replaced it does not expose /metrics yet, so those panels + # were removed rather than left querying series that no longer exist. + # What is left is pod-level: readiness, CPU/memory, restarts, logs. + # + # Restoring request-level and per-model observability means wiring three + # sources, none of which are in place: gateway metrics (RED plus token + # counts and TTFT, which the gateway can measure because it sees the + # response stream), vLLM's own /metrics on reasoning-predictor (rich -- + # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds, + # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's + # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at + # all (verified: /metrics returns 404), so ornith can only ever be observed + # from the gateway side. No ServiceMonitor exists for the llm-serving + # namespace today, so none of the engine metrics are being scraped. data: llm-frontend.json: | - {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]} + {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}