pi-models: fix baseUrl to match homelab-frontend gateway contract

Kong was retired 2026-08-19, replaced by the rock/homelab-frontend Go
gateway (single /v1/chat/completions endpoint, model routed via the
request body's "model" field per API.md). Old per-model baseUrls
(/v1/ornith, /v1/reasoning, /v1/qwen) all 404 against the new gateway.
Also flipping reasoning's supportsTools to true -- confirmed working via
live test now that reasoning runs Qwen3-32B instead of DeepSeek-R1.

k8s/apps/agent-pod/console-ingress.yaml

@@ -1,14 +1,27 @@
 # Exposes agent-hub at api.riotpiao.com/console (WebSocket) and /run
 # (trigger a new session) -- both are routes on the same hub.js service.
+#
+# Was ingressClassName: kong until Kong was retired on 2026-08-19. Pointed
+# straight at nginx rather than through the replacement Go gateway because that
+# gateway has no WebSocket upgrade support yet -- routing /console through it
+# would break the console outright. nginx handles the upgrade natively.
+#
+# Path precedence: the nginx Ingress api/api catch-alls `/` on this same host
+# to the gateway. nginx matches longest prefix first, so these three paths win
+# over `/` and the rest of the host still reaches the gateway.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: console
   namespace: agent-pod
   annotations:
-    konghq.com/strip-path: "false"
+    # A console WebSocket stays open across a whole agent session; nginx's 60s
+    # default read timeout would drop it mid-run.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-buffering: "off"
 spec:
-  ingressClassName: kong
+  ingressClassName: nginx
   rules:
     - host: api.riotpiao.com
       http:

k8s/apps/agent-pod/coordinator-configmap.yaml

@@ -752,10 +752,18 @@ data:
           return finish("branch-crashed");
         }
     
-        if (i === 0 && !resuming) {
+        // Idempotent and run every phase, NOT gated on a fresh (non-resumed)
-          const gitignoreAdditions = [
+        // start -- every run this session was a resume, so the old i===0 &&
-            "",
+        // !resuming gate meant this setup permanently never ran on poiman's
-            "# agent-harness: build artifacts and vendored archives never belong in source control",
+        // branch, and portfolio's PLAN.md stayed tracked from before this rule
+        // ever existed (gitignore has no effect on an already-tracked file --
+        // observed live: it kept getting swept back in by every `git add -A`
+        // regardless of the ignore rule). Check-and-fix on every phase instead
+        // of once-at-genesis so a repo that's missing either self-heals on its
+        // very next run rather than carrying the gap forever.
+        const gitignorePath = path.join(cwd, ".gitignore");
+        const currentGitignore = fs.existsSync(gitignorePath) ? fs.readFileSync(gitignorePath, "utf8").split("\n") : [];
+        const requiredGitignoreLines = [
           "*.tar.gz",
           "*.tgz",
           "*.crate",
@@ -764,19 +772,27 @@ data:
           "*.whl",
           "vendor/",
           "node_modules/",
-            "",
-            "# agent-harness: task/phase completion sentinel files, harness bookkeeping only",
           ".task-result-*",
           ".phase-result-*",
           ".stage-done-*",
-            "",
-            "# agent-harness: PLAN.md is per-task planner scratch state, never a deliverable",
           "PLAN.md",
-          ].join("\n");
+        ];
-          fs.appendFileSync(path.join(cwd, ".gitignore"), gitignoreAdditions + "\n");
+        const missingGitignoreLines = requiredGitignoreLines.filter((line) => !currentGitignore.includes(line));
+        if (missingGitignoreLines.length > 0) {
+          fs.appendFileSync(
+            gitignorePath,
+            "\n# agent-harness: build artifacts, vendored archives, and harness bookkeeping never belong in source control\n" +
+              missingGitignoreLines.join("\n") +
+              "\n"
+          );
           await runGit(cwd, ["add", ".gitignore"]);
           await runGit(cwd, ["commit", "-m", "chore: broaden .gitignore for agent-run artifacts"]);
         }
+        const trackedFiles = await runGit(cwd, ["ls-tree", "-r", "HEAD", "--name-only"]);
+        if (trackedFiles.out.split("\n").includes("PLAN.md")) {
+          await runGit(cwd, ["rm", "--cached", "PLAN.md"]);
+          await runGit(cwd, ["commit", "-m", "chore: untrack PLAN.md (already gitignored, was committed pre-rule)"]);
+        }
     
         await runPhase(cwd, baseBranch, phaseBranch, phaseTasks, pool, repoId, pipelineSession);
     

k8s/apps/api/AUTH-PLAN.md

@@ -1,83 +0,0 @@
-# API Auth Layer — Authentik service account + Kong JWT (model invoke)
-
-Protect the model API (`api.riotpiao.com/*`, Kong OSS 3.9) so only an Authentik
-service account holding a valid **client_credentials** JWT can invoke the KServe
-models. "Invoke role" = **possession of a JWT from the dedicated model-invoke
-OAuth2 provider** (only the service account can obtain one).
-
-## Flow
-
-```
-service account ── client_credentials ──▶ Authentik token endpoint
-                    (client_id + secret)   https://authentik.riotpiao.com/application/o/token/
-        │
-        ▼  RS256 JWT (iss = https://authentik.riotpiao.com/application/o/model-invoke/)
-client ── Authorization: Bearer <jwt> ──▶ Kong (api.riotpiao.com/*)
-                                           jwt plugin: verify RS256 sig via Authentik JWKS,
-                                           check iss/exp → map to KongConsumer → allow
-                                           ▼
-                                        KServe model (reasoning / ornith / ...)
-```
-
-Kong OSS has no enterprise `openid-connect` plugin, so we use the built-in
-**`jwt`** plugin: it validates an RS256 signature against a public key we pin on
-a KongConsumer, keyed by the token's `iss`.
-
-## Changes
-
-### 1. Authentik (k8s/infra/iam/scripts/authentik-provision.py)
-- New **service account** user `model-invoker` (type `service_account`, no
-  password; Authentik issues an app-password/token for M2M).
-- New **OAuth2 provider + application** `model-invoke`:
-  - `client_type: confidential`, `grant_types: ["client_credentials"]`
-  - signing key = existing RS256 keypair (same as other providers)
-  - mappings: `openid` (+ optionally a static `invoke` scope) — no user scopes
-    needed for M2M.
-- Client secret written to k8s Secret `api/model-invoke-oidc`
-  (keys `client-id`, `client-secret`), labelled for whoever consumes it.
-- Bind the service account so it (and only it) can use the provider.
-
-### 2. Kong (k8s/apps/api/, new file `model-auth.yaml`)
-- **KongConsumer** `model-invoker` (ns api).
-- **`jwt` credential** on that consumer (a Secret of type
-  `konghq.com/v1/credential`):
-  - `algorithm: RS256`
-  - `key`  = the token `iss` → `https://authentik.riotpiao.com/application/o/model-invoke/`
-  - `rsa_public_key` = the PEM public key of Authentik's `model-invoke` signing
-    cert (fetched from Authentik JWKS / cert, stored in git or ksops).
-- **KongPlugin** `jwt-auth` (`plugin: jwt`, `config.claims_to_verify: [exp]`).
-
-### 3. Wire onto model routes (k8s/apps/api/llm-routes.yaml)
-- Add `jwt-auth` to each model Ingress's `konghq.com/plugins` annotation
-  (currently e.g. `llm-rewrite-reasoning`) → becomes
-  `llm-rewrite-reasoning,jwt-auth`.
-- Leave `/models` list route open OR protect too (decision).
-
-## Client usage (after build)
-```bash
-TOKEN=$(curl -s https://authentik.riotpiao.com/application/o/token/ \
-  -d grant_type=client_credentials \
-  -d client_id=model-invoke \
-  -d client_secret=<secret> \
-  -d scope=openid | jq -r .access_token)
-
-curl https://api.riotpiao.com/v1/chat/completions \
-  -H "Authorization: Bearer $TOKEN" -d '{...}'
-```
-
-## Test plan
-1. No token → Kong returns 401.
-2. Valid client_credentials token → 200, model responds.
-3. Expired/garbage token → 401.
-4. Confirm the `/models` route behaviour matches the decision.
-
-## Open items / risks
-- Authentik `client_credentials` for a *service account* may require an
-  **app-password / JWT-assertion** flow rather than plain client_secret POST —
-  verify Authentik 2026.x M2M exactly (client_credentials with client_secret vs
-  the SA token). Adjust step 1 accordingly before wiring Kong.
-- Pinning `rsa_public_key`: Authentik key rotation would break it — document a
-  rotation runbook, or have the provision script re-export the cert PEM into the
-  Kong credential on each run (keeps them in sync, same idea as ksops secrets).
-- Kong `jwt` maps token→consumer by the `iss`=`key` match; ensure the provider's
-  issuer is stable.

k8s/apps/api/ingress.yaml

@@ -5,14 +5,17 @@
 #
 # nginx terminates TLS with the wildcard *.riotpiao.com cert (served as its
 # default-ssl-certificate, so no per-rule `tls:` block is needed) and forwards
-# plain HTTP to kong-proxy. Kong then does the real routing, from Ingresses
+# plain HTTP to the gateway.
-# carrying `ingressClassName: kong`.
 #
-# Catch-all `/` on purpose: everything under this host belongs to Kong. Listing
+# Backend was kong-proxy:80 until Kong was retired on 2026-08-19; it is now the
-# per-API paths here would duplicate Kong's routing table inside nginx, and the
+# Go gateway's Service, api-gateway:8080, deployed from rock/homelab-frontend.
-# two copies would drift.
+# Reverting the cutover is a change to these two lines and nothing else.
 #
-# In-cluster callers should prefer http://kong-proxy.api.svc.cluster.local
+# Catch-all `/` on purpose: everything under this host belongs to the gateway.
+# Listing per-API paths here would duplicate the gateway's routing table inside
+# nginx, and the two copies would drift.
+#
+# In-cluster callers should prefer http://api-gateway.api.svc.cluster.local:8080
 # directly. Resolving api.riotpiao.com sends them out to nginx and back in,
 # which is a pointless hairpin unless they need TLS or the public hostname.
 apiVersion: networking.k8s.io/v1
@@ -38,6 +41,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: kong-proxy
+                name: api-gateway
                 port:
-                  number: 80
+                  number: 8080

k8s/apps/api/kong-metrics.yaml

@@ -1,19 +0,0 @@
-# Cluster-wide Kong Prometheus plugin -- `global: "true"` label makes the
-# ingress controller apply it to every route on this Kong instance, so all
-# five LLM routes (ornith/reasoning/qwen/embeddings/rerank) get RED metrics
-# without touching llm-routes.yaml. Scraped via kong-values.yaml's
-# serviceMonitor (status listener, already on by chart default at :8100).
-apiVersion: configuration.konghq.com/v1
-kind: KongClusterPlugin
-metadata:
-  name: prometheus
-  annotations:
-    kubernetes.io/ingress.class: kong
-  labels:
-    global: "true"
-plugin: prometheus
-config:
-  status_code_metrics: true
-  latency_metrics: true
-  bandwidth_metrics: true
-  upstream_health_metrics: true

k8s/apps/api/kong-values.yaml

@@ -1,140 +0,0 @@
-# Kong Gateway — cluster-internal API gateway (namespace `api`).
-#
-# Chart: kong/kong 3.4.1 (appVersion 3.9). Only overrides are listed; every key
-# here was checked against `helm show values kong/kong --version 3.4.1`, because
-# Helm silently ignores unknown keys — a typo is a no-op, not an error.
-#
-# ── Topology ────────────────────────────────────────────────────────────────
-#   external:  client -> nginx (TLS, wildcard *.riotpiao.com) -> kong-proxy:80
-#   internal:  pod    -> kong-proxy.api.svc.cluster.local:80
-#
-# nginx stays the single edge and the only LoadBalancer (192.168.1.160). Kong is
-# the policy/routing layer behind it, so it needs no LB IP and no TLS of its own
-# — hence ClusterIP and proxy.tls disabled. Giving Kong its own IP from
-# homelab-pool would mean duplicating cert-manager wiring and diverging from the
-# CoreDNS convention that sends every *.riotpiao.com host to nginx.
-#
-# ── Routing model ───────────────────────────────────────────────────────────
-# Consumers publish an Ingress with `ingressClassName: kong`; the controller
-# turns it into a Kong route. `nginx` remains the default IngressClass, so this
-# is strictly opt-in and no existing Ingress changes behaviour.
-
-# Without this the release name is prefixed onto everything (`kong-kong-proxy`).
-# Pinning it keeps the Service name stable and independent of the release name,
-# which matters because the nginx Ingress in k8s/bootstrap/ingress/ingress.yaml
-# references it by name.
-fullnameOverride: kong
-
-# Two replicas so a node drain or rollout doesn't take the gateway down. Kong is
-# stateless in DB-less mode, so replicas are pure redundancy.
-replicaCount: 2
-
-# Opt in to the `llm-serving-default-deny` NetworkPolicy, which admits port 8080
-# only from pods carrying this label. That policy is a compensating control, not
-# hygiene: vLLM v0.11.0 is frozen on Volta and will never receive patches for
-# several remote/unauthenticated advisories, so it must not be broadly reachable.
-#
-# Without this label Cilium DROPS the packets rather than refusing them, so the
-# symptom is a request that hangs until the client's timeout — not a connection
-# error. /v1/models still worked while this was missing, because
-# request-termination answers inside Kong and never touches an upstream.
-podLabels:
-  llm-client: "true"
-
-env:
-  # DB-less. Config comes from Kubernetes objects via the ingress controller, so
-  # git stays the source of truth. A Postgres-backed Kong would put live routing
-  # config in a database mutated through the Admin API — state outside git, plus
-  # migration Jobs on every upgrade.
-  database: "off"
-  # `nginx_proxy_<directive>` injects a directive into the proxy location block;
-  # this renders `proxy_buffering off;`.
-  #
-  # Required for LLM streaming. With buffering on (the default) nginx accumulates
-  # the upstream response before forwarding, so an SSE stream from
-  # `"stream": true` arrives in lumps or stalls until the generation finishes —
-  # which defeats the point of streaming. The matching setting is already on the
-  # nginx Ingress in ingress.yaml; both hops have to be unbuffered or the
-  # buffered one dominates.
-  nginx_proxy_proxy_buffering: "off"
-  # Any plugin that rewrites the request body — request-transformer on the
-  # llm-chat-* routes — reads it through `kong.request.get_body()`, and that
-  # returns nothing once nginx has spilled the body past
-  # client_body_buffer_size into a temp file. The plugin then re-serializes a
-  # body with no `messages`, and the upstream answers
-  #   HTTP 400 {"error":{"message":"[] is too short - 'messages'"}}
-  # Measured on /v1/ornith/chat/completions: 10588 B -> 200, 11088 B -> 400.
-  # An agent request carrying tool schemas clears that in one turn, so the
-  # buffer has to hold a whole conversation, not a chat message.
-  nginx_http_client_body_buffer_size: "16m"
-  nginx_http_client_max_body_size: "16m"
-
-ingressController:
-  enabled: true
-  ingressClass: kong
-  # The chart's ingress-class template is gated on
-  # `.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass"`, so a
-  # bare `helm template` renders nothing. ArgoCD passes --api-versions from the
-  # live cluster, so it does render there — verify `kubectl get ingressclass
-  # kong` after the first sync rather than assuming it.
-  createIngressClass: true
-  # Deliberately empty: setting is-default-class here would hijack every Ingress
-  # in the cluster that omits ingressClassName. nginx keeps that role.
-  ingressClassAnnotations: {}
-
-proxy:
-  enabled: true
-  # Chart default is LoadBalancer, which would claim an IP from homelab-pool.
-  type: ClusterIP
-  http:
-    enabled: true
-    servicePort: 80
-    containerPort: 8000
-  # nginx already terminated TLS; a second handshake to the same cluster buys
-  # nothing and would need Kong to hold its own certificate.
-  tls:
-    enabled: false
-
-# No Service for the Admin API. The controller reaches it over localhost inside
-# the pod, so exposing it would only create an unauthenticated write path to the
-# gateway's entire configuration.
-admin:
-  enabled: false
-
-# Kong Manager UI — chart default is `enabled: true` with type NodePort, which
-# would open a port on every node. Not wanted.
-manager:
-  enabled: false
-
-resources:
-  requests:
-    cpu: 200m
-    memory: 256Mi
-  limits:
-    cpu: "2"
-    memory: 1Gi
-
-podDisruptionBudget:
-  enabled: true
-  minAvailable: 1
-
-# Status listener (metrics/health) is on by default at :8100 (chart default,
-# verified via `helm show values`). This just wires the ServiceMonitor the
-# chart already knows how to generate for it, so kong_http_requests_total /
-# kong_latency_* / kong_bandwidth_bytes land in Prometheus. Paired with the
-# cluster-wide `prometheus` KongClusterPlugin in kong-metrics.yaml.
-serviceMonitor:
-  enabled: true
-  labels:
-    release: kube-prometheus-stack
-
-# Spread the two replicas across nodes; `ScheduleAnyway` so a single-node
-# situation degrades to co-location instead of leaving a pod Pending.
-topologySpreadConstraints:
-  - maxSkew: 1
-    topologyKey: kubernetes.io/hostname
-    whenUnsatisfiable: ScheduleAnyway
-    labelSelector:
-      matchLabels:
-        app.kubernetes.io/name: kong
-        app.kubernetes.io/instance: kong

k8s/apps/api/kustomization.yaml

@@ -1,14 +1,14 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-# Explicit allowlist so kong-values.yaml in this directory is NOT treated as a
+# Explicit allowlist. Anything added to this directory and not listed here is
-# manifest — it is Helm input consumed by the chart source of the `kong`
+# silently dropped — no error, no drift shown.
-# Application, not a Kubernetes object. Anything new added here must be listed
+#
-# or it is silently dropped with no error and no drift shown.
+# Down to a single Ingress since Kong was retired (2026-08-19). The Kong Helm
+# values, the KongClusterPlugin for Prometheus, the six KongPlugin CRs behind
+# the path-per-model LLM surface, the KongConsumer and the key-auth plan all
+# went with it.
 resources:
   - ingress.yaml
-  - kong-metrics.yaml
-  - llm-routes.yaml
-  - model-auth.yaml
 # No top-level `namespace:` transformer on purpose: ingress.yaml sets its own
 # namespace, and the transformer rewrites metadata.namespace on every resource
 # it builds, which is a trap for anything cross-namespace added later.

k8s/apps/api/llm-routes.yaml

@@ -1,288 +0,0 @@
-# LLM API surface on the Kong gateway — DeepSeek/OpenAI-shaped.
-#
-# These live in namespace `llm-serving`, not `api`, because a Kubernetes Ingress
-# can only reference a Service in its own namespace and the predictor Services
-# are there. The Kong ingress controller watches all namespaces, so the routes
-# still land on the gateway. They are synced by the `kong` Application (which
-# has a `path: k8s/apps/api` source) so all gateway config stays in one place.
-#
-# ── Model -> upstream map (verified live) ───────────────────────────────────
-#   reasoning              -> reasoning-predictor   vLLM, DeepSeek-R1-Distill-32B, 2 replicas
-#   ornith:35b             -> ornith-predictor      Ollama, 2 replicas (retired verifier-
-#   qwen2.5:3b-instruct    -> ornith-predictor      Ollama   predictor's vLLM PRM slot to get
-#                                                    the 2nd GPU) -- k8s Service load-balances
-#                                                    across both, each replica loads both
-#                                                    models, so 2 concurrent implementer-style
-#                                                    calls each land on an independent instance
-#   nomic-embed-text-v2    -> embeddings-predictor  TEI
-#   bge-reranker-base      -> reranker-predictor    TEI
-#
-# ── Why path-per-model, and why the body is rewritten ───────────────────────
-# Kong matches routes on host, path, method and headers — never on the request
-# body. So a single /v1/chat/completions endpoint that dispatches on the body's
-# `model` field is not expressible in Kong OSS (`ai-proxy-advanced`, which does
-# multi-target model routing, is Enterprise-only).
-#
-# Hence the model is in the path, and each chat route force-overwrites `model`
-# in the body regardless, so a client calling /v1/qwen/... with some other
-# `model` value in the body can't silently get routed to the wrong weights.
-# Callers may omit `model` entirely.
-#
-# ── Timeouts ───────────────────────────────────────────────────────────────
-# Kong's upstream timeouts default to 60000ms. A 32B model generating a long
-# answer on a Volta GPU routinely exceeds that, and the client would see a
-# 504 mid-generation. Raised to 1h on every LLM route. Values are milliseconds.
-
-# ── GET /v1/models ──────────────────────────────────────────────────────────
-# Served entirely by Kong via request-termination: the plugin short-circuits in
-# the access phase, so the backend below is never contacted. It only exists
-# because an Ingress rule requires a backend.
-#
-# The list is static, which means it can drift from what the engines actually
-# serve — notably if the Ollama pull list in the ornith InferenceService
-# changes. Verify with:
-#   curl -s $SVC/v1/models   (against each *-predictor)
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: llm-models-list
-  namespace: llm-serving
-plugin: request-termination
-config:
-  status_code: 200
-  content_type: application/json
-  body: |
-    {"object":"list","data":[
-      {"id":"reasoning","object":"model","owned_by":"homelab","created":0},
-      {"id":"ornith:35b","object":"model","owned_by":"homelab","created":0},
-      {"id":"qwen2.5:3b-instruct","object":"model","owned_by":"homelab","created":0},
-      {"id":"nomic-ai/nomic-embed-text-v2-moe","object":"model","owned_by":"homelab","created":0},
-      {"id":"BAAI/bge-reranker-base","object":"model","owned_by":"homelab","created":0}
-    ]}
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-models
-  namespace: llm-serving
-  annotations:
-    konghq.com/plugins: llm-models-list # model-key-auth stripped -- see model-auth.yaml
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "GET"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/models
-            pathType: Exact
-            backend:
-              # Never actually called — request-termination answers first.
-              service:
-                name: reasoning-predictor
-                port:
-                  number: 80
----
-# ── POST /v1/reasoning/chat/completions ─────────────────────────────────────
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: llm-rewrite-reasoning
-  namespace: llm-serving
-plugin: request-transformer
-config:
-  # `add` only applies when the field is absent, `replace` only when present.
-  # Both are needed to force the value in either case.
-  add:
-    body:
-      - "model:reasoning"
-  replace:
-    body:
-      - "model:reasoning"
-    # The model lives in the path for routing; the upstream still expects the
-    # canonical OpenAI path.
-    uri: /v1/chat/completions
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-chat-reasoning
-  namespace: llm-serving
-  annotations:
-    konghq.com/plugins: llm-rewrite-reasoning # model-key-auth stripped -- see model-auth.yaml
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "3600000"
-    konghq.com/write-timeout: "3600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/reasoning/chat/completions
-            pathType: Prefix
-            backend:
-              service:
-                name: reasoning-predictor
-                port:
-                  number: 80
----
-# ── POST /v1/ornith/chat/completions ────────────────────────────────────────
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: llm-rewrite-ornith
-  namespace: llm-serving
-plugin: request-transformer
-config:
-  add:
-    body:
-      - "model:ornith:35b"
-  replace:
-    body:
-      - "model:ornith:35b"
-    uri: /v1/chat/completions
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-chat-ornith
-  namespace: llm-serving
-  annotations:
-    konghq.com/plugins: llm-rewrite-ornith # model-key-auth stripped -- see model-auth.yaml
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "3600000"
-    konghq.com/write-timeout: "3600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/ornith/chat/completions
-            pathType: Prefix
-            backend:
-              service:
-                name: ornith-predictor
-                port:
-                  number: 80
----
-# ── POST /v1/qwen/chat/completions ──────────────────────────────────────────
-# Same upstream pod as ornith — only the forced body `model` differs. Both stay
-# resident because the engine runs with OLLAMA_MAX_LOADED_MODELS=2 and
-# OLLAMA_KEEP_ALIVE=-1, so this does not trigger a model swap per request.
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: llm-rewrite-qwen
-  namespace: llm-serving
-plugin: request-transformer
-config:
-  add:
-    body:
-      - "model:qwen2.5:3b-instruct"
-  replace:
-    body:
-      - "model:qwen2.5:3b-instruct"
-    uri: /v1/chat/completions
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-chat-qwen
-  namespace: llm-serving
-  annotations:
-    konghq.com/plugins: llm-rewrite-qwen # model-key-auth stripped -- see model-auth.yaml
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "3600000"
-    konghq.com/write-timeout: "3600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/qwen/chat/completions
-            pathType: Prefix
-            backend:
-              service:
-                name: ornith-predictor
-                port:
-                  number: 80
----
-# ── POST /v1/embeddings ─────────────────────────────────────────────────────
-# No path-per-model and no rewrite: there is exactly one embeddings backend, so
-# there is nothing to disambiguate, and TEI already serves the canonical
-# OpenAI path (verified: /v1/embeddings returns 405 to GET, i.e. it exists).
-# That makes an OpenAI SDK a drop-in here.
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-embeddings
-  namespace: llm-serving
-  annotations:
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "600000"
-    konghq.com/write-timeout: "600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/embeddings
-            pathType: Prefix
-            backend:
-              service:
-                name: embeddings-predictor
-                port:
-                  number: 80
----
-# ── POST /v1/rerank ─────────────────────────────────────────────────────────
-# Rerank is not part of the OpenAI spec, and TEI serves it at /rerank — probing
-# /v1/rerank returned 404 while /rerank returned 405, so this one genuinely
-# needs the rewrite that embeddings does not.
-apiVersion: configuration.konghq.com/v1
-kind: KongPlugin
-metadata:
-  name: llm-rewrite-rerank
-  namespace: llm-serving
-plugin: request-transformer
-config:
-  replace:
-    uri: /rerank
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: llm-rerank
-  namespace: llm-serving
-  annotations:
-    konghq.com/plugins: llm-rewrite-rerank # model-key-auth stripped -- see model-auth.yaml
-    konghq.com/strip-path: "false"
-    konghq.com/methods: "POST"
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "600000"
-    konghq.com/write-timeout: "600000"
-spec:
-  ingressClassName: kong
-  rules:
-    - host: api.riotpiao.com
-      http:
-        paths:
-          - path: /v1/rerank
-            pathType: Prefix
-            backend:
-              service:
-                name: reranker-predictor
-                port:
-                  number: 80

k8s/apps/api/model-auth.yaml

@@ -1,45 +0,0 @@
-# API auth layer — Kong key-auth on the model routes.
-#
-# TEMPORARILY RETIRED: verified live that Kong's key-auth here does not accept
-# `Authorization: Bearer <key>` the way the comment below used to claim — a
-# raw `apikey: <key>` header succeeds (200), the same request with only
-# `Authorization: Bearer <key>` fails (401). No OpenAI-SDK-compatible client
-# (pi included) sends a raw apikey header or lets you customize the header
-# name, so every such client was hard-blocked. The KongPlugin below is
-# commented out and every route's `konghq.com/plugins` annotation in
-# llm-routes.yaml has `model-key-auth` stripped, so the model routes are
-# unauthenticated for now. Re-enable once there's a Bearer-compatible fix
-# (e.g. a request-transformer that copies the Bearer token into an `apikey`
-# header before key-auth runs) — do not just uncomment this as-is, that
-# reintroduces the exact block every real client hits.
-#
-# The key itself lives in the ksops-managed Secret model-invoke-apikey
-# (labelled konghq.com/credential: key-auth) and is bound to the KongConsumer
-# below, which stays defined (harmless without the plugin) so re-enabling
-# later is a two-line uncomment instead of a rebuild.
----
-apiVersion: configuration.konghq.com/v1
-kind: KongConsumer
-metadata:
-  name: model-invoker
-  namespace: api
-  annotations:
-    kubernetes.io/ingress.class: kong
-username: model-invoker
-credentials:
-  - model-invoke-apikey
-# ---
-# apiVersion: configuration.konghq.com/v1
-# kind: KongPlugin
-# metadata:
-#   name: model-key-auth
-#   namespace: llm-serving
-# plugin: key-auth
-# config:
-#   key_names:
-#     - apikey
-#     - authorization
-#   key_in_header: true
-#   key_in_query: false
-#   key_in_body: false
-#   hide_credentials: true

k8s/apps/llm-serving/ornith.yaml

@@ -3,20 +3,15 @@ kind: InferenceService
 metadata:
   annotations:
     serving.kserve.io/deploymentMode: RawDeployment
-    # Kong reads its timeouts from the Kubernetes Service, not the Ingress —
+    # The konghq.com/{connect,read,write}-timeout annotations that used to live
-    # Ingress annotations configure Route entities (strip-path, methods,
+    # here went with Kong (retired 2026-08-19). They existed because Kong read
-    # plugins), these configure the Service entity. They were on
+    # its upstream timeouts off the Kubernetes Service, and its 60s default cut
-    # llm-chat-ornith's Ingress and therefore ignored, leaving Kong's 60s
+    # off the first request after any pod restart — a restart flushes VRAM and
-    # default in force. KServe propagates InferenceService annotations to the
+    # reloading ornith:35b takes longer than that. OLLAMA_KEEP_ALIVE=-1 hid the
-    # Service it generates, which is how they reach Kong from here.
+    # problem in steady state.
     #
-    # This was invisible while OLLAMA_KEEP_ALIVE=-1 kept the model resident: no
+    # The equivalent budget now belongs to the Go gateway's per-route timeout
-    # request ever waited on a cold load. A pod restart flushes VRAM, and
+    # config in rock/homelab-frontend, not to an annotation on this object.
-    # loading ornith:35b takes longer than 60s, so the first request after any
-    # restart returned 504.
-    konghq.com/connect-timeout: "10000"
-    konghq.com/read-timeout: "3600000"
-    konghq.com/write-timeout: "3600000"
   labels:
     app.kubernetes.io/name: llm-ornith
     app.kubernetes.io/part-of: llm-serving

k8s/argocd/apps/30-security.yaml

@@ -79,7 +79,59 @@ spec:
       prune: true
       selfHeal: true
 ---
-# Forgejo runner (local chart). Forgejo itself is Phase 0 (bootstrap).
+# Forgejo itself. Was a bootstrap Helm release (phase 3) until it was brought
+# under Argo, because values changes there were inert — a proxy-body-size fix
+# sat committed while the live Ingress kept nginx's 1m default and rejected
+# every OCI push with 413.
+#
+# Wave 3: after databases (wave 2) — Forgejo needs CNPG and Redis up first.
+#
+# Retiring the Helm release: Argo adopts the existing objects on first sync.
+# Delete the release secrets afterwards so helm stops claiming ownership:
+#   kubectl -n cicd delete secret -l owner=helm,name=forgejo
+#
+# automated sync is deliberately absent. This chart owns the Forgejo PVC and
+# the git forge itself; the first sync is manual so its diff can be read before
+# anything is applied. Turn on automated+selfHeal once that diff is clean.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: forgejo
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "3"
+spec:
+  project: homelab
+  sources:
+    - repoURL: https://dl.gitea.com/charts/
+      chart: gitea
+      targetRevision: 12.7.0
+      helm:
+        valueFiles:
+          - $values/k8s/bootstrap/phase3-forgejo/forgejo-values.yaml
+    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: cicd
+  # Reloader injects a STAKATER_* env var carrying a hash of the config Secret,
+  # so the pod rolls when that Secret changes. The chart does not render it, so
+  # Argo would strip it on every sync — and with selfHeal on, Argo and Reloader
+  # would fight over the field and Recreate the forge each round.
+  ignoreDifferences:
+    - group: apps
+      kind: Deployment
+      name: forgejo-gitea
+      jqPathExpressions:
+        - '.spec.template.spec.containers[].env[] | select(.name | startswith("STAKATER_"))'
+  syncPolicy:
+    syncOptions:
+      # Adopt the objects the bootstrap Helm release already created rather
+      # than failing on "already exists".
+      - ServerSideApply=true
+---
+# Forgejo runner (local chart).
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:

k8s/argocd/apps/55-api-gateway.yaml

@@ -1,39 +1,44 @@
-# Wave 7 — Kong, the cluster's internal API gateway (namespace `api`).
+# Wave 7 — api-gw, the cluster's API gateway (namespace `api`).
 #
-# Sits between nginx and the backend services: nginx owns the edge and TLS,
+# Replaces Kong OSS 3.4.1, removed 2026-08-19. Kong existed to route
-# Kong owns routing policy, auth and rate limiting. Wave 7 puts it after the
+# `api.riotpiao.com`, but Kong OSS cannot dispatch on a request body, so the
-# data/messaging tiers it fronts and before the wave-8 applications that
+# LLM surface had to be expressed as one path per model
-# publish routes into it.
+# (`/v1/reasoning/chat/completions`, `/v1/ornith/...`, `/v1/qwen/...`) with a
+# `request-transformer` plugin forcing the body's `model` field on each. The Go
+# gateway reads the body and picks the upstream, so a single canonical
+# `POST /v1/chat/completions` covers every model. See
+# docs/adr/ADR-0001-retire-kong-for-go-gateway.md in the frontend repo.
 #
-# DB-less: routing config comes from Kubernetes objects (Ingress with
+# Two sources:
-# `ingressClassName: kong`, plus KongPlugin/KongConsumer CRDs), so git remains
+#   1. rock/homelab-frontend on the in-cluster Forgejo — the gateway's own
-# the source of truth and there are no migration Jobs on upgrade.
+#      kustomization (Deployment, Service, ConfigMap, RBAC, NetworkPolicy). It
+#      sets `namespace: api` itself, so no transformer is needed here. The
+#      Forgejo host must stay listed in the `homelab` AppProject sourceRepos or
+#      this Application is rejected with "is not permitted in project".
+#   2. k8s/apps/api in this repo — the nginx edge Ingress for
+#      api.riotpiao.com, inherited from the retired `kong` Application. It
+#      cannot move to k8s/bootstrap/ingress/ingress.yaml because that syncs in
+#      wave 1, before namespace `api` exists.
 #
-# CRDs ship in the chart's crds/ directory; ArgoCD applies those by default
+# No resources-finalizer: deleting this Application leaves the workload running
-# (helm.skipCrds is left false).
+# rather than cascading the delete.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
-  name: kong
+  name: api-gw
   namespace: argocd
+  labels:
+    app.kubernetes.io/name: api-gateway
+    app.kubernetes.io/component: gateway
   annotations:
     argocd.argoproj.io/sync-wave: "7"
 spec:
   project: homelab
   revisionHistoryLimit: 3
   sources:
-    - repoURL: https://charts.konghq.com
+    - repoURL: https://forgejo.riotpiao.com/rock/homelab-frontend.git
-      chart: kong
-      targetRevision: "3.4.1"
-      helm:
-        valueFiles:
-          - $values/k8s/apps/api/kong-values.yaml
-    - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
-      ref: values
+      path: k8s
-    # The nginx Ingress for api.riotpiao.com. Kept in this Application rather
-    # than the central k8s/bootstrap/ingress/ingress.yaml because that one syncs
-    # in wave 1, before namespace `api` exists.
     - repoURL: https://github.com/Riotpiaole/riotpiao.homelab.com.git
       targetRevision: main
       path: k8s/apps/api
@@ -46,13 +51,9 @@ spec:
       selfHeal: true
     syncOptions:
       - CreateNamespace=true
-      # The chart's CRDs exceed the annotation size limit that client-side
-      # apply relies on; server-side apply avoids the
-      # "metadata.annotations: Too long" failure CRDs commonly hit.
-      - ServerSideApply=true
     retry:
-      limit: 3
+      limit: 5
       backoff:
-        duration: 10s
+        duration: 5s
         factor: 2
         maxDuration: 3m

k8s/argocd/apps/56-llm-serving.yaml

@@ -1,7 +1,7 @@
 # Wave 6 — the model servers behind api.riotpiao.com (namespace `llm-serving`).
 #
-# Syncs before wave 7 (Kong), so the predictor Services exist before the routes
+# Syncs before wave 7 (api-gw), so the predictor Services exist before the
-# that point at them. KServe itself is part of the substrate; this Application
+# gateway that routes to them. KServe itself is part of the substrate; this Application
 # owns only the InferenceServices.
 #
 # Adopted from live state on 2026-08-15. These five had been `kubectl apply`-ed

k8s/argocd/projects/homelab-project.yaml

@@ -12,6 +12,9 @@ spec:
   description: Homelab GitOps — single-repo, in-cluster destinations only
   sourceRepos:
     - https://github.com/Riotpiaole/riotpiao.homelab.com.git
+    # In-cluster Forgejo — api-gw (k8s/argocd/apps/55-api-gateway.yaml) sources
+    # rock/homelab-frontend from here.
+    - https://forgejo.riotpiao.com/rock/homelab-frontend.git
     # Public Helm chart repos referenced by k8s/argocd/apps/* and bootstrap/*
     - https://cloudnative-pg.github.io/charts
     - https://dl.gitea.com/charts/

k8s/argocd/secrets/agent-pod-models.enc.yaml

@@ -1,23 +1,23 @@
-apiVersion: ENC[AES256_GCM,data:zdc=,iv:VvjvrS5PVNAMIaOE0LaWU+tHcUIYVQDnCANQz6myktY=,tag:xGyWDhRCwwiNny7hPllf5g==,type:str]
+apiVersion: ENC[AES256_GCM,data:C3U=,iv:J6yvL9HYwzrR4AidMrxmTQZAA1AqtAO/nn9AQnS40JY=,tag:WkqaErH6Xfrpd68+4QrfrQ==,type:str]
-kind: ENC[AES256_GCM,data:2bf5Zfy8,iv:5Oz423GzUWmgdaaZHbrtedwRHIAPIuLh4iDMieLL05s=,tag:GNW1ROjlzGvO/4tIOSuH3Q==,type:str]
+kind: ENC[AES256_GCM,data:BJsIWY38,iv:/8AGCpKvtaKoi+iuQNkJbKCSo/jSKi6WRs0f1tJj6d0=,tag:6/Rja8mrZu25kSqg0IzjYQ==,type:str]
 metadata:
-    name: ENC[AES256_GCM,data:7RO0Qxkc+/sA,iv:wKe9A8d7QJSx/6rlEY5H6lU8V24TJqr5IpXQCBc8QgM=,tag:iF6z1MNEXG3pCz2cQ18gLg==,type:str]
+    name: ENC[AES256_GCM,data:lTfngGBtsNA+,iv:shwcjVXeWhFRE+IMYlW6ffPyY4JqVzw5yySqcfyU+4I=,tag:qMmCemQcBjLwubaTnnKqTg==,type:str]
-    namespace: ENC[AES256_GCM,data:RcduxHHLtjH9,iv:opOVx1lL2ltDqQgsleN7NdMAq0TyFr/YQO3FFsHh5AA=,tag:fIwO31apqIatRRzBvamw6g==,type:str]
+    namespace: ENC[AES256_GCM,data:/VJI5GUu+jSX,iv:XCgFTLvytlYl8K09JyhGSsmVaTVCyztY5Xvw4TSkfEg=,tag:3eWHcCiedFHLvcKC8WVnUg==,type:str]
-type: ENC[AES256_GCM,data:C+JjyZt5,iv:xs49Lz6zRzcf3spiPzdUKTm2HZ+VgFahN6wjIe81JI4=,tag:VllJ54MqU+klA8xAIebjrA==,type:str]
+type: ENC[AES256_GCM,data:aibi62c0,iv:MJQRYJ27pTgtaRUKoJI2nb1qKZP47c4Ma+PvjIrCiE0=,tag:jB6mYfAPSnWnZUnY+rC+zQ==,type:str]
 stringData:
-    models.json: ENC[AES256_GCM,data:FvlbdMkcngJchi0GEjIEDjxXpQjRwh+3xnlZNpApgd3v1SpMd0qD2d3TbQr39+feSfsvpxhdmmyU/PmRHZYVdr/QpIKlZBWBD6qTCaOSq62NQbR4ck336muIaWD4AmahWCvRouYEJdEsnG7nedjAivw4cls18vMfnL5pNm2t96foScSyHosWoh3RdT6CWLmJE39+kS6/fwZ1D3Z/GEm/E2zHkVtEbEw/fk1PiqsnmZWt8QcSRAeX8aIDkphV7wOKlkri11r1NuFQKMVIau07VPce1YEi5rsJqsircvDxQelQiMGQt2y2M4GGlXx2NeJgKEtP4Hf9mVGhTeN3CYLcXi8Qyg7k1GIawGVW/Em0kWWfy0GMjvOMfYozNmIpi0YQdYBbr3j6pkixtVM2dWezvld5QIYLnHoGMxw4M6L4IqVIonC3j6tk9pV1DaL5IFskEXyb6ScTrmVY4mQ+VEXvfLYA38jRauWxx/4qf4I7B0RzfGzOx67UJh2jKj0bPWNFHKt5Dz4CK+ul39d1KUPHHMoyaH5gCpOAfLSkjMm44X1Cn+O6uY+QFCugh9MI7bJg3EbovTCZxz95GCupWAvDshv59f7gpGL0T8AuRZHlNp6yZMLdviR9yf79d2rW26PIM1QuNOUWELrOcTs5IAmyuFYr0PI5W5XSf4klOdz3UdhPfMDcyRqtSzVDsSo9oC/WeWOo1+yurtdqqW0wLiFpU/kTaZ4JN5Kl080twqFEqgyPnJrbmlxwWlt4XN2SV9D7pcfE61FzUTwoi5PD/8xt94Fp3XerSgJtwhQ6X4Neo47wCFoMR4Y69mlHrFgSJA4kLFezDc2ISnyAix0W5to0cjwJqQ2EgonIhCXD4xWTvYeApHJMOC2o4B74K5efOZ7klb6PwDMvzM3LyeGzjoJh47aJGvhWN/MpQZC2lSHT8PdMxdGDc+OkEVmsvMncddWMGjQhkTo+69sUexARxLY4H22TCt58azqQGcWa5e7s9NUHb6bRVKHzXh4HMiIhQ0jBev++Jz5gIOyYRnw5xoGbo3ROv8ndczVAqXyAla23EM/VBWT0hLMjr6xxonuLQacP9dPJE9QUofjRMeYblkVvQdIw8WuKGbAlbTR/bokQwGdVp2bywSFQZj7BLlPjzCr0/LJmxYMdq6CKB049PdTlS9/UKBHApddMQ/QRXfOYUCX+lm4k4J19jd7dWk34WNUrTEPSR2d1U/2wTnMvzlrnQHx/zk4/lmI1newfjIbybIbXypX8xVEqzFWXpxUe+iFLP19KmmmHG7gGsV32k1VSGvYFI0HOokZga98QsAbJGbi1gQq8gYpMR/mBdhjJVcnAjOYpg+YaR0vCyrhBN55aSCDoUbT1JOqxvFqmlBHDxTWorD/3KGFxEH/wvSsxussFmUCWdEgYmjGO1ktnQBw1YpOB7mhDUlzvYCSxfpexsYufDYqTbrb/R5gmATC0pEN6EgTth5YqFOL7AiIsogy14RGjRym1wvjVl5SUU9bMQlWGJbQuvNRRwL/TzR/QP210jMsnAh9t8sncDnFPIbhFgeBLsU/ctYoBs0nFVWGWzcuoP9HTnvdWTT2xKmWAeFz8I3DZ5sjm2B787Q4bCrAszTvVLeLiRw/pOE32BeNfG4nNiZDvMQhUomAKpaQypWTu2wLsv3ISm00gud5sRKB6ASJANuSu+EhG9k4OYcBEzAT9PKUxclupFbmsFnXx0u9CXDj3tErrPCUs7DuFSr+fz0ehCLMH+GrcqfytjNDVoTSBRZg+lJYKjAqVCGvq8uRf5V/rdulbt3ggx/uhGSiPzYrBQ0g9iMdjQqpNGBwjlxGkhRE9D2n/PoerAAnHTfwc9Hezffncostjp09xw2CB/xOx9HZFGalQ1R4F5+0lm0iR4lMzc+se+o38foNIiMeMIwAVBJu7IO++UMuKCxpxmdAGqEYj/Ddpzfi3ZcRwe/1HhanVvKRMJsPOhnv0SB2xBrdb7QHnEzLbRTriVRBBdOI77HXWgDt9uQru0tEXCEQ0n+f0BWqHVa/RLN4Yj0dQVWLXU75ddyHnRaCiVdLQnObMvG/QXqY1k/bJTzGNgVF8/KGY51NgdfqskOXC9Fv5IGiyWq+/DL7tQcrmThkzUhZlwDBmwUzPFmzuUC7Z5PkXv2EV+eUrYF16HYqp5KgqTkBHr/JXR8ljb/5gMN929Jk2fBVEuzaEG6W9Yek+Qbx1cnXYrMBI9GWJUJsMsxsAj+1bAjtQeQwt0nXgZesTaV90dfQYq+nTGXMNjqaMhsrJhR6lU1+CDpwY+IObHSttS8AKB+Czsl0nUB44o1a/zKRDNTCGGmPFuv1Q5Ub/1myxorLDDF1mYDpre5TedygtZ/1YSz/YOpKwmWdW32FsnY+m3qjEAcfV5f923N/z8ENVxVpeiWL6mqf7h6F3qTiaSsFq2O8m8tVm70X+oRdig3hPnKqlK/KoxVmzZErhEJ+KmP5I2uvIlb4PViE6imu8kw4VqMClb7jJ7Yk+2Zy84C8RGuDCN8GCDG7FvAahA9xVGxB6rmEpzvqFBxos2EeLOlyfewf7rCw1JAL3C9hZrciA9Jqb/t8B7KqCmCOsKK7xVt1yqyG1dsa364JG/1SKSMKebTheuymZ1+VnGjRZ+pgSAkQGjCtp7B3fTlhjt+WwxW4Ml27yfjokQQ==,iv:8Z39eWukGSMePh/3Dj35e6Zahejil+eeMSqwMYf3snI=,tag:FyBg1y0IQEg/m0mgpf1ESg==,type:str]
+    models.json: ENC[AES256_GCM,data:SZNtmDYpM+ivOATbvUcOGylj5i7RIu6sps3tp63jQcPwrEjM9bNVCcIEdfC8owq3JU2yT3mUMdC5c/ViNuCMhIilLSGf35Da/uTnOTe4e3URNQ2r1LXTBYr5R4ESt6uZrI7llzgc68C7j+k1JEldbPeoyAef0sHHSIGdbQHaVzd/+j6RSnmDcpwNw5fopyeBkZivdnKgY1vW0bz/IEgMUHRpsIeZkENSJvp63pUwsmv7NZ0qaFWk2rgzZiZszIBFYP8K/AcDAaTid8T7F6Ro9A3ClRdHW9K8Zary2TSXMrZJ+Mo9Rcqzcrw5LBiCRCn+lSmILovZfohPHUN6atKbB5YAewo7XBVlJJa5Wl+V7sq3FHZH7+rbLSFQEPovxt8z8SW2k1W/ifz+0HYWLRrtJLADZ3iV6eDKCrkSnDiCvi9pAczwD4jrbQYDdnVb9Djgp+8qhUyjEZT3C7ONs/ZaTAawX434KqsUN4O5LSEKgNlITnvHgmoxMd6eR1Xc48Gb3CBmI9ChZNpXvtgzZX0t9EuKCR0HBoBg9IPz6vADdbEVRGNLVVT1ucDZU7Sp+vMXnaM9ZIw3jDgWnmX+L4sBBpe2H2mC71muYVpF0swHt7+H0o7Nc7vxNsNIqnqWJjTQt/w4joENvbKB4dBs5lNRCeXuWuwyh6/f+MZbz4ZfqtPfvoEsWqiXglGs0eQQttDlXS36R3lsF1brFKbHdO4j7rS7YEeZdKgJIALqp3I+w/v9hAyLh+bu+R6tqXz3QoKWQkDl10Jjip0I/GIkkw9W8ZDBgsy9YcVf9J0ZOa+wm0GZXhBYWyfrE55FiMkCL1IYP2GABb5HdAbMyccIB1dkx9/TAGWeYxsMPmjCvBghf3LawZNa3nDNMnfH7+MxQFfvlHlGJBWbgO7M4V/rOLQu0RhoMIplb1ZyucHSVMWDNt71kteKR7Fme3VdFES+nIoDi8usfUYSQraW99XMB9IsIkPMCz9nFAUoNrXhd/ADkyRmaXe+gb8UohY++7zBy6YllTICHzZFVvucU1YniZLR9i5lVBv16Gcpe0PxnLuCGFwQi+RdgIPglyXTEFh5Woo7ahrr0H/9knrG7p2cqpfJs777ZSWexQL7ncN40kk73fZyAQZFbYw6vhkRGdTHb5Dq2hrLC4FSPev33lWN9v1V4uMo/wa0CGXkxbjTcHeb2NkzfXWFM0+OlqSYTev8fnbZoBGvh/N/UcmdadAG20wPoD4O7kRaZMqZe3vf5s5AsirZtKGMB+FlpBw8f3BQ8GGV9X9bBcFgq42hhcXzGrelqdCwPKFqPpz5ItEasDAdU48dHBFnAn3OWYiKrDn4/uc0XURcMKg5JR0cwJEaAWd6ZPvPy58qYQwWoFA+MVqk1/fql2GjUxW10hTPNMePLq70FYEl5pRqcWAzS2v/9LSM53E/UD/tlyjm61llqPhirDt7QhYmnEawJeQxFzzrgizAdx4jp4SGCpa3envp1oPySJdEyTJkedf/zYxXHLKZ/sc9uuhM/Bl103SeXeuAQrXrFE/kJazdUBgEstOPiDiLSWq9FjMxgMIr9SMcChYCFm7U4irsjWCdboVWKb6iMZH96AFnm2AwP+QvXUSN6dNoRAGY9E7yeiebuAnvCMGQIZ4rxYJ+xsQvRDpIwZZhs1oDAKsvMaozp5o8zZXwU+UCoRfiHVQ+LmAKhDWW0PfLhMVRSsBA9tHDnigwwRNth+A+117IBwnOgpMf/vS2xOTPszAyso24+4np4etTRMWBZLai/sScnkWflGlEwTD2lb38IeCVfTH4Vp1wPfViI3bRGvpj8je6aBg5buRaUO/M2NPXdK9eroM3OQewEtv1QTtuGy7ihsXukeoll8GvOor+1m8qhLeK4/pMH2kEMvE7fFjiG7cVxUYczN1RPl3353BIh65BdSqi8ZbxezHbPuPnoST3teoBdiwVoHcrz80by76n5kIiLKrunuzDVugq3H7NgE8xpnIpttCADiYHL2jtx7Tg7ft3rSMEjep6OSThUWyKyJUUCgyoENmrR7zyWhk50Z1YCkFW/1hOma+chdPY012aGz6nhDBxOOYrI54YlD61jKsheE51IYWKdQXAKfsp0UCGR3Q+t69QuCm+lclZgatuoBWO+tt/WKekO3T4Y1HMx1aoDk/kAKYZwfZXPyCPQ/cIK0GUpS3H8Fa8uDIpaf5KqTj3DTV9qCcrtvrmrf4Qj8YiyN+TBp/wtLiMLtofjI+apTxBqQSGss7oB/JsuH1wacA5EGoKwO8tZAuMWLM6wuPjaSgVRlBezbGsYqKUAiDZkcXLOHco6AbEptMbK91t5Mgw0V8ActHRg+3LbuHo5P/tWwzdF+1Ht7J7EvJIzvz4lfuVGyUA4fh6uVHHeeJG2OrNtjrqr2zq5iSEECJ4tc7hNDnGR7wqHc3TPEi0YIicUNU5701v3wcKgxqiPrrfRS1ONR7SLqiTHijpdFBOZyFrh/bd0216w/6vUk8uZoq/3u0PjUkVG10ofb3farN4GUfOwvQBgSRkRjIe5K4DH1ZKnE0SQtV9UonzWPqU6jWGepPsOlTMiBkdoTeUhaeomAygbJjRuBsKzGGTDDaZwBpVvB0rBVIV5xge1NKVCSTnOyGYi2gYkmEcWgSdgWk+OOI=,iv:9PDXlSUYz+vl2EzVcwMHZTgyamXLNZU7C+XdcAEi9j4=,tag:bRkH77im+qHjDewMW6PsbA==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBBZzB5REh6amhCNzRmY0ta
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4aENRQllpaWh3bnRJYjEw
-            MkN3Y0ZlR29lQ3h2SWo3cW5CUThkL2RnU1NVCmt2ZkhIZTlHN1RQRkFrTjVvbjVw
+            eTFkM250eXBSVUFrQWJXSzJUTGgrNmJqL1NVCklRT0tRWTlRU0duYWNzVFExQllS
-            RGRrTXRoYmdQcnlMSEo3ZWsrZUQ5cHMKLS0tIGU2TGJqZDRxUGJpZzRveEtZankx
+            bWo0TndLVWl2VGllb00zR1c5ZERpWmsKLS0tIDh6Uk5hUFV4bmRkK0lHWWN6L2Jk
-            MzhrT1R2akxxby9QVzd1RXB0RDY1LzQKOF+/e5z5lPX6Y1sMTAHuDj3YqW1m+sBd
+            dlU3cXJlVFZYYi8yMm5kVUJveU91OGcKyin8Tr7OkCocRxf1dzWl/QsC4l2XW4dn
-            u/0R0YnBonYM3wS5nJE3NZMkImaAdQlUjOzQepfBldG+lz++rlnAww==
+            g/it6hJQx1P+23STw9pDZVPqEj4fOdqjnNoRqVCkM8wH4SXJfwrnlA==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-18T20:06:08Z"
+    lastmodified: "2026-08-20T07:26:38Z"
-    mac: ENC[AES256_GCM,data:IM9HkpdwtQE2wCkjwDWOmHH4uP7TlIsrK4TVytiecvYz4SiLk6IRUSIu7I3a+F+dtltC2WtokoATaB69DTXPoI54amzzptirxiFD5FbaU+u2gLjo7KI7V0smYGuKqMYwnod2L/4GdlvP6xjVxFWuA01rQRaYBkFSumS4NABl/I4=,iv:al1MyBmFwni8gap7PPZxWaCwqFKCicfPq6nVmrSn9Xc=,tag:KJGTDQ8xRGF8qMOjoQ+dng==,type:str]
+    mac: ENC[AES256_GCM,data:YIN51aoCGuFrJwxJIGbCf7vY/+S4uHR4HwQ6un084hMKnInh1uK96FxWrQBlcheqBDfoaXHXXmHAd14LVhrVEsj3R1cFPpxiQpqjwt+d+mON4YBeOrC/VcStAU8joKcLsc8H0PF41PGMcBmflQVDX30D/+am60hZ7FMnGavZqgA=,iv:N6tv/3gh9BJvZdWXAQqTwaceR5nLbiA4YwOz01uwbtg=,tag:6VG9ZkEVpE9xLCNy0HLYcw==,type:str]
     unencrypted_suffix: _unencrypted
     version: 3.13.2

k8s/argocd/secrets/forgejo-registry-pull.enc.yaml

@@ -0,0 +1,24 @@
+apiVersion: ENC[AES256_GCM,data:D9Y=,iv:EH+zD6bogxh/h/Oe+RxDCtfO96tkc56ou14V+68nK7k=,tag:xTeuhSzqZyrS/ltqvtHcgw==,type:str]
+data:
+    .dockerconfigjson: ENC[AES256_GCM,data:P8x3bhPbJTFvFIKE8WQY1P7KqPwrxNTYTNR+4/Z7nZSr2YQab/Db0SRcvFO9lr4ImdO6waZ8EZ6CI1RzIYx1WcjGXs8FzX1jpBXSw9EG69vRTZCqRRJR8c2yg7qmNFfSvcPAUhK9tciYCzzFWRHZEDAmQ52cox/sMdxE/YR61dhYCy1E9kPwTnaDmDj5Hu72mbLJDxIYpN3T2hVWlw02lHYHgFnuKsPb0Tf1lxn174j/gMMxUT6ynVhWSEExzHvTqQ0RZzuP+VXAh3N9HasbtfcAabt3FtEjYNZqYJ+OE8oiX/A6YjgPDqUOR3AJFZE3,iv:6YsyIHQc8xp8T8XUWhN/pBeaYVI/VdIHOe/w/hb5e6U=,tag:nmGFdkv2hLgj8Dp8/0MOkw==,type:str]
+kind: ENC[AES256_GCM,data:hOA36Sjr,iv:y0XHfUOUnut8z0yM2g7Beo3qiqxJhLLhffPpNlUhaec=,tag:45s3FjYCr83vYCSXDXu2/g==,type:str]
+metadata:
+    creationTimestamp: null
+    name: ENC[AES256_GCM,data:yD8Fo/fncAA4qkaY7RjhRg==,iv:qwc4WId/kGwradgxFUwG5B5XIVYRlMY2HxsaGJAzrjw=,tag:O8GwNdDHMJreL609G/gykA==,type:str]
+    namespace: ENC[AES256_GCM,data:zg2o,iv:KDpM2L71/LDI4JLTQUtbcv5SV0IraAbKpNEBzSFn/rE=,tag:uFm+3EkgZN3NC2hUPXo9Pg==,type:str]
+type: ENC[AES256_GCM,data:5/a+6lRp9Ea5rU/+gEMIgoDYs72xRWkfwefMcy+h,iv:jYirTXlb4rFwvb+nLcgB5X4x1Q/S+3LsVL+7ypS7mkQ=,tag:5Wc0DrMZxIpC0e39MzlAlw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2ekxNWW9VQ09RTUlMUzVy
+            QlVrRm12Smt2akRaYmkvMHRPRmJvOTdBTzE0CmdhVFNyWDJnWDV3eTFFUm1majk0
+            L01ZWTNZdmcwc1MwMHgzQmdVZy9KcjgKLS0tIHZLVGg5VmNRQ2ZNY1lIRHlzaUlo
+            TWVNelZRcHFveGRiNTVvNzJCNWtDSGMKyV3Puscgx3RqK65KSL6SYaTauxsBY3qd
+            CeFU928hcB86DwAG/Atq2Qtd7S9pzuzOVQmXRZxwpCDTTyRVhU7eVA==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
+    lastmodified: "2026-08-20T04:43:53Z"
+    mac: ENC[AES256_GCM,data:Leus5j38xJwJz3Ge9WggBjQSAh2ESlIMUnX9SylE4oIcAt71f8WadtSCOmnqT3ZK+uN8f+Huq6WetGHWUdLfZjZUngHQHWLoRP1xpTVvB5HwJK4F1ASvmL+u1rC88AG3JsZc3Kc2N1G+m6QruQK/9HSmkMi/HGXy658ijzErEzk=,iv:1ga2dKDqTalj9WnjVT6AubXsL7130CuJp3SbBkTb/64=,tag:YsD9metJdYiJmdVXpPuIsQ==,type:str]
+    unencrypted_suffix: _unencrypted
+    version: 3.13.2

k8s/argocd/secrets/model-invoke-apikey.enc.yaml

@@ -1,25 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:894=,iv:Swg6ADUgmrqwz7wqAZHip9/qwFu0Rn8S2Lx4gBH8LJM=,tag:zbv0tFfRLtwFxBfpsuLt8A==,type:str]
-kind: ENC[AES256_GCM,data:jzVtJHYw,iv:ToZ0orfJqfGF/OnAPeYu/g2f4fXAMOZQDkA1+tmIccs=,tag:pi2skbwIU8qOQVEC86MdAA==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:t7zCeZLvAEbkJqUjWi26yD6UDA==,iv:MS1gq/bwKZdLA1itVDtsrdSOfI7e2CrhjvX5yhs0eQA=,tag:lC1gcoFMI5nfzC56U1WXrg==,type:str]
-    namespace: ENC[AES256_GCM,data:tp+d,iv:gnlet/60mgbSWLXEQpSlcWD98ky7NvlNCzGLTMys0JQ=,tag:PQYj9UeA50YenQESTCl7lg==,type:str]
-    labels:
-        konghq.com/credential: ENC[AES256_GCM,data:SOqQ9bLGLK0=,iv:a7En49UhRDwgHbv5NRB/XilEYIKQdaDqKH86WDrJB5I=,tag:JaDZ6Ko8ovY6AZ1hW9YMGQ==,type:str]
-type: ENC[AES256_GCM,data:r1K5gvop,iv:Gjv4oG2Unyql5rY9RTTljFqyd28xI81CDWtpavuuW5E=,tag:jF8lBs3AQwHPVEu9V+mONw==,type:str]
-stringData:
-    key: ENC[AES256_GCM,data:pm9GmSvX5MAsXO/e6ZcI4NF1Hwr3qjG6LaEEjvV0ihvSWhO23drlAEsrTtzppqgDZbMORf+5+P53mXU=,iv:5AbHNKeiMPoFQP/qTKdA0vEoYPzuaf4kIdGGcMSmfIQ=,tag:LD5w7XK+hiCS5D410+nCfQ==,type:str]
-sops:
-    age:
-        - enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUTmoxQkpuYUp5UlRQMkph
-            a0RaazdvaU5sWkNuL2gvVjlUYXVWV0dUWVVRClZhWDZCN2hpS2hnOG9Pck9zOTkx
-            RGNGMEI3RHpNbzVaaWNGcTNSSEdzZHMKLS0tIFZERnVJWUpreUh3TTlwbGw0dUx4
-            MGlCSkxuWWVEK2RaSDZPUzhNSUlCa28KHN0IsgQc/kBqmjQ6+4sgfb9PJy/45MwN
-            rXaLJ1htpqPZ9MJ8iOukRi0IKnKgQWXsoZengIxGmcOnEctpoH/kyQ==
-            -----END AGE ENCRYPTED FILE-----
-          recipient: age1e5fq3hwxy78psus2nfvmtmua36g0u3suk78ephw6246l974d2utsvn0hla
-    lastmodified: "2026-08-14T01:07:50Z"
-    mac: ENC[AES256_GCM,data:nY+YVwU1GuK8Yz+EZOQKkZKN28tm2L8afflc6hsgVFCFmsep5kVT+zId7AgemvQ+qnrho5N5nqxY2knB0gusFfWNKF3V5A5GBq40WtZCMaAtcwhJSex4kK7ZyaZD6oWDW/RTUumSrivSowkWlqt1XlDKyFLqSlpWQTd7JiGmUv8=,iv:zE8+B5UVYSuuAGYyvXsAwp1N/4vGduCoGyEMCNNEUnM=,tag:6RcrjneV2dOzhmoQi+5HsA==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.13.2

k8s/argocd/secrets/secret-generator.yaml

@@ -11,6 +11,7 @@ files:
   - agent-pod-ssh-key.enc.yaml
   - authentik-secrets.enc.yaml
   - cloudflare-secrets.enc.yaml
+  - forgejo-registry-pull.enc.yaml
   - forgejo-runner-token.enc.yaml
   - forgejo-secrets.enc.yaml
   - grafana-oidc-secrets.enc.yaml
@@ -20,7 +21,6 @@ files:
   - homarr-secrets.enc.yaml
   - homelab-ca-secrets.enc.yaml
   - loki-secrets.enc.yaml
-  - model-invoke-apikey.enc.yaml
   - minio-secrets.enc.yaml
   - vault-secrets.enc.yaml
   - vault-unseal-keys.enc.yaml

k8s/bootstrap/ingress/ingress.yaml

@@ -300,10 +300,11 @@ spec:
                 port:
                   number: 8080
 ---
-# NOTE: api.riotpiao.com (Kong) is deliberately NOT here. Its namespace `api` is
+# NOTE: api.riotpiao.com is deliberately NOT here. Its namespace `api` is
 # created in wave 7, and this Application syncs in wave 1 — an Ingress into a
 # namespace that doesn't exist yet would fail and mark this whole app
-# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced with Kong itself.
+# SyncFailed. It lives in k8s/apps/api/ingress.yaml, synced by the api-gw
+# Application.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

k8s/bootstrap/phase3-forgejo/forgejo-values.yaml

@@ -78,6 +78,18 @@ ingress:
   className: nginx
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
+    # This chart Ingress and the hand-written one in
+    # k8s/bootstrap/ingress/ingress.yaml both claim forgejo.riotpiao.com.
+    # ingress-nginx breaks the tie by oldest creationTimestamp, and the chart's
+    # is older, so it is the one actually serving — the annotations on the other
+    # have never applied. Duplicate should be removed; until then these must
+    # live here or they do nothing.
+    #
+    # proxy-body-size 0 is required for the OCI registry: nginx defaults to 1m,
+    # so any image layer above that fails the push with 413.
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
   hosts:
     - host: forgejo.riotpiao.com
       paths:

k8s/infra/forgejo-runner/templates/deployment.yaml

@@ -91,6 +91,13 @@ spec:
             - name: homelab-ca
               mountPath: /etc/ssl/certs/homelab-ca.pem
               subPath: ca.crt
+            # dockerd resolves per-registry CAs from /etc/docker/certs.d/<host>/
+            # before falling back to the system pool. Mounting it here is what
+            # makes `docker push forgejo.riotpiao.com/...` trust the homelab CA
+            # rather than failing x509: signed by unknown authority.
+            - name: homelab-ca
+              mountPath: /etc/docker/certs.d/forgejo.riotpiao.com/ca.crt
+              subPath: ca.crt
           resources:
             {{- toYaml .Values.dind.resources | nindent 12 }}
 

k8s/infra/forgejo-runner/templates/networkpolicy.yaml

@@ -29,3 +29,13 @@ spec:
             except:
               - 192.168.1.0/24
               - 10.244.0.0/16
+    # Single LAN exception: the ingress-nginx LoadBalancer, which is how
+    # forgejo.riotpiao.com resolves. Image pushes go to that name so the tag
+    # matches what containerd pulls on the nodes; without this the whole /24 is
+    # denied above and `docker push` hangs until it times out.
+    - to:
+        - ipBlock:
+            cidr: {{ .Values.egress.ingressLoadBalancerIP }}/32
+      ports:
+        - protocol: TCP
+          port: 443

k8s/infra/forgejo-runner/values.yaml

@@ -47,3 +47,10 @@ tolerations:
 # attach there.
 nodeSelector:
   topology.kubernetes.io/zone: az-a
+
+# Egress exceptions. The NetworkPolicy denies the whole LAN /24 by default;
+# this is the one address punched back through, because forgejo.riotpiao.com
+# (the image registry) resolves to the ingress-nginx LoadBalancer.
+# Must match the Cilium LB pool allocation — pool is 192.168.1.160/28.
+egress:
+  ingressLoadBalancerIP: 192.168.1.160

k8s/infra/monitoring/dashboards/llm-frontend.yaml

@@ -7,15 +7,22 @@ metadata:
     grafana_dashboard: "1"
   annotations:
     grafana_folder: "LLM"
-    # Request rate/error/latency/bandwidth now come from Kong's prometheus
+    # No request-level panels. The rate/error/latency/bandwidth row used to run
-    # plugin (KongClusterPlugin in kong-metrics.yaml, global: true) via the
+    # on Kong's prometheus plugin; Kong was retired 2026-08-19 and the Go
-    # chart's own ServiceMonitor (kong-values.yaml serviceMonitor.enabled) --
+    # gateway that replaced it does not expose /metrics yet, so those panels
-    # every LLM route runs through Kong, so this covers ornith/reasoning/qwen/
+    # were removed rather than left querying series that no longer exist.
-    # embeddings/rerank uniformly without per-backend instrumentation.
+    # What is left is pod-level: readiness, CPU/memory, restarts, logs.
-    # Token-count metrics are still not available: that needs response-body
+    #
-    # parsing, which Kong only does via ai-proxy-advanced (Enterprise-only).
+    # Restoring request-level and per-model observability means wiring three
-    # Predictor-level metrics (native Ollama/vLLM stats) also still need a
+    # sources, none of which are in place: gateway metrics (RED plus token
-    # dedicated exporter -- not added here.
+    # counts and TTFT, which the gateway can measure because it sees the
+    # response stream), vLLM's own /metrics on reasoning-predictor (rich --
+    # vllm:time_to_first_token_seconds, vllm:inter_token_latency_seconds,
+    # vllm:e2e_request_latency_seconds, vllm:kv_cache_usage_perc), and TEI's
+    # /metrics on embeddings/reranker. Ollama exposes no Prometheus endpoint at
+    # all (verified: /metrics returns 404), so ornith can only ever be observed
+    # from the gateway side. No ServiceMonitor exists for the llm-serving
+    # namespace today, so none of the engine metrics are being scraped.
 data:
   llm-frontend.json: |
-    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"kong (api) pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":15,"title":"Row: Request Rate & Latency (Kong)","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":16,"title":"Request rate by route","type":"timeseries","gridPos":{"h":8,"w":8,"x":0,"y":3},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) by (route)","legendFormat":"{{route}}"}]},{"id":17,"title":"Error rate %","type":"timeseries","gridPos":{"h":8,"w":8,"x":8,"y":3},"fieldConfig":{"defaults":{"unit":"percent"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_http_requests_total{route=~\"llm-.*\",code=~\"5..\"}[5m])) / sum(rate(kong_http_requests_total{route=~\"llm-.*\"}[5m])) * 100"}]},{"id":18,"title":"p95 upstream latency","type":"timeseries","gridPos":{"h":8,"w":8,"x":16,"y":3},"fieldConfig":{"defaults":{"unit":"ms"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"histogram_quantile(0.95, sum(rate(kong_latency_bucket{route=~\"llm-.*\",type=\"upstream\"}[5m])) by (le, route))","legendFormat":"{{route}}"}]},{"id":19,"title":"Bandwidth by route","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":11},"fieldConfig":{"defaults":{"unit":"Bps"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kong_bandwidth_bytes{route=~\"llm-.*\"}[5m])) by (route, direction)","legendFormat":"{{route}}/{{direction}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":3},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api (kong) logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}
+    {"title":"LLM Frontend","uid":"llm-frontend","schemaVersion":39,"timezone":"browser","time":{"from":"now-6h","to":"now"},"refresh":"30s","panels":[{"id":1,"title":"Row: Availability","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"panels":[{"id":2,"title":"llm-serving pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":0,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"llm-serving\",condition=\"true\"})"}]},{"id":3,"title":"agent-pod ready","type":"stat","gridPos":{"h":4,"w":8,"x":8,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"agent-pod\",condition=\"true\"})"}]},{"id":4,"title":"api gateway pods ready","type":"stat","gridPos":{"h":4,"w":8,"x":16,"y":1},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(kube_pod_status_ready{namespace=\"api\",condition=\"true\"})"}]}]},{"id":10,"title":"Row: Resources","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"panels":[{"id":11,"title":"CPU by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":0,"y":2},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"llm-serving|agent-pod|api\"}[5m])) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":12,"title":"Memory by pod","type":"timeseries","gridPos":{"h":8,"w":12,"x":12,"y":2},"fieldConfig":{"defaults":{"unit":"bytes"}},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(container_memory_working_set_bytes{namespace=~\"llm-serving|agent-pod|api\"}) by (namespace, pod)","legendFormat":"{{namespace}}/{{pod}}"}]},{"id":13,"title":"GPU-node predictor restarts","type":"timeseries","gridPos":{"h":8,"w":24,"x":0,"y":10},"datasource":{"type":"prometheus","uid":"prometheus"},"targets":[{"expr":"sum(rate(kube_pod_container_status_restarts_total{namespace=\"llm-serving\"}[15m])) by (pod)","legendFormat":"{{pod}}"}]}]},{"id":20,"title":"Row: Logs","type":"row","collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":2},"panels":[{"id":21,"title":"llm-serving logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":4},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"llm-serving\"}"}]},{"id":22,"title":"agent-pod logs (pi runs)","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":14},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"agent-pod\"}"}]},{"id":23,"title":"api gateway logs","type":"logs","gridPos":{"h":10,"w":24,"x":0,"y":24},"datasource":{"type":"loki","uid":"loki"},"targets":[{"expr":"{namespace=\"api\"}"}]}]}]}

terraform/variables.tf

@@ -179,8 +179,18 @@ variable "cluster_config" {
 
 variable "forgejo_registry_ip" {
   type        = string
-  default     = "10.107.155.96"
+  default     = "192.168.1.160"
-  description = "Forgejo registry (container repo) ClusterIP for host DNS rewrite"
+  description = <<-EOT
+    Address the nodes resolve forgejo.riotpiao.com to, via extraHostEntries.
+    Used by kubelet/containerd for image pulls.
+
+    This is the ingress-nginx LoadBalancer IP, pinned by the Cilium pool
+    192.168.1.160/28, NOT a ClusterIP. It was previously the ingress-nginx
+    ClusterIP 10.107.155.96; the rev-6 Helm upgrade on 2026-08-13 recreated the
+    Service and reassigned that address, leaving every node with a dead hosts
+    entry and every image pull failing on i/o timeout. A ClusterIP rots on any
+    Service recreation; the LB IP does not.
+  EOT
 }
 
 variable "forgejo_hostname" {