Add TTFT & ITL Metrics for LLM Inference (#28)

Time-to-First-Token (TTFT) and Inter-Token Latency (ITL) metrics for LLM inference observability Metrics: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total Closes #31 #32 #33 --------- Co-authored-by: poimen <[email protected]> Reviewed-on: #28
2026-09-15 08:53:58 +00:00
parent 30e0a83a50
commit d439536ca9
22 changed files with 2224 additions and 199 deletions
@@ -80,6 +80,14 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
 		}
 	}

+	// Internal handler: dispatch directly without reverse proxy
+	if adapter.Handler != nil {
+		// Set X-Upstream-Path so the handler knows which method was matched
+		r.Header.Set("X-Upstream-Path", method.UpstreamPath)
+		adapter.Handler.ServeHTTP(w, r)
+		return
+	}
+
 	upstreamURL := adapter.Spec.Upstream.URL
 	if strings.HasPrefix(upstreamURL, "grpc://") {
 		d.dispatchGRPC(w, r, upstreamURL, method, adapter)