Add TTFT & ITL Metrics for LLM Inference (#28)

Time-to-First-Token (TTFT) and Inter-Token Latency (ITL) metrics for LLM inference observability Metrics: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total Closes #31 #32 #33 --------- Co-authored-by: poimen <[email protected]> Reviewed-on: #28
2026-09-15 08:53:58 +00:00
parent 30e0a83a50
commit d439536ca9
22 changed files with 2224 additions and 199 deletions
@@ -37,10 +37,11 @@ type rawRoute struct {

 // rawModel represents a single model entry in the YAML configuration.
 type rawModel struct {
-	Name         string `yaml:"name"`
-	Address      string `yaml:"address"`
-	Path         string `yaml:"path"`
-	AuthRequired *bool  `yaml:"authRequired"`
+	Name          string `yaml:"name"`
+	Address       string `yaml:"address"`
+	Path          string `yaml:"path"`
+	UpstreamModel string `yaml:"upstreamModel"`
+	AuthRequired  *bool  `yaml:"authRequired"`
 }

 // rawAdapter represents a service adapter in the YAML configuration.
@@ -132,10 +133,11 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
 			authRequired = *rawModel.AuthRequired
 		}
 		models[rawModel.Name] = &ModelUpstream{
-			Name:         rawModel.Name,
-			Address:      rawModel.Address,
-			Path:         rawModel.Path,
-			AuthRequired: authRequired,
+			Name:          rawModel.Name,
+			Address:       rawModel.Address,
+			Path:          rawModel.Path,
+			UpstreamModel: rawModel.UpstreamModel,
+			AuthRequired:  authRequired,
 		}
 	}