Add TTFT & ITL Metrics for LLM Inference (#28)

Time-to-First-Token (TTFT) and Inter-Token Latency (ITL) metrics for LLM inference observability

Metrics: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total

Closes #31 #32 #33

---------

Co-authored-by: poimen <[email protected]>
Reviewed-on: #28

internal/serviceadapter/router.go

@@ -80,6 +80,14 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// Internal handler: dispatch directly without reverse proxy
+	if adapter.Handler != nil {
+		// Set X-Upstream-Path so the handler knows which method was matched
+		r.Header.Set("X-Upstream-Path", method.UpstreamPath)
+		adapter.Handler.ServeHTTP(w, r)
+		return
+	}
+
 	upstreamURL := adapter.Spec.Upstream.URL
 	if strings.HasPrefix(upstreamURL, "grpc://") {
 		d.dispatchGRPC(w, r, upstreamURL, method, adapter)

internal/serviceadapter/types.go

@@ -1,6 +1,7 @@
 package serviceadapter
 
 import (
+	"net/http"
 	"time"
 )
 
@@ -49,11 +50,14 @@ type Status struct {
 }
 
 // ServiceAdapter is a gateway service adapter.
+// When Handler is set, the dispatcher routes directly to the internal handler
+// instead of reverse-proxying to Spec.Upstream.URL.
 type ServiceAdapter struct {
-	Name      string // namespace/name
-	Namespace string
+	Name        string // namespace/name
+	Namespace   string
 	ServiceName string
-	Spec      Spec
-	Status    Status
-	CreatedAt time.Time
+	Spec        Spec
+	Status      Status
+	CreatedAt   time.Time
+	Handler     http.Handler `json:"-" yaml:"-"` // internal handler (skip serialization)
 }

internal/serviceadapter/workflow_adapter.go

@@ -84,6 +84,68 @@ func (wa *WorkflowAdapter) HandleUpdate(w http.ResponseWriter, r *http.Request)
 	wa.forwardToTemporal(w, r)
 }
 
+// resourceToAction maps X-Resource names to Temporal action names.
+var resourceToAction = map[string]string{
+	"execute":    "START_WORKFLOW",
+	"describe":   "DESCRIBE_WORKFLOW",
+	"list":       "LIST_WORKFLOWS",
+	"history":    "GET_WORKFLOW_HISTORY",
+	"terminate":  "TERMINATE_WORKFLOW",
+	"cancel":     "CANCEL_WORKFLOW",
+	"signal":     "SIGNAL_WORKFLOW",
+	"query":      "QUERY_WORKFLOW",
+	"reset":      "RESET_WORKFLOW",
+	"update":     "UPDATE_WORKFLOW",
+}
+
+// ServeHTTP implements http.Handler for X-Service: workflow routing.
+// Maps X-Resource header to Temporal action, injects action into body,
+// and forwards to the temporal handler.
+func (wa *WorkflowAdapter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	resource := r.Header.Get("X-Resource")
+	action, ok := resourceToAction[resource]
+	if !ok {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprintf(w, `{"error":"unknown workflow resource: %s"}`, resource)
+		return
+	}
+
+	// Read body, inject action, forward
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprintf(w, `{"error":"failed to read body: %s"}`, err.Error())
+		return
+	}
+
+	var payload map[string]interface{}
+	if len(body) > 0 {
+		if err := json.Unmarshal(body, &payload); err != nil {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":"invalid JSON: %s"}`, err.Error())
+			return
+		}
+	} else {
+		payload = make(map[string]interface{})
+	}
+
+	// Inject action into body for temporal handler
+	payload["action"] = action
+	if _, ok := payload["namespace"]; !ok {
+		payload["namespace"] = "default"
+	}
+
+	newBody, _ := json.Marshal(payload)
+	r.Body = io.NopCloser(bytes.NewReader(newBody))
+	r.ContentLength = int64(len(newBody))
+	r.URL.Path = "/workflow"
+
+	wa.temporalHandler.ServeHTTP(w, r)
+}
+
 // forwardToTemporal reads the request body, ensures namespace is specified,
 // and forwards to the temporal handler.
 func (wa *WorkflowAdapter) forwardToTemporal(w http.ResponseWriter, r *http.Request) {
@@ -134,7 +196,7 @@ func GetWorkflowSpec() *Spec {
 		Retryable: true,
 		Resources: []Resource{
 			{
-				Name: "start",
+				Name: "execute",
 				Methods: []Method{
 					{
 						Verb:           "POST",
@@ -151,6 +213,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "describe",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/DescribeWorkflowExecution",
@@ -166,6 +231,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "list",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/ListWorkflowExecutions",
@@ -181,6 +249,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "history",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/GetWorkflowExecutionHistory",