Add TTFT & ITL Metrics for LLM Inference (#28)

Time-to-First-Token (TTFT) and Inter-Token Latency (ITL) metrics for LLM inference observability Metrics: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total Closes #31 #32 #33 --------- Co-authored-by: poimen <[email protected]> Reviewed-on: #28
2026-09-15 08:53:58 +00:00
parent 30e0a83a50
commit d439536ca9
22 changed files with 2224 additions and 199 deletions
@@ -84,6 +84,68 @@ func (wa *WorkflowAdapter) HandleUpdate(w http.ResponseWriter, r *http.Request)
 	wa.forwardToTemporal(w, r)
 }

+// resourceToAction maps X-Resource names to Temporal action names.
+var resourceToAction = map[string]string{
+	"execute":    "START_WORKFLOW",
+	"describe":   "DESCRIBE_WORKFLOW",
+	"list":       "LIST_WORKFLOWS",
+	"history":    "GET_WORKFLOW_HISTORY",
+	"terminate":  "TERMINATE_WORKFLOW",
+	"cancel":     "CANCEL_WORKFLOW",
+	"signal":     "SIGNAL_WORKFLOW",
+	"query":      "QUERY_WORKFLOW",
+	"reset":      "RESET_WORKFLOW",
+	"update":     "UPDATE_WORKFLOW",
+}
+
+// ServeHTTP implements http.Handler for X-Service: workflow routing.
+// Maps X-Resource header to Temporal action, injects action into body,
+// and forwards to the temporal handler.
+func (wa *WorkflowAdapter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	resource := r.Header.Get("X-Resource")
+	action, ok := resourceToAction[resource]
+	if !ok {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprintf(w, `{"error":"unknown workflow resource: %s"}`, resource)
+		return
+	}
+
+	// Read body, inject action, forward
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusBadRequest)
+		fmt.Fprintf(w, `{"error":"failed to read body: %s"}`, err.Error())
+		return
+	}
+
+	var payload map[string]interface{}
+	if len(body) > 0 {
+		if err := json.Unmarshal(body, &payload); err != nil {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, `{"error":"invalid JSON: %s"}`, err.Error())
+			return
+		}
+	} else {
+		payload = make(map[string]interface{})
+	}
+
+	// Inject action into body for temporal handler
+	payload["action"] = action
+	if _, ok := payload["namespace"]; !ok {
+		payload["namespace"] = "default"
+	}
+
+	newBody, _ := json.Marshal(payload)
+	r.Body = io.NopCloser(bytes.NewReader(newBody))
+	r.ContentLength = int64(len(newBody))
+	r.URL.Path = "/workflow"
+
+	wa.temporalHandler.ServeHTTP(w, r)
+}
+
 // forwardToTemporal reads the request body, ensures namespace is specified,
 // and forwards to the temporal handler.
 func (wa *WorkflowAdapter) forwardToTemporal(w http.ResponseWriter, r *http.Request) {
@@ -134,7 +196,7 @@ func GetWorkflowSpec() *Spec {
 		Retryable: true,
 		Resources: []Resource{
 			{
-				Name: "start",
+				Name: "execute",
 				Methods: []Method{
 					{
 						Verb:           "POST",
@@ -151,6 +213,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "describe",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/DescribeWorkflowExecution",
@@ -166,6 +231,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "list",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/ListWorkflowExecutions",
@@ -181,6 +249,9 @@ func GetWorkflowSpec() *Spec {
 			{
 				Name: "history",
 				Methods: []Method{
+					{
+						Verb:           "GET",
+					},
 					{
 						Verb:           "POST",
 						UpstreamPath:   "/temporal.workflowservice.v1.WorkflowService/GetWorkflowExecutionHistory",