feat: add TTFT/ITL metrics for LLM inference

- RecordTTFT: Time-to-First-Token in milliseconds - RecordITL: Inter-Token Latency in milliseconds - RecordTokenCount: Track total tokens generated - Prometheus exporter for /metrics endpoint - Grafana dashboard ConfigMap (llm-metrics.json) - ResponseWriterWrapper to capture metrics during LLM calls - Metrics exported: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total
2026-09-14 22:33:32 +09:00
parent 6608f1a8d5
commit 44ec3502dc
6 changed files with 922 additions and 17 deletions
@@ -0,0 +1,28 @@
+package observability
+
+import (
+	"net/http"
+)
+
+// MetricsHandler serves Prometheus metrics
+type MetricsHandler struct {
+	exporter *PrometheusExporter
+}
+
+// NewMetricsHandler creates a new metrics handler
+func NewMetricsHandler(m *Metrics) *MetricsHandler {
+	return &MetricsHandler{
+		exporter: NewPrometheusExporter(m),
+	}
+}
+
+// ServeHTTP implements http.Handler for Prometheus /metrics endpoint
+func (h *MetricsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
+	w.Header().Set("Pragma", "no-cache")
+	w.Header().Set("Expires", "0")
+
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(h.exporter.Export()))
+}
@@ -28,6 +28,14 @@ type Metrics struct {
 	// Streaming metrics
 	streamingResponsesTotal map[string]int64
 	streamingByteCount      map[string]int64
+
+	// LLM inference metrics (TTFT and ITL)
+	// ttftMs: Time-to-First-Token in milliseconds
+	ttftMs map[string][]int64 // samples for histogram
+	// itlMs: Inter-Token Latency in milliseconds
+	itlMs map[string][]int64 // samples for histogram
+	// Token counts
+	tokenCount map[string]int64
 }

 // NewMetrics creates a new Metrics instance.
@@ -41,6 +49,9 @@ func NewMetrics() *Metrics {
 		upstreamHealth:         make(map[string]int),
 		streamingResponsesTotal: make(map[string]int64),
 		streamingByteCount:     make(map[string]int64),
+		ttftMs:                 make(map[string][]int64),
+		itlMs:                  make(map[string][]int64),
+		tokenCount:             make(map[string]int64),
 	}
 }

@@ -146,9 +157,113 @@ func (m *Metrics) GetMetrics() map[string]interface{} {
 		"upstream_health":           m.upstreamHealth,
 		"streaming_responses_total": m.streamingResponsesTotal,
 		"streaming_byte_count":      m.streamingByteCount,
+		"llm_ttft_ms":               m.ttftMs,
+		"llm_itl_ms":                m.itlMs,
+		"llm_token_count":           m.tokenCount,
 	}
 }

+// RecordTTFT records Time-to-First-Token in milliseconds
+func (m *Metrics) RecordTTFT(model string, ttftMs int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:ttft:%s", model)
+	m.ttftMs[key] = append(m.ttftMs[key], ttftMs)
+}
+
+// RecordITL records Inter-Token Latency in milliseconds
+func (m *Metrics) RecordITL(model string, itlMs int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:itl:%s", model)
+	m.itlMs[key] = append(m.itlMs[key], itlMs)
+}
+
+// RecordTokenCount records number of tokens in response
+func (m *Metrics) RecordTokenCount(model string, count int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:tokens:%s", model)
+	m.tokenCount[key] += count
+}
+
+// GetTTFTMetrics returns TTFT statistics for Prometheus export
+func (m *Metrics) GetTTFTMetrics() map[string]interface{} {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	result := make(map[string]interface{})
+	for key, samples := range m.ttftMs {
+		if len(samples) > 0 {
+			result[key] = map[string]interface{}{
+				"count": len(samples),
+				"sum":   sumInt64(samples),
+				"avg":   sumInt64(samples) / int64(len(samples)),
+				"min":   minInt64(samples),
+				"max":   maxInt64(samples),
+			}
+		}
+	}
+	return result
+}
+
+// GetITLMetrics returns ITL statistics for Prometheus export
+func (m *Metrics) GetITLMetrics() map[string]interface{} {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	result := make(map[string]interface{})
+	for key, samples := range m.itlMs {
+		if len(samples) > 0 {
+			result[key] = map[string]interface{}{
+				"count": len(samples),
+				"sum":   sumInt64(samples),
+				"avg":   sumInt64(samples) / int64(len(samples)),
+				"min":   minInt64(samples),
+				"max":   maxInt64(samples),
+			}
+		}
+	}
+	return result
+}
+
+func sumInt64(vals []int64) int64 {
+	var s int64
+	for _, v := range vals {
+		s += v
+	}
+	return s
+}
+
+func minInt64(vals []int64) int64 {
+	if len(vals) == 0 {
+		return 0
+	}
+	min := vals[0]
+	for _, v := range vals {
+		if v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func maxInt64(vals []int64) int64 {
+	if len(vals) == 0 {
+		return 0
+	}
+	max := vals[0]
+	for _, v := range vals {
+		if v > max {
+			max = v
+		}
+	}
+	return max
+}
+
 // Reset clears all metrics (for testing).
 func (m *Metrics) Reset() {
 	m.mu.Lock()
@@ -162,4 +277,7 @@ func (m *Metrics) Reset() {
 	m.upstreamHealth = make(map[string]int)
 	m.streamingResponsesTotal = make(map[string]int64)
 	m.streamingByteCount = make(map[string]int64)
+	m.ttftMs = make(map[string][]int64)
+	m.itlMs = make(map[string][]int64)
+	m.tokenCount = make(map[string]int64)
 }
@@ -0,0 +1,199 @@
+package observability
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// PrometheusExporter exports metrics in Prometheus text format
+type PrometheusExporter struct {
+	metrics *Metrics
+}
+
+// NewPrometheusExporter creates a new Prometheus exporter
+func NewPrometheusExporter(m *Metrics) *PrometheusExporter {
+	return &PrometheusExporter{metrics: m}
+}
+
+// Export returns metrics in Prometheus text format
+func (p *PrometheusExporter) Export() string {
+	var lines []string
+
+	lines = append(lines, "# HELP llm_ttft_seconds Time to first token for LLM inference (seconds)")
+	lines = append(lines, "# TYPE llm_ttft_seconds histogram")
+	p.exportTTFT(&lines)
+
+	lines = append(lines, "# HELP llm_itl_seconds Inter-token latency for LLM inference (seconds)")
+	lines = append(lines, "# TYPE llm_itl_seconds histogram")
+	p.exportITL(&lines)
+
+	lines = append(lines, "# HELP llm_tokens_total Total tokens generated")
+	lines = append(lines, "# TYPE llm_tokens_total counter")
+	p.exportTokens(&lines)
+
+	lines = append(lines, "# HELP request_duration_seconds Request latency")
+	lines = append(lines, "# TYPE request_duration_seconds histogram")
+	p.exportRequestDuration(&lines)
+
+	return strings.Join(lines, "\n") + "\n"
+}
+
+func (p *PrometheusExporter) exportTTFT(lines *[]string) {
+	p.metrics.mu.RLock()
+	defer p.metrics.mu.RUnlock()
+
+	// Calculate statistics for each model
+	for key, samples := range p.metrics.ttftMs {
+		if len(samples) == 0 {
+			continue
+		}
+
+		model := extractModel(key)
+		sum := sumInt64(samples)
+		avg := sum / int64(len(samples))
+
+		// Export histogram buckets (in seconds)
+		buckets := []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0}
+		for _, bucket := range buckets {
+			count := countLessOrEqual(samples, int64(bucket*1000))
+			*lines = append(*lines, fmt.Sprintf(
+				`llm_ttft_seconds_bucket{model="%s",le="%.3f"} %d`,
+				model, bucket, count,
+			))
+		}
+
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_ttft_seconds_bucket{model="%s",le="+Inf"} %d`,
+			model, len(samples),
+		))
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_ttft_seconds_sum{model="%s"} %.3f`,
+			model, float64(sum)/1000,
+		))
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_ttft_seconds_count{model="%s"} %d`,
+			model, len(samples),
+		))
+	}
+}
+
+func (p *PrometheusExporter) exportITL(lines *[]string) {
+	p.metrics.mu.RLock()
+	defer p.metrics.mu.RUnlock()
+
+	for key, samples := range p.metrics.itlMs {
+		if len(samples) == 0 {
+			continue
+		}
+
+		model := extractModel(key)
+		sum := sumInt64(samples)
+
+		// Export histogram buckets (in seconds)
+		buckets := []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0}
+		for _, bucket := range buckets {
+			count := countLessOrEqual(samples, int64(bucket*1000))
+			*lines = append(*lines, fmt.Sprintf(
+				`llm_itl_seconds_bucket{model="%s",le="%.3f"} %d`,
+				model, bucket, count,
+			))
+		}
+
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_itl_seconds_bucket{model="%s",le="+Inf"} %d`,
+			model, len(samples),
+		))
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_itl_seconds_sum{model="%s"} %.3f`,
+			model, float64(sum)/1000,
+		))
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_itl_seconds_count{model="%s"} %d`,
+			model, len(samples),
+		))
+	}
+}
+
+func (p *PrometheusExporter) exportTokens(lines *[]string) {
+	p.metrics.mu.RLock()
+	defer p.metrics.mu.RUnlock()
+
+	// Sort keys for consistent output
+	var keys []string
+	for k := range p.metrics.tokenCount {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		model := extractModel(key)
+		count := p.metrics.tokenCount[key]
+		*lines = append(*lines, fmt.Sprintf(
+			`llm_tokens_total{model="%s"} %d`,
+			model, count,
+		))
+	}
+}
+
+func (p *PrometheusExporter) exportRequestDuration(lines *[]string) {
+	p.metrics.mu.RLock()
+	defer p.metrics.mu.RUnlock()
+
+	// Sort keys for consistent output
+	var keys []string
+	for k := range p.metrics.requestDuration {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		route, upstream := parseKey(key)
+		totalMs := p.metrics.requestDuration[key]
+		count := int64(1) // We'd need to track count separately in real impl
+
+		if buckets, ok := p.metrics.requestDurationBuckets[key]; ok {
+			for bucket := range buckets {
+				*lines = append(*lines, fmt.Sprintf(
+					`request_duration_seconds_bucket{route="%s",upstream="%s",le="%.1f"} %d`,
+					route, upstream, bucket, buckets[bucket],
+				))
+			}
+		}
+
+		*lines = append(*lines, fmt.Sprintf(
+			`request_duration_seconds_sum{route="%s",upstream="%s"} %.3f`,
+			route, upstream, float64(totalMs)/1000,
+		))
+		*lines = append(*lines, fmt.Sprintf(
+			`request_duration_seconds_count{route="%s",upstream="%s"} %d`,
+			route, upstream, count,
+		))
+	}
+}
+
+func extractModel(key string) string {
+	parts := strings.Split(key, ":")
+	if len(parts) >= 3 {
+		return parts[2]
+	}
+	return key
+}
+
+func parseKey(key string) (string, string) {
+	parts := strings.Split(key, ":")
+	if len(parts) >= 2 {
+		return parts[0], parts[1]
+	}
+	return key, ""
+}
+
+func countLessOrEqual(samples []int64, threshold int64) int {
+	count := 0
+	for _, s := range samples {
+		if s <= threshold {
+			count++
+		}
+	}
+	return count
+}