feat: add TTFT/ITL metrics for LLM inference

- RecordTTFT: Time-to-First-Token in milliseconds
- RecordITL: Inter-Token Latency in milliseconds
- RecordTokenCount: Track total tokens generated
- Prometheus exporter for /metrics endpoint
- Grafana dashboard ConfigMap (llm-metrics.json)
- ResponseWriterWrapper to capture metrics during LLM calls
- Metrics exported: llm_ttft_seconds, llm_itl_seconds, llm_tokens_total

internal/observability/metrics.go

@@ -28,6 +28,14 @@ type Metrics struct {
 	// Streaming metrics
 	streamingResponsesTotal map[string]int64
 	streamingByteCount      map[string]int64
+
+	// LLM inference metrics (TTFT and ITL)
+	// ttftMs: Time-to-First-Token in milliseconds
+	ttftMs map[string][]int64 // samples for histogram
+	// itlMs: Inter-Token Latency in milliseconds
+	itlMs map[string][]int64 // samples for histogram
+	// Token counts
+	tokenCount map[string]int64
 }
 
 // NewMetrics creates a new Metrics instance.
@@ -41,6 +49,9 @@ func NewMetrics() *Metrics {
 		upstreamHealth:         make(map[string]int),
 		streamingResponsesTotal: make(map[string]int64),
 		streamingByteCount:     make(map[string]int64),
+		ttftMs:                 make(map[string][]int64),
+		itlMs:                  make(map[string][]int64),
+		tokenCount:             make(map[string]int64),
 	}
 }
 
@@ -146,9 +157,113 @@ func (m *Metrics) GetMetrics() map[string]interface{} {
 		"upstream_health":           m.upstreamHealth,
 		"streaming_responses_total": m.streamingResponsesTotal,
 		"streaming_byte_count":      m.streamingByteCount,
+		"llm_ttft_ms":               m.ttftMs,
+		"llm_itl_ms":                m.itlMs,
+		"llm_token_count":           m.tokenCount,
 	}
 }
 
+// RecordTTFT records Time-to-First-Token in milliseconds
+func (m *Metrics) RecordTTFT(model string, ttftMs int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:ttft:%s", model)
+	m.ttftMs[key] = append(m.ttftMs[key], ttftMs)
+}
+
+// RecordITL records Inter-Token Latency in milliseconds
+func (m *Metrics) RecordITL(model string, itlMs int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:itl:%s", model)
+	m.itlMs[key] = append(m.itlMs[key], itlMs)
+}
+
+// RecordTokenCount records number of tokens in response
+func (m *Metrics) RecordTokenCount(model string, count int64) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	key := fmt.Sprintf("llm:tokens:%s", model)
+	m.tokenCount[key] += count
+}
+
+// GetTTFTMetrics returns TTFT statistics for Prometheus export
+func (m *Metrics) GetTTFTMetrics() map[string]interface{} {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	result := make(map[string]interface{})
+	for key, samples := range m.ttftMs {
+		if len(samples) > 0 {
+			result[key] = map[string]interface{}{
+				"count": len(samples),
+				"sum":   sumInt64(samples),
+				"avg":   sumInt64(samples) / int64(len(samples)),
+				"min":   minInt64(samples),
+				"max":   maxInt64(samples),
+			}
+		}
+	}
+	return result
+}
+
+// GetITLMetrics returns ITL statistics for Prometheus export
+func (m *Metrics) GetITLMetrics() map[string]interface{} {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	result := make(map[string]interface{})
+	for key, samples := range m.itlMs {
+		if len(samples) > 0 {
+			result[key] = map[string]interface{}{
+				"count": len(samples),
+				"sum":   sumInt64(samples),
+				"avg":   sumInt64(samples) / int64(len(samples)),
+				"min":   minInt64(samples),
+				"max":   maxInt64(samples),
+			}
+		}
+	}
+	return result
+}
+
+func sumInt64(vals []int64) int64 {
+	var s int64
+	for _, v := range vals {
+		s += v
+	}
+	return s
+}
+
+func minInt64(vals []int64) int64 {
+	if len(vals) == 0 {
+		return 0
+	}
+	min := vals[0]
+	for _, v := range vals {
+		if v < min {
+			min = v
+		}
+	}
+	return min
+}
+
+func maxInt64(vals []int64) int64 {
+	if len(vals) == 0 {
+		return 0
+	}
+	max := vals[0]
+	for _, v := range vals {
+		if v > max {
+			max = v
+		}
+	}
+	return max
+}
+
 // Reset clears all metrics (for testing).
 func (m *Metrics) Reset() {
 	m.mu.Lock()
@@ -162,4 +277,7 @@ func (m *Metrics) Reset() {
 	m.upstreamHealth = make(map[string]int)
 	m.streamingResponsesTotal = make(map[string]int64)
 	m.streamingByteCount = make(map[string]int64)
+	m.ttftMs = make(map[string][]int64)
+	m.itlMs = make(map[string][]int64)
+	m.tokenCount = make(map[string]int64)
 }