feat: phase 8 serviceadapter crd rollout (32/33 tasks)
This commit is contained in:
Admin Bot
@@ -0,0 +1,127 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ExportPrometheus exports metrics in Prometheus text format.
|
||||
func (m *Metrics) ExportPrometheus() string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
var sb strings.Builder
|
||||
|
||||
// Help and type for request_total counter
|
||||
sb.WriteString("# HELP gateway_requests_total Total number of HTTP requests\n")
|
||||
sb.WriteString("# TYPE gateway_requests_total counter\n")
|
||||
for key, count := range m.requestTotal {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 3 {
|
||||
route, upstream, status := parts[0], parts[1], parts[2]
|
||||
sb.WriteString(fmt.Sprintf("gateway_requests_total{route=\"%s\",upstream=\"%s\",status=\"%s\"} %d\n",
|
||||
route, upstream, status, count))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for request_duration_seconds histogram
|
||||
sb.WriteString("# HELP gateway_request_duration_seconds Request latency in seconds\n")
|
||||
sb.WriteString("# TYPE gateway_request_duration_seconds histogram\n")
|
||||
buckets := []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}
|
||||
for key := range m.requestDurationBuckets {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 2 {
|
||||
route, upstream := parts[0], parts[1]
|
||||
|
||||
// Write buckets
|
||||
cumulativeCount := int64(0)
|
||||
for _, bucket := range buckets {
|
||||
if count, ok := m.requestDurationBuckets[key][bucket]; ok {
|
||||
cumulativeCount += count
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("gateway_request_duration_seconds_bucket{route=\"%s\",upstream=\"%s\",le=\"%g\"} %d\n",
|
||||
route, upstream, bucket, cumulativeCount))
|
||||
}
|
||||
|
||||
// Write +Inf bucket
|
||||
totalCount := int64(0)
|
||||
for _, count := range m.requestDurationBuckets[key] {
|
||||
totalCount += count
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("gateway_request_duration_seconds_bucket{route=\"%s\",upstream=\"%s\",le=\"+Inf\"} %d\n",
|
||||
route, upstream, totalCount))
|
||||
|
||||
// Write sum
|
||||
totalDuration := m.requestDuration[key]
|
||||
sb.WriteString(fmt.Sprintf("gateway_request_duration_seconds_sum{route=\"%s\",upstream=\"%s\"} %g\n",
|
||||
route, upstream, float64(totalDuration)/1000.0)) // convert ms to seconds
|
||||
|
||||
// Write count
|
||||
sb.WriteString(fmt.Sprintf("gateway_request_duration_seconds_count{route=\"%s\",upstream=\"%s\"} %d\n",
|
||||
route, upstream, totalCount))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for gateway_bytes_in counter
|
||||
sb.WriteString("# HELP gateway_bytes_in_total Total bytes received from clients\n")
|
||||
sb.WriteString("# TYPE gateway_bytes_in_total counter\n")
|
||||
for key, count := range m.bytesIn {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 2 {
|
||||
route, upstream := parts[0], parts[1]
|
||||
sb.WriteString(fmt.Sprintf("gateway_bytes_in_total{route=\"%s\",upstream=\"%s\"} %d\n",
|
||||
route, upstream, count))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for gateway_bytes_out counter
|
||||
sb.WriteString("# HELP gateway_bytes_out_total Total bytes sent to clients\n")
|
||||
sb.WriteString("# TYPE gateway_bytes_out_total counter\n")
|
||||
for key, count := range m.bytesOut {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 2 {
|
||||
route, upstream := parts[0], parts[1]
|
||||
sb.WriteString(fmt.Sprintf("gateway_bytes_out_total{route=\"%s\",upstream=\"%s\"} %d\n",
|
||||
route, upstream, count))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for upstream_health gauge
|
||||
sb.WriteString("# HELP gateway_upstream_health Upstream health status (1=healthy, 0=unhealthy)\n")
|
||||
sb.WriteString("# TYPE gateway_upstream_health gauge\n")
|
||||
for upstream, health := range m.upstreamHealth {
|
||||
sb.WriteString(fmt.Sprintf("gateway_upstream_health{upstream=\"%s\"} %d\n", upstream, health))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for streaming responses
|
||||
sb.WriteString("# HELP gateway_streaming_responses_total Total streaming responses\n")
|
||||
sb.WriteString("# TYPE gateway_streaming_responses_total counter\n")
|
||||
for key, count := range m.streamingResponsesTotal {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 2 {
|
||||
route, upstream := parts[0], parts[1]
|
||||
sb.WriteString(fmt.Sprintf("gateway_streaming_responses_total{route=\"%s\",upstream=\"%s\"} %d\n",
|
||||
route, upstream, count))
|
||||
}
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Help and type for streaming byte count
|
||||
sb.WriteString("# HELP gateway_streaming_bytes_total Total bytes in streaming responses\n")
|
||||
sb.WriteString("# TYPE gateway_streaming_bytes_total counter\n")
|
||||
for key, count := range m.streamingByteCount {
|
||||
parts := strings.Split(key, ":")
|
||||
if len(parts) == 2 {
|
||||
route, upstream := parts[0], parts[1]
|
||||
sb.WriteString(fmt.Sprintf("gateway_streaming_bytes_total{route=\"%s\",upstream=\"%s\"} %d\n",
|
||||
route, upstream, count))
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Metrics holds all Prometheus metrics for the gateway.
|
||||
type Metrics struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
// Request counters: request_total{route, upstream, status}
|
||||
requestTotal map[string]int64
|
||||
|
||||
// Request latencies: request_duration_seconds (histogram)
|
||||
// Stored as cumulative buckets for Prometheus text format
|
||||
requestDuration map[string]int64 // stores duration samples in milliseconds
|
||||
requestDurationBuckets map[string]map[float64]int64 // histogram buckets
|
||||
|
||||
// Bytes counters: gateway_bytes{direction, route, upstream}
|
||||
bytesIn map[string]int64
|
||||
bytesOut map[string]int64
|
||||
|
||||
// Upstream health: upstream_health{upstream} = 1 or 0
|
||||
upstreamHealth map[string]int
|
||||
|
||||
// Streaming metrics
|
||||
streamingResponsesTotal map[string]int64
|
||||
streamingByteCount map[string]int64
|
||||
}
|
||||
|
||||
// NewMetrics creates a new Metrics instance.
|
||||
func NewMetrics() *Metrics {
|
||||
return &Metrics{
|
||||
requestTotal: make(map[string]int64),
|
||||
requestDuration: make(map[string]int64),
|
||||
requestDurationBuckets: make(map[string]map[float64]int64),
|
||||
bytesIn: make(map[string]int64),
|
||||
bytesOut: make(map[string]int64),
|
||||
upstreamHealth: make(map[string]int),
|
||||
streamingResponsesTotal: make(map[string]int64),
|
||||
streamingByteCount: make(map[string]int64),
|
||||
}
|
||||
}
|
||||
|
||||
// RecordRequest records a request with its route, upstream, status, and duration.
|
||||
func (m *Metrics) RecordRequest(route, upstream string, statusCode int, duration time.Duration) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s:%s:%d", route, upstream, statusCode)
|
||||
m.requestTotal[key]++
|
||||
|
||||
// Record duration in milliseconds
|
||||
durationKey := fmt.Sprintf("%s:%s", route, upstream)
|
||||
m.requestDuration[durationKey] += int64(duration.Milliseconds())
|
||||
|
||||
// Record in histogram buckets
|
||||
if _, ok := m.requestDurationBuckets[durationKey]; !ok {
|
||||
m.requestDurationBuckets[durationKey] = make(map[float64]int64)
|
||||
}
|
||||
|
||||
// Prometheus histogram buckets: .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10
|
||||
buckets := []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}
|
||||
durationSeconds := duration.Seconds()
|
||||
|
||||
for _, bucket := range buckets {
|
||||
if durationSeconds <= bucket {
|
||||
m.requestDurationBuckets[durationKey][bucket]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RecordBytesIn records incoming bytes.
|
||||
func (m *Metrics) RecordBytesIn(route, upstream string, bytes int64) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s:%s", route, upstream)
|
||||
m.bytesIn[key] += bytes
|
||||
}
|
||||
|
||||
// RecordBytesOut records outgoing bytes.
|
||||
func (m *Metrics) RecordBytesOut(route, upstream string, bytes int64) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s:%s", route, upstream)
|
||||
m.bytesOut[key] += bytes
|
||||
}
|
||||
|
||||
// SetUpstreamHealth sets the health status of an upstream (1 = healthy, 0 = unhealthy).
|
||||
func (m *Metrics) SetUpstreamHealth(upstream string, healthy bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if healthy {
|
||||
m.upstreamHealth[upstream] = 1
|
||||
} else {
|
||||
m.upstreamHealth[upstream] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// RecordStreamingResponse records a streaming response with its total byte count and duration.
|
||||
func (m *Metrics) RecordStreamingResponse(route, upstream string, totalBytes int64, duration time.Duration) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
key := fmt.Sprintf("%s:%s", route, upstream)
|
||||
m.streamingResponsesTotal[key]++
|
||||
m.streamingByteCount[key] += totalBytes
|
||||
|
||||
// Also record as request duration
|
||||
m.recordDuration(key, duration)
|
||||
}
|
||||
|
||||
func (m *Metrics) recordDuration(key string, duration time.Duration) {
|
||||
m.requestDuration[key] += int64(duration.Milliseconds())
|
||||
|
||||
if _, ok := m.requestDurationBuckets[key]; !ok {
|
||||
m.requestDurationBuckets[key] = make(map[float64]int64)
|
||||
}
|
||||
|
||||
buckets := []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}
|
||||
durationSeconds := duration.Seconds()
|
||||
|
||||
for _, bucket := range buckets {
|
||||
if durationSeconds <= bucket {
|
||||
m.requestDurationBuckets[key][bucket]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetMetrics returns a copy of current metrics (for testing/export).
|
||||
func (m *Metrics) GetMetrics() map[string]interface{} {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"request_total": m.requestTotal,
|
||||
"request_duration": m.requestDuration,
|
||||
"request_duration_buckets": m.requestDurationBuckets,
|
||||
"bytes_in": m.bytesIn,
|
||||
"bytes_out": m.bytesOut,
|
||||
"upstream_health": m.upstreamHealth,
|
||||
"streaming_responses_total": m.streamingResponsesTotal,
|
||||
"streaming_byte_count": m.streamingByteCount,
|
||||
}
|
||||
}
|
||||
|
||||
// Reset clears all metrics (for testing).
|
||||
func (m *Metrics) Reset() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
m.requestTotal = make(map[string]int64)
|
||||
m.requestDuration = make(map[string]int64)
|
||||
m.requestDurationBuckets = make(map[string]map[float64]int64)
|
||||
m.bytesIn = make(map[string]int64)
|
||||
m.bytesOut = make(map[string]int64)
|
||||
m.upstreamHealth = make(map[string]int)
|
||||
m.streamingResponsesTotal = make(map[string]int64)
|
||||
m.streamingByteCount = make(map[string]int64)
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMetricsRecordRequest(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
// Record some requests
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 500*time.Millisecond)
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 600*time.Millisecond)
|
||||
m.RecordRequest("v1-chat", "reasoning", 500, 100*time.Millisecond)
|
||||
|
||||
metrics := m.GetMetrics()
|
||||
requestTotal := metrics["request_total"].(map[string]int64)
|
||||
|
||||
if requestTotal["v1-chat:reasoning:200"] != 2 {
|
||||
t.Errorf("expected 2 successful requests, got %d", requestTotal["v1-chat:reasoning:200"])
|
||||
}
|
||||
|
||||
if requestTotal["v1-chat:reasoning:500"] != 1 {
|
||||
t.Errorf("expected 1 error request, got %d", requestTotal["v1-chat:reasoning:500"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsRecordBytes(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
m.RecordBytesIn("v1-chat", "reasoning", 1024)
|
||||
m.RecordBytesOut("v1-chat", "reasoning", 2048)
|
||||
|
||||
metrics := m.GetMetrics()
|
||||
bytesIn := metrics["bytes_in"].(map[string]int64)
|
||||
bytesOut := metrics["bytes_out"].(map[string]int64)
|
||||
|
||||
if bytesIn["v1-chat:reasoning"] != 1024 {
|
||||
t.Errorf("expected 1024 bytes in, got %d", bytesIn["v1-chat:reasoning"])
|
||||
}
|
||||
|
||||
if bytesOut["v1-chat:reasoning"] != 2048 {
|
||||
t.Errorf("expected 2048 bytes out, got %d", bytesOut["v1-chat:reasoning"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsUpstreamHealth(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
m.SetUpstreamHealth("reasoning", true)
|
||||
m.SetUpstreamHealth("embedding", false)
|
||||
|
||||
metrics := m.GetMetrics()
|
||||
health := metrics["upstream_health"].(map[string]int)
|
||||
|
||||
if health["reasoning"] != 1 {
|
||||
t.Errorf("expected reasoning upstream healthy (1), got %d", health["reasoning"])
|
||||
}
|
||||
|
||||
if health["embedding"] != 0 {
|
||||
t.Errorf("expected embedding upstream unhealthy (0), got %d", health["embedding"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportPrometheus(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
// Record some data
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 500*time.Millisecond)
|
||||
m.RecordBytesIn("v1-chat", "reasoning", 1024)
|
||||
m.RecordBytesOut("v1-chat", "reasoning", 2048)
|
||||
m.SetUpstreamHealth("reasoning", true)
|
||||
|
||||
export := m.ExportPrometheus()
|
||||
|
||||
// Check for expected metric families
|
||||
if !strings.Contains(export, "# HELP gateway_requests_total") {
|
||||
t.Errorf("missing gateway_requests_total help")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "# TYPE gateway_requests_total counter") {
|
||||
t.Errorf("missing gateway_requests_total type")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_requests_total{route=\"v1-chat\",upstream=\"reasoning\",status=\"200\"} 1") {
|
||||
t.Errorf("missing or incorrect request_total metric")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "# HELP gateway_bytes_in_total") {
|
||||
t.Errorf("missing gateway_bytes_in_total help")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_bytes_in_total{route=\"v1-chat\",upstream=\"reasoning\"} 1024") {
|
||||
t.Errorf("missing or incorrect bytes_in metric")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_bytes_out_total{route=\"v1-chat\",upstream=\"reasoning\"} 2048") {
|
||||
t.Errorf("missing or incorrect bytes_out metric")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_upstream_health{upstream=\"reasoning\"} 1") {
|
||||
t.Errorf("missing or incorrect upstream_health metric")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportPrometheusHistogram(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
// Record requests with different durations
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 50*time.Millisecond)
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 200*time.Millisecond)
|
||||
m.RecordRequest("v1-chat", "reasoning", 200, 1*time.Second)
|
||||
|
||||
export := m.ExportPrometheus()
|
||||
|
||||
// Check for histogram structure
|
||||
if !strings.Contains(export, "# HELP gateway_request_duration_seconds Request latency in seconds") {
|
||||
t.Errorf("missing duration_seconds help")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "# TYPE gateway_request_duration_seconds histogram") {
|
||||
t.Errorf("missing histogram type")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_request_duration_seconds_bucket") {
|
||||
t.Errorf("missing histogram bucket")
|
||||
}
|
||||
|
||||
if !strings.Contains(export, "gateway_request_duration_seconds_count") {
|
||||
t.Errorf("missing histogram count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsThreadSafety(t *testing.T) {
|
||||
m := NewMetrics()
|
||||
|
||||
// Concurrent recordings
|
||||
done := make(chan bool, 2)
|
||||
|
||||
go func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
m.RecordRequest("route1", "upstream1", 200, time.Millisecond)
|
||||
}
|
||||
done <- true
|
||||
}()
|
||||
|
||||
go func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
m.RecordBytesIn("route2", "upstream2", 1024)
|
||||
}
|
||||
done <- true
|
||||
}()
|
||||
|
||||
<-done
|
||||
<-done
|
||||
|
||||
metrics := m.GetMetrics()
|
||||
if len(metrics["request_total"].(map[string]int64)) == 0 {
|
||||
t.Errorf("expected metrics to be recorded")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user