163 lines
4.4 KiB
Go
163 lines
4.4 KiB
Go
package observability
|
|||
|
|
|
||
|
|
import (
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
func TestMetricsRecordRequest(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
// Record some requests
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 500*time.Millisecond)
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 600*time.Millisecond)
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 500, 100*time.Millisecond)
|
||
|
|
|
||
|
|
metrics := m.GetMetrics()
|
||
|
|
requestTotal := metrics["request_total"].(map[string]int64)
|
||
|
|
|
||
|
|
if requestTotal["v1-chat:reasoning:200"] != 2 {
|
||
|
|
t.Errorf("expected 2 successful requests, got %d", requestTotal["v1-chat:reasoning:200"])
|
||
|
|
}
|
||
|
|
|
||
|
|
if requestTotal["v1-chat:reasoning:500"] != 1 {
|
||
|
|
t.Errorf("expected 1 error request, got %d", requestTotal["v1-chat:reasoning:500"])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestMetricsRecordBytes(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
m.RecordBytesIn("v1-chat", "reasoning", 1024)
|
||
|
|
m.RecordBytesOut("v1-chat", "reasoning", 2048)
|
||
|
|
|
||
|
|
metrics := m.GetMetrics()
|
||
|
|
bytesIn := metrics["bytes_in"].(map[string]int64)
|
||
|
|
bytesOut := metrics["bytes_out"].(map[string]int64)
|
||
|
|
|
||
|
|
if bytesIn["v1-chat:reasoning"] != 1024 {
|
||
|
|
t.Errorf("expected 1024 bytes in, got %d", bytesIn["v1-chat:reasoning"])
|
||
|
|
}
|
||
|
|
|
||
|
|
if bytesOut["v1-chat:reasoning"] != 2048 {
|
||
|
|
t.Errorf("expected 2048 bytes out, got %d", bytesOut["v1-chat:reasoning"])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestMetricsUpstreamHealth(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
m.SetUpstreamHealth("reasoning", true)
|
||
|
|
m.SetUpstreamHealth("embedding", false)
|
||
|
|
|
||
|
|
metrics := m.GetMetrics()
|
||
|
|
health := metrics["upstream_health"].(map[string]int)
|
||
|
|
|
||
|
|
if health["reasoning"] != 1 {
|
||
|
|
t.Errorf("expected reasoning upstream healthy (1), got %d", health["reasoning"])
|
||
|
|
}
|
||
|
|
|
||
|
|
if health["embedding"] != 0 {
|
||
|
|
t.Errorf("expected embedding upstream unhealthy (0), got %d", health["embedding"])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestExportPrometheus(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
// Record some data
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 500*time.Millisecond)
|
||
|
|
m.RecordBytesIn("v1-chat", "reasoning", 1024)
|
||
|
|
m.RecordBytesOut("v1-chat", "reasoning", 2048)
|
||
|
|
m.SetUpstreamHealth("reasoning", true)
|
||
|
|
|
||
|
|
export := m.ExportPrometheus()
|
||
|
|
|
||
|
|
// Check for expected metric families
|
||
|
|
if !strings.Contains(export, "# HELP gateway_requests_total") {
|
||
|
|
t.Errorf("missing gateway_requests_total help")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "# TYPE gateway_requests_total counter") {
|
||
|
|
t.Errorf("missing gateway_requests_total type")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_requests_total{route=\"v1-chat\",upstream=\"reasoning\",status=\"200\"} 1") {
|
||
|
|
t.Errorf("missing or incorrect request_total metric")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "# HELP gateway_bytes_in_total") {
|
||
|
|
t.Errorf("missing gateway_bytes_in_total help")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_bytes_in_total{route=\"v1-chat\",upstream=\"reasoning\"} 1024") {
|
||
|
|
t.Errorf("missing or incorrect bytes_in metric")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_bytes_out_total{route=\"v1-chat\",upstream=\"reasoning\"} 2048") {
|
||
|
|
t.Errorf("missing or incorrect bytes_out metric")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_upstream_health{upstream=\"reasoning\"} 1") {
|
||
|
|
t.Errorf("missing or incorrect upstream_health metric")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestExportPrometheusHistogram(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
// Record requests with different durations
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 50*time.Millisecond)
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 200*time.Millisecond)
|
||
|
|
m.RecordRequest("v1-chat", "reasoning", 200, 1*time.Second)
|
||
|
|
|
||
|
|
export := m.ExportPrometheus()
|
||
|
|
|
||
|
|
// Check for histogram structure
|
||
|
|
if !strings.Contains(export, "# HELP gateway_request_duration_seconds Request latency in seconds") {
|
||
|
|
t.Errorf("missing duration_seconds help")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "# TYPE gateway_request_duration_seconds histogram") {
|
||
|
|
t.Errorf("missing histogram type")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_request_duration_seconds_bucket") {
|
||
|
|
t.Errorf("missing histogram bucket")
|
||
|
|
}
|
||
|
|
|
||
|
|
if !strings.Contains(export, "gateway_request_duration_seconds_count") {
|
||
|
|
t.Errorf("missing histogram count")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestMetricsThreadSafety(t *testing.T) {
|
||
|
|
m := NewMetrics()
|
||
|
|
|
||
|
|
// Concurrent recordings
|
||
|
|
done := make(chan bool, 2)
|
||
|
|
|
||
|
|
go func() {
|
||
|
|
for i := 0; i < 100; i++ {
|
||
|
|
m.RecordRequest("route1", "upstream1", 200, time.Millisecond)
|
||
|
|
}
|
||
|
|
done <- true
|
||
|
|
}()
|
||
|
|
|
||
|
|
go func() {
|
||
|
|
for i := 0; i < 100; i++ {
|
||
|
|
m.RecordBytesIn("route2", "upstream2", 1024)
|
||
|
|
}
|
||
|
|
done <- true
|
||
|
|
}()
|
||
|
|
|
||
|
|
<-done
|
||
|
|
<-done
|
||
|
|
|
||
|
|
metrics := m.GetMetrics()
|
||
|
|
if len(metrics["request_total"].(map[string]int64)) == 0 {
|
||
|
|
t.Errorf("expected metrics to be recorded")
|
||
|
|
}
|
||
|
|
}
|