Admin Bot
d71c88498b
- tracing: responseWriter now forwards Flush and exposes Unwrap. Without it w.(http.Flusher) failed for every request passing the tracing middleware, so the parked-stream keep-alive never started: a parked streaming request wrote nothing for up to ADMISSION_MAX_PARK_SECONDS, which is exactly the client timeout the keep-alive was added to prevent. - proxy: once the SSE status line is spent, ModifyResponse frames an upstream response >= 400 as an event: error instead of letting a raw error body land inside the stream, where a streaming client reads it as an empty completion and reports success. net/http keeps the first status, so the upstream status cannot be forwarded after the park has written one. - engine_state: new Snapshot() returns the state and its tick channel under one lock. Get and Tick taken separately can miss the poll that lands between the two calls, parking a request for an extra interval. - engine_state: observer_degraded now tracks current parsability rather than being raised once on the first poll and never cleared, which pinned it at 1 after a single truncated response and never reported drift that appeared later. markDegraded only touches the exporter when the value changes. - deployment: ADMISSION_MAX_INFLIGHT is set explicitly to 4, documenting the coupling to the engine's --max-num-seqs, instead of silently defaulting to VLLM_MAX_NUM_SEQS.
125 lines
3.4 KiB
Go
125 lines
3.4 KiB
Go
package tracing
|
|
|
|
import (
|
|
"net/http"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
const tracerName = "api-gateway"
|
|
|
|
// responseWriter wraps http.ResponseWriter to capture status code.
|
|
type responseWriter struct {
|
|
http.ResponseWriter
|
|
statusCode int
|
|
written int64
|
|
}
|
|
|
|
func newResponseWriter(w http.ResponseWriter) *responseWriter {
|
|
return &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
|
|
}
|
|
|
|
func (rw *responseWriter) WriteHeader(code int) {
|
|
rw.statusCode = code
|
|
rw.ResponseWriter.WriteHeader(code)
|
|
}
|
|
|
|
func (rw *responseWriter) Write(b []byte) (int, error) {
|
|
n, err := rw.ResponseWriter.Write(b)
|
|
rw.written += int64(n)
|
|
return n, err
|
|
}
|
|
|
|
// Flush forwards to the wrapped writer so that streaming handlers can still
|
|
// detect http.Flusher through this wrapper. Without it a bare
|
|
// `w.(http.Flusher)` assertion fails for every request that passes through
|
|
// this middleware, which silently disables SSE flushing and keep-alives.
|
|
func (rw *responseWriter) Flush() {
|
|
if f, ok := rw.ResponseWriter.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
}
|
|
|
|
// Unwrap exposes the wrapped writer to http.ResponseController.
|
|
func (rw *responseWriter) Unwrap() http.ResponseWriter {
|
|
return rw.ResponseWriter
|
|
}
|
|
|
|
// Middleware returns an HTTP middleware that adds tracing to requests.
|
|
func Middleware(next http.Handler) http.Handler {
|
|
tracer := otel.Tracer(tracerName)
|
|
propagator := otel.GetTextMapPropagator()
|
|
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
|
|
// Extract any existing trace context from incoming request
|
|
ctx := propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
|
|
|
|
// Start a new span
|
|
spanName := r.Method + " " + r.URL.Path
|
|
ctx, span := tracer.Start(ctx, spanName,
|
|
trace.WithSpanKind(trace.SpanKindServer),
|
|
trace.WithAttributes(
|
|
semconv.HTTPRequestMethodKey.String(r.Method),
|
|
semconv.URLPath(r.URL.Path),
|
|
semconv.URLScheme(scheme(r)),
|
|
semconv.ServerAddress(r.Host),
|
|
semconv.UserAgentOriginal(r.UserAgent()),
|
|
semconv.NetworkPeerAddress(r.RemoteAddr),
|
|
),
|
|
)
|
|
defer span.End()
|
|
|
|
// Add query parameters if present
|
|
if r.URL.RawQuery != "" {
|
|
span.SetAttributes(semconv.URLQuery(r.URL.RawQuery))
|
|
}
|
|
|
|
// Add model attribute for LLM requests
|
|
if model := r.Header.Get("X-Model"); model != "" {
|
|
span.SetAttributes(attribute.String("llm.model", model))
|
|
}
|
|
|
|
// Wrap response writer to capture status
|
|
rw := newResponseWriter(w)
|
|
|
|
// Inject trace context into response headers (for debugging)
|
|
propagator.Inject(ctx, propagation.HeaderCarrier(w.Header()))
|
|
|
|
// Call the next handler with traced context
|
|
next.ServeHTTP(rw, r.WithContext(ctx))
|
|
|
|
// Record response attributes
|
|
duration := time.Since(start)
|
|
span.SetAttributes(
|
|
semconv.HTTPResponseStatusCode(rw.statusCode),
|
|
attribute.Int64("http.response.body.size", rw.written),
|
|
attribute.Float64("http.request.duration_ms", float64(duration.Milliseconds())),
|
|
)
|
|
|
|
// Set span status based on HTTP status code
|
|
if rw.statusCode >= 400 {
|
|
span.SetStatus(codes.Error, http.StatusText(rw.statusCode))
|
|
} else {
|
|
span.SetStatus(codes.Ok, "")
|
|
}
|
|
})
|
|
}
|
|
|
|
func scheme(r *http.Request) string {
|
|
if r.TLS != nil {
|
|
return "https"
|
|
}
|
|
if s := r.Header.Get("X-Forwarded-Proto"); s != "" {
|
|
return s
|
|
}
|
|
return "http"
|
|
}
|