fix(cag): resolve data races, add queue observability, fix test coverage
Cycle 2 fixes from reviewer-sre + reviewer-quality: - scheduler.go: remove delete() under RLock (data race); expired entries skipped on read, cleaned lazily by next MarkFailed write - proxy.go: pre-populate transports for scheduler backends at startup to prevent concurrent map writes from ServeHTTP goroutines - proxy.go: log + IncrQueueFull() on queue-full 503 path (was silent) - proxy.go: remove stub total_tokens/hit_rate from cache-hit log event - serviceadapter/router.go: Warnf when Auth.Required but jwtValidator nil - observability: add queue_full_requests_total counter + Prometheus export - scheduler_test.go: TestBackendAcquireCancelledContext (ctx cancel branch) - proxy_test.go: implement TestWarmupHeaderHandling (remove t.Skip) - router_test.go: TestAuthenticate_MethodCapabilityOverrideDirect (auth coverage)
This commit is contained in:
+23
-2
@@ -131,6 +131,17 @@ func New(cfg *config.Config) *Handler {
|
||||
h.schedulers[modelName] = NewScheduler(addrs, maxSlots, maxQueue)
|
||||
}
|
||||
}
|
||||
// Pre-populate transports for all scheduler backends at startup (single-threaded).
|
||||
// getOrCreateTransport is not goroutine-safe; pre-populating avoids concurrent writes.
|
||||
for _, model := range cfg.Models {
|
||||
for _, addr := range model.AllAddresses() {
|
||||
h.getOrCreateTransport(addr, &config.Upstream{
|
||||
ConnectTimeout: h.defaultConnectTimeout,
|
||||
ReadTimeout: h.defaultReadTimeout,
|
||||
WriteTimeout: h.defaultWriteTimeout,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
for name, route := range cfg.Routes {
|
||||
// Create a transport per unique upstream address for connection reuse
|
||||
@@ -470,9 +481,21 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if selectedBackend != nil {
|
||||
if err := selectedBackend.Acquire(r.Context()); err != nil {
|
||||
if errors.Is(err, ErrQueueFull) {
|
||||
observability.Global().IncrQueueFull()
|
||||
logging.Warnf("backend queue full", map[string]string{
|
||||
"backend": selectedBackend.Address,
|
||||
"path": r.URL.Path,
|
||||
"model": route.ModelName,
|
||||
})
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
fmt.Fprintf(w, "queue full")
|
||||
} else {
|
||||
logging.Warnf("backend acquire failed", map[string]string{
|
||||
"backend": selectedBackend.Address,
|
||||
"path": r.URL.Path,
|
||||
"model": route.ModelName,
|
||||
"err": err.Error(),
|
||||
})
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
fmt.Fprintf(w, "upstream busy")
|
||||
}
|
||||
@@ -541,8 +564,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
"event": "llm_request",
|
||||
"cache_key": cacheKey,
|
||||
"cached_tokens": n,
|
||||
"total_tokens": 0,
|
||||
"hit_rate": 0.0,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user