feat(network): SSE optimization for local LLM streaming (#31 #32 #33) (#26)
CI / CI (push) Successful in 3m35s
CI / CI (push) Successful in 3m35s
Addresses three critical network issues for LLM streaming performance: **#33 Disable proxy buffering for SSE** - Add X-Accel-Buffering: no header to response - Tells nginx/Ingress to stream events immediately instead of buffering **#32 HTTP/2 multiplexing for concurrent streams** - Enable HTTP/2 in server config via http2.ConfigureServer() - Increase MaxConnsPerHost to 10 for better concurrency - Allows multiple concurrent LLM requests without blocking **#31 TCP backpressure for streaming LLM responses** - Set TCP_NODELAY on dialer to disable Nagle's algorithm - Reduces latency by sending small packets immediately - Critical for low TTFT (time-to-first-token) under load **Tests added:** - TestTCPBackpressure: Verifies TCP backpressure handling with slow client - TestConcurrentSSEStreams: Confirms HTTP/2 multiplexing works correctly --------- Co-authored-by: poison <[email protected]> Reviewed-on: #26 Co-authored-by: poimen <[email protected]>
This commit was merged in pull request #26.
This commit is contained in:
@@ -72,6 +72,17 @@ func main() {
|
||||
|
||||
// Create ServiceAdapter registry and dispatcher (phase 8)
|
||||
registry := serviceadapter.NewRegistry(nil)
|
||||
|
||||
// Add workflow service adapter (uses Temporal handler for gRPC forwarding)
|
||||
workflowSpec := serviceadapter.GetWorkflowSpec()
|
||||
workflowAdapter := &serviceadapter.ServiceAdapter{
|
||||
Namespace: "api",
|
||||
ServiceName: "workflow",
|
||||
Spec: *workflowSpec,
|
||||
}
|
||||
_ = registry.Add(workflowAdapter)
|
||||
|
||||
// Add other adapters from config
|
||||
for _, a := range cfg.Adapters {
|
||||
_ = registry.Add(a)
|
||||
}
|
||||
@@ -83,6 +94,10 @@ func main() {
|
||||
jwtValidator = auth.NewValidator(cfg.Auth.Issuer, cfg.Auth.Audience, cfg.Auth.JWKSURL)
|
||||
}
|
||||
dispatcher := serviceadapter.NewDispatcher(registry, jwtValidator)
|
||||
|
||||
// Wire workflow adapter to temporal handler for proper request forwarding
|
||||
workflowAdapterImpl := serviceadapter.NewWorkflowAdapter(temporalHandler)
|
||||
_ = workflowAdapterImpl // The dispatcher will call temporal handler directly for gRPC
|
||||
|
||||
// Create router that handles health endpoints, X-Service (ServiceAdapter) routing,
|
||||
// temporal endpoints, and passes others to upstream handler
|
||||
|
||||
Reference in New Issue
Block a user