feat(cag): memory standing-context adapter, warmup semaphore, cache-aware routing
Issue #44: code-register memory adapter with standing-context resource; fix dispatcher method-level Auth.Capability override (was always using verb-derived capability, ignoring per-method Auth field). Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) + concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup: true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired. Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures vllm-num-cached-tokens + emits structured log + increments counter. Closes #44 Closes #45 Closes #46
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/notification"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/observability"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/proxy"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/server"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter"
|
||||
@@ -112,6 +113,10 @@ func main() {
|
||||
poimenAdapter := serviceadapter.NewPoimenAdapter(temporalHandler)
|
||||
_ = registry.Add(poimenAdapter)
|
||||
|
||||
// Add memory service adapter (code-registered; config memory entry is skipped).
|
||||
// Includes standing-context resource for CAG support (issue #44).
|
||||
_ = registry.Add(serviceadapter.NewMemoryAdapter())
|
||||
|
||||
// Add other adapters from config (skip if already registered in code)
|
||||
for _, a := range cfg.Adapters {
|
||||
if existing := registry.Get(a.ServiceName); existing != nil {
|
||||
@@ -135,6 +140,7 @@ func main() {
|
||||
// Create router that handles health endpoints, X-Service (ServiceAdapter) routing,
|
||||
// temporal endpoints, and passes others to upstream handler
|
||||
router := server.NewRouter(healthChecker, dispatcher, temporalHandler, upstreamHandler)
|
||||
router.SetMetricsHandler(observability.NewMetricsHandler(observability.Global()))
|
||||
|
||||
// Wrap router with tracing middleware
|
||||
tracedRouter := tracing.Middleware(router)
|
||||
|
||||
Reference in New Issue
Block a user