feat(cag): memory standing-context adapter, warmup semaphore, cache-aware routing
Issue #44: code-register memory adapter with standing-context resource; fix dispatcher method-level Auth.Capability override (was always using verb-derived capability, ignoring per-method Auth field). Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) + concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup: true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired. Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures vllm-num-cached-tokens + emits structured log + increments counter. Closes #44 Closes #45 Closes #46
This commit is contained in:
@@ -33,7 +33,11 @@ type ModelUpstream struct {
|
||||
// Name is the model name clients send (e.g., "reasoning", "ornith:35b").
|
||||
Name string
|
||||
// Address is the upstream server address (host:port).
|
||||
// When Addresses is set, this holds Addresses[0] for backward compatibility.
|
||||
Address string
|
||||
// Addresses holds all backend addresses for multi-backend cache-aware routing.
|
||||
// Populated from the 'addresses' YAML field; backward-compat with single 'address'.
|
||||
Addresses []string
|
||||
// Path is the upstream path for this model (e.g., "/v1/chat/completions").
|
||||
Path string
|
||||
// UpstreamModel is the model name to send to the upstream server.
|
||||
@@ -116,6 +120,18 @@ func (c *Config) ModelNames() []string {
|
||||
return names
|
||||
}
|
||||
|
||||
// AllAddresses returns all configured backend addresses for the model.
|
||||
// Returns Addresses if non-empty; otherwise wraps Address in a slice.
|
||||
func (m *ModelUpstream) AllAddresses() []string {
|
||||
if len(m.Addresses) > 0 {
|
||||
return m.Addresses
|
||||
}
|
||||
if m.Address != "" {
|
||||
return []string{m.Address}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Load reads configuration from environment variables with defaults.
|
||||
func Load() (*Config, error) {
|
||||
listenAddr := "127.0.0.1:8080"
|
||||
|
||||
Reference in New Issue
Block a user