riotpiao-poimen/homelab-frontend
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cag): memory standing-context adapter, warmup semaphore, cache-aware routing

Browse Source 
Issue #44: code-register memory adapter with standing-context resource; fix
dispatcher method-level Auth.Capability override (was always using verb-derived
capability, ignoring per-method Auth field).

Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) +
concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup:
true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired.

Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin
fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures
vllm-num-cached-tokens + emits structured log + increments counter.

Closes #44 Closes #45 Closes #46
This commit is contained in:
Admin Bot
2026-09-27 18:21:41 +08:00
parent ea7bd8336b
commit fc5c0e967d
16 changed files with 861 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/loader.go
+17 -8
View File
@@ -37,9 +37,10 @@ type rawRoute struct {
// rawModel represents a single model entry in the YAML configuration.
type rawModel struct {
Name string `yaml:"name"`
Address string `yaml:"address"`
Path string `yaml:"path"`
Name string `yaml:"name"`
Address string `yaml:"address"`
Addresses []string `yaml:"addresses"`
Path string `yaml:"path"`
UpstreamModel string `yaml:"upstreamModel"`
AuthRequired *bool `yaml:"authRequired"`
}
@@ -121,11 +122,18 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
if _, exists := models[rawModel.Name]; exists {
return nil, nil, nil, AuthConfig{}, fmt.Errorf("duplicate model: \"%s\"", rawModel.Name)
}
if rawModel.Address == "" {
return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": field 'address' is required", rawModel.Name)
// Support both 'address' (single) and 'addresses' (multi-backend) YAML fields.
addresses := rawModel.Addresses
if len(addresses) == 0 && rawModel.Address != "" {
addresses = []string{rawModel.Address}
}
if _, _, err := net.SplitHostPort(rawModel.Address); err != nil {
return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": invalid address \"%s\": %w", rawModel.Name, rawModel.Address, err)
if len(addresses) == 0 {
return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": 'address' or 'addresses' is required", rawModel.Name)
}
for _, addr := range addresses {
if _, _, err := net.SplitHostPort(addr); err != nil {
return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": invalid address \"%s\": %w", rawModel.Name, addr, err)
}
}
// Default authRequired to global auth.enabled if not specified per-model
authRequired := false
@@ -134,7 +142,8 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
}
models[rawModel.Name] = &ModelUpstream{
Name: rawModel.Name,
Address: rawModel.Address,
Address: addresses[0],
Addresses: addresses,
Path: rawModel.Path,
UpstreamModel: rawModel.UpstreamModel,
AuthRequired: authRequired,