Issue #44: code-register memory adapter with standing-context resource; fix dispatcher method-level Auth.Capability override (was always using verb-derived capability, ignoring per-method Auth field). Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) + concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup: true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired. Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures vllm-num-cached-tokens + emits structured log + increments counter. Closes #44 Closes #45 Closes #46
422 lines
9.3 KiB
Go
422 lines
9.3 KiB
Go
package config
|
|
|
|
import (
|
|
"os"
|
|
"testing"
|
|
)
|
|
|
|
func TestLoadModelsValidConfig(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "reasoning"
|
|
address: "reasoning-predictor.llm-serving:80"
|
|
path: "/v1/chat/completions"
|
|
- name: "ornith:35b"
|
|
address: "ornith-predictor.llm-serving:80"
|
|
path: "/v1/chat/completions"
|
|
- name: "qwen2.5:3b-instruct"
|
|
address: "ornith-predictor.llm-serving:80"
|
|
path: "/v1/chat/completions"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, models, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err != nil {
|
|
t.Fatalf("failed to load config: %v", err)
|
|
}
|
|
|
|
if len(models) != 3 {
|
|
t.Errorf("expected 3 models, got %d", len(models))
|
|
}
|
|
|
|
// Test LookupModel
|
|
cfg := &Config{Models: models}
|
|
reasoning := cfg.LookupModel("reasoning")
|
|
if reasoning == nil {
|
|
t.Errorf("expected to find model 'reasoning'")
|
|
}
|
|
if reasoning.Address != "reasoning-predictor.llm-serving:80" {
|
|
t.Errorf("expected address reasoning-predictor.llm-serving:80, got %s", reasoning.Address)
|
|
}
|
|
|
|
// Test case sensitivity
|
|
notFound := cfg.LookupModel("Reasoning")
|
|
if notFound != nil {
|
|
t.Errorf("model lookup should be case-sensitive")
|
|
}
|
|
|
|
// Test ModelNames
|
|
names := cfg.ModelNames()
|
|
if len(names) != 3 {
|
|
t.Errorf("expected 3 model names, got %d", len(names))
|
|
}
|
|
}
|
|
|
|
func TestLoadModelsDuplicateName(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "reasoning"
|
|
address: "upstream1:80"
|
|
path: "/v1/chat"
|
|
- name: "reasoning"
|
|
address: "upstream2:80"
|
|
path: "/v1/chat"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, _, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err == nil {
|
|
t.Errorf("expected error for duplicate model name, got nil")
|
|
}
|
|
if err.Error() != "duplicate model: \"reasoning\"" {
|
|
t.Errorf("expected duplicate model error, got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadModelsEmptyName(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: ""
|
|
address: "upstream:80"
|
|
path: "/v1/chat"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, _, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err == nil {
|
|
t.Errorf("expected error for empty model name, got nil")
|
|
}
|
|
}
|
|
|
|
func TestLoadModelsMissingAddress(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "reasoning"
|
|
address: ""
|
|
path: "/v1/chat"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, _, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err == nil {
|
|
t.Errorf("expected error for missing address, got nil")
|
|
}
|
|
if err.Error() != "model \"reasoning\": 'address' or 'addresses' is required" {
|
|
t.Errorf("expected missing address error, got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadModelsInvalidAddress(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "reasoning"
|
|
address: "invalid-address-no-port"
|
|
path: "/v1/chat"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, _, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err == nil {
|
|
t.Errorf("expected error for invalid address, got nil")
|
|
}
|
|
}
|
|
|
|
func TestLoadModelsOptionalPath(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "reasoning"
|
|
address: "upstream:80"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, models, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err != nil {
|
|
t.Fatalf("failed to load config: %v", err)
|
|
}
|
|
|
|
if len(models) != 1 {
|
|
t.Errorf("expected 1 model, got %d", len(models))
|
|
}
|
|
|
|
model := models["reasoning"]
|
|
if model.Path != "" {
|
|
t.Errorf("expected empty path when not specified, got %s", model.Path)
|
|
}
|
|
}
|
|
|
|
func TestModelSameUpstreamMultipleNames(t *testing.T) {
|
|
data := `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
|
|
models:
|
|
- name: "ornith:35b"
|
|
address: "ollama-pod:80"
|
|
path: "/v1/chat"
|
|
- name: "qwen2.5:3b"
|
|
address: "ollama-pod:80"
|
|
path: "/v1/chat"
|
|
`
|
|
|
|
tmpFile, _ := os.CreateTemp("", "config-*.yaml")
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
routes, models, _, _, err := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
if err != nil {
|
|
t.Fatalf("failed to load config: %v", err)
|
|
}
|
|
|
|
if len(models) != 2 {
|
|
t.Errorf("expected 2 models, got %d", len(models))
|
|
}
|
|
|
|
// Both should resolve independently
|
|
cfg := &Config{Routes: routes, Models: models}
|
|
m1 := cfg.LookupModel("ornith:35b")
|
|
m2 := cfg.LookupModel("qwen2.5:3b")
|
|
|
|
if m1.Address != m2.Address {
|
|
t.Errorf("expected both models to point to same upstream")
|
|
}
|
|
|
|
if m1 == m2 {
|
|
t.Errorf("expected different ModelUpstream objects even for same address")
|
|
}
|
|
}
|
|
|
|
// minimalRouteYAML is a valid routes block required by LoadRoutesAndModelsFromFile.
|
|
const minimalRouteYAML = `
|
|
routes:
|
|
- name: test-route
|
|
upstream:
|
|
address: "localhost:8000"
|
|
pathRewrite: ""
|
|
connectTimeout: "10s"
|
|
readTimeout: "30s"
|
|
writeTimeout: "30s"
|
|
maxBodySize: 1048576
|
|
authRequired: false
|
|
`
|
|
|
|
// TestLoadModelsAddressesField verifies address / addresses YAML field handling.
|
|
func TestLoadModelsAddressesField(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
modelsYAML string
|
|
wantErr bool
|
|
errContains string
|
|
wantAddresses []string
|
|
wantAddress string
|
|
}{
|
|
{
|
|
name: "single address field",
|
|
modelsYAML: `
|
|
models:
|
|
- name: "mymodel"
|
|
address: "host:8080"
|
|
path: "/v1/chat"
|
|
`,
|
|
wantErr: false,
|
|
wantAddresses: []string{"host:8080"},
|
|
wantAddress: "host:8080",
|
|
},
|
|
{
|
|
name: "multi addresses field",
|
|
modelsYAML: `
|
|
models:
|
|
- name: "mymodel"
|
|
addresses:
|
|
- "a:80"
|
|
- "b:80"
|
|
path: "/v1/chat"
|
|
`,
|
|
wantErr: false,
|
|
wantAddresses: []string{"a:80", "b:80"},
|
|
wantAddress: "a:80",
|
|
},
|
|
{
|
|
name: "neither field",
|
|
modelsYAML: `
|
|
models:
|
|
- name: "mymodel"
|
|
path: "/v1/chat"
|
|
`,
|
|
wantErr: true,
|
|
errContains: "mymodel",
|
|
},
|
|
{
|
|
name: "invalid address format",
|
|
modelsYAML: `
|
|
models:
|
|
- name: "mymodel"
|
|
address: "notavalidhostport"
|
|
path: "/v1/chat"
|
|
`,
|
|
wantErr: true,
|
|
errContains: "mymodel",
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
data := minimalRouteYAML + tc.modelsYAML
|
|
|
|
tmpFile, err := os.CreateTemp("", "config-*.yaml")
|
|
if err != nil {
|
|
t.Fatalf("CreateTemp: %v", err)
|
|
}
|
|
defer os.Remove(tmpFile.Name())
|
|
tmpFile.WriteString(data)
|
|
tmpFile.Close()
|
|
|
|
_, models, _, _, loadErr := LoadRoutesAndModelsFromFile(tmpFile.Name())
|
|
|
|
if tc.wantErr {
|
|
if loadErr == nil {
|
|
t.Fatalf("expected error, got nil")
|
|
}
|
|
if tc.errContains != "" && !containsStr(loadErr.Error(), tc.errContains) {
|
|
t.Errorf("error %q does not contain %q", loadErr.Error(), tc.errContains)
|
|
}
|
|
return
|
|
}
|
|
|
|
if loadErr != nil {
|
|
t.Fatalf("unexpected error: %v", loadErr)
|
|
}
|
|
|
|
m, ok := models["mymodel"]
|
|
if !ok {
|
|
t.Fatalf("model 'mymodel' not found in result")
|
|
}
|
|
|
|
if m.Address != tc.wantAddress {
|
|
t.Errorf("Address: want %q, got %q", tc.wantAddress, m.Address)
|
|
}
|
|
|
|
if len(m.Addresses) != len(tc.wantAddresses) {
|
|
t.Errorf("Addresses len: want %d, got %d (%v)", len(tc.wantAddresses), len(m.Addresses), m.Addresses)
|
|
} else {
|
|
for i, a := range tc.wantAddresses {
|
|
if m.Addresses[i] != a {
|
|
t.Errorf("Addresses[%d]: want %q, got %q", i, a, m.Addresses[i])
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// containsStr is a helper to avoid importing strings just for Contains.
|
|
func containsStr(s, sub string) bool {
|
|
return len(s) >= len(sub) && (s == sub || len(sub) == 0 ||
|
|
func() bool {
|
|
for i := 0; i <= len(s)-len(sub); i++ {
|
|
if s[i:i+len(sub)] == sub {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}())
|
|
}
|