feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)

## Summary

Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw.

---

### #44 — Memory ServiceAdapter: standing-context resource

- Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`)
- Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability)
- Auth: `memory:read` required via method-level override; `memory:write` for ingest
- `MEMORY_UPSTREAM_URL` env configures upstream

### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore

- In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20)
- Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4)
- `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low`
- Queue full → 503 immediately, no goroutine pile-up
- Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics`

### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging

- `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`)
- `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU)
- No cache key → round-robin
- Failed backend excluded 30s; next-highest-score fallback
- `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log

---

## Verification

```
go build ./...    ✅
go vet ./...      ✅
go test -race ./... ✅ (all packages, 0 races)
```

## Reviewer verdicts

- reviewer-sre: SIGN-OFF
- reviewer-quality: SIGN-OFF

## Non-blocking findings (🟡🔵 only)

- `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented
- `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO
- OTel trace_id not propagated into CAG log events (existing gap)

Closes #44 Closes #45 Closes #46

---------

Co-authored-by: Admin Bot <[email protected]>
Reviewed-on: #47

internal/config/config.go

@@ -33,7 +33,11 @@ type ModelUpstream struct {
 	// Name is the model name clients send (e.g., "reasoning", "ornith:35b").
 	Name string
 	// Address is the upstream server address (host:port).
+	// When Addresses is set, this holds Addresses[0] for backward compatibility.
 	Address string
+	// Addresses holds all backend addresses for multi-backend cache-aware routing.
+	// Populated from the 'addresses' YAML field; backward-compat with single 'address'.
+	Addresses []string
 	// Path is the upstream path for this model (e.g., "/v1/chat/completions").
 	Path string
 	// UpstreamModel is the model name to send to the upstream server.
@@ -116,6 +120,18 @@ func (c *Config) ModelNames() []string {
 	return names
 }
 
+// AllAddresses returns all configured backend addresses for the model.
+// Returns Addresses if non-empty; otherwise wraps Address in a slice.
+func (m *ModelUpstream) AllAddresses() []string {
+	if len(m.Addresses) > 0 {
+		return m.Addresses
+	}
+	if m.Address != "" {
+		return []string{m.Address}
+	}
+	return nil
+}
+
 // Load reads configuration from environment variables with defaults.
 func Load() (*Config, error) {
 	listenAddr := "127.0.0.1:8080"

internal/config/loader.go

@@ -37,9 +37,10 @@ type rawRoute struct {
 
 // rawModel represents a single model entry in the YAML configuration.
 type rawModel struct {
-	Name          string `yaml:"name"`
-	Address       string `yaml:"address"`
-	Path          string `yaml:"path"`
+	Name          string   `yaml:"name"`
+	Address       string   `yaml:"address"`
+	Addresses     []string `yaml:"addresses"`
+	Path          string   `yaml:"path"`
 	UpstreamModel string `yaml:"upstreamModel"`
 	AuthRequired  *bool  `yaml:"authRequired"`
 }
@@ -121,11 +122,18 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
 		if _, exists := models[rawModel.Name]; exists {
 			return nil, nil, nil, AuthConfig{}, fmt.Errorf("duplicate model: \"%s\"", rawModel.Name)
 		}
-		if rawModel.Address == "" {
-			return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": field 'address' is required", rawModel.Name)
+		// Support both 'address' (single) and 'addresses' (multi-backend) YAML fields.
+		addresses := rawModel.Addresses
+		if len(addresses) == 0 && rawModel.Address != "" {
+			addresses = []string{rawModel.Address}
 		}
-		if _, _, err := net.SplitHostPort(rawModel.Address); err != nil {
-			return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": invalid address \"%s\": %w", rawModel.Name, rawModel.Address, err)
+		if len(addresses) == 0 {
+			return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": 'address' or 'addresses' is required", rawModel.Name)
+		}
+		for _, addr := range addresses {
+			if _, _, err := net.SplitHostPort(addr); err != nil {
+				return nil, nil, nil, AuthConfig{}, fmt.Errorf("model \"%s\": invalid address \"%s\": %w", rawModel.Name, addr, err)
+			}
 		}
 		// Default authRequired to global auth.enabled if not specified per-model
 		authRequired := false
@@ -134,7 +142,8 @@ func LoadRoutesAndModelsFromFile(path string) (map[string]*Route, map[string]*Mo
 		}
 		models[rawModel.Name] = &ModelUpstream{
 			Name:          rawModel.Name,
-			Address:       rawModel.Address,
+			Address:       addresses[0],
+			Addresses:     addresses,
 			Path:          rawModel.Path,
 			UpstreamModel: rawModel.UpstreamModel,
 			AuthRequired:  authRequired,

internal/config/models_test.go

@@ -161,7 +161,7 @@ models:
 	if err == nil {
 		t.Errorf("expected error for missing address, got nil")
 	}
-	if err.Error() != "model \"reasoning\": field 'address' is required" {
+	if err.Error() != "model \"reasoning\": 'address' or 'addresses' is required" {
 		t.Errorf("expected missing address error, got: %v", err)
 	}
 }
@@ -283,3 +283,139 @@ models:
 		t.Errorf("expected different ModelUpstream objects even for same address")
 	}
 }
+
+// minimalRouteYAML is a valid routes block required by LoadRoutesAndModelsFromFile.
+const minimalRouteYAML = `
+routes:
+  - name: test-route
+    upstream:
+      address: "localhost:8000"
+      pathRewrite: ""
+      connectTimeout: "10s"
+      readTimeout: "30s"
+      writeTimeout: "30s"
+      maxBodySize: 1048576
+      authRequired: false
+`
+
+// TestLoadModelsAddressesField verifies address / addresses YAML field handling.
+func TestLoadModelsAddressesField(t *testing.T) {
+	tests := []struct {
+		name          string
+		modelsYAML    string
+		wantErr       bool
+		errContains   string
+		wantAddresses []string
+		wantAddress   string
+	}{
+		{
+			name: "single address field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    address: "host:8080"
+    path: "/v1/chat"
+`,
+			wantErr:       false,
+			wantAddresses: []string{"host:8080"},
+			wantAddress:   "host:8080",
+		},
+		{
+			name: "multi addresses field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    addresses:
+      - "a:80"
+      - "b:80"
+    path: "/v1/chat"
+`,
+			wantErr:       false,
+			wantAddresses: []string{"a:80", "b:80"},
+			wantAddress:   "a:80",
+		},
+		{
+			name: "neither field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    path: "/v1/chat"
+`,
+			wantErr:     true,
+			errContains: "mymodel",
+		},
+		{
+			name: "invalid address format",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    address: "notavalidhostport"
+    path: "/v1/chat"
+`,
+			wantErr:     true,
+			errContains: "mymodel",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			data := minimalRouteYAML + tc.modelsYAML
+
+			tmpFile, err := os.CreateTemp("", "config-*.yaml")
+			if err != nil {
+				t.Fatalf("CreateTemp: %v", err)
+			}
+			defer os.Remove(tmpFile.Name())
+			tmpFile.WriteString(data)
+			tmpFile.Close()
+
+			_, models, _, _, loadErr := LoadRoutesAndModelsFromFile(tmpFile.Name())
+
+			if tc.wantErr {
+				if loadErr == nil {
+					t.Fatalf("expected error, got nil")
+				}
+				if tc.errContains != "" && !containsStr(loadErr.Error(), tc.errContains) {
+					t.Errorf("error %q does not contain %q", loadErr.Error(), tc.errContains)
+				}
+				return
+			}
+
+			if loadErr != nil {
+				t.Fatalf("unexpected error: %v", loadErr)
+			}
+
+			m, ok := models["mymodel"]
+			if !ok {
+				t.Fatalf("model 'mymodel' not found in result")
+			}
+
+			if m.Address != tc.wantAddress {
+				t.Errorf("Address: want %q, got %q", tc.wantAddress, m.Address)
+			}
+
+			if len(m.Addresses) != len(tc.wantAddresses) {
+				t.Errorf("Addresses len: want %d, got %d (%v)", len(tc.wantAddresses), len(m.Addresses), m.Addresses)
+			} else {
+				for i, a := range tc.wantAddresses {
+					if m.Addresses[i] != a {
+						t.Errorf("Addresses[%d]: want %q, got %q", i, a, m.Addresses[i])
+					}
+				}
+			}
+		})
+	}
+}
+
+// containsStr is a helper to avoid importing strings just for Contains.
+func containsStr(s, sub string) bool {
+	return len(s) >= len(sub) && (s == sub || len(sub) == 0 ||
+		func() bool {
+			for i := 0; i <= len(s)-len(sub); i++ {
+				if s[i:i+len(sub)] == sub {
+					return true
+				}
+			}
+			return false
+		}())
+}