feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)

## Summary Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw. --- ### #44 — Memory ServiceAdapter: standing-context resource - Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`) - Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability) - Auth: `memory:read` required via method-level override; `memory:write` for ingest - `MEMORY_UPSTREAM_URL` env configures upstream ### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore - In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20) - Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4) - `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low` - Queue full → 503 immediately, no goroutine pile-up - Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics` ### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging - `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`) - `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU) - No cache key → round-robin - Failed backend excluded 30s; next-highest-score fallback - `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log --- ## Verification ``` go build ./... ✅ go vet ./... ✅ go test -race ./... ✅ (all packages, 0 races) ``` ## Reviewer verdicts - reviewer-sre: SIGN-OFF - reviewer-quality: SIGN-OFF ## Non-blocking findings (🟡🔵 only) - `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented - `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO - OTel trace_id not propagated into CAG log events (existing gap) Closes #44 Closes #45 Closes #46 --------- Co-authored-by: Admin Bot <[email protected]> Reviewed-on: #47
2026-09-27 11:16:40 +00:00
parent aaa9a5d1cb
commit ef5e736673
16 changed files with 1597 additions and 162 deletions
@@ -161,7 +161,7 @@ models:
 	if err == nil {
 		t.Errorf("expected error for missing address, got nil")
 	}
-	if err.Error() != "model \"reasoning\": field 'address' is required" {
+	if err.Error() != "model \"reasoning\": 'address' or 'addresses' is required" {
 		t.Errorf("expected missing address error, got: %v", err)
 	}
 }
@@ -283,3 +283,139 @@ models:
 		t.Errorf("expected different ModelUpstream objects even for same address")
 	}
 }
+
+// minimalRouteYAML is a valid routes block required by LoadRoutesAndModelsFromFile.
+const minimalRouteYAML = `
+routes:
+  - name: test-route
+    upstream:
+      address: "localhost:8000"
+      pathRewrite: ""
+      connectTimeout: "10s"
+      readTimeout: "30s"
+      writeTimeout: "30s"
+      maxBodySize: 1048576
+      authRequired: false
+`
+
+// TestLoadModelsAddressesField verifies address / addresses YAML field handling.
+func TestLoadModelsAddressesField(t *testing.T) {
+	tests := []struct {
+		name          string
+		modelsYAML    string
+		wantErr       bool
+		errContains   string
+		wantAddresses []string
+		wantAddress   string
+	}{
+		{
+			name: "single address field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    address: "host:8080"
+    path: "/v1/chat"
+`,
+			wantErr:       false,
+			wantAddresses: []string{"host:8080"},
+			wantAddress:   "host:8080",
+		},
+		{
+			name: "multi addresses field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    addresses:
+      - "a:80"
+      - "b:80"
+    path: "/v1/chat"
+`,
+			wantErr:       false,
+			wantAddresses: []string{"a:80", "b:80"},
+			wantAddress:   "a:80",
+		},
+		{
+			name: "neither field",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    path: "/v1/chat"
+`,
+			wantErr:     true,
+			errContains: "mymodel",
+		},
+		{
+			name: "invalid address format",
+			modelsYAML: `
+models:
+  - name: "mymodel"
+    address: "notavalidhostport"
+    path: "/v1/chat"
+`,
+			wantErr:     true,
+			errContains: "mymodel",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			data := minimalRouteYAML + tc.modelsYAML
+
+			tmpFile, err := os.CreateTemp("", "config-*.yaml")
+			if err != nil {
+				t.Fatalf("CreateTemp: %v", err)
+			}
+			defer os.Remove(tmpFile.Name())
+			tmpFile.WriteString(data)
+			tmpFile.Close()
+
+			_, models, _, _, loadErr := LoadRoutesAndModelsFromFile(tmpFile.Name())
+
+			if tc.wantErr {
+				if loadErr == nil {
+					t.Fatalf("expected error, got nil")
+				}
+				if tc.errContains != "" && !containsStr(loadErr.Error(), tc.errContains) {
+					t.Errorf("error %q does not contain %q", loadErr.Error(), tc.errContains)
+				}
+				return
+			}
+
+			if loadErr != nil {
+				t.Fatalf("unexpected error: %v", loadErr)
+			}
+
+			m, ok := models["mymodel"]
+			if !ok {
+				t.Fatalf("model 'mymodel' not found in result")
+			}
+
+			if m.Address != tc.wantAddress {
+				t.Errorf("Address: want %q, got %q", tc.wantAddress, m.Address)
+			}
+
+			if len(m.Addresses) != len(tc.wantAddresses) {
+				t.Errorf("Addresses len: want %d, got %d (%v)", len(tc.wantAddresses), len(m.Addresses), m.Addresses)
+			} else {
+				for i, a := range tc.wantAddresses {
+					if m.Addresses[i] != a {
+						t.Errorf("Addresses[%d]: want %q, got %q", i, a, m.Addresses[i])
+					}
+				}
+			}
+		})
+	}
+}
+
+// containsStr is a helper to avoid importing strings just for Contains.
+func containsStr(s, sub string) bool {
+	return len(s) >= len(sub) && (s == sub || len(sub) == 0 ||
+		func() bool {
+			for i := 0; i <= len(s)-len(sub); i++ {
+				if s[i:i+len(sub)] == sub {
+					return true
+				}
+			}
+			return false
+		}())
+}