feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)

## Summary

Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw.

---

### #44 — Memory ServiceAdapter: standing-context resource

- Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`)
- Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability)
- Auth: `memory:read` required via method-level override; `memory:write` for ingest
- `MEMORY_UPSTREAM_URL` env configures upstream

### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore

- In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20)
- Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4)
- `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low`
- Queue full → 503 immediately, no goroutine pile-up
- Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics`

### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging

- `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`)
- `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU)
- No cache key → round-robin
- Failed backend excluded 30s; next-highest-score fallback
- `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log

---

## Verification

```
go build ./...    ✅
go vet ./...      ✅
go test -race ./... ✅ (all packages, 0 races)
```

## Reviewer verdicts

- reviewer-sre: SIGN-OFF
- reviewer-quality: SIGN-OFF

## Non-blocking findings (🟡🔵 only)

- `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented
- `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO
- OTel trace_id not propagated into CAG log events (existing gap)

Closes #44 Closes #45 Closes #46

---------

Co-authored-by: Admin Bot <[email protected]>
Reviewed-on: #47

internal/serviceadapter/memory_adapter.go

@@ -0,0 +1,67 @@
+package serviceadapter
+
+import "os"
+
+// NewMemoryAdapter creates a code-registered ServiceAdapter for the memory service.
+// Takes precedence over any config-file loaded memory adapter (main.go skips config
+// adapters when a code-registered one with the same name exists).
+// Upstream URL configured via MEMORY_UPSTREAM_URL env var.
+func NewMemoryAdapter() *ServiceAdapter {
+	upstreamURL := os.Getenv("MEMORY_UPSTREAM_URL")
+	if upstreamURL == "" {
+		upstreamURL = "http://poimen-memory.poimen.svc.cluster.local:8080"
+	}
+	return &ServiceAdapter{
+		Namespace:   "poimen",
+		ServiceName: "memory",
+		Spec:        *GetMemorySpec(upstreamURL),
+	}
+}
+
+// GetMemorySpec returns the ServiceAdapter spec for the memory service.
+func GetMemorySpec(upstreamURL string) *Spec {
+	return &Spec{
+		ServiceName: "memory",
+		Upstream: Upstream{
+			URL:            upstreamURL,
+			TimeoutSeconds: 30,
+		},
+		Auth: Auth{
+			Required:   true,
+			Capability: "memory:read",
+		},
+		Resources: []Resource{
+			{
+				Name: "ingest",
+				Methods: []Method{{
+					Verb:         "POST",
+					UpstreamPath: "/memory/ingest",
+					Auth:         &Auth{Required: true, Capability: "memory:write"},
+				}},
+			},
+			{
+				Name: "query",
+				Methods: []Method{{
+					Verb:         "POST",
+					UpstreamPath: "/memory/query",
+					Auth:         &Auth{Required: true, Capability: "memory:read"},
+				}},
+			},
+			{
+				Name: "skills",
+				Methods: []Method{{
+					Verb:         "GET",
+					UpstreamPath: "/memory/skills",
+				}},
+			},
+			{
+				Name: "standing-context",
+				Methods: []Method{{
+					Verb:         "POST",
+					UpstreamPath: "/memory/standing-context",
+					Auth:         &Auth{Required: true, Capability: "memory:read"},
+				}},
+			},
+		},
+	}
+}

internal/serviceadapter/router.go

@@ -16,6 +16,7 @@ import (
 
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/identity"
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/logging"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/problem"
 )
 
@@ -74,8 +75,13 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// JWT auth enforcement for adapters that require it
-	if adapter.Spec.Auth.Required && d.jwtValidator != nil {
-		if !d.authenticate(w, r, serviceName, method.Verb) {
+	if adapter.Spec.Auth.Required {
+		if d.jwtValidator == nil {
+			logging.Warnf("auth required but jwtValidator nil; request unauthenticated", map[string]string{
+				"service":  serviceName,
+				"resource": resourceName,
+			})
+		} else if !d.authenticate(w, r, serviceName, method) {
 			return
 		}
 	}
@@ -98,7 +104,7 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
 
 // authenticate validates the JWT and checks service-level capability.
 // Returns false (and writes error response) if auth fails.
-func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, serviceName, verb string) bool {
+func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, serviceName string, method *Method) bool {
 	authHeader := r.Header.Get("Authorization")
 	if authHeader == "" {
 		d.writeError(w, problem.NewProblem(http.StatusUnauthorized,
@@ -115,8 +121,11 @@ func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, servic
 		return false
 	}
 
-	// Check capability: <service>:read for GET/HEAD, <service>:write for mutating verbs
-	required := capabilityForVerb(serviceName, verb)
+	// Check capability: method-level Auth.Capability if set; else derive from verb.
+	required := capabilityForVerb(serviceName, method.Verb)
+	if method.Auth != nil && method.Auth.Capability != "" {
+		required = method.Auth.Capability
+	}
 	if !d.jwtValidator.CheckPermissions(claims, required, "*") {
 		d.writeError(w, problem.NewProblem(http.StatusForbidden,
 			"about:blank#insufficient-permissions", "Insufficient Permissions",

internal/serviceadapter/router_test.go

@@ -1,11 +1,19 @@
 package serviceadapter
 
 import (
+	"crypto/rand"
+	"crypto/rsa"
+	"encoding/base64"
 	"encoding/json"
+	"math/big"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 
+	"github.com/golang-jwt/jwt/v5"
+
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/identity"
 )
 
@@ -263,3 +271,197 @@ func TestDispatch_IdentityHeadersNotSet_WhenNoAuth(t *testing.T) {
 		t.Errorf("X-Auth-Verified should not be set without auth, got %q", gotVerified)
 	}
 }
+
+// TestDispatch_MethodCapabilityOverridesPOSTDefault verifies that a method-level
+// Auth.Capability (e.g. "memory:read" on a POST standing-context endpoint) is
+// respected instead of the verb-derived capability ("memory:write").
+// Because auth.Validator is a concrete struct requiring real JWKS, we pass nil
+// jwtValidator so auth is bypassed and assert only that routing succeeds (not 403).
+func TestDispatch_MethodCapabilityOverridesPOSTDefault(t *testing.T) {
+	// Upstream captures the request so we can verify it was routed.
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer upstream.Close()
+
+	// Memory-like adapter: POST standing-context with method-level Auth.Capability=memory:read.
+	adapter := ServiceAdapter{
+		Name:        "memory",
+		ServiceName: "memory",
+		Spec: Spec{
+			ServiceName: "memory",
+			Upstream:    Upstream{URL: upstream.URL, TimeoutSeconds: 5},
+			Auth:        Auth{Required: false},
+			Resources: []Resource{
+				{
+					Name: "standing-context",
+					Methods: []Method{
+						{
+							Verb:         "POST",
+							UpstreamPath: "/memory/standing-context",
+							// Override: read capability even though verb is POST.
+							Auth: &Auth{Capability: "memory:read"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// nil jwtValidator: auth.Required=false + nil validator → auth skipped entirely.
+	d := NewDispatcher(newTestRegistry(adapter), nil)
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("POST", "/", nil)
+	r.Header.Set("X-Service", "memory")
+	r.Header.Set("X-Resource", "standing-context")
+
+	d.Dispatch(w, r)
+
+	// Must NOT be 403: the method-level memory:read override is not checked
+	// against verb-derived memory:write (auth is bypassed with nil validator).
+	// Must be 200 routed to real upstream.
+	if w.Code == http.StatusForbidden {
+		t.Errorf("got 403 Forbidden; memory:read override should not produce forbidden (auth bypassed)")
+	}
+	if w.Code != http.StatusOK {
+		t.Errorf("expected 200 from upstream, got %d", w.Code)
+	}
+}
+
+// TestAuthenticate_MethodCapabilityOverrideDirect calls d.Dispatch() with a real
+// signed RS256 JWT so authenticate() executes end-to-end and the capability-override
+// branch (router.go lines 124-130) is reached and covered.
+//
+// Sub-test 1: token has "memory:read", method.Auth.Capability="memory:read" → 200.
+// Sub-test 2: token has "memory:read", no method override → POST derives
+//
+//	"memory:write" → CheckPermissions fails → 403.
+func TestAuthenticate_MethodCapabilityOverrideDirect(t *testing.T) {
+	// Generate an RSA key pair for signing test JWTs.
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("rsa.GenerateKey: %v", err)
+	}
+
+	// Build minimal JWKS document from the RSA public key.
+	nBytes := privateKey.PublicKey.N.Bytes()
+	eBig := new(big.Int).SetInt64(int64(privateKey.PublicKey.E))
+	eBytes := eBig.Bytes()
+	jwksDoc, _ := json.Marshal(map[string]interface{}{
+		"keys": []map[string]interface{}{
+			{
+				"kty": "RSA",
+				"use": "sig",
+				"kid": "test-key",
+				"alg": "RS256",
+				"n":   base64.RawURLEncoding.EncodeToString(nBytes),
+				"e":   base64.RawURLEncoding.EncodeToString(eBytes),
+			},
+		},
+	})
+
+	// Serve JWKS from a local test server.
+	jwksSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(jwksDoc)
+	}))
+	defer jwksSrv.Close()
+
+	// Upstream that returns 200 when authenticate() passes.
+	upstreamSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer upstreamSrv.Close()
+
+	// signJWT returns a signed RS256 token with the given permissions slice.
+	signJWT := func(permissions []string) string {
+		tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{
+			"iss":         "https://authentik.riotpiao.com/application/o/test/",
+			"aud":         "test-audience",
+			"sub":         "test-user",
+			"exp":         time.Now().Add(time.Hour).Unix(),
+			"iat":         time.Now().Unix(),
+			"permissions": permissions,
+		})
+		tok.Header["kid"] = "test-key"
+		signed, err := tok.SignedString(privateKey)
+		if err != nil {
+			t.Fatalf("sign JWT: %v", err)
+		}
+		return signed
+	}
+
+	validator := auth.NewValidator("", "", jwksSrv.URL)
+
+	// Sub-test 1: method.Auth.Capability="memory:read"; token has "memory:read".
+	// authenticate() must execute the override branch and pass → upstream returns 200.
+	t.Run("capability_override_match_reaches_upstream", func(t *testing.T) {
+		adapter := ServiceAdapter{
+			Name:        "memory",
+			ServiceName: "memory",
+			Spec: Spec{
+				ServiceName: "memory",
+				Upstream:    Upstream{URL: upstreamSrv.URL, TimeoutSeconds: 5},
+				Auth:        Auth{Required: true},
+				Resources: []Resource{
+					{
+						Name: "standing-context",
+						Methods: []Method{
+							{
+								Verb:         "POST",
+								UpstreamPath: "/memory/standing-context",
+								Auth:         &Auth{Capability: "memory:read"}, // override
+							},
+						},
+					},
+				},
+			},
+		}
+		d := NewDispatcher(newTestRegistry(adapter), validator)
+		w := httptest.NewRecorder()
+		r := httptest.NewRequest("POST", "/", nil)
+		r.Header.Set("X-Service", "memory")
+		r.Header.Set("X-Resource", "standing-context")
+		r.Header.Set("Authorization", "Bearer "+signJWT([]string{"memory:read"}))
+		d.Dispatch(w, r)
+		if w.Code != http.StatusOK {
+			t.Errorf("expected 200 (override memory:read match), got %d body=%s", w.Code, w.Body.String())
+		}
+	})
+
+	// Sub-test 2: no method-level Auth override; POST → capabilityForVerb → "memory:write".
+	// Token has only "memory:read" → CheckPermissions fails → 403.
+	t.Run("verb_derived_capability_mismatch_returns_403", func(t *testing.T) {
+		adapter := ServiceAdapter{
+			Name:        "memory",
+			ServiceName: "memory",
+			Spec: Spec{
+				ServiceName: "memory",
+				Upstream:    Upstream{URL: upstreamSrv.URL, TimeoutSeconds: 5},
+				Auth:        Auth{Required: true},
+				Resources: []Resource{
+					{
+						Name: "data",
+						Methods: []Method{
+							{
+								Verb:         "POST",
+								UpstreamPath: "/memory/data",
+								// No method-level Auth: POST derives "memory:write".
+							},
+						},
+					},
+				},
+			},
+		}
+		d := NewDispatcher(newTestRegistry(adapter), validator)
+		w := httptest.NewRecorder()
+		r := httptest.NewRequest("POST", "/", nil)
+		r.Header.Set("X-Service", "memory")
+		r.Header.Set("X-Resource", "data")
+		r.Header.Set("Authorization", "Bearer "+signJWT([]string{"memory:read"}))
+		d.Dispatch(w, r)
+		if w.Code != http.StatusForbidden {
+			t.Errorf("expected 403 (write required, read only in token), got %d body=%s", w.Code, w.Body.String())
+		}
+	})
+}