feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)
CI / CI (push) Successful in 3m20s
CI / CI (push) Successful in 3m20s
## Summary Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw. --- ### #44 — Memory ServiceAdapter: standing-context resource - Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`) - Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability) - Auth: `memory:read` required via method-level override; `memory:write` for ingest - `MEMORY_UPSTREAM_URL` env configures upstream ### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore - In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20) - Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4) - `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low` - Queue full → 503 immediately, no goroutine pile-up - Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics` ### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging - `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`) - `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU) - No cache key → round-robin - Failed backend excluded 30s; next-highest-score fallback - `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log --- ## Verification ``` go build ./... ✅ go vet ./... ✅ go test -race ./... ✅ (all packages, 0 races) ``` ## Reviewer verdicts - reviewer-sre: SIGN-OFF - reviewer-quality: SIGN-OFF ## Non-blocking findings (🟡🔵 only) - `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented - `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO - OTel trace_id not propagated into CAG log events (existing gap) Closes #44 Closes #45 Closes #46 --------- Co-authored-by: Admin Bot <[email protected]> Reviewed-on: #47
This commit was merged in pull request #47.
This commit is contained in:
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/identity"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/logging"
|
||||
"forgejo.riotpiao.com/rock/homelab-frontend/internal/problem"
|
||||
)
|
||||
|
||||
@@ -74,8 +75,13 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// JWT auth enforcement for adapters that require it
|
||||
if adapter.Spec.Auth.Required && d.jwtValidator != nil {
|
||||
if !d.authenticate(w, r, serviceName, method.Verb) {
|
||||
if adapter.Spec.Auth.Required {
|
||||
if d.jwtValidator == nil {
|
||||
logging.Warnf("auth required but jwtValidator nil; request unauthenticated", map[string]string{
|
||||
"service": serviceName,
|
||||
"resource": resourceName,
|
||||
})
|
||||
} else if !d.authenticate(w, r, serviceName, method) {
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -98,7 +104,7 @@ func (d *Dispatcher) Dispatch(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// authenticate validates the JWT and checks service-level capability.
|
||||
// Returns false (and writes error response) if auth fails.
|
||||
func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, serviceName, verb string) bool {
|
||||
func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, serviceName string, method *Method) bool {
|
||||
authHeader := r.Header.Get("Authorization")
|
||||
if authHeader == "" {
|
||||
d.writeError(w, problem.NewProblem(http.StatusUnauthorized,
|
||||
@@ -115,8 +121,11 @@ func (d *Dispatcher) authenticate(w http.ResponseWriter, r *http.Request, servic
|
||||
return false
|
||||
}
|
||||
|
||||
// Check capability: <service>:read for GET/HEAD, <service>:write for mutating verbs
|
||||
required := capabilityForVerb(serviceName, verb)
|
||||
// Check capability: method-level Auth.Capability if set; else derive from verb.
|
||||
required := capabilityForVerb(serviceName, method.Verb)
|
||||
if method.Auth != nil && method.Auth.Capability != "" {
|
||||
required = method.Auth.Capability
|
||||
}
|
||||
if !d.jwtValidator.CheckPermissions(claims, required, "*") {
|
||||
d.writeError(w, problem.NewProblem(http.StatusForbidden,
|
||||
"about:blank#insufficient-permissions", "Insufficient Permissions",
|
||||
|
||||
Reference in New Issue
Block a user