feat(cag): memory standing-context adapter, warmup semaphore, cache-aware routing

Issue #44: code-register memory adapter with standing-context resource; fix
dispatcher method-level Auth.Capability override (was always using verb-derived
capability, ignoring per-method Auth field).

Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) +
concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup:
true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired.

Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin
fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures
vllm-num-cached-tokens + emits structured log + increments counter.

Closes #44 Closes #45 Closes #46

internal/proxy/proxy.go

@@ -3,13 +3,16 @@ package proxy
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"os"
 	"sort"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -18,6 +21,7 @@ import (
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/identity"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/logging"
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/observability"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing"
 )
 
@@ -37,13 +41,18 @@ type Handler struct {
 	defaultReadTimeout    time.Duration
 	defaultWriteTimeout   time.Duration
 	defaultMaxBodySize    int64
+	// CAG: per-model backend schedulers for cache-aware routing (#45/#46)
+	schedulers  map[string]*Scheduler
+	// CAG: delay applied to warmup requests before queue entry (#45)
+	warmupDelay time.Duration
 }
 
 // Route represents a reverse proxy route.
 type Route struct {
-	Name     string
-	Upstream *config.Upstream
-	Director func(*http.Request)
+	Name      string
+	ModelName string // non-empty for model-based routes; used for scheduler lookup
+	Upstream  *config.Upstream
+	Director  func(*http.Request)
 	Transport *http.Transport
 }
 
@@ -93,6 +102,36 @@ func New(cfg *config.Config) *Handler {
 		h.authHTTP = newAuthClient()
 	}
 
+	// CAG: warmup delay from env (#45)
+	warmupMs := int64(500)
+	if v := os.Getenv("WARMUP_DELAY_MS"); v != "" {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil && n >= 0 {
+			warmupMs = n
+		}
+	}
+	h.warmupDelay = time.Duration(warmupMs) * time.Millisecond
+
+	// CAG: per-model backend schedulers (#45/#46)
+	maxSlots := 4
+	if v := os.Getenv("VLLM_MAX_NUM_SEQS"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			maxSlots = n
+		}
+	}
+	maxQueue := 20
+	if v := os.Getenv("VLLM_MAX_QUEUE_DEPTH"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 {
+			maxQueue = n
+		}
+	}
+	h.schedulers = make(map[string]*Scheduler)
+	for modelName, model := range cfg.Models {
+		addrs := model.AllAddresses()
+		if len(addrs) > 0 {
+			h.schedulers[modelName] = NewScheduler(addrs, maxSlots, maxQueue)
+		}
+	}
+
 	for name, route := range cfg.Routes {
 		// Create a transport per unique upstream address for connection reuse
 		transport := h.getOrCreateTransport(route.Upstream.Address, &route.Upstream)
@@ -414,6 +453,51 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	// CAG: warmup deprioritisation + per-backend semaphore (issues #45/#46)
+	var selectedBackend *Backend
+	cacheKey := r.Header.Get("X-Poimen-Cache-Key")
+	if sched := h.schedulers[route.ModelName]; sched != nil {
+		selectedBackend = sched.Select(cacheKey)
+
+		// Warmup: sleep BEFORE queue entry so agents arriving in that window win slots first.
+		if r.Header.Get("X-Poimen-Warmup") == "true" {
+			r.Header.Del("X-Poimen-Warmup")
+			r.Header.Set("X-Priority", "low")
+			time.Sleep(h.warmupDelay)
+			observability.Global().IncrWarmupDelayed()
+		}
+
+		if selectedBackend != nil {
+			if err := selectedBackend.Acquire(r.Context()); err != nil {
+				if errors.Is(err, ErrQueueFull) {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					fmt.Fprintf(w, "queue full")
+				} else {
+					w.WriteHeader(http.StatusServiceUnavailable)
+					fmt.Fprintf(w, "upstream busy")
+				}
+				return
+			}
+			defer selectedBackend.Release()
+
+			// Patch route to selected backend if it differs from the config default.
+			if selectedBackend.Address != route.Upstream.Address {
+				patchedCfg := *route.Upstream
+				patchedCfg.Address = selectedBackend.Address
+				patchedURL, _ := url.Parse("http://" + selectedBackend.Address)
+				route = &Route{
+					Name:      route.Name,
+					ModelName: route.ModelName,
+					Upstream:  &patchedCfg,
+					Transport: h.getOrCreateTransport(selectedBackend.Address, &patchedCfg),
+					Director: func(req *http.Request) {
+						directorFunc(req, patchedURL, &patchedCfg)
+					},
+				}
+			}
+		}
+	}
+
 	// Create the reverse proxy
 	proxy := httputil.NewSingleHostReverseProxy(&url.URL{
 		Scheme: "http",
@@ -426,8 +510,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Use the connection-pooled transport wrapped with tracing
 	proxy.Transport = tracing.NewTransport(route.Transport)
 
-	// Set error handler to log upstream errors
+	// Set error handler: log upstream errors and exclude failed backend from routing.
 	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {
+		if selectedBackend != nil {
+			if sched := h.schedulers[route.ModelName]; sched != nil {
+				sched.MarkFailed(selectedBackend.Address)
+			}
+		}
 		w.WriteHeader(http.StatusBadGateway)
 		fmt.Fprintf(w, "upstream error")
 		logging.Errorf("upstream error", err, map[string]string{
@@ -436,6 +525,29 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		})
 	}
 
+	// CAG: capture vllm-num-cached-tokens from response headers (issue #46)
+	if cacheKey != "" && r.URL.Path == "/v1/chat/completions" {
+		proxy.ModifyResponse = func(resp *http.Response) error {
+			raw := resp.Header.Get("vllm-num-cached-tokens")
+			if raw == "" {
+				return nil
+			}
+			n, err := strconv.ParseInt(raw, 10, 64)
+			if err != nil || n <= 0 {
+				return nil
+			}
+			observability.Global().IncrPrefixCacheHitTokens(n)
+			logging.Event(map[string]interface{}{
+				"event":         "llm_request",
+				"cache_key":     cacheKey,
+				"cached_tokens": n,
+				"total_tokens":  0,
+				"hit_rate":      0.0,
+			})
+			return nil
+		}
+	}
+
 	// Note: We apply connection timeout via Transport dialer, but NOT read timeout as a context deadline.
 	// Read timeout should apply to inactivity (socket read timeout), not total request duration.
 	// A streaming response that's continuously sending should not be cut off.