Files
homelab-frontend/internal/config/config.go
T
rockandAdmin Bot ef5e736673
CI / CI (push) Successful in 3m20s
feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)
## Summary

Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw.

---

### #44 — Memory ServiceAdapter: standing-context resource

- Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`)
- Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability)
- Auth: `memory:read` required via method-level override; `memory:write` for ingest
- `MEMORY_UPSTREAM_URL` env configures upstream

### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore

- In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20)
- Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4)
- `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low`
- Queue full → 503 immediately, no goroutine pile-up
- Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics`

### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging

- `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`)
- `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU)
- No cache key → round-robin
- Failed backend excluded 30s; next-highest-score fallback
- `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log

---

## Verification

```
go build ./...    ✅
go vet ./...      ✅
go test -race ./... ✅ (all packages, 0 races)
```

## Reviewer verdicts

- reviewer-sre: SIGN-OFF
- reviewer-quality: SIGN-OFF

## Non-blocking findings (🟡🔵 only)

- `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented
- `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO
- OTel trace_id not propagated into CAG log events (existing gap)

Closes #44 Closes #45 Closes #46

---------

Co-authored-by: Admin Bot <[email protected]>
Reviewed-on: #47
2026-09-27 11:16:40 +00:00

187 lines
5.9 KiB
Go

package config
import (
"fmt"
"os"
"time"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter"
)
// Config holds the gateway configuration.
type Config struct {
// ListenAddr is the address to listen on for HTTP traffic.
ListenAddr string
// ShutdownTimeout is the maximum time to wait for in-flight requests
// to complete before forcing shutdown.
ShutdownTimeout time.Duration
// Routes maps route names to their upstream configuration.
Routes map[string]*Route
// Models maps model names to their upstream configuration.
// Multiple models can point to the same upstream address.
Models map[string]*ModelUpstream
// Adapters holds service adapter definitions for X-Service routing.
Adapters []*serviceadapter.ServiceAdapter
// Auth holds JWT authentication configuration for /v1/* endpoints.
Auth AuthConfig
// Temporal holds Temporal server configuration.
Temporal TemporalConfig
}
// ModelUpstream holds upstream configuration for a specific model.
type ModelUpstream struct {
// Name is the model name clients send (e.g., "reasoning", "ornith:35b").
Name string
// Address is the upstream server address (host:port).
// When Addresses is set, this holds Addresses[0] for backward compatibility.
Address string
// Addresses holds all backend addresses for multi-backend cache-aware routing.
// Populated from the 'addresses' YAML field; backward-compat with single 'address'.
Addresses []string
// Path is the upstream path for this model (e.g., "/v1/chat/completions").
Path string
// UpstreamModel is the model name to send to the upstream server.
// If empty, the client-provided model name (Name) is used as-is.
// Use this when the upstream expects a different model name than clients send.
UpstreamModel string
// AuthRequired indicates whether this model requires JWT authentication.
AuthRequired bool
}
// TemporalConfig holds Temporal server configuration.
type TemporalConfig struct {
// HostPort is the address of the Temporal server (host:port).
HostPort string
}
// AuthConfig holds JWT authentication configuration.
type AuthConfig struct {
// Enabled globally enables/disables auth for /v1/* endpoints.
Enabled bool
// Issuer is the expected JWT issuer (iss claim).
Issuer string
// Audience is the expected JWT audience (aud claim).
Audience string
// JWKSURL is the URL to fetch JSON Web Key Set for signature validation.
JWKSURL string
// RequiredCapability is the permission required for LLM inference (e.g., "llm:inference").
RequiredCapability string
// TokenURL is the Authentik token endpoint for password/refresh grants.
TokenURL string
// ClientID is the OAuth2 client ID for token exchange.
ClientID string
// ClientSecret is the OAuth2 client secret (loaded from env, never from config file).
ClientSecret string
}
// Route represents a single route and its upstream configuration.
type Route struct {
// Name is the route identifier.
Name string
// Upstream holds the upstream server configuration.
Upstream Upstream
}
// Upstream holds upstream server configuration for a route.
type Upstream struct {
// Address is the upstream server address (host:port).
Address string
// PathRewrite is an optional path prefix rewrite. Empty string means no rewrite.
PathRewrite string
// ConnectTimeout is the maximum time to establish a connection to the upstream.
ConnectTimeout time.Duration
// ReadTimeout is the maximum time to read a response from the upstream.
ReadTimeout time.Duration
// WriteTimeout is the maximum time to write a request to the upstream.
WriteTimeout time.Duration
// MaxBodySize is the maximum request body size in bytes.
MaxBodySize int64
// AuthRequired indicates whether this route requires authentication.
AuthRequired bool
}
// LookupModel finds a model by name (case-sensitive, exact match).
func (c *Config) LookupModel(name string) *ModelUpstream {
if c == nil || c.Models == nil {
return nil
}
return c.Models[name]
}
// ModelNames returns a sorted list of all known model names.
func (c *Config) ModelNames() []string {
if c == nil || c.Models == nil {
return nil
}
names := make([]string, 0, len(c.Models))
for name := range c.Models {
names = append(names, name)
}
return names
}
// AllAddresses returns all configured backend addresses for the model.
// Returns Addresses if non-empty; otherwise wraps Address in a slice.
func (m *ModelUpstream) AllAddresses() []string {
if len(m.Addresses) > 0 {
return m.Addresses
}
if m.Address != "" {
return []string{m.Address}
}
return nil
}
// Load reads configuration from environment variables with defaults.
func Load() (*Config, error) {
listenAddr := "127.0.0.1:8080"
// Allow override via environment variable
if addr, ok := os.LookupEnv("LISTEN_ADDR"); ok {
listenAddr = addr
}
shutdownTimeout := 30 * time.Second
// Allow override via environment variable
if timeout, ok := os.LookupEnv("SHUTDOWN_TIMEOUT"); ok {
d, err := time.ParseDuration(timeout)
if err != nil {
return nil, fmt.Errorf("invalid SHUTDOWN_TIMEOUT: %w", err)
}
shutdownTimeout = d
}
// Load routes, models, adapters, and auth from config file
routes := make(map[string]*Route)
models := make(map[string]*ModelUpstream)
var adapters []*serviceadapter.ServiceAdapter
var authConfig AuthConfig
if configPath, ok := os.LookupEnv("CONFIG_PATH"); ok {
loadedRoutes, loadedModels, loadedAdapters, loadedAuth, err := LoadRoutesAndModelsFromFile(configPath)
if err != nil {
return nil, err
}
routes = loadedRoutes
models = loadedModels
adapters = loadedAdapters
authConfig = loadedAuth
}
temporalHostPort := "localhost:7233"
// Allow override via environment variable
if hostPort, ok := os.LookupEnv("TEMPORAL_HOST_PORT"); ok {
temporalHostPort = hostPort
}
return &Config{
ListenAddr: listenAddr,
ShutdownTimeout: shutdownTimeout,
Routes: routes,
Models: models,
Adapters: adapters,
Auth: authConfig,
Temporal: TemporalConfig{
HostPort: temporalHostPort,
},
}, nil
}