Files
homelab-frontend/cmd/gateway/main.go
T
rockandAdmin Bot ef5e736673
CI / CI (push) Successful in 3m20s
feat(cag): CAG integration — memory adapter, warmup semaphore, cache-aware routing (#47)
## Summary

Three issues in one PR. Implements Cache-Augmented Generation (CAG) support in the api-gw.

---

### #44 — Memory ServiceAdapter: standing-context resource

- Code-registers `memory` adapter with `standing-context` resource (`POST /memory/standing-context`)
- Fixes dispatcher bug: method-level `Auth.Capability` override now respected (was always using verb-derived capability)
- Auth: `memory:read` required via method-level override; `memory:write` for ingest
- `MEMORY_UPSTREAM_URL` env configures upstream

### #45 — Continuous batching: warmup deprioritisation + per-backend semaphore

- In-memory bounded queue per backend: `VLLM_MAX_QUEUE_DEPTH` slots (default 20)
- Concurrency limiter: `VLLM_MAX_NUM_SEQS` slots (default 4)
- `X-Poimen-Warmup: true` → sleep `WARMUP_DELAY_MS` (default 500ms) before queue entry, strip header, add `X-Priority: low`
- Queue full → 503 immediately, no goroutine pile-up
- Metrics: `warmup_requests_delayed_total`, `queue_full_requests_total` at `/metrics`

### #46 — Cache-aware routing: rendezvous hash + prefix cache hit logging

- `addresses: [...]` YAML field for multi-backend models (backward-compat with `address`)
- `X-Poimen-Cache-Key` header → rendezvous-hash backend selection (same key = same GPU)
- No cache key → round-robin
- Failed backend excluded 30s; next-highest-score fallback
- `vllm-num-cached-tokens` response header captured → `vllm_prefix_cache_hit_tokens_total` counter + structured log

---

## Verification

```
go build ./...    ✅
go vet ./...      ✅
go test -race ./... ✅ (all packages, 0 races)
```

## Reviewer verdicts

- reviewer-sre: SIGN-OFF
- reviewer-quality: SIGN-OFF

## Non-blocking findings (🟡🔵 only)

- `getOrCreateTransport` not goroutine-safe by design; pre-population invariant documented
- `time.Sleep(warmupDelay)` holds goroutine; context-cancel alternative left as TODO
- OTel trace_id not propagated into CAG log events (existing gap)

Closes #44 Closes #45 Closes #46

---------

Co-authored-by: Admin Bot <[email protected]>
Reviewed-on: #47
2026-09-27 11:16:40 +00:00

176 lines
6.4 KiB
Go

package main
import (
"context"
"fmt"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/notification"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/observability"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/proxy"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/server"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/temporal"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing"
)
func main() {
ctx := context.Background()
// Initialize OpenTelemetry tracing
tracingCfg := tracing.DefaultConfig()
shutdownTracer, err := tracing.Init(ctx, tracingCfg)
if err != nil {
log.Printf("warning: failed to initialize tracing: %v", err)
} else {
log.Printf("tracing initialized: service=%s endpoint=%s", tracingCfg.ServiceName, tracingCfg.OTLPEndpoint)
defer func() {
if err := shutdownTracer(ctx); err != nil {
log.Printf("error shutting down tracer: %v", err)
}
}()
}
// Load configuration
cfg, err := config.Load()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err)
os.Exit(1)
}
// Determine if auth is enabled by checking if any route requires it
authEnabled := false
for _, route := range cfg.Routes {
if route.Upstream.AuthRequired {
authEnabled = true
break
}
}
// Create the reverse proxy handler that routes requests based on configuration
upstreamHandler := proxy.New(cfg)
// Create the Temporal workflow handler
// Temporal server address can be configured via environment variable
temporalHostPort := os.Getenv("TEMPORAL_HOST_PORT")
if temporalHostPort == "" {
temporalHostPort = "localhost:7233"
}
log.Printf("Temporal server: %s", temporalHostPort)
temporalHandler := temporal.NewHandler(temporalHostPort)
// Create server with health checker
srv := server.New(cfg.ListenAddr, cfg.ShutdownTimeout, nil)
// Initialize health checker with config validity and auth status
healthChecker := server.NewHealthChecker(true, authEnabled)
srv.SetHealthChecker(healthChecker)
// Create ServiceAdapter registry and dispatcher (phase 8)
registry := serviceadapter.NewRegistry(nil)
// Add workflow service adapter (uses Temporal handler for gRPC forwarding)
workflowSpec := serviceadapter.GetWorkflowSpec()
workflowAdapterHandler := serviceadapter.NewWorkflowAdapter(temporalHandler)
workflowAdapter := &serviceadapter.ServiceAdapter{
Namespace: "temporal",
ServiceName: "workflow",
Handler: workflowAdapterHandler,
Spec: *workflowSpec,
}
_ = registry.Add(workflowAdapter)
// Add notification service adapter (internal handler, no upstream proxy)
notifHandler := notification.NewHandler()
notifAdapter := &serviceadapter.ServiceAdapter{
Namespace: "notification",
ServiceName: "notification",
Handler: notifHandler,
Spec: serviceadapter.Spec{
ServiceName: "notification",
Auth: serviceadapter.Auth{Required: true},
Resources: []serviceadapter.Resource{
{Name: "send-email", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-email"}}},
{Name: "send-message", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-message"}}},
{Name: "list-messages", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-messages"}}},
{Name: "delete-message", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-message"}}},
{Name: "delete-all-messages", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-all-messages"}}},
{Name: "list-applications", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-applications"}}},
{Name: "create-application", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/create-application"}}},
{Name: "delete-application", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-application"}}},
},
},
}
_ = registry.Add(notifAdapter)
// Add Poimen orchestrator adapter (typed REST for AI orchestrator frontend)
poimenAdapter := serviceadapter.NewPoimenAdapter(temporalHandler)
_ = registry.Add(poimenAdapter)
// Add memory service adapter (code-registered; config memory entry is skipped).
// Includes standing-context resource for CAG support (issue #44).
_ = registry.Add(serviceadapter.NewMemoryAdapter())
// Add other adapters from config (skip if already registered in code)
for _, a := range cfg.Adapters {
if existing := registry.Get(a.ServiceName); existing != nil {
log.Printf("skip config adapter '%s': already registered with internal handler", a.ServiceName)
continue
}
_ = registry.Add(a)
}
log.Printf("%d service adapters loaded", registry.Count())
// Create shared JWT validator for X-Service auth enforcement
var jwtValidator *auth.Validator
if cfg.Auth.Enabled && cfg.Auth.JWKSURL != "" {
jwtValidator = auth.NewValidator(cfg.Auth.Issuer, cfg.Auth.Audience, cfg.Auth.JWKSURL)
}
dispatcher := serviceadapter.NewDispatcher(registry, jwtValidator)
// Wire workflow adapter to temporal handler for proper request forwarding
// workflowAdapterHandler (above) handles JSON-to-gRPC translation for workflow service
// Create router that handles health endpoints, X-Service (ServiceAdapter) routing,
// temporal endpoints, and passes others to upstream handler
router := server.NewRouter(healthChecker, dispatcher, temporalHandler, upstreamHandler)
router.SetMetricsHandler(observability.NewMetricsHandler(observability.Global()))
// Wrap router with tracing middleware
tracedRouter := tracing.Middleware(router)
srv.SetHandler(tracedRouter)
// Set up signal handling
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
// Start server in a goroutine
var serverErr error
go func() {
log.Printf("gateway listening on %s", srv.Addr())
serverErr = srv.ListenAndServe()
if serverErr != nil && serverErr != http.ErrServerClosed {
log.Printf("server error: %v", serverErr)
}
}()
// Wait for shutdown signal
sig := <-sigChan
log.Printf("received signal: %v", sig)
// Gracefully shutdown the server
if err := srv.Shutdown(context.Background()); err != nil {
fmt.Fprintf(os.Stderr, "shutdown error: %v\n", err)
os.Exit(1)
}
log.Printf("gateway shutdown complete")
os.Exit(0)
}