Files
homelab-frontend/cmd/gateway/main.go
T
Admin Bot c5a91cf647
CI / CI (pull_request) Successful in 34m5s
feat(proxy): engine-wide LLM admission control via vLLM gauges (#64)
The per-pod semaphore was not a cap: 3 replicas x 4 slots advertised 12
concurrent requests while the engine runs 4, and nothing coordinated the
replicas. Admission is now gated on the engine's own published gauges,
which is global by construction because every replica contends for the
same engine.

- new internal/proxy/engine_state.go: 2s scrape of the backend /metrics,
  hand-rolled exposition parse (no new dependency), unhealthy after 3
  consecutive failed polls, and a startup self-check that raises
  observer_degraded rather than falling back to fail-open silently
- gate: park while kv_cache_usage_perc >= 0.90 or running + waiting >=
  ADMISSION_MAX_INFLIGHT, bounded by ADMISSION_MAX_PARK_SECONDS, then
  503 + Retry-After: 5. Unknown state fails open on purpose
- streaming requests receive SSE keep-alive heartbeats while parked
- router: extract only the top-level model field instead of unmarshalling
  the whole body into a map; body cap from MAX_BODY_BYTES (1 MiB, the
  model path previously inherited the 100 MB default)
- GOMAXPROCS from the cgroup CPU quota: the container is limited to 500m
  while Go defaulted to the host core count and was CFS-throttled
- VLLM_MAX_QUEUE_DEPTH default 20 -> 8
- delete k8s/configmap.yaml: absent from kustomization.yaml, and the live
  ConfigMap of that name carries an unrelated key
- correct the llm-client NetworkPolicy comment; llm-serving-ingress now
  selects by namespace, not by that label

Observers run only for models named in ADMISSION_OBSERVE_MODELS.
reasoning is the only vLLM engine here; ornith and qwen-cpu are Ollama
and embeddings/rerank are TEI, none of which publish vllm:* series, so
observing them would hold observer_degraded at 1 permanently.

Companion change (engine flags + dashboard panels) is
riotpiao-poimen/homelab#51.
2026-09-30 17:39:47 +08:00

238 lines
8.3 KiB
Go

package main
import (
"context"
"fmt"
"log"
"math"
"net/http"
"os"
"os/signal"
"runtime"
"strconv"
"strings"
"syscall"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/notification"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/observability"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/proxy"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/server"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/temporal"
"forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing"
)
func main() {
// Size the Go scheduler to the container's CPU quota before any server
// goroutine starts.
configureGOMAXPROCS()
ctx := context.Background()
// Initialize OpenTelemetry tracing
tracingCfg := tracing.DefaultConfig()
shutdownTracer, err := tracing.Init(ctx, tracingCfg)
if err != nil {
log.Printf("warning: failed to initialize tracing: %v", err)
} else {
log.Printf("tracing initialized: service=%s endpoint=%s", tracingCfg.ServiceName, tracingCfg.OTLPEndpoint)
defer func() {
if err := shutdownTracer(ctx); err != nil {
log.Printf("error shutting down tracer: %v", err)
}
}()
}
// Load configuration
cfg, err := config.Load()
if err != nil {
fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err)
os.Exit(1)
}
// Determine if auth is enabled by checking if any route requires it
authEnabled := false
for _, route := range cfg.Routes {
if route.Upstream.AuthRequired {
authEnabled = true
break
}
}
// Create the reverse proxy handler that routes requests based on configuration
upstreamHandler := proxy.New(cfg)
// Create the Temporal workflow handler
// Temporal server address can be configured via environment variable
temporalHostPort := os.Getenv("TEMPORAL_HOST_PORT")
if temporalHostPort == "" {
temporalHostPort = "localhost:7233"
}
log.Printf("Temporal server: %s", temporalHostPort)
temporalHandler := temporal.NewHandler(temporalHostPort)
// Create server with health checker
srv := server.New(cfg.ListenAddr, cfg.ShutdownTimeout, nil)
// Initialize health checker with config validity and auth status
healthChecker := server.NewHealthChecker(true, authEnabled)
srv.SetHealthChecker(healthChecker)
// Create ServiceAdapter registry and dispatcher (phase 8)
registry := serviceadapter.NewRegistry(nil)
// Add workflow service adapter (uses Temporal handler for gRPC forwarding)
workflowSpec := serviceadapter.GetWorkflowSpec()
workflowAdapterHandler := serviceadapter.NewWorkflowAdapter(temporalHandler)
workflowAdapter := &serviceadapter.ServiceAdapter{
Namespace: "temporal",
ServiceName: "workflow",
Handler: workflowAdapterHandler,
Spec: *workflowSpec,
}
_ = registry.Add(workflowAdapter)
// Add notification service adapter (internal handler, no upstream proxy)
notifHandler := notification.NewHandler()
notifAdapter := &serviceadapter.ServiceAdapter{
Namespace: "notification",
ServiceName: "notification",
Handler: notifHandler,
Spec: serviceadapter.Spec{
ServiceName: "notification",
Auth: serviceadapter.Auth{Required: true},
Resources: []serviceadapter.Resource{
{Name: "send-email", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-email"}}},
{Name: "send-message", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-message"}}},
{Name: "list-messages", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-messages"}}},
{Name: "delete-message", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-message"}}},
{Name: "delete-all-messages", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-all-messages"}}},
{Name: "list-applications", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-applications"}}},
{Name: "create-application", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/create-application"}}},
{Name: "delete-application", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-application"}}},
},
},
}
_ = registry.Add(notifAdapter)
// Add Poimen orchestrator adapter (typed REST for AI orchestrator frontend)
poimenAdapter := serviceadapter.NewPoimenAdapter(temporalHandler)
_ = registry.Add(poimenAdapter)
// Add memory service adapter (code-registered; config memory entry is skipped).
// Includes standing-context resource for CAG support (issue #44).
_ = registry.Add(serviceadapter.NewMemoryAdapter())
// Add sessions service adapter — REST CRUD bridged to poimen-memory ingest
// and poimen-graphql reads. Issue #50.
_ = registry.Add(serviceadapter.NewSessionAdapter())
// Add other adapters from config (skip if already registered in code)
for _, a := range cfg.Adapters {
if existing := registry.Get(a.ServiceName); existing != nil {
log.Printf("skip config adapter '%s': already registered with internal handler", a.ServiceName)
continue
}
_ = registry.Add(a)
}
log.Printf("%d service adapters loaded", registry.Count())
// Create shared JWT validator for X-Service auth enforcement
var jwtValidator *auth.Validator
if cfg.Auth.Enabled && cfg.Auth.JWKSURL != "" {
jwtValidator = auth.NewValidator(cfg.Auth.Issuer, cfg.Auth.Audience, cfg.Auth.JWKSURL)
}
dispatcher := serviceadapter.NewDispatcher(registry, jwtValidator)
// Wire workflow adapter to temporal handler for proper request forwarding
// workflowAdapterHandler (above) handles JSON-to-gRPC translation for workflow service
// Create router that handles health endpoints, X-Service (ServiceAdapter) routing,
// temporal endpoints, and passes others to upstream handler
router := server.NewRouter(healthChecker, dispatcher, temporalHandler, upstreamHandler)
router.SetMetricsHandler(observability.NewMetricsHandler(observability.Global()))
// Wrap router with tracing middleware
tracedRouter := tracing.Middleware(router)
srv.SetHandler(tracedRouter)
// Set up signal handling
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
// Start server in a goroutine
var serverErr error
go func() {
log.Printf("gateway listening on %s", srv.Addr())
serverErr = srv.ListenAndServe()
if serverErr != nil && serverErr != http.ErrServerClosed {
log.Printf("server error: %v", serverErr)
}
}()
// Wait for shutdown signal
sig := <-sigChan
log.Printf("received signal: %v", sig)
// Gracefully shutdown the server
if err := srv.Shutdown(context.Background()); err != nil {
fmt.Fprintf(os.Stderr, "shutdown error: %v\n", err)
os.Exit(1)
}
log.Printf("gateway shutdown complete")
os.Exit(0)
}
// configureGOMAXPROCS sets GOMAXPROCS from the cgroup CPU quota so the runtime
// sizes its scheduler to the container limit rather than the host CPU count.
// The default is left alone when no quota is configured.
func configureGOMAXPROCS() {
quota, period := readCPUQuota()
if quota <= 0 || period <= 0 {
return
}
n := int(math.Ceil(float64(quota) / float64(period)))
if n < 1 {
n = 1
}
runtime.GOMAXPROCS(n)
log.Printf("GOMAXPROCS set to %d (cgroup cpu quota %d/%d)", n, quota, period)
}
// readCPUQuota returns the CPU quota and period from cgroup v2, falling back to
// cgroup v1. A quota of 0 means no limit.
func readCPUQuota() (int64, int64) {
// cgroup v2: "<quota> <period>" or "max <period>".
if data, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil {
fields := strings.Fields(string(data))
if len(fields) >= 2 {
period, perr := strconv.ParseInt(fields[1], 10, 64)
if perr == nil && period > 0 {
if fields[0] == "max" {
return 0, period
}
if quota, qerr := strconv.ParseInt(fields[0], 10, 64); qerr == nil {
return quota, period
}
}
}
}
// cgroup v1: cpu.cfs_quota_us is -1 when unlimited.
period := int64(100000)
if data, err := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); err == nil {
if v, perr := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); perr == nil && v > 0 {
period = v
}
}
if data, err := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); err == nil {
if v, qerr := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); qerr == nil {
return v, period
}
}
return 0, 0
}