CI / CI (pull_request) Successful in 34m5s
The per-pod semaphore was not a cap: 3 replicas x 4 slots advertised 12 concurrent requests while the engine runs 4, and nothing coordinated the replicas. Admission is now gated on the engine's own published gauges, which is global by construction because every replica contends for the same engine. - new internal/proxy/engine_state.go: 2s scrape of the backend /metrics, hand-rolled exposition parse (no new dependency), unhealthy after 3 consecutive failed polls, and a startup self-check that raises observer_degraded rather than falling back to fail-open silently - gate: park while kv_cache_usage_perc >= 0.90 or running + waiting >= ADMISSION_MAX_INFLIGHT, bounded by ADMISSION_MAX_PARK_SECONDS, then 503 + Retry-After: 5. Unknown state fails open on purpose - streaming requests receive SSE keep-alive heartbeats while parked - router: extract only the top-level model field instead of unmarshalling the whole body into a map; body cap from MAX_BODY_BYTES (1 MiB, the model path previously inherited the 100 MB default) - GOMAXPROCS from the cgroup CPU quota: the container is limited to 500m while Go defaulted to the host core count and was CFS-throttled - VLLM_MAX_QUEUE_DEPTH default 20 -> 8 - delete k8s/configmap.yaml: absent from kustomization.yaml, and the live ConfigMap of that name carries an unrelated key - correct the llm-client NetworkPolicy comment; llm-serving-ingress now selects by namespace, not by that label Observers run only for models named in ADMISSION_OBSERVE_MODELS. reasoning is the only vLLM engine here; ornith and qwen-cpu are Ollama and embeddings/rerank are TEI, none of which publish vllm:* series, so observing them would hold observer_degraded at 1 permanently. Companion change (engine flags + dashboard panels) is riotpiao-poimen/homelab#51.
238 lines
8.3 KiB
Go
238 lines
8.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"runtime"
|
|
"strconv"
|
|
"strings"
|
|
"syscall"
|
|
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/auth"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/notification"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/observability"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/proxy"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/server"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/temporal"
|
|
"forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing"
|
|
)
|
|
|
|
func main() {
|
|
// Size the Go scheduler to the container's CPU quota before any server
|
|
// goroutine starts.
|
|
configureGOMAXPROCS()
|
|
|
|
ctx := context.Background()
|
|
|
|
// Initialize OpenTelemetry tracing
|
|
tracingCfg := tracing.DefaultConfig()
|
|
shutdownTracer, err := tracing.Init(ctx, tracingCfg)
|
|
if err != nil {
|
|
log.Printf("warning: failed to initialize tracing: %v", err)
|
|
} else {
|
|
log.Printf("tracing initialized: service=%s endpoint=%s", tracingCfg.ServiceName, tracingCfg.OTLPEndpoint)
|
|
defer func() {
|
|
if err := shutdownTracer(ctx); err != nil {
|
|
log.Printf("error shutting down tracer: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Load configuration
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Determine if auth is enabled by checking if any route requires it
|
|
authEnabled := false
|
|
for _, route := range cfg.Routes {
|
|
if route.Upstream.AuthRequired {
|
|
authEnabled = true
|
|
break
|
|
}
|
|
}
|
|
|
|
// Create the reverse proxy handler that routes requests based on configuration
|
|
upstreamHandler := proxy.New(cfg)
|
|
|
|
// Create the Temporal workflow handler
|
|
// Temporal server address can be configured via environment variable
|
|
temporalHostPort := os.Getenv("TEMPORAL_HOST_PORT")
|
|
if temporalHostPort == "" {
|
|
temporalHostPort = "localhost:7233"
|
|
}
|
|
log.Printf("Temporal server: %s", temporalHostPort)
|
|
temporalHandler := temporal.NewHandler(temporalHostPort)
|
|
|
|
// Create server with health checker
|
|
srv := server.New(cfg.ListenAddr, cfg.ShutdownTimeout, nil)
|
|
|
|
// Initialize health checker with config validity and auth status
|
|
healthChecker := server.NewHealthChecker(true, authEnabled)
|
|
srv.SetHealthChecker(healthChecker)
|
|
|
|
// Create ServiceAdapter registry and dispatcher (phase 8)
|
|
registry := serviceadapter.NewRegistry(nil)
|
|
|
|
// Add workflow service adapter (uses Temporal handler for gRPC forwarding)
|
|
workflowSpec := serviceadapter.GetWorkflowSpec()
|
|
workflowAdapterHandler := serviceadapter.NewWorkflowAdapter(temporalHandler)
|
|
workflowAdapter := &serviceadapter.ServiceAdapter{
|
|
Namespace: "temporal",
|
|
ServiceName: "workflow",
|
|
Handler: workflowAdapterHandler,
|
|
Spec: *workflowSpec,
|
|
}
|
|
_ = registry.Add(workflowAdapter)
|
|
|
|
// Add notification service adapter (internal handler, no upstream proxy)
|
|
notifHandler := notification.NewHandler()
|
|
notifAdapter := &serviceadapter.ServiceAdapter{
|
|
Namespace: "notification",
|
|
ServiceName: "notification",
|
|
Handler: notifHandler,
|
|
Spec: serviceadapter.Spec{
|
|
ServiceName: "notification",
|
|
Auth: serviceadapter.Auth{Required: true},
|
|
Resources: []serviceadapter.Resource{
|
|
{Name: "send-email", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-email"}}},
|
|
{Name: "send-message", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/send-message"}}},
|
|
{Name: "list-messages", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-messages"}}},
|
|
{Name: "delete-message", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-message"}}},
|
|
{Name: "delete-all-messages", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-all-messages"}}},
|
|
{Name: "list-applications", Methods: []serviceadapter.Method{{Verb: "GET", UpstreamPath: "/list-applications"}}},
|
|
{Name: "create-application", Methods: []serviceadapter.Method{{Verb: "POST", UpstreamPath: "/create-application"}}},
|
|
{Name: "delete-application", Methods: []serviceadapter.Method{{Verb: "DELETE", UpstreamPath: "/delete-application"}}},
|
|
},
|
|
},
|
|
}
|
|
_ = registry.Add(notifAdapter)
|
|
|
|
// Add Poimen orchestrator adapter (typed REST for AI orchestrator frontend)
|
|
poimenAdapter := serviceadapter.NewPoimenAdapter(temporalHandler)
|
|
_ = registry.Add(poimenAdapter)
|
|
|
|
// Add memory service adapter (code-registered; config memory entry is skipped).
|
|
// Includes standing-context resource for CAG support (issue #44).
|
|
_ = registry.Add(serviceadapter.NewMemoryAdapter())
|
|
|
|
// Add sessions service adapter — REST CRUD bridged to poimen-memory ingest
|
|
// and poimen-graphql reads. Issue #50.
|
|
_ = registry.Add(serviceadapter.NewSessionAdapter())
|
|
|
|
// Add other adapters from config (skip if already registered in code)
|
|
for _, a := range cfg.Adapters {
|
|
if existing := registry.Get(a.ServiceName); existing != nil {
|
|
log.Printf("skip config adapter '%s': already registered with internal handler", a.ServiceName)
|
|
continue
|
|
}
|
|
_ = registry.Add(a)
|
|
}
|
|
log.Printf("%d service adapters loaded", registry.Count())
|
|
|
|
// Create shared JWT validator for X-Service auth enforcement
|
|
var jwtValidator *auth.Validator
|
|
if cfg.Auth.Enabled && cfg.Auth.JWKSURL != "" {
|
|
jwtValidator = auth.NewValidator(cfg.Auth.Issuer, cfg.Auth.Audience, cfg.Auth.JWKSURL)
|
|
}
|
|
dispatcher := serviceadapter.NewDispatcher(registry, jwtValidator)
|
|
|
|
// Wire workflow adapter to temporal handler for proper request forwarding
|
|
// workflowAdapterHandler (above) handles JSON-to-gRPC translation for workflow service
|
|
|
|
// Create router that handles health endpoints, X-Service (ServiceAdapter) routing,
|
|
// temporal endpoints, and passes others to upstream handler
|
|
router := server.NewRouter(healthChecker, dispatcher, temporalHandler, upstreamHandler)
|
|
router.SetMetricsHandler(observability.NewMetricsHandler(observability.Global()))
|
|
|
|
// Wrap router with tracing middleware
|
|
tracedRouter := tracing.Middleware(router)
|
|
srv.SetHandler(tracedRouter)
|
|
|
|
// Set up signal handling
|
|
sigChan := make(chan os.Signal, 1)
|
|
signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT)
|
|
|
|
// Start server in a goroutine
|
|
var serverErr error
|
|
go func() {
|
|
log.Printf("gateway listening on %s", srv.Addr())
|
|
serverErr = srv.ListenAndServe()
|
|
if serverErr != nil && serverErr != http.ErrServerClosed {
|
|
log.Printf("server error: %v", serverErr)
|
|
}
|
|
}()
|
|
|
|
// Wait for shutdown signal
|
|
sig := <-sigChan
|
|
log.Printf("received signal: %v", sig)
|
|
|
|
// Gracefully shutdown the server
|
|
if err := srv.Shutdown(context.Background()); err != nil {
|
|
fmt.Fprintf(os.Stderr, "shutdown error: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
log.Printf("gateway shutdown complete")
|
|
os.Exit(0)
|
|
}
|
|
|
|
// configureGOMAXPROCS sets GOMAXPROCS from the cgroup CPU quota so the runtime
|
|
// sizes its scheduler to the container limit rather than the host CPU count.
|
|
// The default is left alone when no quota is configured.
|
|
func configureGOMAXPROCS() {
|
|
quota, period := readCPUQuota()
|
|
if quota <= 0 || period <= 0 {
|
|
return
|
|
}
|
|
n := int(math.Ceil(float64(quota) / float64(period)))
|
|
if n < 1 {
|
|
n = 1
|
|
}
|
|
runtime.GOMAXPROCS(n)
|
|
log.Printf("GOMAXPROCS set to %d (cgroup cpu quota %d/%d)", n, quota, period)
|
|
}
|
|
|
|
// readCPUQuota returns the CPU quota and period from cgroup v2, falling back to
|
|
// cgroup v1. A quota of 0 means no limit.
|
|
func readCPUQuota() (int64, int64) {
|
|
// cgroup v2: "<quota> <period>" or "max <period>".
|
|
if data, err := os.ReadFile("/sys/fs/cgroup/cpu.max"); err == nil {
|
|
fields := strings.Fields(string(data))
|
|
if len(fields) >= 2 {
|
|
period, perr := strconv.ParseInt(fields[1], 10, 64)
|
|
if perr == nil && period > 0 {
|
|
if fields[0] == "max" {
|
|
return 0, period
|
|
}
|
|
if quota, qerr := strconv.ParseInt(fields[0], 10, 64); qerr == nil {
|
|
return quota, period
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// cgroup v1: cpu.cfs_quota_us is -1 when unlimited.
|
|
period := int64(100000)
|
|
if data, err := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_period_us"); err == nil {
|
|
if v, perr := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); perr == nil && v > 0 {
|
|
period = v
|
|
}
|
|
}
|
|
if data, err := os.ReadFile("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"); err == nil {
|
|
if v, qerr := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); qerr == nil {
|
|
return v, period
|
|
}
|
|
}
|
|
return 0, 0
|
|
}
|