feat(cag): memory standing-context adapter, warmup semaphore, cache-aware routing
Issue #44: code-register memory adapter with standing-context resource; fix dispatcher method-level Auth.Capability override (was always using verb-derived capability, ignoring per-method Auth field). Issue #45: per-backend bounded in-memory queue (VLLM_MAX_QUEUE_DEPTH=20) + concurrency semaphore (VLLM_MAX_NUM_SEQS=4); warmup requests (X-Poimen-Warmup: true) sleep WARMUP_DELAY_MS before queue entry; /metrics endpoint wired. Issue #46: rendezvous-hash backend selection on X-Poimen-Cache-Key; round-robin fallback; MarkFailed excludes unhealthy backends 30s; ModifyResponse captures vllm-num-cached-tokens + emits structured log + increments counter. Closes #44 Closes #45 Closes #46
This commit is contained in:
@@ -36,6 +36,14 @@ func (p *PrometheusExporter) Export() string {
|
||||
lines = append(lines, "# TYPE request_duration_seconds histogram")
|
||||
p.exportRequestDuration(&lines)
|
||||
|
||||
lines = append(lines, "# HELP warmup_requests_delayed_total LLM warmup requests delayed for vLLM slot management")
|
||||
lines = append(lines, "# TYPE warmup_requests_delayed_total counter")
|
||||
lines = append(lines, fmt.Sprintf("warmup_requests_delayed_total %d", p.metrics.GetWarmupDelayed()))
|
||||
|
||||
lines = append(lines, "# HELP vllm_prefix_cache_hit_tokens_total vLLM KV prefix cache hit tokens")
|
||||
lines = append(lines, "# TYPE vllm_prefix_cache_hit_tokens_total counter")
|
||||
lines = append(lines, fmt.Sprintf("vllm_prefix_cache_hit_tokens_total %d", p.metrics.GetPrefixCacheHitTokens()))
|
||||
|
||||
return strings.Join(lines, "\n") + "\n"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user