homelab-frontend/internal/observability/prometheus.go

package observability

import (
	"fmt"
	"sort"
	"strings"
)

// PrometheusExporter exports metrics in Prometheus text format
type PrometheusExporter struct {
	metrics *Metrics
}

// NewPrometheusExporter creates a new Prometheus exporter
func NewPrometheusExporter(m *Metrics) *PrometheusExporter {
	return &PrometheusExporter{metrics: m}
}

// Export returns metrics in Prometheus text format
func (p *PrometheusExporter) Export() string {
	var lines []string

	lines = append(lines, "# HELP llm_ttft_seconds Time to first token for LLM inference (seconds)")
	lines = append(lines, "# TYPE llm_ttft_seconds histogram")
	p.exportTTFT(&lines)

	lines = append(lines, "# HELP llm_itl_seconds Inter-token latency for LLM inference (seconds)")
	lines = append(lines, "# TYPE llm_itl_seconds histogram")
	p.exportITL(&lines)

	lines = append(lines, "# HELP llm_tokens_total Total tokens generated")
	lines = append(lines, "# TYPE llm_tokens_total counter")
	p.exportTokens(&lines)

	lines = append(lines, "# HELP request_duration_seconds Request latency")
	lines = append(lines, "# TYPE request_duration_seconds histogram")
	p.exportRequestDuration(&lines)

	lines = append(lines, "# HELP queue_full_requests_total Requests rejected due to backend queue saturation")
	lines = append(lines, "# TYPE queue_full_requests_total counter")
	lines = append(lines, fmt.Sprintf("queue_full_requests_total %d", p.metrics.GetQueueFull()))

	lines = append(lines, "# HELP warmup_requests_delayed_total LLM warmup requests delayed for vLLM slot management")
	lines = append(lines, "# TYPE warmup_requests_delayed_total counter")
	lines = append(lines, fmt.Sprintf("warmup_requests_delayed_total %d", p.metrics.GetWarmupDelayed()))

	lines = append(lines, "# HELP vllm_prefix_cache_hit_tokens_total vLLM KV prefix cache hit tokens")
	lines = append(lines, "# TYPE vllm_prefix_cache_hit_tokens_total counter")
	lines = append(lines, fmt.Sprintf("vllm_prefix_cache_hit_tokens_total %d", p.metrics.GetPrefixCacheHitTokens()))

	lines = append(lines, "# HELP admission_parked_total Requests parked waiting for engine admission")
	lines = append(lines, "# TYPE admission_parked_total counter")
	lines = append(lines, fmt.Sprintf("admission_parked_total %d", p.metrics.GetAdmissionParked()))

	lines = append(lines, "# HELP admission_rejected_total Requests rejected by engine admission control")
	lines = append(lines, "# TYPE admission_rejected_total counter")
	rejected := p.metrics.GetAdmissionRejected()
	reasons := make([]string, 0, len(rejected))
	for reason := range rejected {
		reasons = append(reasons, reason)
	}
	sort.Strings(reasons)
	for _, reason := range reasons {
		lines = append(lines, fmt.Sprintf(`admission_rejected_total{reason="%s"} %d`, reason, rejected[reason]))
	}

	lines = append(lines, "# HELP engine_state_stale_total Times the engine metrics feed went stale")
	lines = append(lines, "# TYPE engine_state_stale_total counter")
	lines = append(lines, fmt.Sprintf("engine_state_stale_total %d", p.metrics.GetEngineStateStale()))

	lines = append(lines, "# HELP vllm_requests_running vLLM requests running as read by the gateway")
	lines = append(lines, "# TYPE vllm_requests_running gauge")
	lines = append(lines, fmt.Sprintf("vllm_requests_running %d", p.metrics.GetVLLMRequestsRunning()))

	lines = append(lines, "# HELP vllm_kv_usage_perc vLLM KV cache usage fraction as read by the gateway")
	lines = append(lines, "# TYPE vllm_kv_usage_perc gauge")
	lines = append(lines, fmt.Sprintf("vllm_kv_usage_perc %g", p.metrics.GetVLLMKVUsagePerc()))

	lines = append(lines, "# HELP observer_degraded Engine metrics observer startup self-check failed (1) or healthy (0)")
	lines = append(lines, "# TYPE observer_degraded gauge")
	lines = append(lines, fmt.Sprintf("observer_degraded %d", p.metrics.GetObserverDegraded()))

	return strings.Join(lines, "\n") + "\n"
}

func (p *PrometheusExporter) exportTTFT(lines *[]string) {
	p.metrics.mu.RLock()
	defer p.metrics.mu.RUnlock()

	// Calculate statistics for each model
	for key, samples := range p.metrics.ttftMs {
		if len(samples) == 0 {
			continue
		}

		model := extractModel(key)
		sum := sumInt64(samples)

		// Export histogram buckets (in seconds)
		buckets := []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0}
		for _, bucket := range buckets {
			count := countLessOrEqual(samples, int64(bucket*1000))
			*lines = append(*lines, fmt.Sprintf(
				`llm_ttft_seconds_bucket{model="%s",le="%.3f"} %d`,
				model, bucket, count,
			))
		}

		*lines = append(*lines, fmt.Sprintf(
			`llm_ttft_seconds_bucket{model="%s",le="+Inf"} %d`,
			model, len(samples),
		))
		*lines = append(*lines, fmt.Sprintf(
			`llm_ttft_seconds_sum{model="%s"} %.3f`,
			model, float64(sum)/1000,
		))
		*lines = append(*lines, fmt.Sprintf(
			`llm_ttft_seconds_count{model="%s"} %d`,
			model, len(samples),
		))
	}
}

func (p *PrometheusExporter) exportITL(lines *[]string) {
	p.metrics.mu.RLock()
	defer p.metrics.mu.RUnlock()

	for key, samples := range p.metrics.itlMs {
		if len(samples) == 0 {
			continue
		}

		model := extractModel(key)
		sum := sumInt64(samples)

		// Export histogram buckets (in seconds)
		buckets := []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0}
		for _, bucket := range buckets {
			count := countLessOrEqual(samples, int64(bucket*1000))
			*lines = append(*lines, fmt.Sprintf(
				`llm_itl_seconds_bucket{model="%s",le="%.3f"} %d`,
				model, bucket, count,
			))
		}

		*lines = append(*lines, fmt.Sprintf(
			`llm_itl_seconds_bucket{model="%s",le="+Inf"} %d`,
			model, len(samples),
		))
		*lines = append(*lines, fmt.Sprintf(
			`llm_itl_seconds_sum{model="%s"} %.3f`,
			model, float64(sum)/1000,
		))
		*lines = append(*lines, fmt.Sprintf(
			`llm_itl_seconds_count{model="%s"} %d`,
			model, len(samples),
		))
	}
}

func (p *PrometheusExporter) exportTokens(lines *[]string) {
	p.metrics.mu.RLock()
	defer p.metrics.mu.RUnlock()

	// Sort keys for consistent output
	var keys []string
	for k := range p.metrics.tokenCount {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, key := range keys {
		model := extractModel(key)
		count := p.metrics.tokenCount[key]
		*lines = append(*lines, fmt.Sprintf(
			`llm_tokens_total{model="%s"} %d`,
			model, count,
		))
	}
}

func (p *PrometheusExporter) exportRequestDuration(lines *[]string) {
	p.metrics.mu.RLock()
	defer p.metrics.mu.RUnlock()

	// Sort keys for consistent output
	var keys []string
	for k := range p.metrics.requestDuration {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, key := range keys {
		route, upstream := parseKey(key)
		totalMs := p.metrics.requestDuration[key]
		count := int64(1) // We'd need to track count separately in real impl

		if buckets, ok := p.metrics.requestDurationBuckets[key]; ok {
			for bucket := range buckets {
				*lines = append(*lines, fmt.Sprintf(
					`request_duration_seconds_bucket{route="%s",upstream="%s",le="%.1f"} %d`,
					route, upstream, bucket, buckets[bucket],
				))
			}
		}

		*lines = append(*lines, fmt.Sprintf(
			`request_duration_seconds_sum{route="%s",upstream="%s"} %.3f`,
			route, upstream, float64(totalMs)/1000,
		))
		*lines = append(*lines, fmt.Sprintf(
			`request_duration_seconds_count{route="%s",upstream="%s"} %d`,
			route, upstream, count,
		))
	}
}

func extractModel(key string) string {
	parts := strings.Split(key, ":")
	if len(parts) >= 3 {
		return parts[2]
	}
	return key
}

func parseKey(key string) (string, string) {
	parts := strings.Split(key, ":")
	if len(parts) >= 2 {
		return parts[0], parts[1]
	}
	return key, ""
}

func countLessOrEqual(samples []int64, threshold int64) int {
	count := 0
	for _, s := range samples {
		if s <= threshold {
			count++
		}
	}
	return count
}