feat(tracing): add OpenTelemetry instrumentation to API gateway

- Add internal/tracing package with OTel tracer initialization
- HTTP middleware for server-side tracing (request/response attributes)
- Transport wrapper for client-side upstream call tracing
- Update proxy to use tracing transport
- Add OTEL_* env vars to k8s deployment

Traces flow: api-gateway -> otel-collector -> tempo -> grafana

internal/proxy/proxy.go

@@ -15,6 +15,7 @@ import (
 
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/config"
 	"forgejo.riotpiao.com/rock/homelab-frontend/internal/logging"
+	"forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing"
 )
 
 // Handler is a reverse proxy that routes requests to configured upstreams.
@@ -325,8 +326,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Set the director to apply path rewriting
 	proxy.Director = route.Director
 
-	// Use the connection-pooled transport
-	proxy.Transport = route.Transport
+	// Use the connection-pooled transport wrapped with tracing
+	proxy.Transport = tracing.NewTransport(route.Transport)
 
 	// Set error handler to log upstream errors
 	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) {

internal/tracing/middleware.go

@@ -0,0 +1,109 @@
+package tracing
+
+import (
+	"net/http"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+const tracerName = "api-gateway"
+
+// responseWriter wraps http.ResponseWriter to capture status code.
+type responseWriter struct {
+	http.ResponseWriter
+	statusCode int
+	written    int64
+}
+
+func newResponseWriter(w http.ResponseWriter) *responseWriter {
+	return &responseWriter{ResponseWriter: w, statusCode: http.StatusOK}
+}
+
+func (rw *responseWriter) WriteHeader(code int) {
+	rw.statusCode = code
+	rw.ResponseWriter.WriteHeader(code)
+}
+
+func (rw *responseWriter) Write(b []byte) (int, error) {
+	n, err := rw.ResponseWriter.Write(b)
+	rw.written += int64(n)
+	return n, err
+}
+
+// Middleware returns an HTTP middleware that adds tracing to requests.
+func Middleware(next http.Handler) http.Handler {
+	tracer := otel.Tracer(tracerName)
+	propagator := otel.GetTextMapPropagator()
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+
+		// Extract any existing trace context from incoming request
+		ctx := propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
+
+		// Start a new span
+		spanName := r.Method + " " + r.URL.Path
+		ctx, span := tracer.Start(ctx, spanName,
+			trace.WithSpanKind(trace.SpanKindServer),
+			trace.WithAttributes(
+				semconv.HTTPRequestMethodKey.String(r.Method),
+				semconv.URLPath(r.URL.Path),
+				semconv.URLScheme(scheme(r)),
+				semconv.ServerAddress(r.Host),
+				semconv.UserAgentOriginal(r.UserAgent()),
+				semconv.NetworkPeerAddress(r.RemoteAddr),
+			),
+		)
+		defer span.End()
+
+		// Add query parameters if present
+		if r.URL.RawQuery != "" {
+			span.SetAttributes(semconv.URLQuery(r.URL.RawQuery))
+		}
+
+		// Add model attribute for LLM requests
+		if model := r.Header.Get("X-Model"); model != "" {
+			span.SetAttributes(attribute.String("llm.model", model))
+		}
+
+		// Wrap response writer to capture status
+		rw := newResponseWriter(w)
+
+		// Inject trace context into response headers (for debugging)
+		propagator.Inject(ctx, propagation.HeaderCarrier(w.Header()))
+
+		// Call the next handler with traced context
+		next.ServeHTTP(rw, r.WithContext(ctx))
+
+		// Record response attributes
+		duration := time.Since(start)
+		span.SetAttributes(
+			semconv.HTTPResponseStatusCode(rw.statusCode),
+			attribute.Int64("http.response.body.size", rw.written),
+			attribute.Float64("http.request.duration_ms", float64(duration.Milliseconds())),
+		)
+
+		// Set span status based on HTTP status code
+		if rw.statusCode >= 400 {
+			span.SetStatus(codes.Error, http.StatusText(rw.statusCode))
+		} else {
+			span.SetStatus(codes.Ok, "")
+		}
+	})
+}
+
+func scheme(r *http.Request) string {
+	if r.TLS != nil {
+		return "https"
+	}
+	if s := r.Header.Get("X-Forwarded-Proto"); s != "" {
+		return s
+	}
+	return "http"
+}

internal/tracing/tracing.go

@@ -0,0 +1,94 @@
+// Package tracing provides OpenTelemetry instrumentation for the API gateway.
+package tracing
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+)
+
+// Config holds tracing configuration.
+type Config struct {
+	ServiceName    string
+	ServiceVersion string
+	Environment    string
+	OTLPEndpoint   string
+}
+
+// DefaultConfig returns configuration from environment variables.
+func DefaultConfig() Config {
+	endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "otel-collector.tracing.svc.cluster.local:4317"
+	}
+	return Config{
+		ServiceName:    getEnvOrDefault("OTEL_SERVICE_NAME", "api-gateway"),
+		ServiceVersion: getEnvOrDefault("OTEL_SERVICE_VERSION", "1.0.0"),
+		Environment:    getEnvOrDefault("OTEL_ENVIRONMENT", "production"),
+		OTLPEndpoint:   endpoint,
+	}
+}
+
+func getEnvOrDefault(key, defaultVal string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return defaultVal
+}
+
+// Init initializes the OpenTelemetry tracer provider.
+// Returns a shutdown function that should be called on application exit.
+func Init(ctx context.Context, cfg Config) (func(context.Context) error, error) {
+	// Create OTLP exporter
+	exporter, err := otlptracegrpc.New(ctx,
+		otlptracegrpc.WithEndpoint(cfg.OTLPEndpoint),
+		otlptracegrpc.WithInsecure(),
+		otlptracegrpc.WithTimeout(5*time.Second),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create resource with service information
+	res, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(cfg.ServiceName),
+			semconv.ServiceVersion(cfg.ServiceVersion),
+			attribute.String("deployment.environment", cfg.Environment),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create tracer provider with batch processor
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exporter,
+			sdktrace.WithBatchTimeout(5*time.Second),
+			sdktrace.WithMaxExportBatchSize(512),
+		),
+		sdktrace.WithResource(res),
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+	)
+
+	// Set global tracer provider
+	otel.SetTracerProvider(tp)
+
+	// Set global propagator (W3C Trace Context + Baggage)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	))
+
+	return tp.Shutdown, nil
+}

internal/tracing/transport.go

@@ -0,0 +1,76 @@
+package tracing
+
+import (
+	"net/http"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+// Transport wraps an http.RoundTripper with tracing.
+type Transport struct {
+	base http.RoundTripper
+}
+
+// NewTransport creates a new tracing transport wrapper.
+func NewTransport(base http.RoundTripper) *Transport {
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	return &Transport{base: base}
+}
+
+// RoundTrip implements http.RoundTripper with tracing.
+func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	ctx := req.Context()
+	tracer := otel.Tracer(tracerName)
+	propagator := otel.GetTextMapPropagator()
+
+	// Start client span
+	spanName := "HTTP " + req.Method + " " + req.URL.Host + req.URL.Path
+	ctx, span := tracer.Start(ctx, spanName,
+		trace.WithSpanKind(trace.SpanKindClient),
+		trace.WithAttributes(
+			semconv.HTTPRequestMethodKey.String(req.Method),
+			semconv.URLFull(req.URL.String()),
+			semconv.ServerAddress(req.URL.Host),
+			attribute.String("upstream.name", req.URL.Host),
+		),
+	)
+	defer span.End()
+
+	// Inject trace context into outgoing request headers
+	propagator.Inject(ctx, propagation.HeaderCarrier(req.Header))
+
+	// Perform the request
+	start := time.Now()
+	resp, err := t.base.RoundTrip(req.WithContext(ctx))
+	duration := time.Since(start)
+
+	// Record timing
+	span.SetAttributes(attribute.Float64("http.request.duration_ms", float64(duration.Milliseconds())))
+
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
+	}
+
+	// Record response attributes
+	span.SetAttributes(
+		semconv.HTTPResponseStatusCode(resp.StatusCode),
+	)
+
+	if resp.StatusCode >= 400 {
+		span.SetStatus(codes.Error, http.StatusText(resp.StatusCode))
+	} else {
+		span.SetStatus(codes.Ok, "")
+	}
+
+	return resp, nil
+}