feat(tracing): add OpenTelemetry instrumentation to API gateway

- Add internal/tracing package with OTel tracer initialization
- HTTP middleware for server-side tracing (request/response attributes)
- Transport wrapper for client-side upstream call tracing
- Update proxy to use tracing transport
- Add OTEL_* env vars to k8s deployment

Traces flow: api-gateway -> otel-collector -> tempo -> grafana

internal/tracing/tracing.go

@@ -0,0 +1,94 @@
+// Package tracing provides OpenTelemetry instrumentation for the API gateway.
+package tracing
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+)
+
+// Config holds tracing configuration.
+type Config struct {
+	ServiceName    string
+	ServiceVersion string
+	Environment    string
+	OTLPEndpoint   string
+}
+
+// DefaultConfig returns configuration from environment variables.
+func DefaultConfig() Config {
+	endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "otel-collector.tracing.svc.cluster.local:4317"
+	}
+	return Config{
+		ServiceName:    getEnvOrDefault("OTEL_SERVICE_NAME", "api-gateway"),
+		ServiceVersion: getEnvOrDefault("OTEL_SERVICE_VERSION", "1.0.0"),
+		Environment:    getEnvOrDefault("OTEL_ENVIRONMENT", "production"),
+		OTLPEndpoint:   endpoint,
+	}
+}
+
+func getEnvOrDefault(key, defaultVal string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return defaultVal
+}
+
+// Init initializes the OpenTelemetry tracer provider.
+// Returns a shutdown function that should be called on application exit.
+func Init(ctx context.Context, cfg Config) (func(context.Context) error, error) {
+	// Create OTLP exporter
+	exporter, err := otlptracegrpc.New(ctx,
+		otlptracegrpc.WithEndpoint(cfg.OTLPEndpoint),
+		otlptracegrpc.WithInsecure(),
+		otlptracegrpc.WithTimeout(5*time.Second),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create resource with service information
+	res, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName(cfg.ServiceName),
+			semconv.ServiceVersion(cfg.ServiceVersion),
+			attribute.String("deployment.environment", cfg.Environment),
+		),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create tracer provider with batch processor
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exporter,
+			sdktrace.WithBatchTimeout(5*time.Second),
+			sdktrace.WithMaxExportBatchSize(512),
+		),
+		sdktrace.WithResource(res),
+		sdktrace.WithSampler(sdktrace.AlwaysSample()),
+	)
+
+	// Set global tracer provider
+	otel.SetTracerProvider(tp)
+
+	// Set global propagator (W3C Trace Context + Baggage)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	))
+
+	return tp.Shutdown, nil
+}