From f9addf945d58c9ff6108f6e583d8c44ca84ae01a Mon Sep 17 00:00:00 2001 From: Admin Bot Date: Mon, 31 Aug 2026 15:01:56 -0700 Subject: [PATCH] feat(tracing): add OpenTelemetry instrumentation to API gateway - Add internal/tracing package with OTel tracer initialization - HTTP middleware for server-side tracing (request/response attributes) - Transport wrapper for client-side upstream call tracing - Update proxy to use tracing transport - Add OTEL_* env vars to k8s deployment Traces flow: api-gateway -> otel-collector -> tempo -> grafana --- cmd/gateway/main.go | 22 ++++++- go.mod | 23 +++++-- go.sum | 61 ++++++++++-------- internal/proxy/proxy.go | 5 +- internal/tracing/middleware.go | 109 +++++++++++++++++++++++++++++++++ internal/tracing/tracing.go | 94 ++++++++++++++++++++++++++++ internal/tracing/transport.go | 76 +++++++++++++++++++++++ k8s/deployment.yaml | 9 +++ 8 files changed, 367 insertions(+), 32 deletions(-) create mode 100644 internal/tracing/middleware.go create mode 100644 internal/tracing/tracing.go create mode 100644 internal/tracing/transport.go diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 1a18c4c..3e33d78 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -14,9 +14,26 @@ import ( "forgejo.riotpiao.com/rock/homelab-frontend/internal/server" "forgejo.riotpiao.com/rock/homelab-frontend/internal/serviceadapter" "forgejo.riotpiao.com/rock/homelab-frontend/internal/temporal" + "forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing" ) func main() { + ctx := context.Background() + + // Initialize OpenTelemetry tracing + tracingCfg := tracing.DefaultConfig() + shutdownTracer, err := tracing.Init(ctx, tracingCfg) + if err != nil { + log.Printf("warning: failed to initialize tracing: %v", err) + } else { + log.Printf("tracing initialized: service=%s endpoint=%s", tracingCfg.ServiceName, tracingCfg.OTLPEndpoint) + defer func() { + if err := shutdownTracer(ctx); err != nil { + log.Printf("error shutting down tracer: %v", err) + } + }() + } + // Load configuration cfg, err := config.Load() if err != nil { @@ -63,7 +80,10 @@ func main() { // Create router that handles health endpoints, X-Service (ServiceAdapter) routing, // temporal endpoints, and passes others to upstream handler router := server.NewRouter(healthChecker, dispatcher, temporalHandler, upstreamHandler) - srv.SetHandler(router) + + // Wrap router with tracing middleware + tracedRouter := tracing.Middleware(router) + srv.SetHandler(tracedRouter) // Set up signal handling sigChan := make(chan os.Signal, 1) diff --git a/go.mod b/go.mod index 580a669..372b95c 100644 --- a/go.mod +++ b/go.mod @@ -5,30 +5,43 @@ go 1.26.0 require ( github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/golang-jwt/jwt/v5 v5.3.1 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 go.temporal.io/api v1.63.5 go.temporal.io/sdk v1.48.0 google.golang.org/grpc v1.83.2 - google.golang.org/protobuf v1.36.11 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/mock v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/nexus-rpc/nexus-proto-annotations v0.1.0 // indirect github.com/nexus-rpc/sdk-go v0.7.0 // indirect github.com/robfig/cron v1.2.0 // indirect github.com/stretchr/objx v0.5.3 // indirect - github.com/stretchr/testify v1.12.0 // indirect + github.com/stretchr/testify v1.12.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.3.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect ) diff --git a/go.sum b/go.sum index f531759..e3f05ca 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,14 @@ github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -22,8 +25,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 h1:sGm2vDRFUrQJO/Veii4h4zG2vvqG6uWNkBHSTqXOZk0= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2/go.mod h1:wd1YpapPLivG6nQgbf7ZkG1hhSOXDhhn4MLTknx2aAc= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -36,31 +39,41 @@ github.com/nexus-rpc/sdk-go v0.7.0 h1:38NrfY5rLnZAiMMs2ZfCKI/CSDzdfJG+27iAgfA8bU github.com/nexus-rpc/sdk-go v0.7.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI= -github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.temporal.io/api v1.63.5 h1:c11+kPYHkXXL3UiShPdbMD+xtvqGsbTibUA9ypmiCa4= go.temporal.io/api v1.63.5/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= go.temporal.io/sdk v1.48.0/go.mod h1:SHv3+fLzD0GGZAwf0xNSvu8UmO1nFgG9WBSYoowApIk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -106,14 +119,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= -google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 11df0f2..56f9344 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -15,6 +15,7 @@ import ( "forgejo.riotpiao.com/rock/homelab-frontend/internal/config" "forgejo.riotpiao.com/rock/homelab-frontend/internal/logging" + "forgejo.riotpiao.com/rock/homelab-frontend/internal/tracing" ) // Handler is a reverse proxy that routes requests to configured upstreams. @@ -325,8 +326,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Set the director to apply path rewriting proxy.Director = route.Director - // Use the connection-pooled transport - proxy.Transport = route.Transport + // Use the connection-pooled transport wrapped with tracing + proxy.Transport = tracing.NewTransport(route.Transport) // Set error handler to log upstream errors proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { diff --git a/internal/tracing/middleware.go b/internal/tracing/middleware.go new file mode 100644 index 0000000..af7c4a7 --- /dev/null +++ b/internal/tracing/middleware.go @@ -0,0 +1,109 @@ +package tracing + +import ( + "net/http" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "api-gateway" + +// responseWriter wraps http.ResponseWriter to capture status code. +type responseWriter struct { + http.ResponseWriter + statusCode int + written int64 +} + +func newResponseWriter(w http.ResponseWriter) *responseWriter { + return &responseWriter{ResponseWriter: w, statusCode: http.StatusOK} +} + +func (rw *responseWriter) WriteHeader(code int) { + rw.statusCode = code + rw.ResponseWriter.WriteHeader(code) +} + +func (rw *responseWriter) Write(b []byte) (int, error) { + n, err := rw.ResponseWriter.Write(b) + rw.written += int64(n) + return n, err +} + +// Middleware returns an HTTP middleware that adds tracing to requests. +func Middleware(next http.Handler) http.Handler { + tracer := otel.Tracer(tracerName) + propagator := otel.GetTextMapPropagator() + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + // Extract any existing trace context from incoming request + ctx := propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header)) + + // Start a new span + spanName := r.Method + " " + r.URL.Path + ctx, span := tracer.Start(ctx, spanName, + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes( + semconv.HTTPRequestMethodKey.String(r.Method), + semconv.URLPath(r.URL.Path), + semconv.URLScheme(scheme(r)), + semconv.ServerAddress(r.Host), + semconv.UserAgentOriginal(r.UserAgent()), + semconv.NetworkPeerAddress(r.RemoteAddr), + ), + ) + defer span.End() + + // Add query parameters if present + if r.URL.RawQuery != "" { + span.SetAttributes(semconv.URLQuery(r.URL.RawQuery)) + } + + // Add model attribute for LLM requests + if model := r.Header.Get("X-Model"); model != "" { + span.SetAttributes(attribute.String("llm.model", model)) + } + + // Wrap response writer to capture status + rw := newResponseWriter(w) + + // Inject trace context into response headers (for debugging) + propagator.Inject(ctx, propagation.HeaderCarrier(w.Header())) + + // Call the next handler with traced context + next.ServeHTTP(rw, r.WithContext(ctx)) + + // Record response attributes + duration := time.Since(start) + span.SetAttributes( + semconv.HTTPResponseStatusCode(rw.statusCode), + attribute.Int64("http.response.body.size", rw.written), + attribute.Float64("http.request.duration_ms", float64(duration.Milliseconds())), + ) + + // Set span status based on HTTP status code + if rw.statusCode >= 400 { + span.SetStatus(codes.Error, http.StatusText(rw.statusCode)) + } else { + span.SetStatus(codes.Ok, "") + } + }) +} + +func scheme(r *http.Request) string { + if r.TLS != nil { + return "https" + } + if s := r.Header.Get("X-Forwarded-Proto"); s != "" { + return s + } + return "http" +} diff --git a/internal/tracing/tracing.go b/internal/tracing/tracing.go new file mode 100644 index 0000000..d03a490 --- /dev/null +++ b/internal/tracing/tracing.go @@ -0,0 +1,94 @@ +// Package tracing provides OpenTelemetry instrumentation for the API gateway. +package tracing + +import ( + "context" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// Config holds tracing configuration. +type Config struct { + ServiceName string + ServiceVersion string + Environment string + OTLPEndpoint string +} + +// DefaultConfig returns configuration from environment variables. +func DefaultConfig() Config { + endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if endpoint == "" { + endpoint = "otel-collector.tracing.svc.cluster.local:4317" + } + return Config{ + ServiceName: getEnvOrDefault("OTEL_SERVICE_NAME", "api-gateway"), + ServiceVersion: getEnvOrDefault("OTEL_SERVICE_VERSION", "1.0.0"), + Environment: getEnvOrDefault("OTEL_ENVIRONMENT", "production"), + OTLPEndpoint: endpoint, + } +} + +func getEnvOrDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + +// Init initializes the OpenTelemetry tracer provider. +// Returns a shutdown function that should be called on application exit. +func Init(ctx context.Context, cfg Config) (func(context.Context) error, error) { + // Create OTLP exporter + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithEndpoint(cfg.OTLPEndpoint), + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithTimeout(5*time.Second), + ) + if err != nil { + return nil, err + } + + // Create resource with service information + res, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(cfg.ServiceName), + semconv.ServiceVersion(cfg.ServiceVersion), + attribute.String("deployment.environment", cfg.Environment), + ), + ) + if err != nil { + return nil, err + } + + // Create tracer provider with batch processor + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter, + sdktrace.WithBatchTimeout(5*time.Second), + sdktrace.WithMaxExportBatchSize(512), + ), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) + + // Set global tracer provider + otel.SetTracerProvider(tp) + + // Set global propagator (W3C Trace Context + Baggage) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + return tp.Shutdown, nil +} diff --git a/internal/tracing/transport.go b/internal/tracing/transport.go new file mode 100644 index 0000000..6c947b1 --- /dev/null +++ b/internal/tracing/transport.go @@ -0,0 +1,76 @@ +package tracing + +import ( + "net/http" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" +) + +// Transport wraps an http.RoundTripper with tracing. +type Transport struct { + base http.RoundTripper +} + +// NewTransport creates a new tracing transport wrapper. +func NewTransport(base http.RoundTripper) *Transport { + if base == nil { + base = http.DefaultTransport + } + return &Transport{base: base} +} + +// RoundTrip implements http.RoundTripper with tracing. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + ctx := req.Context() + tracer := otel.Tracer(tracerName) + propagator := otel.GetTextMapPropagator() + + // Start client span + spanName := "HTTP " + req.Method + " " + req.URL.Host + req.URL.Path + ctx, span := tracer.Start(ctx, spanName, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + semconv.HTTPRequestMethodKey.String(req.Method), + semconv.URLFull(req.URL.String()), + semconv.ServerAddress(req.URL.Host), + attribute.String("upstream.name", req.URL.Host), + ), + ) + defer span.End() + + // Inject trace context into outgoing request headers + propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) + + // Perform the request + start := time.Now() + resp, err := t.base.RoundTrip(req.WithContext(ctx)) + duration := time.Since(start) + + // Record timing + span.SetAttributes(attribute.Float64("http.request.duration_ms", float64(duration.Milliseconds()))) + + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + return nil, err + } + + // Record response attributes + span.SetAttributes( + semconv.HTTPResponseStatusCode(resp.StatusCode), + ) + + if resp.StatusCode >= 400 { + span.SetStatus(codes.Error, http.StatusText(resp.StatusCode)) + } else { + span.SetStatus(codes.Ok, "") + } + + return resp, nil +} diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index 5a8d4a9..c73d196 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -61,6 +61,15 @@ spec: value: "5m" - name: LOG_LEVEL value: "info" + # OpenTelemetry tracing configuration + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "otel-collector.tracing.svc.cluster.local:4317" + - name: OTEL_SERVICE_NAME + value: "api-gateway" + - name: OTEL_SERVICE_VERSION + value: "1.0.0" + - name: OTEL_ENVIRONMENT + value: "production" volumeMounts: - name: config mountPath: /etc/gateway