TT-7594 OTel instrumentation (TykTechnologies#5269)

Adding OpenTelemetry instrumentation on our middleware chain and RoundTripper to propagate context.

gateway/api_loader.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/coprocess"
+	"github.com/TykTechnologies/tyk/internal/otel"
 	"github.com/TykTechnologies/tyk/storage"
 	"github.com/TykTechnologies/tyk/trace"
 )
@@ -496,8 +497,10 @@ func (gw *Gateway) processSpec(spec *APISpec, apisByListen map[string]int,
 
 	logger.Debug("Setting Listen Path: ", spec.Proxy.ListenPath)
 
-	if trace.IsEnabled() {
+	if trace.IsEnabled() { // trace.IsEnabled = check if opentracing is enabled
 		chainDef.ThisHandler = trace.Handle(spec.Name, chain)
+	} else if gw.GetConfig().OpenTelemetry.Enabled { // check if opentelemetry is enabled
+		chainDef.ThisHandler = otel.HTTPHandler(spec.Name, chain, gw.TracerProvider)
 	} else {
 		chainDef.ThisHandler = chain
 	}

gateway/api_loader_test.go

@@ -2,14 +2,18 @@ package gateway
 
 import (
 	"fmt"
+	"io"
+	"net"
 	"net/http"
+	"net/http/httptest"
 	"path"
 	_ "path"
 	"sync/atomic"
 	"testing"
 
 	"github.com/TykTechnologies/storage/persistent/model"
 	"github.com/TykTechnologies/tyk/apidef"
+	"github.com/TykTechnologies/tyk/config"
 
 	"github.com/stretchr/testify/assert"
 
@@ -426,3 +430,97 @@ func TestAllApisAreMTLS(t *testing.T) {
 		t.Errorf("Expected AllApisAreMTLS to return %v, but got %v", expected, result)
 	}
 }
+
+func TestOpenTelemetry(t *testing.T) {
+
+	t.Run("Opentelemetry enabled - check if we are sending traces", func(t *testing.T) {
+		otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) {
+			//check the body
+			body, err := io.ReadAll(r.Body)
+			assert.Nil(t, err)
+
+			assert.NotEmpty(t, body)
+
+			// check the user agent
+			agent, ok := r.Header["User-Agent"]
+			assert.True(t, ok)
+			assert.Len(t, agent, 1)
+			assert.Contains(t, agent[0], "OTLP")
+
+			//check if we are sending the traces to the right endpoint
+			assert.Contains(t, r.URL.Path, "/v1/traces")
+
+			// Here you can check the request and return a response
+			w.WriteHeader(http.StatusOK)
+		}, ":0")
+
+		// Start the server.
+		otelCollectorMock.Start()
+		// Stop the server on return from the function.
+		defer otelCollectorMock.Close()
+
+		ts := StartTest(func(globalConf *config.Config) {
+			globalConf.OpenTelemetry.Enabled = true
+			globalConf.OpenTelemetry.Exporter = "http"
+			globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL
+			globalConf.OpenTelemetry.SpanProcessorType = "simple"
+		})
+		defer ts.Close()
+
+		ts.Gw.BuildAndLoadAPI(func(spec *APISpec) {
+			spec.APIID = "test"
+			spec.Proxy.ListenPath = "/my-api/"
+			spec.UseKeylessAccess = true
+		})
+
+		_, _ = ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK})
+		assert.Equal(t, "otel", ts.Gw.TracerProvider.Type())
+	})
+
+	t.Run("Opentelemetry disabled - check if we are not sending traces", func(t *testing.T) {
+
+		otelCollectorMock := httpCollectorMock(t, func(w http.ResponseWriter, r *http.Request) {
+			t.Fail()
+		}, ":0")
+
+		// Start the server.
+		otelCollectorMock.Start()
+		// Stop the server on return from the function.
+		defer otelCollectorMock.Close()
+
+		ts := StartTest(func(globalConf *config.Config) {
+			globalConf.OpenTelemetry.Enabled = false
+			globalConf.OpenTelemetry.Exporter = "http"
+			globalConf.OpenTelemetry.Endpoint = otelCollectorMock.URL
+			globalConf.OpenTelemetry.SpanProcessorType = "simple"
+		})
+		defer ts.Close()
+
+		ts.Gw.BuildAndLoadAPI(func(spec *APISpec) {
+			spec.APIID = "test"
+			spec.Proxy.ListenPath = "/my-api/"
+			spec.UseKeylessAccess = true
+		})
+
+		_, _ = ts.Run(t, test.TestCase{Path: "/my-api/", Code: http.StatusOK})
+		assert.Equal(t, "noop", ts.Gw.TracerProvider.Type())
+	})
+}
+
+func httpCollectorMock(t *testing.T, fn http.HandlerFunc, address string) *httptest.Server {
+	t.Helper()
+
+	l, err := net.Listen("tcp", address)
+	if err != nil {
+		t.Fatalf("error setting up collector mock: %s", err.Error())
+	}
+
+	otelCollectorMock := httptest.NewUnstartedServer(http.HandlerFunc(fn))
+
+	// NewUnstartedServer creates a listener. Close that listener and replace
+	// with the one we created.
+	otelCollectorMock.Listener.Close()
+	otelCollectorMock.Listener = l
+
+	return otelCollectorMock
+}

gateway/middleware.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/TykTechnologies/tyk/internal/cache"
+	"github.com/TykTechnologies/tyk/internal/otel"
 	"github.com/TykTechnologies/tyk/rpc"
 
 	"github.com/TykTechnologies/tyk/header"
@@ -64,6 +65,22 @@ func (tr TraceMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Request,
 		defer span.Finish()
 		setContext(r, ctx)
 		return tr.TykMiddleware.ProcessRequest(w, r, conf)
+	} else if baseMw := tr.Base(); baseMw != nil {
+		if cfg := baseMw.Gw.GetConfig(); cfg.OpenTelemetry.Enabled {
+			ctx, span := baseMw.Gw.TracerProvider.Tracer().Start(r.Context(), tr.Name())
+			defer span.End()
+
+			setContext(r, ctx)
+
+			err, i := tr.TykMiddleware.ProcessRequest(w, r, conf)
+			if err != nil {
+				span.SetStatus(otel.SPAN_STATUS_ERROR, err.Error())
+			} else {
+				span.SetStatus(otel.SPAN_STATUS_OK, "")
+			}
+
+			return err, i
+		}
 	}
 
 	return tr.TykMiddleware.ProcessRequest(w, r, conf)

gateway/reverse_proxy.go

@@ -47,6 +47,7 @@ import (
 	"github.com/TykTechnologies/tyk/apidef"
 	"github.com/TykTechnologies/tyk/ctx"
 	"github.com/TykTechnologies/tyk/header"
+	"github.com/TykTechnologies/tyk/internal/otel"
 	"github.com/TykTechnologies/tyk/regexp"
 	"github.com/TykTechnologies/tyk/trace"
 	"github.com/TykTechnologies/tyk/user"
@@ -814,9 +815,19 @@ func (rt *TykRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
 		return handleInMemoryLoop(handler, r)
 	}
 
+	if rt.Gw.GetConfig().OpenTelemetry.Enabled {
+		var baseRoundTripper http.RoundTripper = rt.transport
+		if rt.h2ctransport != nil {
+			baseRoundTripper = rt.h2ctransport
+		}
+
+		tr := otel.HTTPRoundTripper(baseRoundTripper)
+		return tr.RoundTrip(r)
+	}
 	if rt.h2ctransport != nil {
 		return rt.h2ctransport.RoundTrip(r)
 	}
+
 	return rt.transport.RoundTrip(r)
 }
 

gateway/testutil.go

@@ -36,6 +36,7 @@ import (
 	"github.com/gorilla/websocket"
 	"golang.org/x/net/context"
 
+	"github.com/TykTechnologies/tyk/internal/otel"
 	"github.com/TykTechnologies/tyk/internal/uuid"
 
 	"github.com/TykTechnologies/graphql-go-tools/pkg/execution/datasource"
@@ -1200,6 +1201,8 @@ func (s *Test) newGateway(genConf func(globalConf *config.Config)) *Gateway {
 
 	go s.reloadSimulation(s.ctx, gw)
 
+	gw.TracerProvider = otel.InitOpenTelemetry(gw.ctx, mainLog.Logger, &gwConfig.OpenTelemetry)
+
 	return gw
 }
 

go.mod

@@ -77,7 +77,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require github.com/TykTechnologies/opentelemetry v0.0.3
+require github.com/TykTechnologies/opentelemetry v0.0.4
 
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
@@ -103,6 +103,7 @@ require (
 	github.com/eclipse/paho.mqtt.golang v1.2.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
 	github.com/fatih/structs v1.1.0 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/getsentry/raven-go v0.2.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -172,6 +173,7 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 // indirect

go.sum

@@ -97,8 +97,8 @@ github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632 h1:T5NWziF
 github.com/TykTechnologies/murmur3 v0.0.0-20230310161213-aad17efd5632/go.mod h1:UsPYgOFBpNzDXLEti7MKOwHLpVSqdzuNGkVFPspQmnQ=
 github.com/TykTechnologies/openid2go v0.1.2 h1:WXctksOahA/epTVVvbn9iNUuMXKRr0ksrF4dY9KW8o8=
 github.com/TykTechnologies/openid2go v0.1.2/go.mod h1:gYfkqeWa+lY3Xz/Z2xYtIzmYXynlgKZaBIbPCqdcdMA=
-github.com/TykTechnologies/opentelemetry v0.0.3 h1:S7TkDXozHo5oAC1ZbxMq7F+CMVqz3MOe2pjkSiE5D3Q=
-github.com/TykTechnologies/opentelemetry v0.0.3/go.mod h1:O/kFyW5Jrj3vHxv2xDNHsc1c0P5u3WTPigLit5wHa1M=
+github.com/TykTechnologies/opentelemetry v0.0.4 h1:eI+rkpkUJykr3w2CKxSX5Z1bDIUZkjEihJEmo7+M5l4=
+github.com/TykTechnologies/opentelemetry v0.0.4/go.mod h1:qZ718eI1wOZ4EppSa/KnJpMhuhCm/1zHU/NhGlDWYEU=
 github.com/TykTechnologies/storage v1.0.5 h1:lfMljPueySAW7Mpc70g1/qC5n2LKNcKgQs+Xw30apP8=
 github.com/TykTechnologies/storage v1.0.5/go.mod h1:+0S3KuNlLGBTMTSFREuZFm315zzXjuuCO4QSAPy+d3M=
 github.com/TykTechnologies/tyk-pump v1.8.0-rc4 h1:odhM/skFNYWBdSL+atx49f6/PleNRq6l4E9SyeDfmTg=
@@ -257,6 +257,8 @@ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40eg
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -1006,6 +1008,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8=
 go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
 go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s=
 go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4=

internal/otel/otel.go

@@ -13,6 +13,16 @@ type TracerProvider = tyktrace.Provider
 
 type Config = otelconfig.OpenTelemetry
 
+var HTTPHandler = tyktrace.NewHTTPHandler
+
+var HTTPRoundTripper = tyktrace.NewHTTPTransport
+
+const (
+	SPAN_STATUS_OK    = tyktrace.SPAN_STATUS_OK
+	SPAN_STATUS_ERROR = tyktrace.SPAN_STATUS_ERROR
+	SPAN_STATUS_UNSET = tyktrace.SPAN_STATUS_UNSET
+)
+
 // InitOpenTelemetry initializes OpenTelemetry - it returns a TracerProvider
 // which can be used to create a tracer. If OpenTelemetry is disabled or misconfigured,
 // a NoopProvider is returned.