Instrumenting observability with opencensus (gomods#604)

* Initial chanages for observability

* Fix some panics to start testing

* Export tracing properly

* First example of child spans using opencensus

* Add spans to download protocol

* Add url to traces

* Remove opentracing

* Remove gopkg.* files

* Start deprecating opentracing

* Resolve stupid build errors

* Use observability package

* Fix test errors

* Convert buffalo spans to observercontext

* change package name

* defer flush to the end of the app execution

* Change op names to the correct package

* Rename pkg/observability to pkg/observ

* Show traces for the package

* Keep tracing in the earlier way

* Add info from request headers

* Remove whitespace

* Move exporter url to env var

* Add to env file for documentation

* Remove opentracing stuff

* Use stdlib

* Shorten service name

* Add a service name to olympus as well

* Add test to test if there is a recursion or not

* Add Ops Suite

* Move around code

* Make sure the service is not instantiated if the exporter is not found

cmd/olympus/actions/app.go

@@ -54,6 +54,9 @@ var (
 	T *i18n.Translator
 )
 
+// Service is the name of the service that we want to tag our processes with
+const Service = "olympus"
+
 // App is where all routes and middleware for buffalo should be defined.
 // This is the nerve center of your application.
 func App(config *AppConfig) (*buffalo.App, error) {
@@ -97,7 +100,6 @@ func App(config *AppConfig) (*buffalo.App, error) {
 	if ENV == "development" {
 		app.Use(middleware.ParameterLogger)
 	}
-	initializeTracing(app)
 	// Protect against CSRF attacks. https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)
 	// Remove to disable this.
 	if env.EnableCSRFProtection() {

cmd/proxy/.env

@@ -1,2 +1,4 @@
 ATHENS_STORAGE_TYPE=mongo
-GO_ENV=test
+GO_ENV=development
+TRACE_EXPORTER=http://0.0.0.0:14268
+

cmd/proxy/actions/app.go

@@ -3,6 +3,8 @@ package actions
 import (
 	"fmt"
 
+	"github.com/gomods/athens/pkg/observ"
+
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/middleware"
 	"github.com/gobuffalo/buffalo/middleware/csrf"
@@ -18,6 +20,9 @@ import (
 	"github.com/unrolled/secure"
 )
 
+// Service is the name of the service that we want to tag our processes with
+const Service = "proxy"
+
 // ENV is used to help switch settings based on where the
 // application is being run. Default is "development".
 var ENV = env.GoEnvironmentWithDefault("development")
@@ -84,6 +89,15 @@ func App() (*buffalo.App, error) {
 		app = app.Group(prefix)
 	}
 
+	// Register exporter to export traces
+	exporter, err := observ.RegisterTraceExporter(Service)
+	if err != nil {
+		lggr.SystemErr(err)
+	} else {
+		defer exporter.Flush()
+		app.Use(observ.Tracer(Service))
+	}
+
 	// Automatically redirect to SSL
 	app.Use(ssl.ForceSSL(secure.Options{
 		SSLRedirect:     env.ProxyForceSSL(),
@@ -93,7 +107,7 @@ func App() (*buffalo.App, error) {
 	if ENV == "development" {
 		app.Use(middleware.ParameterLogger)
 	}
-	initializeTracing(app)
+
 	initializeAuth(app)
 	// Protect against CSRF attacks. https://www.owasp.org/index.php/Cross-Site_Request_Forgery_(CSRF)
 	// Remove to disable this.

docker-compose.yml

@@ -3,7 +3,7 @@ services:
   redis:
     image: redis:alpine
     command: ["redis-server", "--appendonly", "yes"]
-    ports: 
+    ports:
       - "6379:6379"
   mongo:
     image: mongo:3.7.9-jessie

go.mod

@@ -3,12 +3,12 @@ module github.com/gomods/athens
 require (
 	cloud.google.com/go v0.26.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.6.0 // indirect
+	git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999 // indirect
 	github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876 // indirect
 	github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b
 	github.com/BurntSushi/toml v0.3.0
 	github.com/aws/aws-sdk-go v1.15.24
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
-	github.com/bketelsen/buffet v0.1.5
 	github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
 	github.com/codegangsta/negroni v0.3.0 // indirect
 	github.com/fatih/color v1.7.0
@@ -42,7 +42,6 @@ require (
 	github.com/mitchellh/go-homedir v1.0.0
 	github.com/onsi/ginkgo v1.6.0 // indirect
 	github.com/onsi/gomega v1.4.1 // indirect
-	github.com/opentracing/opentracing-go v1.0.2
 	github.com/prometheus/client_golang v0.8.0 // indirect
 	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
 	github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect
@@ -59,7 +58,7 @@ require (
 	github.com/uber/jaeger-client-go v2.14.0+incompatible
 	github.com/uber/jaeger-lib v1.5.0
 	github.com/unrolled/secure v0.0.0-20180618144512-8287f3899c8e
-	go.opencensus.io v0.15.0 // indirect
+	go.opencensus.io v0.15.0
 	go.uber.org/atomic v1.3.2 // indirect
 	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
 	golang.org/x/text v0.3.0 // indirect

go.sum

@@ -3,6 +3,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 contrib.go.opencensus.io/exporter/stackdriver v0.6.0 h1:U0FQWsZU3aO8W+BrZc88T8fdd24qe3Phawa9V9oaVUE=
 contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw=
 dmitri.shuralyov.com/text/kebabcase v0.0.0-20180217051803-40e40b42552a/go.mod h1:3YpR/7A6nvWHA/oFH66Hp/dJ5A2gM63I3xkA/3FV6tY=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999 h1:sihTnRgTOUSCQz0iS0pjZuFQy/z7GXCJgSBg3+rZKHw=
+git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876 h1:3c3mGlhASTJh6H6Ba9EHv2FDSmEUyJuJHR6UD7b+YuE=
 github.com/Azure/azure-pipeline-go v0.0.0-20180607212504-7571e8eb0876/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
 github.com/Azure/azure-storage-blob-go v0.0.0-20180727221336-197d1c0aea1b h1:7cOe9XtL/0qFd/jb0whfm5NoRmhEcVU3bZ65zUZUz54=
@@ -15,8 +17,6 @@ github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4R
 github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/bketelsen/buffet v0.1.5 h1:KQ+f1YK8APG9R3V/aBibRptLOkIJmDtHF/cvXDHCdOw=
-github.com/bketelsen/buffet v0.1.5/go.mod h1:dN5bkZP+hUjYa/ky3N6xfKg3S/94wr5MeaKIgBaEIqY=
 github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7 h1:XFqp7VFIbbJO1hlpGbzo45NVYWVIM2eMD9MAxrOTVzU=
 github.com/cockroachdb/cockroach-go v0.0.0-20180212155653-59c0560478b7/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
@@ -171,8 +171,6 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1 h1:PZSj/UFNaVp3KxrzHOcS7oyuWA7LoOY/77yCTEFu21U=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg=
-github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=

pkg/config/env/trace.go

@@ -0,0 +1,8 @@
+package env
+
+import "os"
+
+// TraceExporterURL returns where the trace is stored to
+func TraceExporterURL() string {
+	return os.Getenv("TRACE_EXPORTER")
+}

pkg/download/get_module_params.go

@@ -6,7 +6,7 @@ import (
 	"github.com/gomods/athens/pkg/paths"
 )
 
-func getModuleParams(op errors.Op, c buffalo.Context) (mod string, vers string, err error) {
+func getModuleParams(c buffalo.Context, op errors.Op) (mod string, vers string, err error) {
 	params, err := paths.GetAllParams(c)
 	if err != nil {
 		return "", "", errors.E(op, err, errors.KindBadRequest)

pkg/download/latest.go

@@ -3,7 +3,6 @@ package download
 import (
 	"net/http"
 
-	"github.com/bketelsen/buffet"
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/render"
 	"github.com/gomods/athens/pkg/errors"
@@ -18,9 +17,6 @@ const PathLatest = "/{module:.+}/@latest"
 func LatestHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
 	const op errors.Op = "download.LatestHandler"
 	return func(c buffalo.Context) error {
-		sp := buffet.SpanFromContext(c)
-		sp.SetOperationName("latestHandler")
-		defer sp.Finish()
 		mod, err := paths.GetModule(c)
 		if err != nil {
 			lggr.SystemErr(errors.E(op, err))

pkg/download/list.go

@@ -4,7 +4,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/bketelsen/buffet"
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/render"
 	"github.com/gomods/athens/pkg/errors"
@@ -19,8 +18,6 @@ const PathList = "/{module:.+}/@v/list"
 func ListHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
 	const op errors.Op = "download.ListHandler"
 	return func(c buffalo.Context) error {
-		sp := buffet.SpanFromContext(c).SetOperationName("listHandler")
-		defer sp.Finish()
 		mod, err := paths.GetModule(c)
 		if err != nil {
 			lggr.SystemErr(errors.E(op, err))

pkg/download/protocol.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/module"
+	"github.com/gomods/athens/pkg/observ"
 	"github.com/gomods/athens/pkg/stash"
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/spf13/afero"
@@ -69,7 +70,9 @@ type protocol struct {
 }
 
 func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
-	const op errors.Op = "protocol.List"
+	const op errors.Op = "download.protocol.List"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
 	lr, err := p.list(op, mod)
 	if err != nil {
 		return nil, err
@@ -79,7 +82,9 @@ func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
 }
 
 func (p *protocol) Latest(ctx context.Context, mod string) (*storage.RevInfo, error) {
-	const op errors.Op = "protocol.Latest"
+	const op errors.Op = "download.protocol.Latest"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
 	lr, err := p.list(op, mod)
 	if err != nil {
 		return nil, errors.E(op, err)
@@ -143,7 +148,9 @@ func (p *protocol) list(op errors.Op, mod string) (*listResp, error) {
 }
 
 func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
-	const op errors.Op = "protocol.Info"
+	const op errors.Op = "download.protocol.Info"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
 	info, err := p.s.Info(ctx, mod, ver)
 	if errors.IsNotFoundErr(err) {
 		err = p.stasher.Stash(mod, ver)
@@ -160,7 +167,9 @@ func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
 }
 
 func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
-	const op errors.Op = "protocol.GoMod"
+	const op errors.Op = "download.protocol.GoMod"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
 	goMod, err := p.s.GoMod(ctx, mod, ver)
 	if errors.IsNotFoundErr(err) {
 		err = p.stasher.Stash(mod, ver)
@@ -177,7 +186,9 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
 }
 
 func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
-	const op errors.Op = "protocol.Zip"
+	const op errors.Op = "download.protocol.Zip"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
 	zip, err := p.s.Zip(ctx, mod, ver)
 	if errors.IsNotFoundErr(err) {
 		err = p.stasher.Stash(mod, ver)

pkg/download/version_info.go

@@ -3,7 +3,8 @@ package download
 import (
 	"net/http"
 
-	"github.com/bketelsen/buffet"
+	"github.com/gomods/athens/pkg/observ"
+
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/render"
 	"github.com/gomods/athens/pkg/errors"
@@ -17,14 +18,14 @@ const PathVersionInfo = "/{module:.+}/@v/{version}.info"
 func VersionInfoHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
 	const op errors.Op = "download.versionInfoHandler"
 	return func(c buffalo.Context) error {
-		sp := buffet.SpanFromContext(c).SetOperationName("versionInfoHandler")
-		defer sp.Finish()
-		mod, ver, err := getModuleParams(op, c)
+		ctx, span := observ.StartSpan(c, op.String())
+		defer span.End()
+		mod, ver, err := getModuleParams(c, op)
 		if err != nil {
 			lggr.SystemErr(err)
 			return c.Render(errors.Kind(err), nil)
 		}
-		info, err := dp.Info(c, mod, ver)
+		info, err := dp.Info(ctx, mod, ver)
 		if err != nil {
 			lggr.SystemErr(errors.E(op, err, errors.M(mod), errors.V(ver)))
 			return c.Render(errors.Kind(err), nil)

pkg/download/version_module.go

@@ -1,11 +1,11 @@
 package download
 
 import (
-	"github.com/bketelsen/buffet"
 	"github.com/gobuffalo/buffalo"
 	"github.com/gobuffalo/buffalo/render"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/observ"
 )
 
 // PathVersionModule URL.
@@ -15,14 +15,14 @@ const PathVersionModule = "/{module:.+}/@v/{version}.mod"
 func VersionModuleHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
 	const op errors.Op = "download.VersionModuleHandler"
 	return func(c buffalo.Context) error {
-		sp := buffet.SpanFromContext(c).SetOperationName("VersionModuleHandler")
-		defer sp.Finish()
-		mod, ver, err := getModuleParams(op, c)
+		ctx, span := observ.StartSpan(c, op.String())
+		defer span.End()
+		mod, ver, err := getModuleParams(c, op)
 		if err != nil {
 			lggr.SystemErr(err)
 			return c.Render(errors.Kind(err), nil)
 		}
-		modBts, err := dp.GoMod(c, mod, ver)
+		modBts, err := dp.GoMod(ctx, mod, ver)
 		if err != nil {
 			err = errors.E(op, errors.M(mod), errors.V(ver), err)
 			lggr.SystemErr(err)