Benchmark storages (gomods#426)

* Adding benchmark for mongo and fs

* Adding delete, exists, save with other storages

* Fixing typos, adding nonexistent module exits benchmark

* Fixing delete operation in delete benchmark

* Separating non testing existing modules benchmark, shortening the benchmark name

* running benchmark in ci

* Revert "running benchmark in ci" - mongo index make duplicate records to
fail
idempotent, will fix in a separate PR.

This reverts commit f7d7826.

* Adding reset timer, fixing duplicate record issue

* Revert "Revert "running benchmark in ci" - mongo index make duplicate records to"

This reverts commit 40a7b33.

* removing benchmark from ci

* Adding operation for returning errors

Makefile

@@ -39,6 +39,8 @@ test-e2e:
 olympus-docker:
 	docker build -t gopackages/olympus -f cmd/olympus/Dockerfile .
 
+bench:
+	./scripts/benchmark.sh
 
 .PHONY: alldeps
 alldeps:

pkg/config/env/mongo.go

@@ -2,6 +2,8 @@ package env
 
 import (
 	"fmt"
+	"strconv"
+	"time"
 
 	"github.com/gobuffalo/envy"
 )
@@ -56,10 +58,15 @@ func MongoPassword() (string, error) {
 	return env, nil
 }
 
-// MongoConnectionTimeoutWithDefault returns Athens Mongo Storage connection timeout defined by MONGO_CONN_TIMEOUT_SEC.
+// MongoConnectionTimeoutSecWithDefault returns Athens Mongo Storage connection timeout defined by MONGO_CONN_TIMEOUT_SEC.
 // Values are in seconds.
-func MongoConnectionTimeoutWithDefault(value string) string {
-	return envy.Get("MONGO_CONN_TIMEOUT_SEC", value)
+func MongoConnectionTimeoutSecWithDefault(defTimeout int) time.Duration {
+	timeoutConf := envy.Get("MONGO_CONN_TIMEOUT_SEC", strconv.Itoa(defTimeout))
+	timeout, err := strconv.ParseInt(timeoutConf, 10, 32)
+	if err != nil {
+		return time.Duration(defTimeout) * time.Second
+	}
+	return time.Duration(timeout) * time.Second
 }
 
 // MongoSSLWithDefault returns Athens Mongo Storage SSL flag defined by MONGO_SSL.

pkg/storage/fs/test_suite.go

@@ -46,6 +46,6 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 }
 
 // Cleanup tears down test
-func (ts *TestSuite) Cleanup() {
-	ts.Require().NoError(ts.fs.RemoveAll(ts.rootDir))
+func (ts *TestSuite) Cleanup() error {
+	return ts.fs.RemoveAll(ts.rootDir)
 }

pkg/storage/mem/test_suite.go

@@ -32,5 +32,6 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 }
 
 // Cleanup tears down test
-func (ts *TestSuite) Cleanup() {
+func (ts *TestSuite) Cleanup() error {
+	return nil
 }

pkg/storage/minio/test_suite.go

@@ -42,12 +42,16 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 }
 
 // Cleanup tears down test
-func (ts *TestSuite) Cleanup() {
+func (ts *TestSuite) Cleanup() error {
 	minioClient, _ := minio.New(ts.endpoint, ts.accessKeyID, ts.secretAccessKey, false)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 	objectCh := minioClient.ListObjectsV2(ts.bucketName, "", true, doneCh)
 	for object := range objectCh {
-		ts.Require().NoError(minioClient.RemoveObject(ts.bucketName, object.Key))
+		//TODO: could return multi error and clean other objects
+		if err := minioClient.RemoveObject(ts.bucketName, object.Key); err != nil {
+			return err
+		}
 	}
+	return nil
 }

pkg/storage/mongo/mongo.go

@@ -4,6 +4,8 @@ import (
 	"strings"
 
 	"github.com/globalsign/mgo"
+	"github.com/gomods/athens/pkg/config/env"
+	"github.com/gomods/athens/pkg/errors"
 )
 
 // ModuleStore represents a mongo backed storage backend.
@@ -23,9 +25,11 @@ func NewStorage(url string) *ModuleStore {
 
 // Connect conntect the the newly created mongo backend.
 func (m *ModuleStore) Connect() error {
-	s, err := mgo.Dial(m.url)
+	const op errors.Op = "mongo.Connect"
+	timeout := env.MongoConnectionTimeoutSecWithDefault(1)
+	s, err := mgo.DialWithTimeout(m.url, timeout)
 	if err != nil {
-		return err
+		return errors.E(op, err)
 	}
 	m.s = s
 

pkg/storage/mongo/test_suite.go

@@ -3,6 +3,7 @@ package mongo
 import (
 	"fmt"
 
+	"github.com/globalsign/mgo"
 	"github.com/gobuffalo/suite"
 	"github.com/gomods/athens/pkg/config/env"
 	"github.com/gomods/athens/pkg/storage"
@@ -11,11 +12,22 @@ import (
 // TestSuite implements storage.TestSuite interface
 type TestSuite struct {
 	*suite.Model
-	storage storage.Backend
+	storage *ModuleStore
 }
 
 // NewTestSuite creates a common test suite
 func NewTestSuite(model *suite.Model) (storage.TestSuite, error) {
+	ms, err := newTestStore()
+	if err != nil {
+		return nil, err
+	}
+	return &TestSuite{
+		storage: ms,
+		Model:   model,
+	}, err
+}
+
+func newTestStore() (*ModuleStore, error) {
 	muri, err := env.MongoURI()
 	if err != nil {
 		return nil, err
@@ -26,12 +38,7 @@ func NewTestSuite(model *suite.Model) (storage.TestSuite, error) {
 		return nil, fmt.Errorf("Mongo storage is nil")
 	}
 
-	err = mongoStore.Connect()
-
-	return &TestSuite{
-		storage: mongoStore,
-		Model:   model,
-	}, err
+	return mongoStore, mongoStore.Connect()
 }
 
 // Storage retrieves initialized storage backend
@@ -45,5 +52,16 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 }
 
 // Cleanup tears down test
-func (ts *TestSuite) Cleanup() {
+func (ts *TestSuite) Cleanup() error {
+	muri, err := env.MongoURI()
+	if err != nil {
+		return err
+	}
+	timeout := env.MongoConnectionTimeoutSecWithDefault(1)
+	s, err := mgo.DialWithTimeout(muri, timeout)
+	defer s.Close()
+	if err != nil {
+		return err
+	}
+	return s.DB("athens").C("modules").DropCollection()
 }

pkg/storage/rdbms/test_suite.go

@@ -33,5 +33,6 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 }
 
 // Cleanup tears down test
-func (ts *TestSuite) Cleanup() {
+func (ts *TestSuite) Cleanup() error {
+	return nil
 }

pkg/storage/storage_tests/module_storage/storage_benchmark_test.go

@@ -0,0 +1,149 @@
+package modulestorage
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/gobuffalo/suite"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/gomods/athens/pkg/storage/fs"
+	"github.com/gomods/athens/pkg/storage/mem"
+	"github.com/gomods/athens/pkg/storage/minio"
+	"github.com/gomods/athens/pkg/storage/mongo"
+	"github.com/gomods/athens/pkg/storage/rdbms"
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkStorageList(b *testing.B) {
+	module, version := "athens_module", "1.0.1"
+	zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
+
+	for _, store := range getStores(b) {
+
+		backend := store.Storage()
+
+		require.NoError(b, backend.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info), "Save for storage %s failed", backend)
+
+		b.ResetTimer()
+		b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				_, err := backend.List(context.Background(), module)
+				require.NoError(b, err, "Error in listing module")
+			}
+		})
+
+		require.NoError(b, store.Cleanup())
+	}
+}
+
+func BenchmarkStorageSave(b *testing.B) {
+	module, version := "athens_module", "1.0.1"
+	zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
+
+	for _, store := range getStores(b) {
+
+		backend := store.Storage()
+
+		b.ResetTimer()
+		mi := 0
+		b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				err := backend.Save(context.Background(), fmt.Sprintf("save-%s-%d", module, mi), version, mod, bytes.NewReader(zip), info)
+				require.NoError(b, err)
+				mi++
+			}
+		})
+
+		require.NoError(b, store.Cleanup())
+	}
+}
+
+func BenchmarkStorageSaveAndDelete(b *testing.B) {
+	module, version := "athens_module", "1.0.1"
+	zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
+
+	for _, store := range getStores(b) {
+
+		backend := store.Storage()
+
+		b.ResetTimer()
+		b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				name := fmt.Sprintf("del-%s-%d", module, i)
+				err := backend.Save(context.Background(), name, version, mod, bytes.NewReader(zip), info)
+				require.NoError(b, err, "save for storage %s module: %s failed", backend, name)
+				err = backend.Delete(context.Background(), name, version)
+				require.NoError(b, err, "delete failed: %s", name)
+			}
+		})
+
+		require.NoError(b, store.Cleanup())
+	}
+}
+
+func BenchmarkStorageDeleteNonExistingModules(b *testing.B) {
+	module, version := "random-module", "version"
+	for _, store := range getStores(b) {
+		backend := store.Storage()
+
+		b.ResetTimer()
+		b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				err := backend.Delete(context.Background(), fmt.Sprintf("del-%s-%d", module, i), version)
+				require.Equal(b, errors.KindNotFound, errors.Kind(err))
+			}
+		})
+	}
+}
+
+func BenchmarkStorageExists(b *testing.B) {
+	module, version := "athens_module", "1.0.1"
+	zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
+	moduleName := fmt.Sprintf("existing-%s", module)
+
+	for _, store := range getStores(b) {
+		backend := store.Storage()
+		err := backend.Save(context.Background(), moduleName, version, mod, bytes.NewReader(zip), info)
+		require.NoError(b, err, "exists for storage %s failed", backend)
+
+		b.ResetTimer()
+		b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				require.True(b, backend.Exists(context.Background(), moduleName, version))
+			}
+		})
+
+		require.NoError(b, store.Cleanup())
+	}
+}
+
+func getStores(b *testing.B) []storage.TestSuite {
+	var stores []storage.TestSuite
+
+	//TODO: create the instance without model or TestSuite
+	model := suite.NewModel()
+	fsStore, err := fs.NewTestSuite(model)
+	require.NoError(b, err, "couldn't create filesystem store")
+	stores = append(stores, fsStore)
+
+	mongoStore, err := mongo.NewTestSuite(model)
+	require.NoError(b, err, "couldn't create mongo store")
+	stores = append(stores, mongoStore)
+
+	rdbmsStore, err := rdbms.NewTestSuite(model)
+	require.NoError(b, err, "couldn't create mongo store")
+	stores = append(stores, rdbmsStore)
+
+	memStore, err := mem.NewTestSuite(model)
+	require.NoError(b, err)
+	stores = append(stores, memStore)
+
+	minioStore, err := minio.NewTestSuite(model)
+	require.NoError(b, err)
+	stores = append(stores, minioStore)
+
+	return stores
+}

pkg/storage/test_suite.go

@@ -4,5 +4,5 @@ package storage
 type TestSuite interface {
 	Storage() Backend
 	StorageHumanReadableName() string
-	Cleanup()
+	Cleanup() error
 }

scripts/benchmark.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+go test -v -bench=. $(find . -iname '*storage*test.go' -not -path '/vendor/') -run=^$