[Metricbeat] Run Go integration tests sequentially (elastic#15114)

auditbeat/magefile.go

@@ -31,6 +31,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/integtest"
 )
 
 func init() {
@@ -149,13 +151,6 @@ func Docs() {
 	mg.Deps(auditbeat.ModuleDocs, auditbeat.FieldDocs)
 }
 
-// IntegTest executes integration tests (it uses Docker to run the tests).
-func IntegTest() {
-	devtools.AddIntegTestUsage()
-	defer devtools.StopIntegTestEnv()
-	mg.SerialDeps(GoIntegTest, PythonIntegTest)
-}
-
 // UnitTest executes the unit tests.
 func UnitTest() {
 	mg.SerialDeps(GoUnitTest, PythonUnitTest)
@@ -169,29 +164,8 @@ func GoUnitTest(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
 }
 
-// GoIntegTest executes the Go integration tests.
-// Use TEST_COVERAGE=true to enable code coverage profiling.
-// Use RACE_DETECTOR=true to enable the race detector.
-func GoIntegTest(ctx context.Context) error {
-	mg.Deps(Fields)
-	return devtools.RunIntegTest("goIntegTest", func() error {
-		return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-	})
-}
-
 // PythonUnitTest executes the python system tests.
 func PythonUnitTest() error {
 	mg.Deps(devtools.BuildSystemTestBinary)
 	return devtools.PythonNoseTest(devtools.DefaultPythonTestUnitArgs())
 }
-
-// PythonIntegTest executes the python system tests in the integration environment (Docker).
-func PythonIntegTest(ctx context.Context) error {
-	if !devtools.IsInIntegTestEnv() {
-		mg.SerialDeps(Fields, Dashboards)
-	}
-	return devtools.RunIntegTest("pythonIntegTest", func() error {
-		mg.Deps(devtools.BuildSystemTestBinary)
-		return devtools.PythonNoseTest(devtools.DefaultPythonTestIntegrationArgs())
-	})
-}

dev-tools/mage/gotest.go

@@ -74,6 +74,22 @@ func makeGoTestArgs(name string) GoTestArgs {
 	return params
 }
 
+func makeGoTestArgsForModule(name, module string) GoTestArgs {
+	fileName := fmt.Sprintf("build/TEST-go-%s-%s", strings.Replace(strings.ToLower(name), " ", "_", -1),
+		strings.Replace(strings.ToLower(module), " ", "_", -1))
+	params := GoTestArgs{
+		TestName:        fmt.Sprintf("%s-%s", name, module),
+		Race:            RaceDetector,
+		Packages:        []string{fmt.Sprintf("./module/%s/...", module)},
+		OutputFile:      fileName + ".out",
+		JUnitReportFile: fileName + ".xml",
+	}
+	if TestCoverage {
+		params.CoverageProfileFile = fileName + ".cov"
+	}
+	return params
+}
+
 // DefaultGoTestUnitArgs returns a default set of arguments for running
 // all unit tests. We tag unit test files with '!integration'.
 func DefaultGoTestUnitArgs() GoTestArgs { return makeGoTestArgs("Unit") }
@@ -86,6 +102,14 @@ func DefaultGoTestIntegrationArgs() GoTestArgs {
 	return args
 }
 
+// GoTestIntegrationArgsForModule returns a default set of arguments for running
+// module integration tests. We tag integration test files with 'integration'.
+func GoTestIntegrationArgsForModule(module string) GoTestArgs {
+	args := makeGoTestArgsForModule("Integration", module)
+	args.Tags = append(args.Tags, "integration")
+	return args
+}
+
 // DefaultTestBinaryArgs returns the default arguments for building
 // a binary for testing.
 func DefaultTestBinaryArgs() TestBinaryArgs {
@@ -94,6 +118,39 @@ func DefaultTestBinaryArgs() TestBinaryArgs {
 	}
 }
 
+// GoTestIntegrationForModule executes the Go integration tests sequentially.
+// Currently all test cases must be present under "./module" directory.
+//
+// Motivation: previous implementation executed all integration tests at once,
+// causing high CPU load, high memory usage and resulted in timeouts.
+//
+// This method executes integration tests for a single module at a time.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoTestIntegrationForModule(ctx context.Context) error {
+	return RunIntegTest("goIntegTest", func() error {
+		modulesFileInfo, err := ioutil.ReadDir("./module")
+		if err != nil {
+			return err
+		}
+
+		var failed bool
+		for _, fi := range modulesFileInfo {
+			if !fi.IsDir() {
+				continue
+			}
+			err := GoTest(ctx, GoTestIntegrationArgsForModule(fi.Name()))
+			if err != nil {
+				failed = true
+			}
+		}
+		if failed {
+			return errors.New("integration tests failed")
+		}
+		return nil
+	})
+}
+
 // GoTest invokes "go test" and reports the results to stdout. It returns an
 // error if there was any failure executing the tests or if there were any
 // test failures.
@@ -102,9 +159,8 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 
 	// Build args list to Go.
 	args := []string{"test"}
-	if mg.Verbose() {
-		args = append(args, "-v")
-	}
+	args = append(args, "-v")
+
 	if params.Race {
 		args = append(args, "-race")
 	}

dev-tools/mage/target/integtest/integtest.go

@@ -62,12 +62,7 @@ func IntegTest() {
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
 func GoIntegTest(ctx context.Context) error {
-	if !devtools.IsInIntegTestEnv() {
-		mg.SerialDeps(goTestDeps...)
-	}
-	return devtools.RunIntegTest("goIntegTest", func() error {
-		return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-	}, whitelistedEnvVars...)
+	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
 }
 
 // PythonIntegTest executes the python system tests in the integration environment (Docker).

dev-tools/mage/target/integtest/notests/integtest.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package notests
+
+import "fmt"
+
+// IntegTest method fallbacks to GoIntegTest()
+func IntegTest() {
+	GoIntegTest()
+}
+
+// GoIntegTest method informs that no integration tests will be executed.
+func GoIntegTest() {
+	fmt.Println(">> integTest: Complete (no tests require the integ test environment)")
+}

dev-tools/make/xpack.mk

@@ -44,7 +44,7 @@ stop-environment:
 
 .PHONY: testsuite
 testsuite: mage
-	-rm build/TEST-go-integration.out
+	rm -f build/TEST-go-integration.out
 	mage update build unitTest integTest || ( cat build/TEST-go-integration.out && false )
 
 .PHONY: update

filebeat/magefile.go

@@ -196,9 +196,7 @@ func GoUnitTest(ctx context.Context) error {
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
 func GoIntegTest(ctx context.Context) error {
-	return devtools.RunIntegTest("goIntegTest", func() error {
-		return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-	})
+	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
 }
 
 // PythonUnitTest executes the python system tests.

heartbeat/magefile.go

@@ -33,6 +33,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/integtest/notests"
 )
 
 func init() {
@@ -127,13 +129,6 @@ func GoTestUnit(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
 }
 
-// GoTestIntegration executes the Go integration tests.
-// Use TEST_COVERAGE=true to enable code coverage profiling.
-// Use RACE_DETECTOR=true to enable the race detector.
-func GoTestIntegration(ctx context.Context) error {
-	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-}
-
 func customizePackaging() {
 	monitorsDTarget := "monitors.d"
 	unixMonitorsDir := "/etc/{{.BeatName}}/monitors.d"

journalbeat/magefile.go

@@ -33,6 +33,8 @@ import (
 
 	// mage:import
 	"github.com/elastic/beats/dev-tools/mage/target/common"
+	// mage:import
+	_ "github.com/elastic/beats/dev-tools/mage/target/integtest/notests"
 )
 
 func init() {
@@ -119,13 +121,6 @@ func GoTestUnit(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
 }
 
-// GoTestIntegration executes the Go integration tests.
-// Use TEST_COVERAGE=true to enable code coverage profiling.
-// Use RACE_DETECTOR=true to enable the race detector.
-func GoTestIntegration(ctx context.Context) error {
-	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-}
-
 // -----------------------------------------------------------------------------
 // Customizations specific to Journalbeat.
 // - Install required headers on builders for different architectures.

libbeat/magefile.go

@@ -45,14 +45,14 @@ func GoTestUnit(ctx context.Context) error {
 	return devtools.GoTest(ctx, devtools.DefaultGoTestUnitArgs())
 }
 
-// GoTestIntegration executes the Go integration tests.
-// Use TEST_COVERAGE=true to enable code coverage profiling.
-// Use RACE_DETECTOR=true to enable the race detector.
-func GoTestIntegration(ctx context.Context) error {
-	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
-}
-
 // Config generates example and reference configuration for libbeat.
 func Config() error {
 	return devtools.Config(devtools.ShortConfigType|devtools.ReferenceConfigType, devtools.ConfigFileParams{}, ".")
 }
+
+// GoIntegTest executes the Go integration tests.
+// Use TEST_COVERAGE=true to enable code coverage profiling.
+// Use RACE_DETECTOR=true to enable the race detector.
+func GoIntegTest(ctx context.Context) error {
+	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+}

libbeat/outputs/kafka/client.go

@@ -192,9 +192,6 @@ func (c *client) getEventMessage(data *publisher.Event) (*message, error) {
 			return nil, errNoTopicsSelected
 		}
 		msg.topic = topic
-		if logp.IsDebug(debugSelector) {
-			debugf("setting event.Meta[\"topic\"] = %v", topic)
-		}
 		if _, err := data.Cache.Put("topic", topic); err != nil {
 			return nil, fmt.Errorf("setting kafka topic in publisher event failed: %v", err)
 		}

libbeat/outputs/kafka/partition.go

@@ -128,9 +128,6 @@ func (p *messagePartitioner) Partition(
 
 	msg.partition = partition
 
-	if logp.IsDebug(debugSelector) {
-		debugf("setting event.Meta[\"partition\"] = %v", partition)
-	}
 	if _, err := msg.data.Cache.Put("partition", partition); err != nil {
 		return 0, fmt.Errorf("setting kafka partition in publisher event failed: %v", err)
 	}

libbeat/outputs/redis/redis_integration_test.go

@@ -217,8 +217,6 @@ func testPublishChannel(t *testing.T, cfg map[string]interface{}) {
 	var messages [][]byte
 	assert.NoError(t, conn.Err())
 	for conn.Err() == nil {
-		t.Logf("try collect message")
-
 		switch v := psc.Receive().(type) {
 		case redis.Message:
 			messages = append(messages, v.Data)

libbeat/processors/script/javascript/session.go

@@ -96,8 +96,7 @@ func newSession(p *goja.Program, conf Config, test bool) (*session, error) {
 	}
 
 	// Register modules.
-	for name, registerModule := range sessionHooks {
-		s.log.Debugf("Registering module %v with the Javascript runtime.", name)
+	for _, registerModule := range sessionHooks {
 		registerModule(s)
 	}
 

libbeat/publisher/queue/spool/spool_test.go

@@ -44,6 +44,8 @@ type testLogger struct {
 	t *testing.T
 }
 
+type silentLogger struct{}
+
 func init() {
 	flag.Int64Var(&seed, "seed", time.Now().UnixNano(), "test random seed")
 	flag.BoolVar(&debug, "noisy", false, "print test logs to console")
@@ -95,7 +97,14 @@ func makeTestQueue(
 			}
 		}()
 
-		spool, err := NewSpool(&testLogger{t}, path, Settings{
+		var logger logger
+		if debug {
+			logger = &testLogger{t}
+		} else {
+			logger = new(silentLogger)
+		}
+
+		spool, err := NewSpool(logger, path, Settings{
 			WriteBuffer:       writeBuffer,
 			WriteFlushTimeout: flushTimeout,
 			Codec:             codecCBORL,
@@ -133,15 +142,18 @@ func (l *testLogger) Errorf(fmt string, vs ...interface{}) { l.reportf("Error",
 func (l *testLogger) report(level string, vs []interface{}) {
 	args := append([]interface{}{level, ":"}, vs...)
 	l.t.Log(args...)
-	if debug {
-		fmt.Println(args...)
-	}
+	fmt.Println(args...)
 }
 
 func (l *testLogger) reportf(level string, str string, vs []interface{}) {
 	str = level + ": " + str
 	l.t.Logf(str, vs...)
-	if debug {
-		fmt.Printf(str, vs...)
-	}
+	fmt.Printf(str, vs...)
 }
+
+func (*silentLogger) Debug(vs ...interface{})              {}
+func (*silentLogger) Debugf(fmt string, vs ...interface{}) {}
+func (*silentLogger) Info(vs ...interface{})               {}
+func (*silentLogger) Infof(fmt string, vs ...interface{})  {}
+func (*silentLogger) Error(vs ...interface{})              {}
+func (*silentLogger) Errorf(fmt string, vs ...interface{}) {}

libbeat/scripts/Makefile

@@ -195,11 +195,10 @@ unit: ## @testing Runs the unit tests without coverage reports.
 
 .PHONY: integration-tests
 integration-tests: ## @testing Run integration tests. Unit tests are run as part of the integration tests.
-integration-tests: prepare-tests
+integration-tests: prepare-tests mage
 	rm -f docker-compose.yml.lock
-	$(COVERAGE_TOOL) -tags=integration $(RACE) -coverprofile=${COVERAGE_DIR}/integration.cov ${GOPACKAGES}
+	mage goIntegTest
 
-#
 .PHONY: integration-tests-environment
 integration-tests-environment:  ## @testing Runs the integration inside a virtual environment. This can be run on any docker-machine (local, remote)
 integration-tests-environment: prepare-tests build-image