Code coverage tracking (vitessio#11)

* added code coverage using sonar and codecov

Signed-off-by: Arindam Nayak <[email protected]>
Signed-off-by: Arindam Nayak <[email protected]>

.github/workflows/sonar_analysis.yml

@@ -0,0 +1,55 @@
+name: sonar_analysis
+on:
+  push:
+    branches:
+      - 'master'
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Get dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+          sudo service mysql stop
+          sudo service etcd stop
+          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+          go mod download
+
+      - name: Execute unit test and cluster endtoend test
+        run: |
+          eatmydata -- ./tools/all_test_for_coverage.sh
+          mkdir report
+          cp /tmp/*.out ./report/.
+
+      - name: Analyse sonar
+        run: |
+          export SONAR_SCANNER_VERSION=4.2.0.1873
+          export SONAR_SCANNER_HOME=$HOME/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux
+          rm -rf $SONAR_SCANNER_HOME
+          mkdir -p $SONAR_SCANNER_HOME
+          curl -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip
+          unzip $HOME/.sonar/sonar-sct Analysisanner.zip -d $HOME/.sonar/
+          rm $HOME/.sonar/sonar-scanner.zip
+          export PATH=$SONAR_SCANNER_HOME/bin:$PATH
+          export SONAR_SCANNER_OPTS="-server"
+          sonar-scanner \
+            -Dsonar.projectKey=vitessio \
+            -Dsonar.organization=vitess \
+            -Dsonar.host.url=https://sonarcloud.io \
+            -Dsonar.login=${SONAR_TOKEN} \
+            -Dsonar.go.coverage.reportPaths=report/*.out
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -85,3 +85,6 @@ releases
 /vthook/
 /bin/
 /vtdataroot/
+
+.scannerwork
+report

README.md

@@ -4,6 +4,7 @@
 [![Go Report Card](https://goreportcard.com/badge/vitess.io/vitess)](https://goreportcard.com/report/vitess.io/vitess)
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fvitessio%2Fvitess.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fvitessio%2Fvitess?ref=badge_shield)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1724/badge)](https://bestpractices.coreinfrastructure.org/projects/1724)
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=vitessio&metric=coverage)](https://sonarcloud.io/dashboard?id=vitessio)
 
 # Vitess 
 

go/test/endtoend/cluster/cluster_process.go

@@ -39,6 +39,7 @@ const (
 var (
 	keepData   = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
 	topoFlavor = flag.String("topo-flavor", "etcd2", "choose a topo server from etcd2, zk2 or consul")
+	isCoverage = flag.Bool("is-coverage", false, "whether coverage is required")
 )
 
 // LocalProcessCluster Testcases need to use this to iniate a cluster
@@ -639,3 +640,11 @@ func (cluster *LocalProcessCluster) StartVttablet(tablet *Vttablet, servingStatu
 	tablet.VttabletProcess.ServingStatus = servingStatus
 	return tablet.VttabletProcess.Setup()
 }
+
+func getCoveragePath(fileName string) string {
+	covDir := os.Getenv("COV_DIR")
+	if covDir == "" {
+		covDir = os.TempDir()
+	}
+	return path.Join(covDir, fileName)
+}

go/test/endtoend/cluster/cluster_util.go

@@ -120,3 +120,18 @@ func getTablet(tabletGrpcPort int, hostname string) *tabletpb.Tablet {
 	portMap["grpc"] = int32(tabletGrpcPort)
 	return &tabletpb.Tablet{Hostname: hostname, PortMap: portMap}
 }
+
+func filterResultWhenRunsForCoverage(input string) string {
+	lines := strings.Split(input, "\n")
+	var result string
+	for _, line := range lines {
+		if strings.Contains(line, "=== RUN") {
+			continue
+		}
+		if strings.Contains(line, "--- PASS:") || strings.Contains(line, "PASS") {
+			break
+		}
+		result = result + line + "\n"
+	}
+	return result
+}

go/test/endtoend/cluster/mysqlctl_process.go

@@ -43,14 +43,17 @@ type MysqlctlProcess struct {
 
 // InitDb executes mysqlctl command to add cell info
 func (mysqlctl *MysqlctlProcess) InitDb() (err error) {
-	tmpProcess := exec.Command(
-		mysqlctl.Binary,
-		"-log_dir", mysqlctl.LogDirectory,
+	args := []string{"-log_dir", mysqlctl.LogDirectory,
 		"-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID),
 		"-mysql_port", fmt.Sprintf("%d", mysqlctl.MySQLPort),
 		"init",
-		"-init_db_sql_file", mysqlctl.InitDBFile,
-	)
+		"-init_db_sql_file", mysqlctl.InitDBFile}
+	if *isCoverage {
+		args = append([]string{"-test.coverprofile=" + getCoveragePath("mysql-initdb.out"), "-test.v"}, args...)
+	}
+	tmpProcess := exec.Command(
+		mysqlctl.Binary,
+		args...)
 	return tmpProcess.Run()
 }
 
@@ -71,6 +74,9 @@ func (mysqlctl *MysqlctlProcess) StartProcess() (*exec.Cmd, error) {
 		"-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID),
 		"-mysql_port", fmt.Sprintf("%d", mysqlctl.MySQLPort),
 	)
+	if *isCoverage {
+		tmpProcess.Args = append(tmpProcess.Args, []string{"-test.coverprofile=" + getCoveragePath("mysql-start.out")}...)
+	}
 
 	if len(mysqlctl.ExtraArgs) > 0 {
 		tmpProcess.Args = append(tmpProcess.Args, mysqlctl.ExtraArgs...)
@@ -100,6 +106,9 @@ func (mysqlctl *MysqlctlProcess) StopProcess() (*exec.Cmd, error) {
 		mysqlctl.Binary,
 		"-tablet_uid", fmt.Sprintf("%d", mysqlctl.TabletUID),
 	)
+	if *isCoverage {
+		tmpProcess.Args = append(tmpProcess.Args, []string{"-test.coverprofile=" + getCoveragePath("mysql-stop.out")}...)
+	}
 	if len(mysqlctl.ExtraArgs) > 0 {
 		tmpProcess.Args = append(tmpProcess.Args, mysqlctl.ExtraArgs...)
 	}

go/test/endtoend/cluster/vtctl_process.go

@@ -43,11 +43,15 @@ func (vtctl *VtctlProcess) AddCellInfo(Cell string) (err error) {
 		"-topo_implementation", vtctl.TopoImplementation,
 		"-topo_global_server_address", vtctl.TopoGlobalAddress,
 		"-topo_global_root", vtctl.TopoGlobalRoot,
+	)
+	if *isCoverage {
+		tmpProcess.Args = append(tmpProcess.Args, "-test.coverprofile="+getCoveragePath("vtctl-addcell.out"))
+	}
+	tmpProcess.Args = append(tmpProcess.Args,
 		"AddCellInfo",
 		"-root", vtctl.TopoRootPath+Cell,
 		"-server_address", vtctl.TopoServerAddress,
-		Cell,
-	)
+		Cell)
 	log.Info(fmt.Sprintf("Adding Cell into Keyspace with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	fmt.Println(fmt.Sprintf("Adding Cell into Keyspace with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	return tmpProcess.Run()
@@ -60,8 +64,12 @@ func (vtctl *VtctlProcess) CreateKeyspace(keyspace string) (err error) {
 		"-topo_implementation", vtctl.TopoImplementation,
 		"-topo_global_server_address", vtctl.TopoGlobalAddress,
 		"-topo_global_root", vtctl.TopoGlobalRoot,
-		"CreateKeyspace", keyspace,
 	)
+	if *isCoverage {
+		tmpProcess.Args = append(tmpProcess.Args, "-test.coverprofile="+getCoveragePath("vtctl-create-ks.out"))
+	}
+	tmpProcess.Args = append(tmpProcess.Args,
+		"CreateKeyspace", keyspace)
 	log.Info(fmt.Sprintf("Starting CreateKeyspace with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	return tmpProcess.Run()
 }
@@ -73,13 +81,16 @@ func (vtctl *VtctlProcess) ExecuteCommandWithOutput(args ...string) (result stri
 		"-topo_implementation", vtctl.TopoImplementation,
 		"-topo_global_server_address", vtctl.TopoGlobalAddress,
 		"-topo_global_root", vtctl.TopoGlobalRoot}, args...)
+	if *isCoverage {
+		args = append([]string{"-test.coverprofile=" + getCoveragePath("vtctl-o-"+args[0]+".out"), "-test.v"}, args...)
+	}
 	tmpProcess := exec.Command(
 		vtctl.Binary,
 		args...,
 	)
 	log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	resultByte, err := tmpProcess.CombinedOutput()
-	return string(resultByte), err
+	return filterResultWhenRunsForCoverage(string(resultByte)), err
 }
 
 // ExecuteCommand executes any vtctlclient command
@@ -89,6 +100,9 @@ func (vtctl *VtctlProcess) ExecuteCommand(args ...string) (err error) {
 		"-topo_implementation", vtctl.TopoImplementation,
 		"-topo_global_server_address", vtctl.TopoGlobalAddress,
 		"-topo_global_root", vtctl.TopoGlobalRoot}, args...)
+	if *isCoverage {
+		args = append([]string{"-test.coverprofile=" + getCoveragePath("vtctl-"+args[0]+".out"), "-test.v"}, args...)
+	}
 	tmpProcess := exec.Command(
 		vtctl.Binary,
 		args...,

go/test/endtoend/cluster/vtctlclient_process.go

@@ -62,10 +62,15 @@ func (vtctlclient *VtctlClientProcess) ApplyVSchema(Keyspace string, JSON string
 
 // ExecuteCommand executes any vtctlclient command
 func (vtctlclient *VtctlClientProcess) ExecuteCommand(args ...string) (err error) {
-	args = append([]string{"-server", vtctlclient.Server}, args...)
+	pArgs := []string{"-server", vtctlclient.Server}
+
+	if *isCoverage {
+		pArgs = append(pArgs, "-test.coverprofile="+getCoveragePath("vtctlclient-"+args[0]+".out"), "-test.v")
+	}
+	pArgs = append(pArgs, args...)
 	tmpProcess := exec.Command(
 		vtctlclient.Binary,
-		args...,
+		pArgs...,
 	)
 	println(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
@@ -74,15 +79,19 @@ func (vtctlclient *VtctlClientProcess) ExecuteCommand(args ...string) (err error
 
 // ExecuteCommandWithOutput executes any vtctlclient command and returns output
 func (vtctlclient *VtctlClientProcess) ExecuteCommandWithOutput(args ...string) (result string, err error) {
-	args = append([]string{"-server", vtctlclient.Server}, args...)
+	pArgs := []string{"-server", vtctlclient.Server}
+	if *isCoverage {
+		pArgs = append(pArgs, "-test.coverprofile="+getCoveragePath("vtctlclient-"+args[0]+".out"), "-test.v")
+	}
+	pArgs = append(pArgs, args...)
 	tmpProcess := exec.Command(
 		vtctlclient.Binary,
-		args...,
+		pArgs...,
 	)
 	println(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	log.Info(fmt.Sprintf("Executing vtctlclient with arguments %v", strings.Join(tmpProcess.Args, " ")))
 	resultByte, err := tmpProcess.CombinedOutput()
-	return string(resultByte), err
+	return filterResultWhenRunsForCoverage(string(resultByte)), err
 }
 
 // VtctlClientProcessInstance returns a VtctlProcess handle for vtctlclient process

go/test/endtoend/cluster/vtctld_process.go

@@ -70,6 +70,9 @@ func (vtctld *VtctldProcess) Setup(cell string, extraArgs ...string) (err error)
 		"-grpc_port", fmt.Sprintf("%d", vtctld.GrpcPort),
 		"-pid_file", vtctld.PidFile,
 	)
+	if *isCoverage {
+		vtctld.proc.Args = append(vtctld.proc.Args, "-test.coverprofile="+getCoveragePath("vtctld.out"))
+	}
 	vtctld.proc.Args = append(vtctld.proc.Args, extraArgs...)
 
 	errFile, _ := os.Create(path.Join(vtctld.LogDir, "vtctld-stderr.txt"))

go/test/endtoend/cluster/vtgate_process.go

@@ -83,6 +83,10 @@ func (vtgate *VtgateProcess) Setup() (err error) {
 		"-mysql_auth_server_impl", vtgate.MySQLAuthServerImpl,
 		"-pid_file", vtgate.PidFile,
 	)
+	if *isCoverage {
+		vtgate.proc.Args = append(vtgate.proc.Args, "-test.coverprofile="+getCoveragePath("vtgate.out"))
+	}
+
 	vtgate.proc.Args = append(vtgate.proc.Args, vtgate.ExtraArgs...)
 
 	errFile, _ := os.Create(path.Join(vtgate.LogDir, "vtgate-stderr.txt"))

go/test/endtoend/cluster/vttablet_process.go

@@ -99,6 +99,9 @@ func (vttablet *VttabletProcess) Setup() (err error) {
 		"-service_map", vttablet.ServiceMap,
 		"-vtctld_addr", vttablet.VtctldAddress,
 	)
+	if *isCoverage {
+		vttablet.proc.Args = append(vttablet.proc.Args, "-test.coverprofile="+getCoveragePath("vttablet.out"))
+	}
 
 	if vttablet.SupportsBackup {
 		vttablet.proc.Args = append(vttablet.proc.Args, "-restore_from_backup")

go/test/endtoend/cluster/vtworker_process.go

@@ -68,6 +68,9 @@ func (vtworker *VtworkerProcess) Setup(cell string) (err error) {
 		"-cell", cell,
 		"-command_display_interval", "10ms",
 	)
+	if *isCoverage {
+		vtworker.proc.Args = append(vtworker.proc.Args, "-test.coverprofile=vtworker.out", "-test.v")
+	}
 	vtworker.proc.Args = append(vtworker.proc.Args, vtworker.ExtraArgs...)
 
 	vtworker.proc.Stderr = os.Stderr
@@ -140,6 +143,9 @@ func (vtworker *VtworkerProcess) TearDown() error {
 func (vtworker *VtworkerProcess) ExecuteCommand(args ...string) (err error) {
 	args = append([]string{"-vtworker_client_protocol", "grpc",
 		"-server", vtworker.Server, "-log_dir", vtworker.LogDir, "-stderrthreshold", "info"}, args...)
+	if *isCoverage {
+		args = append([]string{"-test.coverprofile=" + getCoveragePath("vtworkerclient-exec-cmd.out")}, args...)
+	}
 	tmpProcess := exec.Command(
 		"vtworkerclient",
 		args...,
@@ -162,6 +168,9 @@ func (vtworker *VtworkerProcess) ExecuteVtworkerCommand(port int, grpcPort int,
 		"-grpc_port", fmt.Sprintf("%d", grpcPort),
 		"-cell", vtworker.Cell,
 		"-log_dir", vtworker.LogDir, "-stderrthreshold", "1"}, args...)
+	if *isCoverage {
+		args = append([]string{"-test.coverprofile=" + getCoveragePath("vtworker-exec-cmd.out")}, args...)
+	}
 	tmpProcess := exec.Command(
 		"vtworker",
 		args...,

sonar-project.properties

@@ -0,0 +1,12 @@
+sonar.projectKey=vitessio
+sonar.projectName=vitessio
+sonar.sources=go
+sonar.exclusions=**/*_test.go,**/vendor/**,**/java/**,go/test/endtoend/**,**/*.pb.go
+sonar.tests=.
+sonar.test.inclusions=**/*_test.go
+sonar.test.exclusions=**/vendor/**
+
+
+# Change the host.url to point to the
+# sonarqube server (default localhost)
+sonar.host.url=http://localhost:9000

tools/all_test_for_coverage.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# Copyright 2019 The Vitess Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These test uses excutables and launch them as process
+# After that all tests run, here we are testing those
+
+# All Go packages with test files.
+# Output per line: <full Go package name> <all _test.go files in the package>*
+
+
+### Execute unit testcase ###
+source build.env
+make tools
+make build
+echo "--------- executing unit testcases ---------"
+packages_with_all_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/... | sort)
+all_except_endtoend_tests=$(echo "$packages_with_all_tests" | grep -v "endtoend" | cut -d" " -f1 )
+
+counter=0
+for pkg in $all_except_endtoend_tests
+do
+   go test -coverpkg=vitess.io/vitess/go/... -coverprofile "/tmp/unit_$counter.out" $pkg -v -p=1 || :
+   counter=$((counter+1))
+done
+
+## Copy the test files to get instrumented binaries ###
+cp ./tools/coverage-go/vtctl_test.go ./go/cmd/vtctl/vtctl_test.go
+cp ./tools/coverage-go/vtctld_test.go ./go/cmd/vtctld/vtctld_test.go
+cp ./tools/coverage-go/mysqlctl_test.go ./go/cmd/mysqlctl/mysqlctl_test.go
+cp ./tools/coverage-go/vtctlclient_test.go ./go/cmd/vtctlclient/vtctlclient_test.go
+cp ./tools/coverage-go/vttablet_test.go ./go/cmd/vttablet/vttablet_test.go
+cp ./tools/coverage-go/vtgate_test.go ./go/cmd/vtgate/vtgate_test.go
+cp ./tools/coverage-go/vtworker_test.go ./go/cmd/vtworker/vtworker_test.go
+cp ./tools/coverage-go/vtworkerclient_test.go ./go/cmd/vtworkerclient/vtworkerclient_test.go
+
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtctl -o ./bin/vtctl
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtctld -o ./bin/vtctld
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/mysqlctl -o ./bin/mysqlctl
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtctlclient -o ./bin/vtctlclient
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vttablet -o ./bin/vttablet
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtgate -o ./bin/vtgate
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtworker -o ./bin/vtworker
+go test -coverpkg=vitess.io/vitess/go/...  -c vitess.io/vitess/go/cmd/vtworkerclient -o ./bin/vtworkerclient
+
+### Execute go/test/endtoend testcase ###
+echo "--------- executing endtoend testcases ---------"
+cluster_tests=$(echo "$packages_with_all_tests" | grep -E "go/test/endtoend" | cut -d" " -f1)
+
+
+# Run cluster test sequentially
+for i in $cluster_tests
+do
+   echo "starting test for $i"
+   go test  $i -v -p=1 -is-coverage=true || :
+done
+