Add descriptive names to vrep shards. Update test generator script

Signed-off-by: Rohit Nayak <[email protected]>

.github/workflows/cluster_endtoend_11.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (11)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_12.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (12)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_13.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (13)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_14.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (14)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_15.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (15)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_16.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (16)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_17.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (17)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_18.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (18)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_19.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (19)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_20.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (20)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_21.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (21)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_22.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (22)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_23.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (23)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_24.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (24)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_26.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (26)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_27.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (27)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_vreplication_basic.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Cluster (vreplication_basic)
 on: [push, pull_request]
 jobs:

.github/workflows/cluster_endtoend_vreplication_cellalias.yml

@@ -0,0 +1,40 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
+name: Cluster (vreplication_cellalias)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (vreplication_cellalias)
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vreplication_cellalias

.github/workflows/cluster_endtoend_vreplication_multicell.yml

@@ -0,0 +1,40 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
+name: Cluster (vreplication_multicell)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (vreplication_multicell)
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vreplication_multicell

.github/workflows/cluster_endtoend_vreplication_v2.yml

@@ -0,0 +1,40 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
+name: Cluster (vreplication_v2)
+on: [push, pull_request]
+jobs:
+
+  build:
+    name: Run endtoend tests on Cluster (vreplication_v2)
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Set up Go
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.15
+
+    - name: Check out code
+      uses: actions/checkout@v2
+
+    - name: Get dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
+        sudo service mysql stop
+        sudo service etcd stop
+        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
+        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
+        go mod download
+
+        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get install -y gnupg2
+        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+        sudo apt-get update
+        sudo apt-get install percona-xtrabackup-24
+
+    - name: Run cluster endtoend test
+      timeout-minutes: 30
+      run: |
+        source build.env
+        eatmydata -- go run test.go -docker=false -print-log -follow -shard vreplication_v2

.github/workflows/unit_test_mariadb101.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (mariadb101)
 on: [push, pull_request]
 jobs:

.github/workflows/unit_test_mariadb102.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (mariadb102)
 on: [push, pull_request]
 jobs:

.github/workflows/unit_test_mariadb103.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (mariadb103)
 on: [push, pull_request]
 jobs:

.github/workflows/unit_test_mysql57.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (mysql57)
 on: [push, pull_request]
 jobs:

.github/workflows/unit_test_mysql80.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (mysql80)
 on: [push, pull_request]
 jobs:

.github/workflows/unit_test_percona56.yml

@@ -1,3 +1,5 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go
+
 name: Unit Test (percona56)
 on: [push, pull_request]
 jobs:

test/ci_workflow_gen.go

@@ -31,17 +31,16 @@ const (
 	unitTestTemplate  = "templates/unit_test.tpl"
 	unitTestDatabases = "percona56, mysql57, mysql80, mariadb101, mariadb102, mariadb103"
 
-	clusterTestTemplate        = "templates/cluster_endtoend_test.tpl"
-	clusterList                = "11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 27, vreplication_basic"
-	clustersRequiringMakeTools = "18,24"
-
+	clusterTestTemplate         = "templates/cluster_endtoend_test.tpl"
+	clusterList                 = "11,12,13,14,15,16,17,18,19,20,21,22,23,24,26,27,vreplication_basic,vreplication_multicell,vreplication_cellalias,vreplication_v2"
 	// TODO: currently some percona tools including xtrabackup are installed on all clusters, we can possibly optimize
 	// this by only installing them in the required clusters
 	clustersRequiringXtraBackup = clusterList
+	clustersRequiringMakeTools  = "18,24"
 )
 
 type unitTest struct {
-	Name, Database string
+	Name, Platform string
 }
 
 type clusterTest struct {
@@ -113,7 +112,7 @@ func generateUnitTestWorkflows() {
 	for _, platform := range platforms {
 		test := &unitTest{
 			Name:     fmt.Sprintf("Unit Test (%s)", platform),
-			Database: platform,
+			Platform: platform,
 		}
 		path := fmt.Sprintf("%s/unit_test_%s.yml", workflowConfigDir, platform)
 		generateWorkflowFile(unitTestTemplate, path, test)
@@ -139,6 +138,7 @@ func generateWorkflowFile(templateFile, path string, test interface{}) {
 		log.Println("Error creating file: ", err)
 		return
 	}
+	f.WriteString("# DO NOT MODIFY: THIS FILE IS GENERATED USING test/ci_workflow_gen.go\n\n")
 	f.WriteString(mergeBlankLines(buf))
 	fmt.Printf("Generated %s\n", path)
 

test/config.json

@@ -594,17 +594,17 @@
 			"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "MultiCell"],
 			"Command": [],
 			"Manual": false,
-			"Shard": "22",
-			"RetryMax": 3,
+			"Shard": "vreplication_multicell",
+			"RetryMax": 0,
 			"Tags": []
 		},
 		"vreplication_cellalias": {
 			"File": "unused.go",
 			"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "CellAlias"],
 			"Command": [],
 			"Manual": false,
-			"Shard": "23",
-			"RetryMax": 3,
+			"Shard": "vreplication_cellalias",
+			"RetryMax": 0,
 			"Tags": []
 		},
 		"vreplication_basic": {
@@ -613,7 +613,7 @@
 			"Command": [],
 			"Manual": false,
 			"Shard": "vreplication_basic",
-			"RetryMax": 3,
+			"RetryMax": 0,
 			"Tags": []
 		},
 		"orchestrator": {
@@ -639,8 +639,8 @@
 			"Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestBasicV2Workflows"],
 			"Command": [],
 			"Manual": false,
-			"Shard": "21",
-			"RetryMax": 3,
+			"Shard": "vreplication_v2",
+			"RetryMax": 0,
 			"Tags": []
 		}
 	}