CI updates

- Add support for running individual test in a tests suite - building integration and longhaul test binaries - Addiing support for skipping CI or running just smoke test on a PR.
devyanikota · Sep 21, 2017 · 21228d4 · 21228d4
1 parent c1e2675
commit 21228d4
Show file tree

Hide file tree

Showing 15 changed files with 160 additions and 300 deletions.
diff --git a/INSTALL.md b/INSTALL.md
@@ -165,3 +165,20 @@ To prune the number of cached images run `make prune`. There are two options tha
 
 - `PRUNE_HOURS` - the number of hours from when an image was last used (a cache hit) for it to be considered for pruning. The default is 48 hrs.
 - `PRUNE_KEEP` - the minimum number of cached images to keep in the cache. Default is 24 images.
+
+## CI workflow and options
+Every PR and every merge to master triggers the CI process in [Jenkins](http://jenkins.rook.io).
+The Jenkins CI will build, run unit tests, run integration tests and Publish artifacts- On every commit to PR and master.
+If any of the CI stages fail, then the process is aborted and no artifacts are published. 
+On every successful build Artifacts are pushed to a [s3 bucket](https://release.rook.io/). On every successful master build, 
+images are uploaded to quay and docker hub in addition.  
+
+During Integration tests phase, all End to End Integration tests under [/tests/integration](/tests/integration) are run.
+It may take a while to run all Integration tests. Based on nature of the PR, it may not be required to run full regression 
+Or users may want to skip build all together for trivial changes like documentation changes. Based on the PR body text,Jenkins will skip the build or skip some tests
+
+1. [skip ci] - if this text is found in the body of PR, then Jenkins will skip the build process and accept the commit
+2. [smoke only] - if this text is found in the body of PR, then Jenkins will only run Smoke Test during integration test phase
+
+The above flags work only on PRs,The full regression is run on every merge to master.
+
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,6 +1,11 @@
 // Rook build for Jenkins Pipelines
 
 pipeline {
+   parameters {
+         booleanParam(defaultValue: true, description: 'Execute pipeline?', name: 'shouldBuild')
+         booleanParam(defaultValue: false, description: 'Run Only Smoke Test', name: 'smokeOnly')
+
+      }
     agent { label 'ec2-stateful' }
 
     options {
@@ -10,23 +15,56 @@ pipeline {
     }
 
     stages {
+        stage('Pre Build check'){
+            when { branch "PR-*" }
+            steps {
+                script {
+                    pr_number = sh (script: "echo ${env.BRANCH_NAME} | grep -o -E '[0-9]+' ",returnStdout: true)
+                    def json = sh (script: "curl -s https://api.github.com/repos/rook/rook/pulls/${pr_number}", returnStdout: true).trim()
+                    def body = evaluateJson(json,'${json.body}')
+                    if (body.contains("[skip ci]")) {
+                         echo ("'[skip ci]' spotted in PR body text. Aborting.")
+                         env.shouldBuild = "false"
+
+                    }
+                    if (body.contains("[smoke only]")) {
+                          env.smokeOnly = "true"
+
+                    }
+                }
+            }
+        }
         stage('Build') {
+            when {
+                expression {
+                    return env.shouldBuild == "true"
+                }
+            }
             steps {
                 sh 'build/run make -j\$(nproc) build.all'
             }
         }
         stage('Unit Tests') {
+            when {
+                expression {
+                    return env.shouldBuild == "true"
+                }
+            }
             steps {
                 sh 'build/run make -j\$(nproc) test'
             }
         }
 
        stage('Integration Tests') {
+            when {
+                expression {
+                    return env.shouldBuild == "true"
+                }
+            }
             steps{
                 sh 'cat _output/version | xargs tests/scripts/makeTestImages.sh  save amd64'
                 stash name: 'repo-amd64',includes: 'rook-amd64.tar,build/common.sh,_output/tests/linux_amd64/,_output/charts/,tests/scripts/'
                 script{
-
                     def data = [
                         "aws_ci": "v1.6.7",
                         "gce_ci": "v1.7.5"
@@ -50,6 +88,11 @@ pipeline {
             }
         }
         stage('Publish') {
+            when {
+                expression {
+                    return env.shouldBuild == "true"
+                }
+            }
             environment {
                 DOCKER = credentials('rook-docker-hub')
                 QUAY = credentials('rook-quay-io')
@@ -91,11 +134,22 @@ def RunIntegrationTest(k, v) {
                           export KUBECONFIG=$HOME/admin.conf
                           tests/scripts/helm.sh up'''
                     try{
+                        if ("${env.smokeOnly}" == "true") {
+                            echo "Running Smoke Tests"
+                            sh '''#!/bin/bash
+                                  set -o pipefail
+                                  export KUBECONFIG=$HOME/admin.conf
+                                  kubectl config view
+                                  _output/tests/linux_amd64/integration -test.v -test.timeout 600s -test.run SmokeSuite 2>&1 | tee _output/tests/integrationTests.log'''
+                        }
+                        else {
+                        echo "Running full regression"
                         sh '''#!/bin/bash
-                        set -o pipefail
-                        export KUBECONFIG=$HOME/admin.conf
-                        kubectl config view
-                        _output/tests/linux_amd64/smoke -test.v -test.timeout 1200s 2>&1 | tee _output/tests/integrationTests.log'''
+                              set -o pipefail
+                              export KUBECONFIG=$HOME/admin.conf
+                              kubectl config view
+                              _output/tests/linux_amd64/integration -test.v -test.timeout 1800s 2>&1 | tee _output/tests/integrationTests.log'''
+                         }
                     }
                     finally{
                         sh '''#!/bin/bash
@@ -119,4 +173,12 @@ def RunIntegrationTest(k, v) {
 @NonCPS
 List<List<?>> mapToList(Map map) {
   return map.collect { it ->[it.key, it.value]}
+}
+
+@NonCPS
+def evaluateJson(String json, String gpath){
+    //parse json
+    def ojson = new groovy.json.JsonSlurper().parseText(json)
+    //evaluate gpath as a gstring template where $json is a parsed json parameter
+    return new groovy.text.GStringTemplateEngine().createTemplate(gpath).make(json:ojson).toString()
 }
diff --git a/Makefile b/Makefile
@@ -57,7 +57,8 @@ CLIENT_PACKAGES = $(GO_PROJECT)/cmd/rookctl
 SERVER_PACKAGES = $(GO_PROJECT)/cmd/rook
 
 # tests packages that will be compiled into binaries
-TEST_PACKAGES = $(GO_PROJECT)/tests/smoke
+TEST_PACKAGES = $(GO_PROJECT)/tests/integration
+LONGHAUL_TEST_PACKAGES = $(GO_PROJECT)/tests/longhaul
 
 # the root go project
 GO_PROJECT=github.com/rook/rook
@@ -82,8 +83,10 @@ GO_LDFLAGS=$(LDFLAGS)
 GO_TAGS=$(TAGS)
 
 GO_TEST_PACKAGES=$(TEST_PACKAGES)
+GO_LONGHAUL_TEST_PACKAGES=$(LONGHAUL_TEST_PACKAGES)
 GO_TEST_FLAGS=$(TESTFLAGS)
 GO_TEST_SUITE=$(SUITE)
+GO_TEST_FILTER=$(TESTFILTER)
 
 include build/makelib/golang.mk
 
@@ -184,6 +187,7 @@ help:
 	@echo '    DEBUG        Whether to generate debug symbols. Default is 0.'
 	@echo '    PLATFORM     The platform to build.'
 	@echo '    SUITE        The test suite to run.'
+	@echo '    TESTFILTER   Tests to run in a suite.'
 	@echo '    VERSION      The version information compiled into binaries.'
 	@echo '                 The default is obtained from git.'
 	@echo '    V            Set to 1 enable verbose build. Default is 0.'
diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk
@@ -52,12 +52,16 @@ GO_TEST_FLAGS ?=
 GO_SUPPORTED_VERSIONS ?= 1.7|1.8|1.9
 
 GO_PACKAGES := $(foreach t,$(GO_SUBDIRS),$(GO_PROJECT)/$(t)/...)
-GO_INTEGRATION_TEST_PACKAGES := $(foreach t,$(GO_INTEGRATION_TESTS_SUBDIRS),$(GO_PROJECT)/$(t)/...)
+GO_INTEGRATION_TEST_PACKAGES := $(foreach t,$(GO_INTEGRATION_TESTS_SUBDIRS),$(GO_PROJECT)/$(t)/integration)
 
 ifneq ($(GO_TEST_SUITE),)
 GO_TEST_FLAGS += -run '$(GO_TEST_SUITE)'
 endif
 
+ifneq ($(GO_TEST_FILTER),)
+TEST_FILTER_PARAM := -testify.m '$(GO_TEST_FILTER)'
+endif
+
 GOPATH := $(shell go env GOPATH)
 
 # setup tools used during the build
@@ -136,6 +140,8 @@ go.build.test.packages.$(1):
 go.build.test.packages: go.build.test.packages.$(1)
 endef
 $(foreach p,$(GO_TEST_PACKAGES),$(eval $(call go.test.project,$(lastword $(subst /, ,$(p))),$(p),CGO_ENABLED=0,$(GO_STATIC_FLAGS))))
+$(foreach p,$(GO_LONGHAUL_TEST_PACKAGES),$(eval $(call go.test.project,$(lastword $(subst /, ,$(p))),$(p),CGO_ENABLED=0,$(GO_STATIC_FLAGS))))
+
 
 .PHONY: go.build
 go.build: go.build.packages go.build.test.packages
@@ -156,7 +162,7 @@ go.test.integration: $(GOJUNIT)
 	@echo === go test integration-tests
 	@mkdir -p $(GO_TEST_OUTPUT)
 	@CGO_ENABLED=0 $(GOHOST) test -v -i $(GO_STATIC_FLAGS) $(GO_INTEGRATION_TEST_PACKAGES)
-	@CGO_ENABLED=0 $(GOHOST) test -v $(GO_TEST_FLAGS) $(GO_STATIC_FLAGS) $(GO_INTEGRATION_TEST_PACKAGES) 2>&1 | tee $(GO_TEST_OUTPUT)/integration-tests.log
+	@CGO_ENABLED=0 $(GOHOST) test -v $(GO_TEST_FLAGS) $(GO_STATIC_FLAGS) $(GO_INTEGRATION_TEST_PACKAGES) $(TEST_FILTER_PARAM) 2>&1 | tee $(GO_TEST_OUTPUT)/integration-tests.log
 	@cat $(GO_TEST_OUTPUT)/integration-tests.log | $(GOJUNIT) -set-exit-code > $(GO_TEST_OUTPUT)/integration-tests.xml
 
 .PHONY: go.lint

diff --git a/tests/README.md b/tests/README.md
@@ -47,28 +47,33 @@ Use [helm.sh](/tests/scripts/helm.sh) to install helm and set up rook charts def
 **Note:**  *kubeadm.sh, minikube.sh and helm.sh scripts depend on some artifacts under _output dir generated during build time, these scripts 
 should be run from project root. eg* `tests/script/kubeadm.sh up`
 
-### Run Tests
-From the root do the following:
-1. Build rook: `make build`
-2. Start kubernetes using one of the following:
+## Run Tests
+From the project root do the following:
+#### 1. Build rook: 
+ run `make build`
+
+#### 2. Start kubernetes 
+using one of the following:
 
    -  using Kubeadm  
    ```
     tests/scripts/kubeadm.sh up
-    tests/scripts/helm.sh up
-    ```
+    tests/scripts/helm.sh up  
+  ```   
    - using minikube
    ```
    tests/scripts/minikube.sh up
-   tests/scripts/minikube.sh update
    tests/scripts/minikube.sh helm
    tests/scripts/helm.sh up
    ```
-3. Run integration tests: `make test-integration`
 
+#### 3. Run integration tests: 
+Integration tests can be run using tests binary `_output/tests/${platform}/integration` that is generated during build time
+eg ` ~/integration -test.v `
+
 
 ### Test parameters
-The following parameters are available while running tests
+In addition to standard go tests parameters, the following custom parameters are available while running tests
 
  Parameter | Description | Possible values | Default
  --- |--- | --- | ---
@@ -83,34 +88,28 @@ If the `install_rook` flag is set to false, then all the other flags are ignored
 and tests are run without rook being installed and setup. Use this flag to run tests against
 a pre-installed/configured rook.
 
+
 ### Running Tests with parameters.
 
 #### To run all integration tests run 
 ```
-make test-integration
+go test -v -timeout 1800s github.com/rook/rook/tests/integration
 ```
 
-#### To run all integration Tests on a specific suite (uses regex)
+#### To run a specific suite (uses regex)
 ```
-make test-integration SUITE=SmokeSuite
+go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration
 ```
 
-#### To run all tests in a package:
-```
-go test github.com/rook/rook/e2e/tests/smoke
+#### To run specific tests inside a suite:
 ```
-runs all tests under /tests/smoke folder. 
+go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration -testify.m TestRookClusterInstallation_smokeTest
 
-#### To run specific tests: 
 ```
-go test -run SmokeSuite github.com/rook/rook/tests/smoke
-```
-which runs all tests that match regex SmokeSuite in /tests/smoke folder 
-
 
-##### To run specific without installing rook: 
+##### To run specific without installing rook
 ```
-go test -run SmokeSuite github.com/rook/rook/tests/smoke --skip_install_rook=true
+go test -v -timeout 1800s -run SmokeSuite github.com/rook/rook/tests/integration --skip_install_rook=true
 ```
 If the `skip_install_rook` flag is set to true, then rook is not uninstalled either. 
 

diff --git a/tests/smoke/base_block_test.go → tests/integration/base_block_test.go b/tests/smoke/base_block_test.go → tests/integration/base_block_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package smoke
+package integration
 
 import (
 	"time"

diff --git a/tests/smoke/base_deploy_test.go → tests/integration/base_deploy_test.go b/tests/smoke/base_deploy_test.go → tests/integration/base_deploy_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package smoke
+package integration
 
 import (
 	"github.com/coreos/pkg/capnslog"

diff --git a/tests/smoke/base_file_test.go → tests/integration/base_file_test.go b/tests/smoke/base_file_test.go → tests/integration/base_file_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package smoke
+package integration
 
 import (
 	"fmt"

diff --git a/tests/smoke/base_object_test.go → tests/integration/base_object_test.go b/tests/smoke/base_object_test.go → tests/integration/base_object_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package smoke
+package integration
 
 import (
 	"errors"