Fix integration tests caused by recent helm change (istio#9619)

* helm fix * Fix codecov * Update makefile. * Address comments.
AmitKumarDas · Nov 1, 2018 · 6a9bd94 · 6a9bd94
1 parent 7b0ec8a
commit 6a9bd94
Show file tree

Hide file tree

Showing 5 changed files with 166 additions and 43 deletions.
diff --git a/codecov.skip b/codecov.skip
@@ -11,9 +11,11 @@ istio.io/istio/pkg/mcp/testing
 istio.io/istio/pkg/test
 istio.io/istio/security/tests/integration
 istio.io/istio/tests/e2e
-istio.io/istio/tests/integration/
-istio.io/istio/tests/integration_old/
-istio.io/istio/tests/util/
+istio.io/istio/tests/integration2/examples
+istio.io/istio/tests/integration2/qualification
+istio.io/istio/tests/integration_old
+istio.io/istio/tests/local
+istio.io/istio/tests/util
 istio.io/istio/tools/githubContrib
 istio.io/istio/tools/hyperistio
 istio.io/istio/tools/license
diff --git a/pkg/test/deployment/helm.go b/pkg/test/deployment/helm.go
@@ -108,39 +108,38 @@ func HelmTemplate(deploymentName, namespace, chartDir, workDir, valuesFile strin
 		valuesFileString = fmt.Sprintf(" --values %s", valuesFile)
 	}
 
-	err := initCharts(chartDir, workDir)
-	if err != nil {
+	helmRepoDir := filepath.Join(workDir, "helmrepo")
+	chartBuildDir := filepath.Join(workDir, "charts")
+	if err := os.MkdirAll(helmRepoDir, os.ModePerm); err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(chartBuildDir, os.ModePerm); err != nil {
 		return "", err
 	}
 
-	return shell.Execute(
-		"helm template %s --name %s --namespace %s%s%s",
-		chartDir, deploymentName, namespace, valuesFileString, valuesString)
-}
-
-func initCharts(chartDir, workDir string) error {
 	// Initialize the helm (but do not install tiller).
-	if err := exec("helm init --client-only"); err != nil {
-		return err
+	if _, err := exec(fmt.Sprintf("helm --home %s init --client-only", helmRepoDir)); err != nil {
+		return "", err
 	}
 
-	// Create a dir for the packaged charts.
-	chartBuildDir, err := ioutil.TempDir(workDir, "helm-charts-")
-	if err != nil {
-		return err
+	// Adding cni dependency as a workaround for now.
+	if _, err := exec(fmt.Sprintf("helm --home %s repo add istio.io %s",
+		helmRepoDir, "https://raw.githubusercontent.com/istio/istio.io/master/static/charts")); err != nil {
+		return "", err
 	}
 
 	// Package the chart dir.
-	if err := exec(fmt.Sprintf("helm package -u %s -d %s", chartDir, chartBuildDir)); err != nil {
-		return err
+	if _, err := exec(fmt.Sprintf("helm --home %s package -u %s -d %s", helmRepoDir, chartDir, chartBuildDir)); err != nil {
+		return "", err
 	}
-	return nil
+	return exec(fmt.Sprintf("helm --home %s template %s --name %s --namespace %s%s%s",
+		helmRepoDir, chartDir, deploymentName, namespace, valuesFileString, valuesString))
 }
 
-func exec(cmd string) error {
+func exec(cmd string) (string, error) {
 	str, err := shell.Execute(cmd)
 	if err != nil {
-		return fmt.Errorf("failed executing command (%s): %v: %s", cmd, err, str)
+		scopes.CI.Errorf("failed executing command (%s): %v: %s", cmd, err, str)
 	}
-	return nil
+	return str, err
 }
diff --git a/prow/istio-integ-k8s-tests.sh b/prow/istio-integ-k8s-tests.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Copyright 2017 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+WD=$(dirname "$0")
+WD=$(cd "$WD"; pwd)
+ROOT=$(dirname "$WD")
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+# Check https://github.com/istio/test-infra/blob/master/boskos/configs.yaml
+# for existing resources types
+RESOURCE_TYPE="${RESOURCE_TYPE:-gke-e2e-test}"
+OWNER="${OWNER:-integ}"
+CLEAN_CLUSTERS="${CLEAN_CLUSTERS:-True}"
+
+
+# shellcheck source=prow/lib.sh
+source "${ROOT}/prow/lib.sh"
+# shellcheck source=prow/mason_lib.sh
+source "${ROOT}/prow/mason_lib.sh"
+# shellcheck source=prow/cluster_lib.sh
+source "${ROOT}/prow/cluster_lib.sh"
+
+function cleanup() {
+  if [[ "${CLEAN_CLUSTERS}" == "True" ]]; then
+    unsetup_clusters
+  fi
+  if [[ "${USE_MASON_RESOURCE}" == "True" ]]; then
+    mason_cleanup
+    cat "${FILE_LOG}"
+  fi
+}
+
+trap cleanup EXIT
+
+INFO_PATH="$(mktemp /tmp/XXXXX.boskos.info)"
+FILE_LOG="$(mktemp /tmp/XXXXX.boskos.log)"
+
+setup_and_export_git_sha
+
+get_resource "${RESOURCE_TYPE}" "${OWNER}" "${INFO_PATH}" "${FILE_LOG}"
+
+
+if [ "${CI:-}" == 'bootstrap' ]; then
+  # bootsrap upload all artifacts in _artifacts to the log bucket.
+  ARTIFACTS_DIR=${ARTIFACTS_DIR:-"${GOPATH}/src/istio.io/istio/_artifacts"}
+fi
+
+export HUB=${HUB:-"gcr.io/istio-testing"}
+export TAG="${TAG:-${GIT_SHA}}"
+
+make init
+
+setup_cluster
+
+time make test.integration.kube T=-v
+
diff --git a/prow/istio-integ-local-tests.sh b/prow/istio-integ-local-tests.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+WD=$(dirname "$0")
+WD=$(cd "$WD"; pwd)
+ROOT=$(dirname "$WD")
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+# shellcheck source=prow/lib.sh
+source "${ROOT}/prow/lib.sh"
+setup_and_export_git_sha
+
+cd "${ROOT}"
+
+# Unit tests are run against a local apiserver and etcd.
+# Integration/e2e tests in the other scripts are run against GKE or real clusters.
+JUNIT_UNIT_TEST_XML="${ARTIFACTS_DIR}/junit_unit-tests.xml" \
+T="-v" \
+make test.integration
diff --git a/tests/integration2/tests.mk b/tests/integration2/tests.mk
@@ -29,33 +29,16 @@ ifneq ($(KUBECONFIG),)
     INTEGRATION_TEST_KUBECONFIG = $(KUBECONFIG)
 endif
 
-# The names of the integration test folders at ROOT/tests/integration2/*.
-_INTEGRATION_TEST_NAMES = galley mixer citadel
-
-# Generate the names of the integration test targets that use local environment (i.e. test.integration.galley)
-_INTEGRATION_TESTS_LOCAL = $(addprefix test.integration., $(_INTEGRATION_TEST_NAMES))
-
-# Generate the names of the integration test targets that use kubernetes environment (i.e. test.integration.galley.kube)
-_INTEGRATION_TESTS_KUBE  = $(addsuffix .kube, $(addprefix test.integration., $(_INTEGRATION_TEST_NAMES)))
-
 # This is a useful debugging target for testing everything.
-.PHONY: test.integration.all
+#.PHONY: test.integration.all
 test.integration.all: test.integration test.integration.kube
 
-# All integration tests targeting local environment.
-.PHONY: test.integration
-test.integration: $(_INTEGRATION_TESTS_LOCAL)
-
-# All integration tests targeting Kubernetes environment.
-.PHONY: test.integration.kube
-test.integration.kube: $(_INTEGRATION_TESTS_KUBE)
-
 # Generate integration test targets for local environment.
-$(_INTEGRATION_TESTS_LOCAL): test.integration.%:
+test.integration.%:
 	$(GO) test -p 1 ${T} ./tests/integration2/$*/... --istio.test.env local
 
 # Generate integration test targets for kubernetes environment.
-$(_INTEGRATION_TESTS_KUBE): test.integration.%.kube:
+test.integration.%.kube:
 	$(GO) test -p 1 ${T} ./tests/integration2/$*/... ${_INTEGRATION_TEST_WORKDIR_FLAG} ${_INTEGRATION_TEST_LOGGING_FLAG} \
 	--istio.test.env kubernetes \
 	--istio.test.kube.config ${INTEGRATION_TEST_KUBECONFIG} \
@@ -64,4 +47,30 @@ $(_INTEGRATION_TESTS_KUBE): test.integration.%.kube:
 	${_INTEGRATION_TEST_INGRESS_FLAG}
 
 
+JUNIT_UNIT_TEST_XML ?= $(ISTIO_OUT)/junit_unit-tests.xml
+JUNIT_REPORT = $(shell which go-junit-report 2> /dev/null || echo "${ISTIO_BIN}/go-junit-report")
+
+# TODO: Exclude examples and qualification since they are very flaky.
+TEST_PACKAGES = $(shell go list ./tests/integration2/... | grep -v /qualification | grep -v /examples)
+
+# All integration tests targeting local environment.
+.PHONY: test.integration
+test.integration: | $(JUNIT_REPORT)
+	mkdir -p $(dir $(JUNIT_UNIT_TEST_XML))
+	set -o pipefail; \
+	$(GO) test -p 1 ${T} ${TEST_PACKAGES} --istio.test.env local \
+	2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_UNIT_TEST_XML))
+
+# All integration tests targeting Kubernetes environment.
+.PHONY: test.integration.kube
+test.integration.kube: | $(JUNIT_REPORT)
+	mkdir -p $(dir $(JUNIT_UNIT_TEST_XML))
+	set -o pipefail; \
+	$(GO) test -p 1 ${T} ${TEST_PACKAGES} ${_INTEGRATION_TEST_WORKDIR_FLAG} ${_INTEGRATION_TEST_LOGGING_FLAG} \
+	--istio.test.env kubernetes \
+	--istio.test.kube.config ${INTEGRATION_TEST_KUBECONFIG} \
+	--istio.test.kube.deploy \
+	--istio.test.kube.helm.values global.hub=${HUB},global.tag=${TAG} \
+	${_INTEGRATION_TEST_INGRESS_FLAG} \
+	2>&1 | tee >($(JUNIT_REPORT) > $(JUNIT_UNIT_TEST_XML))