Allow url-map tests to run concurrently (grpc#26764)

* Allow url-map tests to run concurrently

* Clean-up client namespace if everything goes smoothly

* Make isort happy

* Update tools/internal_ci/linux/grpc_xds_url_map_python.sh

Co-authored-by: Sergii Tkachenko <[email protected]>

* Add logging && change suffix generation

* Move the suffix generation to helpers

* Remove unused import

Co-authored-by: Sergii Tkachenko <[email protected]>

tools/internal_ci/linux/grpc_xds_url_map.cfg

@@ -16,7 +16,7 @@
 
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_xds_url_map.sh"
-timeout_mins: 120
+timeout_mins: 60
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"

tools/internal_ci/linux/grpc_xds_url_map_python.cfg

@@ -0,0 +1,26 @@
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_xds_url_map_python.sh"
+timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*sponge_log.log"
+    strip_prefix: "artifacts"
+  }
+}

tools/internal_ci/linux/grpc_xds_url_map_python.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+# Copyright 2021 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+# Constants
+readonly GITHUB_REPOSITORY_NAME="grpc"
+# GKE Cluster
+readonly GKE_CLUSTER_NAME="interop-test-psm-sec-v2-us-central1-a"
+readonly GKE_CLUSTER_ZONE="us-central1-a"
+## xDS test client Docker images
+readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/python-client"
+readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}"
+readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing"
+readonly LANGUAGE_NAME="Python"
+
+#######################################
+# Builds test app Docker images and pushes them to GCR
+# Globals:
+#   BUILD_APP_PATH
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of `gcloud builds submit` to stdout, stderr
+#######################################
+build_test_app_docker_images() {
+  echo "Building ${LANGUAGE_NAME} xDS interop test app Docker images"
+
+  pushd "${SRC_DIR}"
+  docker build \
+    -f src/python/grpcio_tests/tests_py3_only/interop/Dockerfile.client \
+    -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
+    .
+
+  popd
+
+  gcloud -q auth configure-docker
+
+  docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}"
+}
+
+#######################################
+# Builds test app and its docker images unless they already exist
+# Globals:
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+#   FORCE_IMAGE_BUILD
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output to stdout, stderr
+#######################################
+build_docker_images_if_needed() {
+  # Check if images already exist
+  client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")"
+  printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}"
+  echo "${client_tags:-Client image not found}"
+
+  # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1
+  if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${client_tags}" ]]; then
+    build_test_app_docker_images
+  else
+    echo "Skipping ${LANGUAGE_NAME} test app build"
+  fi
+}
+
+#######################################
+# Executes the test case
+# Globals:
+#   TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile
+#   KUBE_CONTEXT: The name of kubectl context with GKE cluster access
+#   TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+# Arguments:
+#   Test case name
+# Outputs:
+#   Writes the output of test execution to stdout, stderr
+#   Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml
+#######################################
+run_test() {
+  # Test driver usage:
+  # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
+  local test_name="${1:?Usage: run_test test_name}"
+  set -x
+  python -m "tests.${test_name}" \
+    --flagfile="${TEST_DRIVER_FLAGFILE}" \
+    --kube_context="${KUBE_CONTEXT}" \
+    --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
+    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
+    --flagfile="config/url-map.cfg"
+  set +x
+}
+
+#######################################
+# Main function: provision software necessary to execute tests, and run them
+# Globals:
+#   KOKORO_ARTIFACTS_DIR
+#   GITHUB_REPOSITORY_NAME
+#   SRC_DIR: Populated with absolute path to the source repo
+#   TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing
+#                         the test driver
+#   TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code
+#   TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile
+#   TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report
+#   GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build
+#   GIT_COMMIT: Populated with the SHA-1 of git commit being built
+#   GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built
+#   KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access
+# Arguments:
+#   None
+# Outputs:
+#   Writes the output of test execution to stdout, stderr
+#######################################
+main() {
+  local script_dir
+  script_dir="$(dirname "$0")"
+  # shellcheck source=tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh
+  source "${script_dir}/grpc_xds_k8s_install_test_driver.sh"
+  set -x
+  if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then
+    kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}"
+  else
+    local_setup_test_driver "${script_dir}"
+  fi
+  build_docker_images_if_needed
+  # Run tests
+  cd "${TEST_DRIVER_FULL_DIR}"
+  run_test url_map
+}
+
+main "$@"

tools/run_tests/xds_k8s_test_driver/config/url-map.cfg

@@ -1,6 +1,5 @@
 --resource_prefix=interop-psm-url-map
 --strategy=reuse
-# TODO(lidiz): Remove the next line when xds port randomization supported.
 --server_xds_port=8848
 # NOTE(lidiz) we pin the server image to java-server because:
 # 1. Only Java server understands the rpc-behavior metadata.

tools/run_tests/xds_k8s_test_driver/framework/helpers/rand.py

@@ -15,6 +15,8 @@
 import random
 import string
 
+import framework.helpers.datetime
+
 # Alphanumeric characters, similar to regex [:alnum:] class, [a-zA-Z0-9]
 ALPHANUM = string.ascii_letters + string.digits
 # Lowercase alphanumeric characters: [a-z0-9]
@@ -25,9 +27,23 @@
 def rand_string(length: int = 8, *, lowercase: bool = False) -> str:
     """Return random alphanumeric string of given length.
 
-    Space for default arguments: alphabet^length
-       lowercase and uppercase = (26*2 + 10)^8 = 2.18e14 = 218 trillion.
-       lowercase only = (26 + 10)^8 = 2.8e12 = 2.8 trillion.
-    """
+   Space for default arguments: alphabet^length
+      lowercase and uppercase = (26*2 + 10)^8 = 2.18e14 = 218 trillion.
+      lowercase only = (26 + 10)^8 = 2.8e12 = 2.8 trillion.
+   """
     alphabet = ALPHANUM_LOWERCASE if lowercase else ALPHANUM
     return ''.join(random.choices(population=alphabet, k=length))
+
+
+def random_resource_suffix() -> str:
+    """Return a ready-to-use resource suffix with datetime and nonce."""
+    # Date and time suffix for debugging. Seconds skipped, not as relevant
+    # Format example: 20210626-1859
+    datetime_suffix: str = framework.helpers.datetime.datetime_suffix()
+    # Use lowercase chars because some resource names won't allow uppercase.
+    # For len 5, total (26 + 10)^5 = 60,466,176 combinations.
+    # Approx. number of test runs needed to start at the same minute to
+    # produce a collision: math.sqrt(math.pi/2 * (26+10)**5) ≈ 9745.
+    # https://en.wikipedia.org/wiki/Birthday_attack#Mathematics
+    unique_hash: str = rand_string(5, lowercase=True)
+    return f'{datetime_suffix}-{unique_hash}'

tools/run_tests/xds_k8s_test_driver/framework/xds_k8s_testcase.py

@@ -26,7 +26,6 @@
 from framework import xds_flags
 from framework import xds_k8s_flags
 from framework.helpers import retryers
-import framework.helpers.datetime
 import framework.helpers.rand
 from framework.infrastructure import gcp
 from framework.infrastructure import k8s
@@ -130,7 +129,8 @@ def setUp(self):
         super().setUp()
 
         if self._resource_suffix_randomize:
-            self.resource_suffix = self._random_resource_suffix()
+            self.resource_suffix = framework.helpers.rand.random_resource_suffix(
+            )
         logger.info('Test run resource prefix: %s, suffix: %s',
                     self.resource_prefix, self.resource_suffix)
 
@@ -195,19 +195,6 @@ def _cleanup(self):
         self.server_runner.cleanup(force=self.force_cleanup,
                                    force_namespace=self.force_cleanup)
 
-    @staticmethod
-    def _random_resource_suffix() -> str:
-        # Date and time suffix for debugging. Seconds skipped, not as relevant
-        # Format example: 20210626-1859
-        datetime_suffix: str = framework.helpers.datetime.datetime_suffix()
-        # Use lowercase chars because some resource names won't allow uppercase.
-        # For len 5, total (26 + 10)^5 = 60,466,176 combinations.
-        # Approx. number of test runs needed to start at the same minute to
-        # produce a collision: math.sqrt(math.pi/2 * (26+10)**5) ≈ 9745.
-        # https://en.wikipedia.org/wiki/Birthday_attack#Mathematics
-        unique_hash: str = framework.helpers.rand.rand_string(5, lowercase=True)
-        return f'{datetime_suffix}-{unique_hash}'
-
     def setupTrafficDirectorGrpc(self):
         self.td.setup_for_grpc(self.server_xds_host,
                                self.server_xds_port,

tools/run_tests/xds_k8s_test_driver/framework/xds_url_map_test_resources.py

@@ -15,13 +15,15 @@
 
 import functools
 import inspect
+import time
 from typing import Any, Iterable, List, Mapping, Tuple
 
 from absl import flags
 from absl import logging
 
 from framework import xds_flags
 from framework import xds_k8s_flags
+import framework.helpers.rand
 from framework.infrastructure import gcp
 from framework.infrastructure import k8s
 from framework.infrastructure import traffic_director
@@ -140,6 +142,17 @@ def __init__(self, absl_flags: Mapping[str, Any] = None):
         if absl_flags is not None:
             for key in absl_flags:
                 setattr(self, key, absl_flags[key])
+        # Pick a client_namespace_suffix if not set
+        if self.resource_suffix is None:
+            self.resource_suffix = ""
+            self.client_namespace_suffix = framework.helpers.rand.random_resource_suffix(
+            )
+        else:
+            self.client_namespace_suffix = self.resource_suffix
+        logging.info(
+            'GcpResourceManager: resource prefix=%s, suffix=%s, client_namespace_suffix=%s',
+            self.resource_prefix, self.resource_suffix,
+            self.client_namespace_suffix)
         # API managers
         self.k8s_api_manager = k8s.KubernetesApiManager(self.kube_context)
         self.gcp_api_manager = gcp.api.GcpApiManager()
@@ -153,9 +166,16 @@ def __init__(self, absl_flags: Mapping[str, Any] = None):
         # Kubernetes namespace
         self.k8s_namespace = k8s.KubernetesNamespace(self.k8s_api_manager,
                                                      self.resource_prefix)
+        if self.client_namespace_suffix != self.resource_suffix:
+            self.k8s_client_namespace = k8s.KubernetesNamespace(
+                self.k8s_api_manager,
+                client_app.KubernetesClientRunner.make_namespace_name(
+                    self.resource_prefix, self.client_namespace_suffix))
+        else:
+            self.k8s_client_namespace = self.k8s_namespace
         # Kubernetes Test Client
         self.test_client_runner = client_app.KubernetesClientRunner(
-            self.k8s_namespace,
+            self.k8s_client_namespace,
             deployment_name=self.client_name,
             image_name=self.client_image,
             gcp_project=self.project,
@@ -203,6 +223,7 @@ def _pre_cleanup(self):
         logging.info('GcpResourceManager: pre clean-up')
         self.td.cleanup(force=True)
         self.test_client_runner.delete_namespace()
+        self.test_server_runner.delete_namespace()
 
     def setup(self, test_case_classes: 'Iterable[XdsUrlMapTestCase]') -> None:
         if self.strategy not in ['create', 'keep']:
@@ -276,6 +297,11 @@ def setup(self, test_case_classes: 'Iterable[XdsUrlMapTestCase]') -> None:
         self.td.wait_for_affinity_backends_healthy_status()
 
     def cleanup(self) -> None:
+        if hasattr(self, 'test_client_runner'):
+            self.test_client_runner.cleanup(
+                force=True,
+                # Clean-up ephemeral client namespace
+                force_namespace=self.k8s_client_namespace != self.k8s_namespace)
         if self.strategy not in ['create']:
             logging.info(
                 'GcpResourceManager: skipping tear down for strategy [%s]',
@@ -284,8 +310,6 @@ def cleanup(self) -> None:
         logging.info('GcpResourceManager: start tear down')
         if hasattr(self, 'td'):
             self.td.cleanup(force=True)
-        if hasattr(self, 'test_client_runner'):
-            self.test_client_runner.cleanup(force=True)
         if hasattr(self, 'test_server_runner'):
             self.test_server_runner.cleanup(force=True)
         if hasattr(self, 'test_server_alternative_runner'):