Merge pull request mesonbuild#5621 from mesonbuild/paralleltests

Run tests in paralled with pytest when installed

.travis.yml

@@ -16,7 +16,7 @@ compiler:
 
 env:
   - MESON_ARGS=""
-  - MESON_ARGS="--unity=on"
+  - RUN_TESTS_ARGS="--no-unittests" MESON_ARGS="--unity=on"
 
 language:
   - cpp
@@ -63,4 +63,4 @@ script:
           /bin/sh -c "cd /root && mkdir -p tools; wget -c http://nirbheek.in/files/binaries/ninja/linux-amd64/ninja -O /root/tools/ninja; chmod +x /root/tools/ninja; CC=$CC CXX=$CXX OBJC=$CC OBJCXX=$CXX PATH=/root/tools:$PATH MESON_FIXED_NINJA=1 ./run_tests.py $RUN_TESTS_ARGS -- $MESON_ARGS && chmod -R a+rwX .coverage"
     fi
   # Ensure that llvm is added after $PATH, otherwise the clang from that llvm install will be used instead of the native apple clang.
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) CPPFLAGS=-I/usr/local/include LDFLAGS=-L/usr/local/lib OBJC=$CC OBJCXX=$CXX PATH=$HOME/tools:/usr/local/opt/qt/bin:$PATH:$(brew --prefix llvm)/bin MESON_FIXED_NINJA=1 ./run_tests.py --backend=ninja -- $MESON_ARGS ; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then SDKROOT=$(xcodebuild -version -sdk macosx Path) CPPFLAGS=-I/usr/local/include LDFLAGS=-L/usr/local/lib OBJC=$CC OBJCXX=$CXX PATH=$HOME/tools:/usr/local/opt/qt/bin:$PATH:$(brew --prefix llvm)/bin MESON_FIXED_NINJA=1 ./run_tests.py $RUN_TESTS_ARGS --backend=ninja -- $MESON_ARGS ; fi

ci/azure-steps.yml

@@ -154,6 +154,10 @@ steps:
      where.exe python
      python --version
 
+     # Needed for running unit tests in parallel.
+     python -m pip install --upgrade pytest-xdist
+
+
      echo ""
      echo "Locating cl, rc:"
      where.exe cl

ciimage/Dockerfile

@@ -7,6 +7,7 @@ ENV DC=gdc
 RUN sed -i '/^#\sdeb-src /s/^#//' "/etc/apt/sources.list" \
 && apt-get -y update && apt-get -y upgrade \
 && apt-get -y build-dep meson \
+&& apt-get -y install python3-pytest-xdist \
 && apt-get -y install python3-pip libxml2-dev libxslt1-dev cmake libyaml-dev \
 && python3 -m pip install hotdoc codecov \
 && apt-get -y install wget unzip \

run_project_tests.py

@@ -803,6 +803,8 @@ def check_format():
     for (root, _, files) in os.walk('.'):
         if '.dub' in root: # external deps are here
             continue
+        if '.pytest_cache' in root:
+            continue
         if 'meson-logs' in root or 'meson-private' in root:
             continue
         for fname in files:
@@ -870,6 +872,8 @@ def detect_system_compiler():
                         choices=backendlist)
     parser.add_argument('--failfast', action='store_true',
                         help='Stop running if test case fails')
+    parser.add_argument('--no-unittests', action='store_true',
+                        help='Not used, only here to simplify run_tests.py')
     parser.add_argument('--only', help='name of test(s) to run', nargs='+')
     options = parser.parse_args()
     setup_commands(options.backend)

run_tests.py

@@ -262,6 +262,7 @@ def main():
                         choices=backendlist)
     parser.add_argument('--cross', default=False, dest='cross', action='store_true')
     parser.add_argument('--failfast', action='store_true')
+    parser.add_argument('--no-unittests', action='store_true', default=False)
     (options, _) = parser.parse_known_args()
     # Enable coverage early...
     enable_coverage = options.cov
@@ -273,6 +274,7 @@ def main():
     returncode = 0
     cross = options.cross
     backend, _ = guess_backend(options.backend, shutil.which('msbuild'))
+    no_unittests = options.no_unittests
     # Running on a developer machine? Be nice!
     if not mesonlib.is_windows() and not mesonlib.is_haiku() and 'CI' not in os.environ:
         os.nice(20)
@@ -314,12 +316,16 @@ def main():
             returncode += subprocess.call(cmd, env=env)
             if options.failfast and returncode != 0:
                 return returncode
-            cmd = mesonlib.python_command + ['run_unittests.py', '-v']
-            if options.failfast:
-                cmd += ['--failfast']
-            returncode += subprocess.call(cmd, env=env)
-            if options.failfast and returncode != 0:
-                return returncode
+            if no_unittests:
+                print('Skipping all unit tests.')
+                returncode = 0
+            else:
+                cmd = mesonlib.python_command + ['run_unittests.py', '-v']
+                if options.failfast:
+                    cmd += ['--failfast']
+                returncode += subprocess.call(cmd, env=env)
+                if options.failfast and returncode != 0:
+                    return returncode
             cmd = mesonlib.python_command + ['run_project_tests.py'] + sys.argv[1:]
             returncode += subprocess.call(cmd, env=env)
         else:

run_unittests.py

@@ -3398,7 +3398,7 @@ def test_introspection_target_subproject(self):
 
         for entry in res:
             name = entry['name']
-            self.assertEquals(entry['subproject'], expected[name])
+            self.assertEqual(entry['subproject'], expected[name])
 
     def test_introspect_projectinfo_subproject_dir(self):
         testdir = os.path.join(self.common_test_dir, '79 custom subproject dir')
@@ -6480,6 +6480,17 @@ def unset_envs():
 
 def main():
     unset_envs()
+    pytest_args = ['-n', 'auto', './run_unittests.py']
+    if shutil.which('pytest-3'):
+        return subprocess.run(['pytest-3'] + pytest_args).returncode
+    elif shutil.which('pytest'):
+        return subprocess.run(['pytest'] + pytest_args).returncode
+    try:
+        import pytest # noqa: F401
+        return subprocess.run(python_command + ['-m', 'pytest'] + pytest_args).returncode
+    except ImportError:
+        pass
+    # All attempts at locating pytest failed, fall back to plain unittest.
     cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests',
              'PythonTests', 'NativeFileTests', 'RewriterTests', 'CrossFileTests',
              'TAPParserTests',