ICU-22349 Use -l with make -j to limit jobs based on load average.

It has been proposed that make running too many parallel jobs recently has led to resource exhaustion in our CI, so that some kind of limit would be helpful to set. The load average 2.5 limit choosen here is simply the limit used as an example in the make documentation, as we don't really have any way of picking an initial value that's certain to be better. https://www.gnu.org/software/make/manual/html_node/Parallel.html See unicode-org#2421 See unicode-org#2422
cyrusimap · May 10, 2023 · e1e1c5f · e1e1c5f
1 parent 5618203
commit e1e1c5f
Show file tree

Hide file tree

Showing 12 changed files with 70 additions and 70 deletions.
diff --git a/.ci-builds/.azure-exhaustive-tests.yml b/.ci-builds/.azure-exhaustive-tests.yml
@@ -62,7 +62,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5
       displayName: 'Build'
     - script: |
         cd icu4c/source && make check-exhaustive

diff --git a/.ci-builds/.azure-pipelines.yml b/.ci-builds/.azure-pipelines.yml
@@ -33,7 +33,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: clang
@@ -57,7 +57,7 @@ jobs:
     - script: |
         cd icu4c/source && \
         ICU_DATA_FILTER_FILE=../../.ci-builds/data-filter.json ./runConfigureICU Linux && \
-        make -j2 tests && \
+        make -j -l2.5 tests && \
         \[ ! -d data/out/build/icudt66l/translit \] && \
         (cd test/intltest && LD_LIBRARY_PATH=../../lib:../../tools/ctestfw ./intltest translit/TransliteratorTest/TestBasicTransliteratorEvenWithoutData)
       displayName: 'Build with Data Filter'
@@ -75,7 +75,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        export CXXFLAGS="-std=c++17 -Winvalid-constexpr" && cd icu4c/source && ./runConfigureICU --enable-debug --disable-release Linux && make -j2 check
+        export CXXFLAGS="-std=c++17 -Winvalid-constexpr" && cd icu4c/source && ./runConfigureICU --enable-debug --disable-release Linux && make -j -l2.5 check
       displayName: 'Build and Test C++17'
       env:
         CC: clang
@@ -91,7 +91,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: gcc
@@ -112,7 +112,7 @@ jobs:
         sudo apt install gcc-11 g++-11
       displayName: 'Install GCC-11'
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: gcc-11
@@ -129,7 +129,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: clang
@@ -153,7 +153,7 @@ jobs:
         sudo apt install -qy clang-14 lld-14 libc++-14-dev libc++abi-14-dev
       displayName: 'Install Clang-14'
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 tests
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 tests
       displayName: 'Build only (WarningsAsErrors)'
       env:
         CC: clang-14
@@ -177,7 +177,7 @@ jobs:
         sudo apt install -qy clang-14 lld-14 libc++-14-dev libc++abi-14-dev
       displayName: 'Install Clang-14'
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: clang-14
@@ -202,7 +202,7 @@ jobs:
         sudo apt install -qy clang-14 lld-14 libc++-14-dev libc++abi-14-dev
       displayName: 'Install Clang-14'
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2 check
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5 check
       displayName: 'Build and Test'
       env:
         CC: clang-14
@@ -556,7 +556,7 @@ jobs:
       displayName: 'Install mingw-w64-x86_64-toolchain'
     - script: |
        set MSYSTEM=MINGW64
-       c:\tools\msys64\usr\bin\bash.exe -lc "cd $BUILD_SOURCESDIRECTORY && cd icu4c/source && ./runConfigureICU MinGW --prefix='/tmp/build-icu' && make -j2 check"
+       c:\tools\msys64\usr\bin\bash.exe -lc "cd $BUILD_SOURCESDIRECTORY && cd icu4c/source && ./runConfigureICU MinGW --prefix='/tmp/build-icu' && make -j -l2.5 check"
       displayName: 'Build and Test'
     - script: |
        set MSYSTEM=MINGW64
@@ -578,7 +578,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        export CPPFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wundef -Wnon-virtual-dtor" && cd icu4c/source && ./runConfigureICU MacOSX && make -j2 check
+        export CPPFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wundef -Wnon-virtual-dtor" && cd icu4c/source && ./runConfigureICU MacOSX && make -j -l2.5 check
       displayName: 'Build and Test (WarningsAsErrors)'
       env:
         CC: clang
@@ -640,13 +640,13 @@ jobs:
         %CYG_ROOT%\\bin\\sh -lc 'echo Hello' && %CYG_ROOT%\\bin\\sh -lc 'uname -a'
       displayName: 'Check Cygwin environment'
     - script: |
-        %CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && ./runConfigureICU Cygwin && make tests -j "
+        %CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && ./runConfigureICU Cygwin && make tests -j -l2.5"
       displayName: 'Build ICU (source and test)'
       env:
         CC: gcc
         CXX: g++
     - script: |
-        %CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && make -j check"
+        %CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && make -j -l2.5 check"
       displayName: 'Run Tests'
       env:
         CC: gcc
@@ -662,7 +662,7 @@ jobs:
       lfs: true
       fetchDepth: 10
     - script: |
-        cd icu4c/source && ./runConfigureICU Linux && make -j2
+        cd icu4c/source && ./runConfigureICU Linux && make -j -l2.5
       displayName: 'Build'
       env:
         CC: clang

diff --git a/.github/workflows/icu_ci.yml b/.github/workflows/icu_ci.yml
@@ -139,7 +139,7 @@ jobs:
           cd icu4c/source;
           ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data ICU_DATA_FILTER_FILE=../../.github/lstm_for_th_my.json ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
           make clean;
-          make -j ICU4J_ROOT=../../../icu4j icu4j-data-install;
+          make -j -l2.5 ICU4J_ROOT=../../../icu4j icu4j-data-install;
           cd ../..
       - name: ICU4J
         run: |
@@ -171,7 +171,7 @@ jobs:
           cd icu4c/source;
           ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data ICU_DATA_FILTER_FILE=../../.github/adaboost.json CPPFLAGS=-DUCONFIG_USE_ML_PHRASE_BREAKING=1 ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
           make clean;
-          make -j ICU4J_ROOT=../../../icu4j icu4j-data-install;
+          make -j -l2.5 ICU4J_ROOT=../../../icu4j icu4j-data-install;
           cd ../..
       - name: ICU4J
         run: |
@@ -205,7 +205,7 @@ jobs:
           mkdir build;
           cd build;
           ../icu4c/source/runConfigureICU --enable-debug --disable-release Linux/gcc --prefix=$PREFIX --enable-tracing;
-          make -j check;
+          make -j -l2.5 check;
           ( cd ../icu4c/source/test/depstest && ./depstest.py ../../../../build/ );
           make install;
           PATH=$PREFIX/bin:$PATH make -C test/hdrtst check
@@ -232,7 +232,7 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU Linux;
-          make -j check;
+          make -j -l2.5 check;
       - name: Test Dependency
         run: |
           cd icu4c/source/test/depstest;
@@ -266,8 +266,8 @@ jobs:
           mkdir build;
           cd build;
           ../icu4c/source/runConfigureICU Linux --disable-layout --disable-layoutex --prefix=$PREFIX;
-          make -j check;
-          make -j install;
+          make -j -l2.5 check;
+          make -j -l2.5 install;
           cd $PREFIX/bin;
           LD_LIBRARY_PATH=../lib ./icuinfo
 
@@ -281,7 +281,7 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming --enable-tracing;
-          make -j check;
+          make -j -l2.5 check;
         env:
           CPPFLAGS: -fsanitize=address
           LDFLAGS: -fsanitize=address
@@ -295,7 +295,7 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming;
-          make -j check;
+          make -j -l2.5 check;
         env:
           CPPFLAGS: -fsanitize=undefined -fsanitize=alignment -fno-sanitize-recover=undefined,alignment
           CFLAGS: -fsanitize=undefined -fsanitize=alignment -fno-sanitize-recover=undefined,alignment
@@ -310,7 +310,7 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU Linux --disable-renaming;
-          make -j check;
+          make -j -l2.5 check;
         env:
           CC: clang
           CXX: clang++
@@ -329,9 +329,9 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming
-          make -j ;
-          make -j -C test;
-          make -j -C test/intltest check
+          make -j -l2.5;
+          make -j -l2.5 -C test;
+          make -j -l2.5 -C test/intltest check
         env:
           INTLTEST_OPTS: utility/MultithreadTest
           CPPFLAGS: -fsanitize=thread
@@ -347,7 +347,7 @@ jobs:
         run: |
           cd icu4c/source;
           PYTHON=python3 ./runConfigureICU MacOSX;
-          make -j check
+          make -j -l2.5 check
 
   # Run ICU4C tests with stubdata.
   run-with-stubdata:
@@ -359,14 +359,14 @@ jobs:
         run: |
           cd icu4c/source;
           ./runConfigureICU Linux;
-          make -j check;
+          make -j -l2.5 check;
           rm lib/libicudata.so*;
           cp -P stubdata/libicudata.so* lib;
 
           cd test/cintltst;
           echo 'Running ICU4C cintltst with stubdata.';
           # Note: 'Elapsed Time: ' is printed by makefile upon final success.
-          CINTLTST_OPTS=-w make -j check 2>&1 | tee stubdata_ctest.log;
+          CINTLTST_OPTS=-w make -j -l2.5 check 2>&1 | tee stubdata_ctest.log;
           if ! grep 'Elapsed Time: ' stubdata_ctest.log
           then
             echo
@@ -380,7 +380,7 @@ jobs:
 
           cd ../intltest;
           echo 'Running ICU4C intltest with stubdata.';
-          INTLTEST_OPTS=-w make -j check 2>&1 | tee stubdata_intltest.log;
+          INTLTEST_OPTS=-w make -j -l2.5 check 2>&1 | tee stubdata_intltest.log;
           if ! grep 'Elapsed Time: ' stubdata_intltest.log
           then
             echo
@@ -400,7 +400,7 @@ jobs:
       - run: |
           cd icu4c/source;
           ./runConfigureICU Linux CPPFLAGS="-DU_CHARSET_IS_UTF8=1";
-          make -j check
+          make -j -l2.5 check
 
   # Test U_OVERRIDE_CXX_ALLOCATION-is-0-test
   u-override-cxx-allocation-is-0-test:
@@ -411,7 +411,7 @@ jobs:
           cd icu4c/source;
           ./runConfigureICU Linux CPPFLAGS="-DU_OVERRIDE_CXX_ALLOCATION=0";
           make clean;
-          make -j check
+          make -j -l2.5 check
 
   # Test LSTM
   lstm-test:
@@ -422,7 +422,7 @@ jobs:
           cd icu4c/source;
           ICU_DATA_FILTER_FILE=../../.github/lstm_for_th_my.json ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
           make clean;
-          make -j check
+          make -j -l2.5 check
 
   # Test adaboost
   adaboost-test:
@@ -433,7 +433,7 @@ jobs:
           cd icu4c/source;
           ICU_DATA_FILTER_FILE=../../.github/adaboost.json CPPFLAGS=-DUCONFIG_USE_ML_PHRASE_BREAKING=1 ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
           make clean;
-          make -j  check
+          make -j -l2.5 check
 
   # Build and run testmap
   testmap:
@@ -443,10 +443,10 @@ jobs:
       - run: |
           cd icu4c/source;
           ./runConfigureICU Linux;
-          make -j check;
+          make -j -l2.5 check;
           CONFIG_FILES=test/testmap/Makefile ./config.status;
           cd test/testmap;
-          make -j check | grep '*** PASS PASS PASS, test PASSED!!!!!!!!'
+          make -j -l2.5 check | grep '*** PASS PASS PASS, test PASSED!!!!!!!!'
 
   # Copyright scan
   copyright-scan:
@@ -520,7 +520,7 @@ jobs:
         run: |
           cd icu4c/source/;
           ./runConfigureICU Linux;
-          make -j tests;
+          make -j -l2.5 tests;
 
   # Run header tests with UCONFIG_NO_XXX variations.
   uconfig-header-tests:
@@ -564,7 +564,7 @@ jobs:
           cd icu4c/source/;
           mkdir /tmp/icu_cnfg;
           ./runConfigureICU Linux --prefix=/tmp/icu_cnfg;
-          make -j install;
+          make -j -l2.5 install;
       - name: Test
         env:
           UCONFIG_NO: ${{ matrix.uconfig_cppflags }}
@@ -674,7 +674,7 @@ jobs:
           CINTLTST_OPTS: -w
         run: |
           cd icu4c/source
-          make -j check
+          make -j -l2.5 check
   
   # https://unicode-org.github.io/icu/processes/release/tasks/healthy-code.html#test-uconfig_no_conversion
   icu4c-uconfig-no-version:
@@ -690,8 +690,8 @@ jobs:
       - name: Run make for stubdata, common, i18n
         run: |
           cd icu4c/source
-          pushd stubdata && make -j && popd
+          pushd stubdata && make -j -l2.5 && popd
           # Ensure lib directory for output object file exists
           mkdir -p lib
-          pushd common && make -j && popd
-          pushd i18n && make -j && popd
+          pushd common && make -j -l2.5 && popd
+          pushd i18n && make -j -l2.5 && popd
diff --git a/.github/workflows/icu_envtest.yml b/.github/workflows/icu_envtest.yml
@@ -45,7 +45,7 @@ jobs:
         run: |
           cd icu4c/source/;
           ./runConfigureICU Linux;
-          make -j tests;
+          make -j -l2.5 tests;
       - name: Test different locales as LC_ALL
         run: |
           echo "This job run the environment test of the following Locales";
@@ -62,7 +62,7 @@ jobs:
               for test_dir in iotest cintltst intltest
               do
                   cd $test_dir;
-                  LC_ALL=$loc make -j check;
+                  LC_ALL=$loc make -j -l2.5 check;
                   cd ..;
               done
 
@@ -104,7 +104,7 @@ jobs:
         run: |
           cd icu4c/source/;
           ./runConfigureICU Linux;
-          make -j tests;
+          make -j -l2.5 tests;
       - name: Test different locales as TZ
         run: |
           echo "This job run the environment test of the following TimeZones";
@@ -122,7 +122,7 @@ jobs:
               for test_dir in iotest cintltst intltest
               do
                   cd $test_dir;
-                  TZ=$tz make -j check;
+                  TZ=$tz make -j -l2.5 check;
                   cd ..;
               done