From b707816f9cb64913a7d713d05f69c23a81a5b216 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Tue, 24 Jun 2025 09:07:04 -0700
Subject: [PATCH 1/8] remove cu118 support

---
 MODULE.bazel                               | 2 +-
 packaging/pre_build_script.sh              | 8 --------
 packaging/pre_build_script_windows.sh      | 8 --------
 toolchains/ci_workspaces/MODULE.bazel.tmpl | 2 +-
 4 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/MODULE.bazel b/MODULE.bazel
index a22e70f071..a5cbca77ba 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -66,7 +66,7 @@ http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/nightly/cu129/libtorch-shared-with-deps-latest.zip"],
 )
 
 # in aarch64 platform you can get libtorch via either local or wheel file
diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh
index 707edcd718..1f1a2120a9 100755
--- a/packaging/pre_build_script.sh
+++ b/packaging/pre_build_script.sh
@@ -66,14 +66,6 @@ if [[ ${TENSORRT_VERSION} != "" ]]; then
          pyproject.toml
 fi
 
-if [[ "${CU_VERSION::4}" < "cu12" ]]; then
-  # replace dependencies from tensorrt-cu12-bindings/libs to tensorrt-cu11-bindings/libs
-  sed -i -e "s/tensorrt-cu12/tensorrt-${CU_VERSION::4}/g" \
-         -e "s/tensorrt-cu12-bindings/tensorrt-${CU_VERSION::4}-bindings/g" \
-         -e "s/tensorrt-cu12-libs/tensorrt-${CU_VERSION::4}-libs/g" \
-         pyproject.toml
-fi
-
 cat toolchains/ci_workspaces/MODULE.bazel.tmpl | envsubst > MODULE.bazel
 
 if [[ ${TENSORRT_VERSION} != "" ]]; then
diff --git a/packaging/pre_build_script_windows.sh b/packaging/pre_build_script_windows.sh
index 7d089b602a..b5b62ebf05 100644
--- a/packaging/pre_build_script_windows.sh
+++ b/packaging/pre_build_script_windows.sh
@@ -17,14 +17,6 @@ if [[ ${TENSORRT_VERSION} != "" ]]; then
          pyproject.toml
 fi
 
-if [[ "${CU_VERSION::4}" < "cu12" ]]; then
-  # replace dependencies from tensorrt-cu12-bindings/libs to tensorrt-cu11-bindings/libs
-  sed -i -e "s/tensorrt-cu12/tensorrt-${CU_VERSION::4}/g" \
-         -e "s/tensorrt-cu12-bindings/tensorrt-${CU_VERSION::4}-bindings/g" \
-         -e "s/tensorrt-cu12-libs/tensorrt-${CU_VERSION::4}-libs/g" \
-         pyproject.toml
-fi
-
 TORCH=$(grep "^torch>" py/requirements.txt)
 INDEX_URL=https://download.pytorch.org/whl/${CHANNEL}/${CU_VERSION}
 
diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl
index 562b55c6ce..4f03473c08 100644
--- a/toolchains/ci_workspaces/MODULE.bazel.tmpl
+++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl
@@ -65,7 +65,7 @@ http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "ht
 #    name = "libtorch",
 #    build_file = "@//third_party/libtorch:BUILD",
 #    strip_prefix = "libtorch",
-#    urls = ["https://download.pytorch.org/libtorch/${CHANNEL}/${CU_VERSION}/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
+#    urls = ["https://download.pytorch.org/libtorch/${CHANNEL}/${CU_VERSION}/libtorch-shared-with-deps-latest.zip"],
 #)
 
 # Download these tarballs manually from the NVIDIA website

From 350f3980e779526d9b81647f96e586c2b3f4eb77 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Tue, 24 Jun 2025 15:14:01 -0700
Subject: [PATCH 2/8] no deps for torchvision

---
 .github/scripts/install-torch-tensorrt.sh     |  4 +---
 .github/workflows/build-test-linux-x86_64.yml | 17 +----------------
 MODULE.bazel                                  |  2 +-
 py/requirements.txt                           |  3 +--
 tests/py/requirements.txt                     |  3 +--
 tests/py/requirements_no_deps.txt             |  7 +++++++
 6 files changed, 12 insertions(+), 24 deletions(-)
 create mode 100644 tests/py/requirements_no_deps.txt

diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh
index 109b9565f3..3c58aa820e 100755
--- a/.github/scripts/install-torch-tensorrt.sh
+++ b/.github/scripts/install-torch-tensorrt.sh
@@ -2,7 +2,6 @@
 set -x
 
 TORCH=$(grep "^torch>" ${PWD}/py/requirements.txt)
-TORCHVISION=$(grep "^torchvision" ${PWD}/py/requirements.txt)
 INDEX_URL=https://download.pytorch.org/whl/${CHANNEL}/${CU_VERSION}
 PLATFORM=$(python -c "import sys; print(sys.platform)")
 
@@ -14,9 +13,8 @@ fi
 
 # Install all the dependencies required for Torch-TensorRT
 pip install --pre -r ${PWD}/tests/py/requirements.txt
+pip install --pre -r ${PWD}/tests/py/requirements_no_deps.txt --no-deps --extra-index-url ${INDEX_URL}
 pip install --force-reinstall --pre ${TORCH} --index-url ${INDEX_URL}
-pip install --force-reinstall --pre ${TORCHVISION} --index-url ${INDEX_URL}
-
 
 # Install Torch-TensorRT
 if [[ ${PLATFORM} == win32 ]]; then
diff --git a/.github/workflows/build-test-linux-x86_64.yml b/.github/workflows/build-test-linux-x86_64.yml
index f788f88dc9..51f3730d02 100644
--- a/.github/workflows/build-test-linux-x86_64.yml
+++ b/.github/workflows/build-test-linux-x86_64.yml
@@ -138,22 +138,7 @@ jobs:
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
-        cd tests/py
-        major=${PYTHON_VERSION%%.*}
-        minor=${PYTHON_VERSION#*.}
-        minor=${minor%%.*}
-        if (( major > 3 || (major == 3 && minor >= 13) )); then
-          echo "flashinfer-python is not supported for python version 3.13 or higher"
-        else
-          echo "Installing flashinfer-python"
-          # flashinfer-python is broken on python 3.9 at the moment, so we skip it for now
-          if (major == 3 && minor == 9); then
-            echo "Skipping flashinfer-python for python 3.9"
-          else
-            python -m pip install flashinfer-python --no-deps
-          fi
-        fi
-        cd dynamo
+        cd tests/py/dynamo
         python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml -n 4 conversion/
         python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin.py
         python -m pytest -ra --junitxml=${RUNNER_TEST_RESULTS_DIR}/dynamo_converters_test_results.xml automatic_plugin/test_automatic_plugin_with_attrs.py
diff --git a/MODULE.bazel b/MODULE.bazel
index a5cbca77ba..a22e70f071 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -66,7 +66,7 @@ http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu129/libtorch-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
 )
 
 # in aarch64 platform you can get libtorch via either local or wheel file
diff --git a/py/requirements.txt b/py/requirements.txt
index 00cb832331..302b7e92af 100644
--- a/py/requirements.txt
+++ b/py/requirements.txt
@@ -1,9 +1,8 @@
 numpy
 packaging
 pybind11==2.6.2
---extra-index-url https://download.pytorch.org/whl/nightly/cu128
+--extra-index-url https://download.pytorch.org/whl/nightly/cu129
 torch>=2.8.0.dev,<2.9.0
-torchvision>=0.22.0.dev,<0.23.0
 --extra-index-url https://pypi.ngc.nvidia.com
 pyyaml
 dllist
\ No newline at end of file
diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt
index dd217bfd5c..1b83803592 100644
--- a/tests/py/requirements.txt
+++ b/tests/py/requirements.txt
@@ -7,7 +7,6 @@ parameterized>=0.2.0
 pytest>=8.2.1
 pytest-xdist>=3.6.1
 pyyaml
-timm>=1.0.3
 transformers==4.49.0
-nvidia-modelopt[all]~=0.27.0; python_version >'3.9' and python_version <'3.13'
+nvidia-modelopt[all]; python_version >'3.9' and python_version <'3.13'
 --extra-index-url https://pypi.nvidia.com
\ No newline at end of file
diff --git a/tests/py/requirements_no_deps.txt b/tests/py/requirements_no_deps.txt
new file mode 100644
index 0000000000..2215449d52
--- /dev/null
+++ b/tests/py/requirements_no_deps.txt
@@ -0,0 +1,7 @@
+# flashinfer-python is not supported for python version 3.13 or higher
+# flashinfer-python is broken on python 3.9 at the moment, so skip it for now
+flashinfer-python; python_version >'3.9' and python_version <'3.13'
+--extra-index-url https://download.pytorch.org/whl/nightly/cu129
+torchvision>=0.23.0.dev,<0.24.0
+timm>=1.0.3
+

From 7a3a7e738170968d7d795f76960d29daf8613317 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Tue, 24 Jun 2025 16:41:19 -0700
Subject: [PATCH 3/8] test

---
 .github/scripts/install-cuda-aarch64.sh   | 7 +++++--
 .github/scripts/install-torch-tensorrt.sh | 6 +++++-
 MODULE.bazel                              | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/install-cuda-aarch64.sh b/.github/scripts/install-cuda-aarch64.sh
index 8301b333e4..2a9b6e0286 100755
--- a/.github/scripts/install-cuda-aarch64.sh
+++ b/.github/scripts/install-cuda-aarch64.sh
@@ -3,12 +3,15 @@ install_cuda_aarch64() {
     echo "install cuda ${CU_VERSION}"
     # CU_VERSION: cu128 --> CU_VER: 12-8
     CU_VER=${CU_VERSION:2:2}-${CU_VERSION:4:1}
+    # CU_VERSION: cu129 --> CU_DOT_VER: 12.9
+    CU_DOT_VER=${CU_VERSION:2:2}.${CU_VERSION:4:1}
     dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo
     dnf -y install cuda-compiler-${CU_VER}.aarch64 \
                    cuda-libraries-${CU_VER}.aarch64 \
-                   cuda-libraries-devel-${CU_VER}.aarch64
+                   cuda-libraries-devel-${CU_VER}.aarch64 \
+                   libnccl-2.26.5-1+cuda${CU_DOT_VER} libnccl-devel-2.26.5-1+cuda${CU_DOT_VER} libnccl-static-2.26.5-1+cuda${CU_DOT_VER}
     dnf clean all
-    export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+    export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/lib64:$LD_LIBRARY_PATH
     ls -lart /usr/local/
     nvcc --version
     echo "cuda ${CU_VER} installed successfully"
diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh
index 3c58aa820e..2ea9bc53ea 100755
--- a/.github/scripts/install-torch-tensorrt.sh
+++ b/.github/scripts/install-torch-tensorrt.sh
@@ -13,9 +13,13 @@ fi
 
 # Install all the dependencies required for Torch-TensorRT
 pip install --pre -r ${PWD}/tests/py/requirements.txt
-pip install --pre -r ${PWD}/tests/py/requirements_no_deps.txt --no-deps --extra-index-url ${INDEX_URL}
+# intentionally not using --no-deps here, because we need to install the dependencies
+pip install --pre -r ${PWD}/tests/py/requirements_no_deps.txt --extra-index-url ${INDEX_URL}
+# reinstall torch to make sure we have the correct version
+pip uninstall -y torch
 pip install --force-reinstall --pre ${TORCH} --index-url ${INDEX_URL}
 
+
 # Install Torch-TensorRT
 if [[ ${PLATFORM} == win32 ]]; then
     pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl
diff --git a/MODULE.bazel b/MODULE.bazel
index a22e70f071..1ecaebba28 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -66,7 +66,7 @@ http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-shared-with-deps-latest.zip"],
 )
 
 # in aarch64 platform you can get libtorch via either local or wheel file

From 3d98b42d0445263841442cdf75f4289b1fadc350 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Tue, 24 Jun 2025 17:12:09 -0700
Subject: [PATCH 4/8] test

---
 .github/scripts/install-torch-tensorrt.sh | 10 ++++++----
 tests/py/requirements.txt                 |  8 +++++++-
 tests/py/requirements_no_deps.txt         |  7 -------
 3 files changed, 13 insertions(+), 12 deletions(-)
 delete mode 100644 tests/py/requirements_no_deps.txt

diff --git a/.github/scripts/install-torch-tensorrt.sh b/.github/scripts/install-torch-tensorrt.sh
index 2ea9bc53ea..c567a6f172 100755
--- a/.github/scripts/install-torch-tensorrt.sh
+++ b/.github/scripts/install-torch-tensorrt.sh
@@ -2,6 +2,7 @@
 set -x
 
 TORCH=$(grep "^torch>" ${PWD}/py/requirements.txt)
+TORCHVISION=$(grep "^torchvision>" ${PWD}/tests/py/requirements.txt)
 INDEX_URL=https://download.pytorch.org/whl/${CHANNEL}/${CU_VERSION}
 PLATFORM=$(python -c "import sys; print(sys.platform)")
 
@@ -13,10 +14,11 @@ fi
 
 # Install all the dependencies required for Torch-TensorRT
 pip install --pre -r ${PWD}/tests/py/requirements.txt
-# intentionally not using --no-deps here, because we need to install the dependencies
-pip install --pre -r ${PWD}/tests/py/requirements_no_deps.txt --extra-index-url ${INDEX_URL}
-# reinstall torch to make sure we have the correct version
-pip uninstall -y torch
+# dependencies in the tests/py/requirements.txt might install a different version of torch or torchvision
+# eg. timm will install the latest torchvision, however we want to use the torchvision from nightly
+# reinstall torch torchvisionto make sure we have the correct version
+pip uninstall -y torch torchvision
+pip install --force-reinstall --pre ${TORCHVISION} --index-url ${INDEX_URL} --no-deps
 pip install --force-reinstall --pre ${TORCH} --index-url ${INDEX_URL}
 
 
diff --git a/tests/py/requirements.txt b/tests/py/requirements.txt
index 1b83803592..b806a668db 100644
--- a/tests/py/requirements.txt
+++ b/tests/py/requirements.txt
@@ -9,4 +9,10 @@ pytest-xdist>=3.6.1
 pyyaml
 transformers==4.49.0
 nvidia-modelopt[all]; python_version >'3.9' and python_version <'3.13'
---extra-index-url https://pypi.nvidia.com
\ No newline at end of file
+--extra-index-url https://pypi.nvidia.com
+# flashinfer-python is not supported for python version 3.13 or higher
+# flashinfer-python is broken on python 3.9 at the moment, so skip it for now
+flashinfer-python; python_version >'3.9' and python_version <'3.13'
+--extra-index-url https://download.pytorch.org/whl/nightly/cu129
+torchvision>=0.23.0.dev,<0.24.0
+timm>=1.0.3
\ No newline at end of file
diff --git a/tests/py/requirements_no_deps.txt b/tests/py/requirements_no_deps.txt
deleted file mode 100644
index 2215449d52..0000000000
--- a/tests/py/requirements_no_deps.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# flashinfer-python is not supported for python version 3.13 or higher
-# flashinfer-python is broken on python 3.9 at the moment, so skip it for now
-flashinfer-python; python_version >'3.9' and python_version <'3.13'
---extra-index-url https://download.pytorch.org/whl/nightly/cu129
-torchvision>=0.23.0.dev,<0.24.0
-timm>=1.0.3
-

From b1f390071c73ecde83e7d3a4d158e6f44e0e5c62 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Wed, 25 Jun 2025 08:20:26 -0700
Subject: [PATCH 5/8] fix windows issue

---
 .github/workflows/windows-test.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml
index a8b27c0aa9..a0bac5f0dc 100644
--- a/.github/workflows/windows-test.yml
+++ b/.github/workflows/windows-test.yml
@@ -39,7 +39,10 @@ on:
         description: "Prevents a job from failing when a step fails. Set to true to allow a job to pass when exec script step fails."
         default: false
         type: boolean
-
+      architecture:
+        description: 'CPU architecture to build for'
+        default: "x64"
+        type: string
 jobs:
   test:
     strategy:
@@ -107,7 +110,7 @@ jobs:
         if: ${{ matrix.tensorrt == '' }}
         uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ARTIFACT_NAME }}
+          name: ${{ env.ARTIFACT_NAME }}_${{ inputs.architecture }}
           path: ${{ runner.temp }}/artifacts/
       - name: Download artifacts
         if: ${{ matrix.tensorrt != '' }}

From c3597fb1116cde4ff65cc78c604a5198ad7cc354 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Wed, 25 Jun 2025 09:16:03 -0700
Subject: [PATCH 6/8] fix nccl version

---
 .github/scripts/install-cuda-aarch64.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/install-cuda-aarch64.sh b/.github/scripts/install-cuda-aarch64.sh
index 2a9b6e0286..3f0b3b2178 100755
--- a/.github/scripts/install-cuda-aarch64.sh
+++ b/.github/scripts/install-cuda-aarch64.sh
@@ -6,10 +6,11 @@ install_cuda_aarch64() {
     # CU_VERSION: cu129 --> CU_DOT_VER: 12.9
     CU_DOT_VER=${CU_VERSION:2:2}.${CU_VERSION:4:1}
     dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo
+    # nccl version must match libtorch_cuda.so was built with https://github.com/pytorch/pytorch/blob/main/.ci/docker/ci_commit_pins/nccl-cu12.txt
     dnf -y install cuda-compiler-${CU_VER}.aarch64 \
                    cuda-libraries-${CU_VER}.aarch64 \
                    cuda-libraries-devel-${CU_VER}.aarch64 \
-                   libnccl-2.26.5-1+cuda${CU_DOT_VER} libnccl-devel-2.26.5-1+cuda${CU_DOT_VER} libnccl-static-2.26.5-1+cuda${CU_DOT_VER}
+                   libnccl-2.27.3-1+cuda${CU_DOT_VER} libnccl-devel-2.27.3-1+cuda${CU_DOT_VER} libnccl-static-2.27.3-1+cuda${CU_DOT_VER}
     dnf clean all
     export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/lib64:$LD_LIBRARY_PATH
     ls -lart /usr/local/

From 08806a6f9b124e742fef40eaeb58a4cc4cfb05b8 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Wed, 25 Jun 2025 09:20:08 -0700
Subject: [PATCH 7/8] test

---
 .github/workflows/windows-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/windows-test.yml b/.github/workflows/windows-test.yml
index a0bac5f0dc..8dc1b107d3 100644
--- a/.github/workflows/windows-test.yml
+++ b/.github/workflows/windows-test.yml
@@ -110,7 +110,7 @@ jobs:
         if: ${{ matrix.tensorrt == '' }}
         uses: actions/download-artifact@v4
         with:
-          name: ${{ env.ARTIFACT_NAME }}_${{ inputs.architecture }}
+          name: ${{ env.ARTIFACT_NAME }}${{ inputs.architecture }}
           path: ${{ runner.temp }}/artifacts/
       - name: Download artifacts
         if: ${{ matrix.tensorrt != '' }}

From f9e18e08c4715f30979ad6d5bdb981edae2194a0 Mon Sep 17 00:00:00 2001
From: lanluo-nvidia <lanl@nvidia.com>
Date: Wed, 25 Jun 2025 10:15:16 -0700
Subject: [PATCH 8/8] test

---
 .github/workflows/build-test-tensorrt-linux.yml   | 8 ++++++++
 .github/workflows/build-test-tensorrt-windows.yml | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/.github/workflows/build-test-tensorrt-linux.yml b/.github/workflows/build-test-tensorrt-linux.yml
index 0cf9eab4fd..81814ea719 100644
--- a/.github/workflows/build-test-tensorrt-linux.yml
+++ b/.github/workflows/build-test-tensorrt-linux.yml
@@ -94,6 +94,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         export LD_LIBRARY_PATH=/usr/lib64:$LD_LIBRARY_PATH
@@ -130,6 +131,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -160,6 +162,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -190,6 +193,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -220,6 +224,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -252,6 +257,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -284,6 +290,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -316,6 +323,7 @@ jobs:
       build-matrix: ${{ needs.generate-tensorrt-matrix.outputs.matrix }}
       pre-script: ${{ matrix.pre-script }}
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
diff --git a/.github/workflows/build-test-tensorrt-windows.yml b/.github/workflows/build-test-tensorrt-windows.yml
index 0c847a459c..427b689656 100644
--- a/.github/workflows/build-test-tensorrt-windows.yml
+++ b/.github/workflows/build-test-tensorrt-windows.yml
@@ -101,6 +101,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -133,6 +134,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -160,6 +162,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -187,6 +190,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -214,6 +218,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -243,6 +248,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -272,6 +278,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .
@@ -300,6 +307,7 @@ jobs:
       build-matrix: ${{ needs.substitute-runner.outputs.matrix }}
       pre-script: packaging/driver_upgrade.bat
       script: |
+        set -euo pipefail
         export USE_HOST_DEPS=1
         export CI_BUILD=1
         pushd .