Switch to docker image based linux bots (google#582)

* Update scripts
* Switch to clang-8
* use ${PYTHON_EXECUTABLE} for lit tests

README.md

@@ -65,7 +65,7 @@ Show help:
 You will need:
 
 * CMake
-* Python
+* Python3
 * A C++ compiler
 * git
 
@@ -75,7 +75,7 @@ Clspv depends on the sources for other projects, at specific commits.
 Run the following command to download those dependencies, and place them
 in the `third_party` directory:
 
-    python utils/fetch_sources.py
+    python3 utils/fetch_sources.py
 
 ### Building
 

kokoro/amber-linux-gcc-release/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build-amber.sh RELEASE gcc
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="gcc" \
+  --env BUILD_TYPE="RelWithDebInfo" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build-amber.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/check-format/build-docker.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Copyright 2020 The Clspv Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Fail on any error.
+set -e
+
+. /bin/using.sh # Declare the bash 'using' function
+
+# Display commands being run.
+set -x
+
+BUILD_ROOT=$PWD
+SRC=$PWD/github/clspv
+
+using clang-8.0.0
+which clang-format
+
+cd $SRC
+/usr/bin/python3 utils/fetch_sources.py
+cp third_party/llvm/clang/tools/clang-format/clang-format-diff.py utils/clang-format-diff.py
+
+echo $(date): Check formatting...
+./utils/check_code_format.sh
+echo $(date): check completed.

kokoro/check-format/build.sh

@@ -18,18 +18,13 @@ set -e
 # Display commands being run.
 set -x
 
-BUILD_ROOT=$PWD
-SRC=$PWD/github/clspv
+ROOT_DIR=`pwd`
+SCRIPT_DIR=`dirname "$BASH_SOURCE"`
 
-# Get clang-format-5.0.0.
-# Once kokoro upgrades the Ubuntu VMs, we can use 'apt-get install clang-format'
-curl -L http://releases.llvm.org/5.0.0/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04.tar.xz -o clang-llvm.tar.xz
-tar xf clang-llvm.tar.xz
-export PATH=$PWD/clang+llvm-5.0.0-linux-x86_64-ubuntu14.04/bin:$PATH
-
-cd $SRC
-curl -L http://llvm.org/svn/llvm-project/cfe/trunk/tools/clang-format/clang-format-diff.py -o utils/clang-format-diff.py;
-
-echo $(date): Check formatting...
-./utils/check_code_format.sh $1
-echo $(date): check completed.
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="clang" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/build-docker.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/clvk-linux-clang-debug/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build-clvk.sh DEBUG clang
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="clang" \
+  --env BUILD_TYPE="Debug" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build-clvk.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/linux-clang-debug/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG clang
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="clang" \
+  --env BUILD_TYPE="Debug" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/linux-clang-release/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE clang
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="clang" \
+  --env BUILD_TYPE="RelWithDebInfo" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/linux-gcc-debug/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build.sh DEBUG gcc
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="gcc" \
+  --env BUILD_TYPE="Debug" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/linux-gcc-release/build.sh

@@ -18,5 +18,14 @@ set -e
 # Display commands being run.
 set -x
 
+ROOT_DIR=`pwd`
 SCRIPT_DIR=`dirname "$BASH_SOURCE"`
-source $SCRIPT_DIR/../scripts/linux/build.sh RELEASE gcc
+
+docker run --rm -i \
+  --volume "${ROOT_DIR}:${ROOT_DIR}" \
+  --volume "${KOKORO_ARTIFACTS_DIR}:/mnt/artifacts" \
+  --workdir "${ROOT_DIR}" \
+  --env BUILD_TOOLCHAIN="gcc" \
+  --env BUILD_TYPE="RelWithDebInfo" \
+  --entrypoint "${ROOT_DIR}/${SCRIPT_DIR}/../scripts/linux/build.sh" \
+  "gcr.io/shaderc-build/radial-build:latest"

kokoro/scripts/linux/build-amber.sh

@@ -15,90 +15,43 @@
 
 # Fail on any error.
 set -e
+
+. /bin/using.sh # Declare the bash 'using' function.
+
 # Display commands being run.
 set -x
 
+# Common tools.
+using cmake-3.17.2
+using gcc-7
+using ninja-1.10.0
+
 BUILD_ROOT=$PWD
 SRC=$PWD/github/clspv
 AMBER_SRC=$PWD/github/amber
-CONFIG=$1
-COMPILER=$2
-
-BUILD_TYPE="Debug"
-
-CMAKE_C_CXX_COMPILER=""
-if [ $COMPILER = "clang" ]; then
-  # Use newer clang than default.
-  CMAKE_C_CXX_COMPILER="-DCMAKE_C_COMPILER=/usr/bin/clang-5.0 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-5.0"
-else
-  # Specify we want to build with GCC 7 (which supports C++14)
-  sudo add-apt-repository ppa:ubuntu-toolchain-r/test
-  sudo apt-get update -qq
-  sudo aptitude install -y gcc-7 g++-7
-  sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 100 --slave /usr/bin/g++ g++ /usr/bin/g++-7
-  sudo update-alternatives --set gcc "/usr/bin/gcc-7"
-  CMAKE_C_CXX_COMPILER="-DCMAKE_C_COMPILER=/usr/bin/gcc-7 -DCMAKE_CXX_COMPILER=/usr/bin/g++-7"
-fi
-
-# Possible configurations are:
-# ASAN, COVERAGE, RELEASE, DEBUG, DEBUG_EXCEPTION, RELEASE_MINGW
-
-if [ $CONFIG = "RELEASE" ] || [ $CONFIG = "RELEASE_MINGW" ]
-then
-  BUILD_TYPE="RelWithDebInfo"
-fi
 
-ADDITIONAL_CMAKE_FLAGS=""
-if [ $CONFIG = "ASAN" ]
-then
-  ADDITIONAL_CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=-fsanitize=address -DCMAKE_C_FLAGS=-fsanitize=address"
-  export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer-3.4
-elif [ $CONFIG = "COVERAGE" ]
-then
-  ADDITIONAL_CMAKE_FLAGS="-DENABLE_CODE_COVERAGE=ON"
-  SKIP_TESTS="True"
-elif [ $CONFIG = "DEBUG_EXCEPTION" ]
-then
-  ADDITIONAL_CMAKE_FLAGS="-DDISABLE_EXCEPTIONS=ON -DDISABLE_RTTI=ON"
-elif [ $CONFIG = "RELEASE_MINGW" ]
-then
-  ADDITIONAL_CMAKE_FLAGS="-Dgtest_disable_pthreads=ON -DCMAKE_TOOLCHAIN_FILE=$SRC/cmake/linux-mingw-toolchain.cmake"
-  SKIP_TESTS="True"
-fi
-
-# removing the old version
-sudo apt-get purge -y --auto-remove cmake
-
-# Get newer version of CMake
-wget -q https://github.com/Kitware/CMake/releases/download/v3.16.0/cmake-3.16.0-Linux-x86_64.tar.gz
-tar xf cmake-3.16.0-Linux-x86_64.tar.gz
-export PATH="$PWD/cmake-3.16.0-Linux-x86_64/bin:$PATH"
-
-echo $(date): $(cmake --version)
-
-# Get NINJA.
-wget -q https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip
-unzip -q ninja-linux.zip
-export PATH="$PWD:$PATH"
-
-# Get dependencies.
 cd $SRC
 /usr/bin/python3 utils/fetch_sources.py
 
 cd $BUILD_ROOT/github
 git clone https://github.com/google/amber.git amber
 cd $BUILD_ROOT/github/amber
-/usr/bin/python3 tools/git-sync-deps --with-clspv --with-swiftshader
+/usr/bin/python3 tools/git-sync-deps --with-swiftshader
 
 mkdir build && cd $BUILD_ROOT/github/amber/build
 
+if [ "$BUILD_TOOLCHAIN" == "clang" ]; then
+  using clang-8.0.0
+fi
+
 # Invoke the build.
 BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
 echo $(date): Starting build...
 cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python3 -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
   -DAMBER_ENABLE_SWIFTSHADER=TRUE -DAMBER_USE_CLSPV=TRUE \
+  -DBUILD_WSI_XCB_SUPPORT=OFF \
   -DAMBER_CLSPV_DIR=$SRC -DAMBER_CLSPV_LLVM_DIR=$SRC/third_party/llvm \
-  $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
+   ..
 
 echo $(date): Build everything...
 ninja
@@ -111,4 +64,3 @@ export VK_ICD_FILENAMES=$BUILD_ROOT/github/amber/build/Linux/vk_swiftshader_icd.
   --dir $SRC/amber \
   --swiftshader
 echo $(date): Unit tests completed.
-