win32_build.sh: mingw-llvm support

winpthreads is a library that emulates the pthreads API using Windows primitives. It's also used by the mingw/gcc libstdc++ for std::thread. The issue is that winpthreads isn't well maintained. There have been numerous bugs that haven't been addressed in years. Specifically, we've been hitting deadlocks because of the winpthreads rw lock implementation. This change will allow building Ceph for Windows using mingw/llvm, which uses libc++ and doesn't rely on winpthreads. Signed-off-by: Lucian Petrut <[email protected]>
sajibreadd · Aug 30, 2023 · 3ce1e4a · 3ce1e4a
1 parent 82ef35d
commit 3ce1e4a
Show file tree

Hide file tree

Showing 4 changed files with 161 additions and 52 deletions.
diff --git a/README.windows.rst b/README.windows.rst
@@ -29,6 +29,8 @@ Flag               Description                      Default value
 =================  ===============================  ===============================
 OS                 Host OS distribution, for mingw  ubuntu (also valid: suse)
                    and other OS specific settings.
+TOOLCHAIN          Mingw toolchain: mingw-llvm or   mingw-llvm
+                   mingw-gcc.
 CEPH_DIR           The Ceph source code directory.  The same as the script.
 BUILD_DIR          The directory where the          $CEPH_DIR/build
                    generated artifacts will be

diff --git a/mingw_conf.sh b/mingw_conf.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 # MINGW Settings:
 # Due to inconsistencies between distributions, mingw versions, binaries,
 # and directories must be determined (or defined) prior to building.
@@ -8,6 +10,16 @@
 # * MINGW_CMAKE_FILE - if set, a cmake toolchain file will be created
 # * MINGW_POSIX_FLAGS - if set, Mingw Posix compatibility mode will be
 #                       enabled by defining the according flags.
+# * USE_MINGW_LLVM - allows using the mingw llvm toolchain
+# * MINGW_LLVM_DIR - allows specifying the mingw-llvm toolchain location.
+#                    If unset, we'll use the default path from build.deps.
+
+SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
+SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    MINGW_LLVM_DIR=${MINGW_LLVM_DIR:-"$SCRIPT_DIR/build.deps/mingw-llvm"}
+fi
 
 # -Common mingw settings-
 MINGW_PREFIX="x86_64-w64-mingw32-"
@@ -17,60 +29,82 @@ MINGW_DLLTOOL="${MINGW_BASE}-dlltool"
 MINGW_WINDRES="${MINGW_BASE}-windres"
 MINGW_STRIP="${MINGW_BASE}-strip"
 MINGW_OBJCOPY="${MINGW_BASE}-objcopy"
-# -Distribution specific mingw settings-
-case "$OS" in
-    ubuntu)
-        mingwPosix="-posix"
-        mingwLibDir="/usr/lib/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
-        mingwLibpthreadDir="/usr/${MINGW_BASE}/lib"
-        PTW32Include=/usr/share/mingw-w64/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/lib
-       ;;
-    rhel)
-        mingwPosix=""
-        mingwLibDir="/usr/lib64/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
-        mingwLibpthreadDir="$mingwTargetLibDir"
-        PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
-       ;;
-    suse)
-        mingwPosix=""
-        mingwLibDir="/usr/lib64/gcc"
-        mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
-        mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
-        mingwLibpthreadDir="$mingwTargetLibDir"
-        PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
-        PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
-        ;;
-    *)
-        echo "$ID is unknown, automatic mingw configuration is not possible."
-        exit 1
-        ;;
-esac
-# -Common mingw settings, dependent upon distribution specific settings-
-MINGW_FIND_ROOT_LIB_PATH="${mingwLibDir}/\${TOOLCHAIN_PREFIX}/${mingwVersion}"
-MINGW_CC="${MINGW_BASE}-gcc${mingwPosix}"
-MINGW_CXX="${MINGW_BASE}-g++${mingwPosix}"
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    # This package isn't currently provided by Linux distributions, we're
+    # fetching it from Github.
+    export PATH="$MINGW_LLVM_DIR/bin:$PATH"
+    mingwPosix=""
+    mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+    mingwX64IncludeDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/include"
+    mingwX64BinDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/bin"
+    mingwX64LibDir="$MINGW_LLVM_DIR/x86_64-w64-mingw32/lib"
+    mingwTargetLibDir="$mingwX64BinDir"
+    mingwLibpthreadDir="$mingwX64BinDir"
+    PTW32Include="$mingwX64IncludeDir"
+    PTW32Lib="$mingwX64LibDir"
+
+    MINGW_CC="${MINGW_BASE}-clang${mingwPosix}"
+    MINGW_CXX="${MINGW_BASE}-clang++${mingwPosix}"
+
+    MINGW_FIND_ROOT_PATH="$MINGW_LLVM_DIR/x86_64-w64-mingw32"
+else
+    # -Distribution specific mingw settings-
+    case "$OS" in
+        ubuntu)
+            mingwPosix="-posix"
+            mingwLibDir="/usr/lib/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
+            mingwLibpthreadDir="/usr/${MINGW_BASE}/lib"
+            PTW32Include=/usr/share/mingw-w64/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/lib
+            ;;
+        rhel)
+            mingwPosix=""
+            mingwLibDir="/usr/lib64/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
+            mingwLibpthreadDir="$mingwTargetLibDir"
+            PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
+            ;;
+        suse)
+            mingwPosix=""
+            mingwLibDir="/usr/lib64/gcc"
+            mingwVersion="$(${MINGW_CPP}${mingwPosix} -dumpversion)"
+            mingwTargetLibDir="/usr/${MINGW_BASE}/sys-root/mingw/bin"
+            mingwLibpthreadDir="$mingwTargetLibDir"
+            PTW32Include=/usr/x86_64-w64-mingw32/sys-root/mingw/include
+            PTW32Lib=/usr/x86_64-w64-mingw32/sys-root/mingw/lib
+            ;;
+        *)
+            echo "$ID is unknown, automatic mingw configuration is not possible."
+            exit 1
+            ;;
+    esac
+    MINGW_CC="${MINGW_BASE}-gcc${mingwPosix}"
+    MINGW_CXX="${MINGW_BASE}-g++${mingwPosix}"
+
+    # -Common mingw settings, dependent upon distribution specific settings-
+    MINGW_FIND_ROOT_LIB_PATH="${mingwLibDir}/${MINGW_BASE}/${mingwVersion}"
+    MINGW_FIND_ROOT_PATH="/usr/${MINGW_BASE} ${MINGW_FIND_ROOT_LIB_PATH}"
+fi
 # End MINGW configuration
 
 
 if [[ -n $MINGW_CMAKE_FILE ]]; then
     cat > $MINGW_CMAKE_FILE <<EOL
 set(CMAKE_SYSTEM_NAME Windows)
-set(TOOLCHAIN_PREFIX ${MINGW_BASE})
 set(CMAKE_SYSTEM_PROCESSOR x86_64)
 
 # We'll need to use posix threads in order to use
 # C++11 features, such as std::thread.
-set(CMAKE_C_COMPILER \${TOOLCHAIN_PREFIX}-gcc${mingwPosix})
-set(CMAKE_CXX_COMPILER \${TOOLCHAIN_PREFIX}-g++${mingwPosix})
-set(CMAKE_RC_COMPILER \${TOOLCHAIN_PREFIX}-windres)
+set(CMAKE_C_COMPILER ${MINGW_CC})
+set(CMAKE_CXX_COMPILER ${MINGW_CXX})
+set(CMAKE_RC_COMPILER ${MINGW_WINDRES})
 
-set(CMAKE_FIND_ROOT_PATH /usr/\${TOOLCHAIN_PREFIX} ${MINGW_FIND_ROOT_LIB_PATH})
+set(CMAKE_FIND_ROOT_PATH ${MINGW_FIND_ROOT_PATH})
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 # TODO: consider switching this to "ONLY". The issue with
 # that is that all our libs should then be under
@@ -88,4 +122,12 @@ add_definitions(-D_POSIX_=1)
 add_definitions(-D_POSIX_THREADS=1)
 EOL
     fi
-fi
+
+    if [[ -n $USE_MINGW_LLVM ]]; then
+        cat >> $MINGW_CMAKE_FILE <<EOL
+add_definitions(-I$mingwX64IncludeDir)
+add_definitions(-march=native)
+add_definitions(-Wno-unknown-attributes)
+EOL
+    fi
+fi
diff --git a/win32_build.sh b/win32_build.sh
@@ -60,6 +60,25 @@ if [[ -z $OS ]]; then
 fi
 export OS="$OS"
 
+# The main advantages of mingw-llvm:
+# * not affected by the libstdc++/winpthread rw lock bugs
+# * can generate pdb debug symbols, which are compatible with WinDBG
+TOOLCHAIN=${TOOLCHAIN:-"mingw-llvm"}
+
+case "$TOOLCHAIN" in
+    mingw-llvm)
+        echo "Using mingw-llvm."
+        export USE_MINGW_LLVM=1
+        ;;
+    mingw-gcc)
+        echo "Using mingw-gcc"
+        ;;
+    *)
+        echo "Unsupported toolchain: $TOOLCHAIN."
+        echo "Allowed toolchains: mingw-llvm or mingw-gcc."
+esac
+
+
 # We'll have to be explicit here since auto-detecting doesn't work
 # properly when cross compiling.
 ALLOCATOR=${ALLOCATOR:-libc}
@@ -212,11 +231,18 @@ if [[ -z $SKIP_DLL_COPY ]]; then
         $lz4Dir/lib/dll/liblz4-1.dll
         $sslDir/bin/libcrypto-1_1-x64.dll
         $sslDir/bin/libssl-1_1-x64.dll
-        $mingwTargetLibDir/libstdc++-6.dll
-        $mingwTargetLibDir/libgcc_s_seh-1.dll
-        $mingwTargetLibDir/libssp*.dll
         $mingwLibpthreadDir/libwinpthread-1.dll
         $boostDir/lib/*.dll)
+    if [[ -n $USE_MINGW_LLVM ]]; then
+        required_dlls+=(
+            $mingwTargetLibDir/libc++.dll
+            $mingwTargetLibDir/libunwind.dll)
+    else
+        required_dlls+=(
+            $mingwTargetLibDir/libstdc++-6.dll
+            $mingwTargetLibDir/libssp*.dll
+            $mingwTargetLibDir/libgcc_s_seh-1.dll)
+    fi
     echo "Copying required dlls to $binDir."
     cp ${required_dlls[@]} $binDir
 fi

diff --git a/win32_deps_build.sh b/win32_deps_build.sh
@@ -5,6 +5,8 @@ set -e
 SCRIPT_DIR="$(dirname "$BASH_SOURCE")"
 SCRIPT_DIR="$(realpath "$SCRIPT_DIR")"
 
+USE_MINGW_LLVM=${USE_MINGW_LLVM:-}
+
 num_vcpus=$(nproc)
 NUM_WORKERS=${NUM_WORKERS:-$num_vcpus}
 
@@ -21,6 +23,7 @@ sslSrcDir="${depsSrcDir}/openssl"
 
 # For now, we'll keep the version number within the file path when not using git.
 boostUrl="https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+boostSha256Sum="66a469b6e608a51f8347236f4912e27dc5c60c60d7d53ae9bfe4683316c6f04c"
 boostSrcDir="${depsSrcDir}/boost_1_82_0"
 boostDir="${depsToolsetDir}/boost"
 zlibDir="${depsToolsetDir}/zlib"
@@ -43,6 +46,10 @@ dokanTag="v2.0.5.1000"
 dokanSrcDir="${depsSrcDir}/dokany"
 dokanLibDir="${depsToolsetDir}/dokany/lib"
 
+mingwLlvmUrl="https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-msvcrt-ubuntu-18.04-x86_64.tar.xz"
+mingwLlvmSha256Sum="bc97745e702fb9e8f2a16f7d09dd5061ceeef16554dd12e542f619ce937e8d7a"
+mingwLlvmDir="${DEPS_DIR}/mingw-llvm"
+
 # Allow for OS specific customizations through the OS flag (normally
 # passed through from win32_build).
 # Valid options are currently "ubuntu", "rhel", and "suse".
@@ -100,6 +107,22 @@ case "$OS" in
         ;;
 esac
 
+if [[ -n $USE_MINGW_LLVM && ! -d $mingwLlvmDir ]]; then
+    echo "Fetching mingw-llvm"
+    cd $DEPS_DIR
+    wget -q -O mingw-llvm.tar.xz $mingwLlvmUrl
+    checksum=`sha256sum mingw-llvm.tar.xz | cut -d ' ' -f 1`
+    if [[ "$mingwLlvmSha256Sum" != "$checksum" ]]; then
+        echo "Invalid mingw-llvm checksum: $checksum" >&2
+        exit 1
+    fi
+    tar xJf mingw-llvm.tar.xz
+    rm mingw-llvm.tar.xz
+    # Remove the version from the mingw-llvm dirname, making it easier to locate
+    # and avoiding MAX_PATH issues with WSL.
+    mv `basename $mingwLlvmUrl | sed 's/\.tar\..*//g'` $mingwLlvmDir
+fi
+
 MINGW_CMAKE_FILE="$DEPS_DIR/mingw.cmake"
 source "$SCRIPT_DIR/mingw_conf.sh"
 
@@ -148,17 +171,33 @@ echo "Building boost."
 cd $depsSrcDir
 if [[ ! -d $boostSrcDir ]]; then
     echo "Downloading boost."
-    wget -qO- $boostUrl | tar xz
+    wget -q -O boost.tar.gz $boostUrl
+    checksum=`sha256sum boost.tar.gz | cut -d ' ' -f 1`
+    if [[ "$boostSha256Sum" != "$checksum" ]]; then
+        echo "Invalid boost checksum: $checksum" >&2
+        exit 1
+    fi
+    tar xzf boost.tar.gz
+    rm boost.tar.gz
 fi
 
 cd $boostSrcDir
-echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
+
+if [[ -n $USE_MINGW_LLVM ]]; then
+    b2toolset="clang"
+    echo "using clang :  : ${MINGW_CXX} ;" > user-config.jam
+else
+    b2toolset="gcc-mingw32"
+    echo "using gcc : mingw32 : ${MINGW_CXX} ;" > user-config.jam
+fi
 
 # Workaround for https://github.com/boostorg/thread/issues/156
 # Older versions of mingw provided a different pthread lib.
 sed -i 's/lib$(libname)GC2.a/lib$(libname).a/g' ./libs/thread/build/Jamfile.v2
-sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
-sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
+if [[ -z $USE_MINGW_LLVM ]]; then
+    sed -i 's/mthreads/pthreads/g' ./tools/build/src/tools/gcc.jam
+    sed -i 's/pthreads/mthreads/g' ./tools/build/src/tools/gcc.jam
+fi
 
 export PTW32_INCLUDE=${PTW32Include}
 export PTW32_LIB=${PTW32Lib}
@@ -197,7 +236,7 @@ EOL
 
 ./bootstrap.sh
 
-./b2 install --user-config=user-config.jam toolset=gcc-mingw32 \
+./b2 install --user-config=user-config.jam toolset=$b2toolset \
     target-os=windows release \
     link=static,shared \
     threadapi=win32 --prefix=$boostDir \