Merge pull request beeware#22 from paulproteus/openssl-upgrade

Upgrade OpenSSL to 1.1.1g

Author: freakboy3742

main.sh

@@ -105,34 +105,43 @@ function build_one_abi() {
     fix_permissions
 }
 
-# Download a file and verify its sha256sum.
-function download_verify_sha256() {
-    local url="$1"
-    local sha256="$2"
-    local filename_prefix="${3:-}"
-    local DOWNLOAD_CACHE="$PWD/downloads"
-    local DOWNLOAD_CACHE_TMP="$PWD/downloads.tmp"
-    local expected_filename="${filename_prefix}$(echo "$url" | tr '/' '\n' | tail -n1)"
+# Download a file into downloads/$name/$filename and verify its sha256sum.
+# If any files exist under downloads/$name, remove them. In the Dockerfile,
+# we refer to the tarball as downloads/$name/* , allowing the Dockerfile
+# to avoid redundantly stating the version number.
+function download() {
+    local name="$1"
+    local url="$2"
+    local sha256="$3"
+    local download_dir="${PWD}/downloads/$name"
+    local base_filename="$(echo "$url" | tr '/' '\n' | tail -n1)"
+    local full_filename="$download_dir/$base_filename"
+    local full_filename_tmp="${full_filename}.tmp"
 
     # Check existing file.
-    if [ -f "${DOWNLOAD_CACHE}/${expected_filename}" ] ; then
-        echo "Using ${expected_filename} from downloads/"
+    if [ -f "${full_filename}" ] ; then
+        echo "Using $name (${full_filename})"
         return
     fi
 
-    echo "Downloading $expected_filename"
-    rm -rf downloads.tmp && mkdir -p downloads.tmp
-    curl -L "$url" -o "downloads.tmp/$expected_filename"
+    echo "Downloading $name ($full_filename)"
+    rm -rf "$download_dir"
+    mkdir -p "$download_dir"
+    curl -L "$url" -o "$full_filename_tmp"
     local OK="no"
-    shasum -a 256 "${DOWNLOAD_CACHE_TMP}/${expected_filename}" | grep -q "$sha256" && OK="yes"
+    local actual_sha256=$(shasum -a 256 "${full_filename_tmp}")
+    echo $actual_sha256 | grep -q "$sha256" && OK="yes"
     if [ "$OK" = "yes" ] ; then
-        mkdir -p "$DOWNLOAD_CACHE"
-        mv "${DOWNLOAD_CACHE_TMP}/${expected_filename}" "${DOWNLOAD_CACHE}/${expected_filename}"
-        rmdir "${DOWNLOAD_CACHE_TMP}"
+        mv "${full_filename_tmp}" "${full_filename}"
     else
-        echo "Checksum mismatch while downloading: $url"
+        echo "Checksum mismatch while downloading $name <$url>"
+        echo "Expected: $sha256"
+        echo "     Got: $actual_sha256"
         echo ""
-        echo "Maybe your Internet connection got disconnected during the download. Please re-run the script."
+        echo "Maybe your Internet connection got disconnected during the download. Re-run"
+        echo "the script to re-download. If you're updating the version of this package"
+        echo "update the expected SHA in this script."
+        echo "Partial file remains in: ${full_filename_tmp}"
         echo "Aborting."
         exit 1
     fi
@@ -149,8 +158,8 @@ fix_permissions() {
 function main() {
     # Interpret argv for settings; first, set defaults. For some settings, create
     # DEFAULT_* variables for inclusion into help output.
-    local DEFAULT_VERSIONS="3.6,3.7"
-    local VERSIONS="$DEFAULT_VERSIONS"
+    local DEFAULT_VERSION="3.7"
+    local VERSION="$DEFAULT_VERSION"
     local DEFAULT_TARGET_ABIS="x86,x86_64,armeabi-v7a,arm64-v8a"
     local TARGET_ABIS="$DEFAULT_TARGET_ABIS"
     local DEFAULT_COMPRESS_LEVEL="8"
@@ -159,7 +168,7 @@ function main() {
     while getopts ":v:a:n:z:" opt; do
         case "${opt}" in
             v) # process Python version
-                VERSIONS="$OPTARG"
+                VERSION="$OPTARG"
                 ;;
             a) # process Android ABIs
                 TARGET_ABIS="$OPTARG"
@@ -174,12 +183,12 @@ function main() {
                 echo "Invalid option: $OPTARG requires an argument" 1>&2
                 ;;
             \? )
-                echo "Usage: main.sh [-v versions] [-a ABIs] [-n build_number] [-z compression_level]
+                echo "Usage: main.sh [-v version] [-a ABIs] [-n build_number] [-z compression_level]
 
 Build ZIP file of Python resources for Android, including CPython compiled as a .so.
 
--v: Specify Python versions to build, separated by commas. For example: -v 3.6,3.7
-    Default: ${DEFAULT_VERSIONS}
+-v: Specify Python version to build. For example: -v 3.6
+    Default: ${DEFAULT_VERSION}
 
 -a: Specify Android ABIs to build, separated by commas. For example: -a x86,arm64-v8a
     Default: ${TARGET_ABIS}
@@ -207,58 +216,50 @@ Build ZIP file of Python resources for Android, including CPython compiled as a
 
     echo "Downloading compile-time dependencies."
 
-    local build_dependencies=(
-        "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u242-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz=f39b523c724d0e0047d238eb2bb17a9565a60574cf651206c867ee5fc000ab43"
-        "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip=8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c"
-        "https://www.openssl.org/source/openssl-1.1.1f.tar.gz=186c6bfe6ecfba7a5b48c47f8a1673d0f3b0e5ba2e25602dd23b629975da3f35"
-        "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz=72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056"
-        "https://tukaani.org/xz/xz-5.2.4.tar.gz=b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145"
-        "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269"
-        "http://archive.ubuntu.com/ubuntu/pool/main/s/sqlite3/sqlite3_3.11.0.orig.tar.xz=79fb8800b8744337d5317270899a5a40612bb76f81517e131bf496c26b044490"
-    )
-    for build_dependency in "${build_dependencies[@]}" ; do
-        download_verify_sha256 ${build_dependency/=/ }
-    done
-
-    # Download rubicon-java source tarball with a rubicon-java-* filename prefix. This allows the
-    # Dockerfile to find it as rubicon-java-*.tar.gz . Other tarballs don't need this treatment
-    # because they have the project name in the filename.
-    download_verify_sha256 "https://github.com/beeware/rubicon-java/archive/v0.2.0.tar.gz" "b0d3d9ad4988c2d0e6995e2cbec085a5ef49b15e1be0d325b6141fb90fccccf7" "rubicon-java-"
+    download jdk "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u242-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz" "f39b523c724d0e0047d238eb2bb17a9565a60574cf651206c867ee5fc000ab43"
+    download ndk "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip" "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c"
+    download openssl "https://www.openssl.org/source/openssl-1.1.1g.tar.gz" "ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46"
+    download libffi "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz" "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056"
+    download xz "https://tukaani.org/xz/xz-5.2.5.tar.gz" "f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10"
+    download bzip2 "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz" "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269"
+    download sqlite3 "http://archive.ubuntu.com/ubuntu/pool/main/s/sqlite3/sqlite3_3.11.0.orig.tar.xz" "79fb8800b8744337d5317270899a5a40612bb76f81517e131bf496c26b044490"
+    download rubicon-java "https://github.com/beeware/rubicon-java/archive/v0.2.1.tar.gz" "a1d1c6edccbd75631a0c3cc129239e10f7b6d8f221a393b96fbdc83293636f8b"
 
-    echo "Downloading Python versions, as needed."
-    for version in ${VERSIONS//,/ } ; do
-        if [ "$version" = "3.7" ] ; then
-            download_verify_sha256 "https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tar.xz" "55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f"
-        elif [ "$version" = "3.6" ] ; then
-            download_verify_sha256 "https://www.python.org/ftp/python/3.6.10/Python-3.6.10.tar.xz" "0a833c398ac8cd7c5538f7232d8531afef943c60495c504484f308dac3af40de"
-        else
-            echo "Unknown Python version: $version. Aborting."
+    echo "Downloading Python version."
+    case "$VERSION" in
+        3.6)
+            download "python-3.6" "https://www.python.org/ftp/python/3.6.10/Python-3.6.10.tar.xz" "0a833c398ac8cd7c5538f7232d8531afef943c60495c504484f308dac3af40de"
+            ;;
+        3.7)
+            download "python-3.7" "https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tar.xz" "55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f"
+            ;;
+        *)
+            echo "Invalid Python version: $VERSION"
             exit 1
-        fi
-    done
+            ;;
+    esac
 
     echo 'Starting Docker builds.'
-    for VERSION in ${VERSIONS//,/ } ; do
-        # Clear the build directory.
-        mkdir -p build
-        mkdir -p dist
-        fix_permissions
-        rm -rf ./build/"$VERSION"
-        mkdir -p build/"$VERSION"
 
-        # Build each ABI.
-        for TARGET_ABI_SHORTNAME in ${TARGET_ABIS//,/ }; do
-            echo "Building Python $VERSION for $TARGET_ABI_SHORTNAME"
-            build_one_abi "$TARGET_ABI_SHORTNAME" "$VERSION" "$COMPRESS_LEVEL"
-        done
+    # Clear the build directory.
+    mkdir -p build
+    mkdir -p dist
+    fix_permissions
+    rm -rf ./build/"$VERSION"
+    mkdir -p build/"$VERSION"
 
-        # Make a ZIP file, writing it first to `.tmp` so that we atomically clobber an
-        # existing ZIP file rather than attempt to merge the new contents with old.
-        pushd build/"$VERSION"/app > /dev/null
-        zip -x@../../../excludes/all/excludes -r -"${COMPRESS_LEVEL}" "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip".tmp .
-        mv "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip".tmp "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip"
-        popd
+    # Build each ABI.
+    for TARGET_ABI_SHORTNAME in ${TARGET_ABIS//,/ }; do
+        echo "Building Python $VERSION for $TARGET_ABI_SHORTNAME"
+        build_one_abi "$TARGET_ABI_SHORTNAME" "$VERSION" "$COMPRESS_LEVEL"
     done
+
+    # Make a ZIP file, writing it first to `.tmp` so that we atomically clobber an
+    # existing ZIP file rather than attempt to merge the new contents with old.
+    pushd build/"$VERSION"/app > /dev/null
+    zip -x@../../../excludes/all/excludes -r -"${COMPRESS_LEVEL}" "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip".tmp .
+    mv "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip".tmp "../../../dist/Python-$VERSION-Android-support${BUILD_TAG}.zip"
+    popd
 }
 
 main "$@"

python.Dockerfile

@@ -6,13 +6,14 @@ RUN apt-get update -qq && apt-get -qq install unzip rsync
 
 # Install toolchains: Android NDK & Java JDK.
 WORKDIR /opt/ndk
-ADD downloads/android-ndk-r20b-linux-x86_64.zip .
-RUN unzip -q android-ndk-r20b-linux-x86_64.zip && rm android-ndk-r20b-linux-x86_64.zip
-ENV NDK /opt/ndk/android-ndk-r20b
+ADD downloads/ndk/* .
+RUN unzip -q android-ndk-*-linux-x86_64.zip && rm android-ndk-*-linux-x86_64.zip && mv android-ndk-* android-ndk
+ENV NDK /opt/ndk/android-ndk
 WORKDIR /opt/jdk
-ADD downloads/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz .
-ENV JAVA_HOME /opt/jdk/jdk8u242-b08/
-ENV PATH "/opt/jdk/jdk8u242-b08/bin:${PATH}"
+ADD downloads/jdk/* .
+RUN mv jdk* jdk_home
+ENV JAVA_HOME /opt/jdk/jdk_home/
+ENV PATH "/opt/jdk/jdk_home/bin:${PATH}"
 
 # Store output here; the directory structure corresponds to our Android app template.
 ENV APPROOT /opt/python-build/approot
@@ -49,51 +50,56 @@ ENV AR=$TOOLCHAIN/bin/$TOOLCHAIN_TRIPLE-ar \
 # We hard-code avoid_version=yes into libtool so that libsqlite3.so is the SONAME.
 FROM toolchain as build_sqlite
 RUN apt-get update -qq && apt-get -qq install make autoconf autotools-dev tcl8.6-dev build-essential
-ADD downloads/sqlite3_3.11.0.orig.tar.xz .
-RUN cd sqlite3-3.11.0 && autoreconf && cp -f /usr/share/misc/config.sub . && cp -f /usr/share/misc/config.guess .
-RUN cd sqlite3-3.11.0 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$BUILD_HOME/built/sqlite"
-RUN cd sqlite3-3.11.0 && sed -i -E 's,avoid_version=no,avoid_version=yes,' ltmain.sh libtool
-RUN cd sqlite3-3.11.0 && make install
+ADD downloads/sqlite3/* .
+RUN mv sqlite3-* sqlite3-src
+RUN cd sqlite3-src && autoreconf && cp -f /usr/share/misc/config.sub . && cp -f /usr/share/misc/config.guess .
+RUN cd sqlite3-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$BUILD_HOME/built/sqlite"
+RUN cd sqlite3-src && sed -i -E 's,avoid_version=no,avoid_version=yes,' ltmain.sh libtool
+RUN cd sqlite3-src && make install
 
 # Install bzip2 & lzma libraries, for stdlib's _bzip2 and _lzma modules.
 FROM toolchain as build_xz
 RUN apt-get update -qq && apt-get -qq install make
-ADD downloads/xz-5.2.4.tar.gz .
+ADD downloads/xz/* .
+RUN mv xz-* xz-src
 ENV LIBXZ_INSTALL_DIR="$BUILD_HOME/built/xz"
 RUN mkdir -p "$LIBXZ_INSTALL_DIR"
-RUN cd xz-5.2.4 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBXZ_INSTALL_DIR"
-RUN cd xz-5.2.4 && make install
+RUN cd xz-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBXZ_INSTALL_DIR"
+RUN cd xz-src && make install
 
 FROM toolchain as build_bz2
 RUN apt-get update -qq && apt-get -qq install make
 ENV LIBBZ2_INSTALL_DIR="$BUILD_HOME/built/libbz2"
-ADD downloads/bzip2-1.0.8.tar.gz .
+ADD downloads/bzip2/* .
+RUN mv bzip2-* bzip2-src
 RUN mkdir -p "$LIBBZ2_INSTALL_DIR" && \
-    cd bzip2-1.0.8 && \
+    cd bzip2-src && \
     sed -i -e 's,[.]1[.]0.8,,' -e 's,[.]1[.]0,,' -e 's,ln -s,#ln -s,' -e 's,rm -f libbz2.so,#rm -f libbz2.so,' -e 's,^CC=,#CC=,' Makefile-libbz2_so
-RUN cd bzip2-1.0.8 && make -f Makefile-libbz2_so
+RUN cd bzip2-src && make -f Makefile-libbz2_so
 RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/lib"
-RUN cp bzip2-1.0.8/libbz2.so "${LIBBZ2_INSTALL_DIR}/lib"
+RUN cp bzip2-src/libbz2.so "${LIBBZ2_INSTALL_DIR}/lib"
 RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/include"
-RUN cp bzip2-1.0.8/bzlib.h "${LIBBZ2_INSTALL_DIR}/include"
+RUN cp bzip2-src/bzlib.h "${LIBBZ2_INSTALL_DIR}/include"
 
 # libffi is required by ctypes
 FROM toolchain as build_libffi
 RUN apt-get update -qq && apt-get -qq install file make
-ADD downloads/libffi-3.3.tar.gz .
+ADD downloads/libffi/* .
+RUN mv libffi-* libffi-src
 ENV LIBFFI_INSTALL_DIR="$BUILD_HOME/built/libffi"
 RUN mkdir -p "$LIBFFI_INSTALL_DIR"
-RUN cd libffi-3.3 && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBFFI_INSTALL_DIR"
-RUN cd libffi-3.3 && make install
+RUN cd libffi-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBFFI_INSTALL_DIR"
+RUN cd libffi-src && make install
 
 FROM toolchain as build_openssl
 # OpenSSL requires libfindlibs-libs-perl. make is nice, too.
 RUN apt-get update -qq && apt-get -qq install libfindbin-libs-perl make
-ADD downloads/openssl-1.1.1f.tar.gz .
+ADD downloads/openssl/* .
+RUN mv openssl-* openssl-src
 ARG OPENSSL_BUILD_TARGET
-RUN cd openssl-1.1.1f && ANDROID_NDK_HOME="$NDK" ./Configure ${OPENSSL_BUILD_TARGET} -D__ANDROID_API__="$ANDROID_API_LEVEL" --prefix="$BUILD_HOME/built/openssl" --openssldir="$BUILD_HOME/built/openssl"
-RUN cd openssl-1.1.1f && make SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
-RUN cd openssl-1.1.1f && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
+RUN cd openssl-src && ANDROID_NDK_HOME="$NDK" ./Configure ${OPENSSL_BUILD_TARGET} -D__ANDROID_API__="$ANDROID_API_LEVEL" --prefix="$BUILD_HOME/built/openssl" --openssldir="$BUILD_HOME/built/openssl"
+RUN cd openssl-src && make SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
+RUN cd openssl-src && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
 
 # This build container builds Python, rubicon-java, and any dependencies. Each Python version
 # requires itself to be installed globally during a cross-compile, and Python 3.6 additionally
@@ -116,7 +122,7 @@ ENV PKG_CONFIG_PATH="/opt/python-build/built/libffi/lib/pkgconfig:/opt/python-bu
 
 # Download & patch Python. We assume that there is only one Python-${VERSION}.*.tar.xz file.
 ARG PYTHON_VERSION
-ADD downloads/Python-${PYTHON_VERSION}.*.tar.xz .
+ADD downloads/python-${PYTHON_VERSION}/* .
 RUN mv Python-* python-src
 # Modify ./configure so that, even though this is Linux, it does not append .1.0 to the .so file.
 RUN sed -i -e 's,INSTSONAME="$LDLIBRARY".$SOVERSION,,' python-src/configure
@@ -205,7 +211,7 @@ RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython${PYTHON_VERSION}m.so "$JNI_LIBS"
 # Download & install rubicon-java's Java & C parts. The *.py files in rubicon-java are
 # incorporated into apps via app dependency management and are ABI-independent since
 # they access the C library via `ctypes`.
-ADD downloads/rubicon-java-* .
+ADD downloads/rubicon-java/* .
 RUN mv rubicon-java-* rubicon-java-src
 RUN cd rubicon-java-src && \
     LDFLAGS='-landroid -llog' PYTHON_CONFIG=$PYTHON_INSTALL_DIR/bin/python3-config make