Simplify image tagging, and allow for build log exfiltration.

freakboy3742 · freakboy3742 · commit 075407428c86 · 2022-05-25T14:01:22.000+08:00
diff --git a/main.sh b/main.sh
@@ -100,9 +100,10 @@ function build_one_abi() {
     # We do also use a temporary tag name so that we can use `rsync` to pull some
     # data out of the image.
     #
-    # For debugging the Docker image, you can use the name python-android-support-local:latest.
-    TAG_NAME="python-android-support-local:$(python3 -c 'import random; print(random.randint(0, 1e16))')"
-    DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --tag ${TAG_NAME} --tag python-android-support-local:latest \
+    # For debugging the Docker image, you can use the name python-android-support-<version>:<abi>.
+    # e.g. python-android-support-3.7:arm64-v8a
+    TAG_NAME="python-android-support-${PYTHON_VERSION}:${TARGET_ABI_SHORTNAME}"
+    DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --tag ${TAG_NAME} \
         --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg PYTHON_SOVERSION="${PYTHON_SOVERSION}" \
         --build-arg COMPRESS_LEVEL="${COMPRESS_LEVEL}" --build-arg COMPILER_TRIPLE="${COMPILER_TRIPLE}" \
         --build-arg OPENSSL_BUILD_TARGET="$OPENSSL_BUILD_TARGET" --build-arg TARGET_ABI_SHORTNAME="$TARGET_ABI_SHORTNAME" \
@@ -113,6 +114,8 @@ function build_one_abi() {
     docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/approot/. /mnt/.
     # Extract header files
     docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/app/include/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/built/python/include/ /mnt/
+    # Extract log files
+    docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/logs/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/logs/ /mnt/
 
     # Docker creates files as root; reown as the local user
     fix_permissions
@@ -121,8 +124,6 @@ function build_one_abi() {
     mv "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/pyconfig.h "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/pyconfig-${TARGET_ABI_SHORTNAME}.h
     # Inject a platform-agnostic pyconfig.h wrapper.
     cp "${PWD}/patches/all/pyconfig.h" "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/
-    # Remove temporary local tag.
-    docker rmi "$TAG_NAME" > /dev/null
 }
 
 # Download a file into downloads/$name/$filename and verify its sha256sum.
diff --git a/patches/3.10/Setup.local b/patches/3.10/Setup.local
@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp
diff --git a/patches/3.11/Setup.local b/patches/3.11/Setup.local
@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp
diff --git a/patches/3.7/Setup.local b/patches/3.7/Setup.local
@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp
diff --git a/patches/3.8/Setup.local b/patches/3.8/Setup.local
@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp
diff --git a/patches/3.9/Setup.local b/patches/3.9/Setup.local
@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp
diff --git a/python.Dockerfile b/python.Dockerfile
@@ -12,7 +12,7 @@ ENV NDK /opt/ndk/android-ndk
 WORKDIR /opt/jdk
 ADD downloads/jdk/* .
 RUN mv jdk* jdk_home
-ENV JAVA_HOME /opt/jdk/jdk_home/
+ENV JAVA_HOME /opt/jdk/jdk_home
 ENV PATH "/opt/jdk/jdk_home/bin:${PATH}"
 
 # Store output here; the directory structure corresponds to our Android app template.
@@ -44,18 +44,21 @@ ENV AR=$TOOLCHAIN/bin/llvm-ar \
     READELF=$TOOLCHAIN/bin/llvm-readelf \
     CFLAGS="-fPIC -Wall -O0 -g"
 
-# We build sqlite using a tarball from Ubuntu. We need to patch config.sub & config.guess so
-# autoconf can accept our weird TOOLCHAIN_TRIPLE value. It requires tcl8.6-dev and build-essential
+# Set up a directory for logs.
+ENV LOGS_DIR=${BUILD_HOME}/logs/${TARGET_ABI_SHORTNAME}
+RUN mkdir -p ${LOGS_DIR}
+
+# We build sqlite from official sources. It requires tcl8.6-dev and build-essential
 # because the compile process build and executes some commands on the host as part of the build process.
 # We hard-code avoid_version=yes into libtool so that libsqlite3.so is the SONAME.
 FROM toolchain as build_sqlite
 RUN apt-get update -qq && apt-get -qq install make autoconf autotools-dev tcl8.6-dev build-essential
 ADD downloads/sqlite3/* .
 RUN unzip -q version-*.zip && mv sqlite-* sqlite3-src
 RUN cd sqlite3-src && autoreconf
-RUN cd sqlite3-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$BUILD_HOME/built/sqlite"
+RUN cd sqlite3-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$BUILD_HOME/built/sqlite" | tee -a $LOGS_DIR/sqlite3.configure.log
 RUN cd sqlite3-src && sed -i -E 's,avoid_version=no,avoid_version=yes,' ltmain.sh libtool
-RUN cd sqlite3-src && make install
+RUN cd sqlite3-src && make install | tee -a $LOGS_DIR/sqlite3.install.log
 
 # Install bzip2 & lzma libraries, for stdlib's _bzip2 and _lzma modules.
 FROM toolchain as build_xz
@@ -64,8 +67,8 @@ ADD downloads/xz/* .
 RUN mv xz-* xz-src
 ENV LIBXZ_INSTALL_DIR="$BUILD_HOME/built/xz"
 RUN mkdir -p "$LIBXZ_INSTALL_DIR"
-RUN cd xz-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBXZ_INSTALL_DIR"
-RUN cd xz-src && make install
+RUN cd xz-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBXZ_INSTALL_DIR" | tee -a $LOGS_DIR/xz.configure.log
+RUN cd xz-src && make install | tee -a $LOGS_DIR/xz.install.log
 
 FROM toolchain as build_bz2
 RUN apt-get update -qq && apt-get -qq install make
@@ -75,7 +78,7 @@ RUN mv bzip2-* bzip2-src
 RUN mkdir -p "$LIBBZ2_INSTALL_DIR" && \
     cd bzip2-src && \
     sed -i -e 's,[.]1[.]0.8,,' -e 's,[.]1[.]0,,' -e 's,ln -s,#ln -s,' -e 's,rm -f libbz2.so,#rm -f libbz2.so,' -e 's,^CC=,#CC=,' Makefile-libbz2_so
-RUN cd bzip2-src && make -f Makefile-libbz2_so
+RUN cd bzip2-src && make -f Makefile-libbz2_so | tee -a $LOGS_DIR/bz2.log
 RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/lib"
 RUN cp bzip2-src/libbz2.so "${LIBBZ2_INSTALL_DIR}/lib"
 RUN mkdir -p "${LIBBZ2_INSTALL_DIR}/include"
@@ -88,18 +91,18 @@ ADD downloads/libffi/* .
 RUN mv libffi-* libffi-src
 ENV LIBFFI_INSTALL_DIR="$BUILD_HOME/built/libffi"
 RUN mkdir -p "$LIBFFI_INSTALL_DIR"
-RUN cd libffi-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBFFI_INSTALL_DIR"
-RUN cd libffi-src && make install
+RUN cd libffi-src && ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --prefix="$LIBFFI_INSTALL_DIR" | tee -a $LOGS_DIR/libffi.configure.log
+RUN cd libffi-src && make install | tee -a $LOGS_DIR/libffi.install.log
 
 FROM toolchain as build_openssl
 # OpenSSL requires libfindlibs-libs-perl. make is nice, too.
 RUN apt-get update -qq && apt-get -qq install libfindbin-libs-perl make
 ADD downloads/openssl/* .
 RUN mv openssl-* openssl-src
 ARG OPENSSL_BUILD_TARGET
-RUN cd openssl-src && ANDROID_NDK_HOME="$NDK" ./Configure ${OPENSSL_BUILD_TARGET} -D__ANDROID_API__="$ANDROID_API_LEVEL" --prefix="$BUILD_HOME/built/openssl" --openssldir="$BUILD_HOME/built/openssl"
-RUN cd openssl-src && make SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
-RUN cd openssl-src && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
+RUN cd openssl-src && ANDROID_NDK_HOME="$NDK" ./Configure ${OPENSSL_BUILD_TARGET} -D__ANDROID_API__="$ANDROID_API_LEVEL" --prefix="$BUILD_HOME/built/openssl" --openssldir="$BUILD_HOME/built/openssl" | tee -a $LOGS_DIR/openssl.configure.log
+RUN cd openssl-src && make SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so' | tee -a $LOGS_DIR/openssl.build.log
+RUN cd openssl-src && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so' | tee -a $LOGS_DIR/openssl.install.log
 
 # This build container builds Python, rubicon-java, and any dependencies. Each Python version
 # requires itself to be installed globally during a cross-compile.
@@ -108,18 +111,25 @@ RUN apt-get update -qq && apt-get -qq install software-properties-common dirmngr
 RUN apt-add-repository ppa:deadsnakes/ppa
 RUN apt-get update -qq && apt-get -qq install python3.7 python3.8 python3.9 python3.10 python3.11 pkg-config zip quilt
 
-# Get libs & vars
+# Get libs & vars from the build stages
 COPY --from=build_openssl /opt/python-build/built/openssl /opt/python-build/built/openssl
 COPY --from=build_bz2 /opt/python-build/built/libbz2 /opt/python-build/built/libbz2
 COPY --from=build_xz /opt/python-build/built/xz /opt/python-build/built/xz
 COPY --from=build_libffi /opt/python-build/built/libffi /opt/python-build/built/libffi
 COPY --from=build_sqlite /opt/python-build/built/sqlite /opt/python-build/built/sqlite
 
+# Copy logs from the build stages
+COPY --from=build_openssl $LOGS_DIR/* $LOGS_DIR
+COPY --from=build_bz2 $LOGS_DIR/* $LOGS_DIR
+COPY --from=build_xz $LOGS_DIR/* $LOGS_DIR
+COPY --from=build_libffi $LOGS_DIR/* $LOGS_DIR
+COPY --from=build_sqlite $LOGS_DIR/* $LOGS_DIR
+
 ENV OPENSSL_INSTALL_DIR=/opt/python-build/built/openssl
 ENV LIBBZ2_INSTALL_DIR="$BUILD_HOME/built/libbz2"
 ENV LIBXZ_INSTALL_DIR="$BUILD_HOME/built/xz"
 RUN mkdir -p "$JNI_LIBS" && cp -a "$OPENSSL_INSTALL_DIR"/lib/*.so "$LIBBZ2_INSTALL_DIR"/lib/*.so /opt/python-build/built/libffi/lib/*.so /opt/python-build/built/xz/lib/*.so /opt/python-build/built/sqlite/lib/*.so "$JNI_LIBS"
-ENV PKG_CONFIG_PATH="/opt/python-build/built/libffi/lib/pkgconfig:/opt/python-build/built/xz/lib/pkgconfig"
+ENV PKG_CONFIG_PATH="/opt/python-build/built/libffi/lib/pkgconfig:/opt/python-build/built/openssl/lib/pkgconfig:/opt/python-build/built/sqlite/lib/pkgconfig:/opt/python-build/built/xz/lib/pkgconfig"
 
 # Download & patch Python. We assume that there is only one Python-${VERSION}.*.tar.xz file.
 ARG PYTHON_VERSION
@@ -136,9 +146,9 @@ RUN sed -i -e s,'test $(INSTSONAME) != $(LDLIBRARY)',true, -e s,'$(LN) -f $(INST
 ARG PYTHON_SOVERSION
 # Apply a C extensions linker hack; already fixed in Python 3.8+; see https://github.com/python/cpython/commit/254b309c801f82509597e3d7d4be56885ef94c11
 RUN sed -i -e s,'libraries or \[\],\["pythonPYTHON_SOVERSION"] + libraries if libraries else \["pythonPYTHON_SOVERSION"\],' -e  "s,pythonPYTHON_SOVERSION,python${PYTHON_SOVERSION},g" python-src/Lib/distutils/extension.py
-# Apply a hack to get the NDK library paths into the Python build. Python 3.6 (but not 3.7+) needs OpenSSL here as well.
+# Apply a hack to get the NDK library paths into the Python build.
 # TODO(someday): Discuss with e.g. Kivy and see how to remove this.
-RUN sed -i -e "s# dirs = \[\]# dirs = \[os.environ.get('SYSROOT_INCLUDE'), os.environ.get('SYSROOT_LIB'), os.environ.get('OPENSSL_INSTALL_DIR') + '/include', os.environ.get('OPENSSL_INSTALL_DIR') + '/lib' \]#" python-src/setup.py
+RUN sed -i -e "s# dirs = \[\]# dirs = \[os.environ.get('SYSROOT_INCLUDE'), os.environ.get('SYSROOT_LIB') \]#" python-src/setup.py
 # Apply a hack to get the sqlite include path into setup.py. TODO(someday): Discuss with upstream Python if we can use pkg-config for sqlite.
 RUN sed -i -E 's,sqlite_inc_paths = [[][]],sqlite_inc_paths = ["/opt/python-build/built/sqlite/include"],' python-src/setup.py
 # Apply a hack to make platform.py stop looking for a libc version.
@@ -148,27 +158,50 @@ RUN sed -i -e "s#Linux#DisabledLinuxCheck#" python-src/Lib/platform.py
 ADD patches/${PYTHON_VERSION} python-src/patches
 RUN cd python-src && if [ "$(wc -l < patches/series)" != "0" ] ; then quilt push -a; else echo "No patches." ; fi
 
+# Add a Setup.local configuration
+RUN cp python-src/patches/Setup.local python-src/Modules
+
 # Build Python, pre-configuring some values so it doesn't check if those exist.
 ENV SYSROOT_LIB=${TOOLCHAIN}/sysroot/usr/lib/${TOOLCHAIN_TRIPLE}/${ANDROID_API_LEVEL}/ \
-    SYSROOT_INCLUDE=${NDK}/sysroot/usr/include/
+    SYSROOT_INCLUDE=${TOOLCHAIN}/sysroot/usr/include/
 # Add any version-specific configuration flags
 ARG PYTHON_EXTRA_CONFIGURE_FLAGS
 ENV PYTHON_EXTRA_CONFIGURE_FLAGS $PYTHON_EXTRA_CONFIGURE_FLAGS
-# Call ./configure with enough parameters to work properly on Python 3.6 and 3.7.
-# Python 3.6 needs OpenSSL's headers in CFLAGS; Python 3.7+ consumes it from --with-openssl=.
-# Python 3.6 needs ac_cv_header_langinfo_h=no because it lacks a more specific check for nl_langinfo
-# (which Android lacks).
+# Call ./configure with enough parameters to work.
 RUN cd python-src && LDFLAGS="$(pkg-config --libs-only-L libffi) $(pkg-config --libs-only-L liblzma) -L${LIBBZ2_INSTALL_DIR}/lib -L$OPENSSL_INSTALL_DIR/lib" \
-    CFLAGS="${CFLAGS} -I${LIBBZ2_INSTALL_DIR}/include $(pkg-config --cflags-only-I libffi) $(pkg-config --cflags-only-I liblzma) -I$OPENSSL_INSTALL_DIR/include" \
-    ./configure --host "$TOOLCHAIN_TRIPLE" --build "$COMPILER_TRIPLE" --enable-shared \
-    --enable-ipv6 ac_cv_file__dev_ptmx=yes \
-    --with-openssl=$OPENSSL_INSTALL_DIR \
-    ac_cv_header_langinfo_h=no \
-    ac_cv_file__dev_ptc=no --without-ensurepip ac_cv_little_endian_double=yes \
+    CFLAGS="${CFLAGS} -I${LIBBZ2_INSTALL_DIR}/include $(pkg-config --cflags-only-I libffi) $(pkg-config --cflags-only-I liblzma)" \
+    ./configure \
+    --host "$TOOLCHAIN_TRIPLE" \
+    --build "$COMPILER_TRIPLE" \
     --prefix="$PYTHON_INSTALL_DIR" \
+    --enable-shared \
+    --enable-ipv6 \
+    --with-openssl=$OPENSSL_INSTALL_DIR \
+    --without-ensurepip \
+    ac_cv_file__dev_ptmx=yes \
+    ac_cv_file__dev_ptc=no \
+    ac_cv_little_endian_double=yes \
     $PYTHON_EXTRA_CONFIGURE_FLAGS \
-    ac_cv_func_setuid=no ac_cv_func_seteuid=no ac_cv_func_setegid=no ac_cv_func_getresuid=no ac_cv_func_setresgid=no ac_cv_func_setgid=no ac_cv_func_sethostname=no ac_cv_func_setresuid=no ac_cv_func_setregid=no ac_cv_func_setreuid=no ac_cv_func_getresgid=no ac_cv_func_setregid=no ac_cv_func_clock_settime=no ac_cv_header_termios_h=no ac_cv_func_sendfile=no ac_cv_header_spawn_h=no ac_cv_func_posix_spawn=no \
-    ac_cv_func_setlocale=no ac_cv_working_tzset=no ac_cv_member_struct_tm_tm_zone=no ac_cv_func_sched_setscheduler=no
+    ac_cv_func_setuid=no \
+    ac_cv_func_seteuid=no \
+    ac_cv_func_setegid=no \
+    ac_cv_func_getresuid=no \
+    ac_cv_func_setresgid=no \
+    ac_cv_func_setgid=no \
+    ac_cv_func_sethostname=no \
+    ac_cv_func_setresuid=no \
+    ac_cv_func_setregid=no \
+    ac_cv_func_setreuid=no \
+    ac_cv_func_getresgid=no \
+    ac_cv_func_setregid=no \
+    ac_cv_func_clock_settime=no \
+    ac_cv_func_sendfile=no \
+    ac_cv_header_spawn_h=no \
+    ac_cv_func_posix_spawn=no \
+    ac_cv_func_setlocale=no \
+    ac_cv_working_tzset=no \
+    ac_cv_member_struct_tm_tm_zone=no \
+    ac_cv_func_sched_setscheduler=no | tee -a $LOGS_DIR/python.configure.log
 # Override ./configure results to futher force Python not to use some libc calls that trigger blocked syscalls.
 # TODO(someday): See if HAVE_INITGROUPS has another way to disable it.
 RUN cd python-src && sed -i -E 's,#define (HAVE_CHROOT|HAVE_SETGROUPS|HAVE_INITGROUPS) 1,,' pyconfig.h
@@ -180,7 +213,7 @@ RUN cd python-src && sed -i -E 's,#define.*(HAVE_EXECV|HAVE_FORK).*1,,' Modules/
 # Copy libbz2 into the SYSROOT_LIB. This is the IMHO the easiest way for setup.py to find it.
 RUN cp "${LIBBZ2_INSTALL_DIR}/lib/libbz2.so" $SYSROOT_LIB
 # Compile Python. We can still remove some tests from the test suite before `make install`.
-RUN cd python-src && make
+RUN cd python-src && make | tee -a $LOGS_DIR/python.build.log
 
 # Modify stdlib & test suite before `make install`.
 
@@ -216,7 +249,7 @@ RUN cd python-src && rm Lib/test/test_xmlrpc.py
 RUN cd python-src && rm Lib/test/test_wsgiref.py
 
 # Install Python.
-RUN cd python-src && make install
+RUN cd python-src && make install | tee -a $LOGS_DIR/python.install.log
 RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython${PYTHON_SOVERSION}.so "$JNI_LIBS"
 
 # Download & install rubicon-java's Java & C parts. The *.py files in rubicon-java are
@@ -225,7 +258,7 @@ RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython${PYTHON_SOVERSION}.so "$JNI_LIBS"
 ADD downloads/rubicon-java/* .
 RUN mv rubicon-java-* rubicon-java-src
 RUN cd rubicon-java-src && \
-    LDFLAGS='-landroid -llog' PYTHON_CONFIG=$PYTHON_INSTALL_DIR/bin/python3-config make
+    LDFLAGS='-landroid -llog' PYTHON_CONFIG=$PYTHON_INSTALL_DIR/bin/python3-config make | tee -a $LOGS_DIR/rubicon.log
 RUN mv rubicon-java-src/build/librubicon.so $JNI_LIBS
 RUN mkdir -p /opt/python-build/app/libs/ && mv rubicon-java-src/build/rubicon.jar $APPROOT/app/libs/
 

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +*disabled*
++
 +_gdbm
 +_dbm
 +grp