Merge pull request beeware#23 from paulproteus/add-3.8

Add Python 3.8

Author: freakboy3742

main.sh

@@ -81,6 +81,12 @@ function build_one_abi() {
         ;;
     esac
 
+    local PYTHON_SOVERSION="${PYTHON_VERSION}"
+    if [ "$PYTHON_VERSION" = "3.7" ] || [ "$PYTHON_VERSION" = "3.6" ] ; then
+        # 3.6 and 3.7 use 3.6m/3.7m
+        PYTHON_SOVERSION="${PYTHON_VERSION}m"
+    fi
+
     # We use Docker to run the build. We rely on Docker's build cache to allow the
     # build to be speedy if nothing changed, approx 1-2 seconds for a no-op build.
     # The cache persists even if no Docker "tags" point to the image.
@@ -91,15 +97,15 @@ function build_one_abi() {
     # For debugging the Docker image, you can use the name python-android-support-local:latest.
     TAG_NAME="python-android-support-local:$(python3 -c 'import random; print(random.randint(0, 1e16))')"
     DOCKER_BUILDKIT=1 docker build --tag ${TAG_NAME} --tag python-android-support-local:latest \
-        --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
+        --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg PYTHON_SOVERSION="${PYTHON_SOVERSION}" \
         --build-arg COMPRESS_LEVEL="${COMPRESS_LEVEL}" --build-arg COMPILER_TRIPLE="${COMPILER_TRIPLE}" \
         --build-arg OPENSSL_BUILD_TARGET="$OPENSSL_BUILD_TARGET" --build-arg TARGET_ABI_SHORTNAME="$TARGET_ABI_SHORTNAME" \
         --build-arg TOOLCHAIN_TRIPLE="$TOOLCHAIN_TRIPLE" --build-arg ANDROID_API_LEVEL="$ANDROID_API_LEVEL" \
         -f python.Dockerfile .
     # Extract the build artifacts we need to create our zip file.
     docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/approot/. /mnt/.
     # Extract pyconfig.h for debugging ./configure strangeness.
-    docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/built/python/include/python"${PYTHON_VERSION}"m/pyconfig.h /mnt/
+    docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/built/python/include/python"${PYTHON_SOVERSION}"/pyconfig.h /mnt/
     # Remove temporary local tag.
     docker rmi "$TAG_NAME" > /dev/null
     fix_permissions
@@ -233,6 +239,9 @@ Build ZIP file of Python resources for Android, including CPython compiled as a
         3.7)
             download "python-3.7" "https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tar.xz" "55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f"
             ;;
+        3.8)
+            download "python-3.8" "https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tar.xz" "dfab5ec723c218082fe3d5d7ae17ecbdebffa9a1aea4d64aa3a2ecdd2e795864"
+            ;;
         *)
             echo "Invalid Python version: $VERSION"
             exit 1

patches/3.8/01_python_ssl_module_add_android_certificates

@@ -0,0 +1,48 @@
+From 86b7aee31eb66b99e2e9ff55f83cb2e6393dd8d3 Mon Sep 17 00:00:00 2001
+From: Asheesh Laroia <[email protected]>
+Date: Fri, 19 Jun 2020 16:47:25 -0700
+Subject: [PATCH] Parse certificate authority certs from Android
+ /etc/security/cacerts directory
+
+---
+ Lib/ssl.py           | 11 +++++++++++
+ Lib/test/test_ssl.py |  1 +
+ 2 files changed, 12 insertions(+)
+
+diff --git a/Lib/ssl.py b/Lib/ssl.py
+index 0726caee49..3878bdd8fa 100644
+--- a/Lib/ssl.py
++++ b/Lib/ssl.py
+@@ -572,6 +572,17 @@ class SSLContext(_SSLContext):
+         if sys.platform == "win32":
+             for storename in self._windows_cert_stores:
+                 self._load_windows_store_certs(storename, purpose)
++        if os.path.exists('/etc/security/cacerts'):
++            certs = []
++            for basename in os.listdir('/etc/security/cacerts'):
++                with open('/etc/security/cacerts/' + basename) as fd:
++                    s = fd.read()
++                    if 'END CERTIFICATE' not in s:
++                        continue
++                    lines = s.split('\n')
++                    line_end_certificate = [i for i, line in enumerate(lines) if 'END CERTIFICATE' in line][0]
++                    certs.append('\n'.join(lines[0:line_end_certificate+1]))
++            self.load_verify_locations(None, None, '\n'.join(certs))
+         self.set_default_verify_paths()
+ 
+     if hasattr(_SSLContext, 'minimum_version'):
+diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
+index 0bc0a8c452..3c79debf3e 100644
+--- a/Lib/test/test_ssl.py
++++ b/Lib/test/test_ssl.py
+@@ -1580,6 +1580,7 @@ class ContextTests(unittest.TestCase):
+     @unittest.skipIf(sys.platform == "win32", "not-Windows specific")
+     @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars")
+     def test_load_default_certs_env(self):
++        raise unittest.SkipTest("Skipping this test for Python within an Android app")
+         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+         with support.EnvironmentVarGuard() as env:
+             env["SSL_CERT_DIR"] = CAPATH
+-- 
+2.27.0
+

patches/3.8/series

@@ -0,0 +1 @@
+01_python_ssl_module_add_android_certificates

python.Dockerfile

@@ -105,7 +105,7 @@ RUN cd openssl-src && make install SHLIB_EXT='${SHLIB_VERSION_NUMBER}.so'
 # requires itself to be installed globally during a cross-compile, and Python 3.6 additionally
 # requires Python 2 to be installed at build time.
 FROM toolchain as build_python
-RUN apt-get update -qq && apt-get -qq install python python3.6 python3.7 pkg-config zip quilt
+RUN apt-get update -qq && apt-get -qq install python python3.6 python3.7 python3.8 pkg-config zip quilt
 
 # Get libs & vars
 COPY --from=build_openssl /opt/python-build/built/openssl /opt/python-build/built/openssl
@@ -126,8 +126,9 @@ ADD downloads/python-${PYTHON_VERSION}/* .
 RUN mv Python-* python-src
 # Modify ./configure so that, even though this is Linux, it does not append .1.0 to the .so file.
 RUN sed -i -e 's,INSTSONAME="$LDLIBRARY".$SOVERSION,,' python-src/configure
+ARG PYTHON_SOVERSION
 # Apply a C extensions linker hack; already fixed in Python 3.8+; see https://github.com/python/cpython/commit/254b309c801f82509597e3d7d4be56885ef94c11
-RUN sed -i -e s,'libraries or \[\],\["pythonX.Ym"] + libraries if libraries else \["pythonX.Ym"\],' -e  "s,pythonX.Ym,python${PYTHON_VERSION}m,g" python-src/Lib/distutils/extension.py
+RUN sed -i -e s,'libraries or \[\],\["pythonPYTHON_SOVERSION"] + libraries if libraries else \["pythonPYTHON_SOVERSION"\],' -e  "s,pythonPYTHON_SOVERSION,python${PYTHON_SOVERSION},g" python-src/Lib/distutils/extension.py
 # Apply a hack to get the NDK library paths into the Python build. Python 3.6 (but not 3.7+) needs OpenSSL here as well.
 # TODO(someday): Discuss with e.g. Kivy and see how to remove this.
 RUN sed -i -e "s# dirs = \[\]# dirs = \[os.environ.get('SYSROOT_INCLUDE'), os.environ.get('SYSROOT_LIB'), os.environ.get('OPENSSL_INSTALL_DIR') + '/include', os.environ.get('OPENSSL_INSTALL_DIR') + '/lib' \]#" python-src/setup.py
@@ -138,7 +139,7 @@ RUN sed -i -e "s#Linux#DisabledLinuxCheck#" python-src/Lib/platform.py
 
 # Apply our patches to Python. See patches/3.*/* for details.
 ADD patches/${PYTHON_VERSION} python-src/patches
-RUN cd python-src && quilt push -a
+RUN cd python-src && if [ "$(wc -l < patches/series)" != "0" ] ; then quilt push -a; else echo "No patches." ; fi
 
 # Build Python, pre-configuring some values so it doesn't check if those exist.
 ENV SYSROOT_LIB=${TOOLCHAIN}/sysroot/usr/lib/${TOOLCHAIN_TRIPLE}/${ANDROID_API_LEVEL}/ \
@@ -174,7 +175,7 @@ RUN cd python-src && make
 # Modify stdlib & test suite before `make install`.
 
 # Apply a hack to ctypes so that it loads libpython.so, even though this isn't Windows.
-RUN sed -i -e 's,pythonapi = PyDLL(None),pythonapi = PyDLL("libpythonX.Ym.so"),'  -e "s,libpythonX.Ym,libpython${PYTHON_VERSION}m,g" python-src/Lib/ctypes/__init__.py
+RUN sed -i -e 's,pythonapi = PyDLL(None),pythonapi = PyDLL("libpythonPYTHON_SOVERSION.so"),'  -e "s,libpythonPYTHON_SOVERSION,libpython${PYTHON_SOVERSION},g" python-src/Lib/ctypes/__init__.py
 # Hack the test suite so that when it tries to remove files, if it can't remove them, the error passes silently.
 # To see if ths is still an issue, run `test_bdb`.
 RUN sed -i -e "s#NotADirectoryError#NotADirectoryError, OSError#" python-src/Lib/test/support/__init__.py
@@ -206,7 +207,7 @@ RUN cd python-src && rm Lib/test/test_wsgiref.py
 
 # Install Python.
 RUN cd python-src && make install
-RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython${PYTHON_VERSION}m.so "$JNI_LIBS"
+RUN cp -a $PYTHON_INSTALL_DIR/lib/libpython${PYTHON_SOVERSION}.so "$JNI_LIBS"
 
 # Download & install rubicon-java's Java & C parts. The *.py files in rubicon-java are
 # incorporated into apps via app dependency management and are ABI-independent since