Modernize and relocate iree/runtime Python package to runtime/binding…

…s/python. (iree-org#8912)

* Modernize and relocate iree/runtime Python package to iree/runtime/python.

Non-functional changes:

* Moves `bindings/python/iree/runtime` -> `iree/runtime/python/iree/runtime`.
* Fixes dash vs undescore inconsistency in compile install path. Now both use underscores (python_packages/iree_compiler and python_packages/iree_runtime).
* Moves build directory for iree/compiler/python to iree/compiler/python (was outputting to bindings/python). Updates locations that were hard-coded.

Functional changes:

* Removes the old build-dir only setup.py in favor of an iree/runtime/setup.py that works from either the source or build dir.
* Reworks the releases to use the new setup.py as-is vs scripting the build manually.
* iree.runtime.version is now generated in the same way as iree.compiler.version.
* Users can now run iree/runtime/setup.py with pip themselves to generate a wheel (i.e. `pip wheel iree/runtime`).

It is now possible to integrate python package testing into the presubmit and have build jobs that generated Python installable binaries for subsequent steps.

The only file left in bindings/python is build_requirements.txt. It is referred to by some docs and CI jobs so leaving as-is for the moment (will find it a new home in a followup).

.github/workflows/build_package.yml

@@ -69,7 +69,7 @@ jobs:
       # Note that on Linux, we run under docker with an altered path.
       # Note that on Windows, we need to configure the compiler api project to
       # but its CMake build directory on a short path to avoid path overflow.
-      CIBW_ENVIRONMENT_LINUX: "REPO_DIR=/project/main_checkout BINDIST_DIR=/output CMAKE_GENERATOR=Ninja IREE_TARGET_BACKEND_CUDA=ON"
+      CIBW_ENVIRONMENT_LINUX: "REPO_DIR=/project/main_checkout BINDIST_DIR=/output CMAKE_GENERATOR=Ninja IREE_TARGET_BACKEND_CUDA=ON IREE_HAL_DRIVER_CUDA=ON"
       CIBW_ENVIRONMENT_MACOS: "REPO_DIR=${{ github.workspace }}/main_checkout CMAKE_GENERATOR=Ninja"
       CIBW_ENVIRONMENT_WINDOWS: "REPO_DIR='${{ github.workspace }}\\main_checkout' CMAKE_GENERATOR=Ninja IREE_COMPILER_API_CMAKE_BUILD_DIR=D:/b"
 
@@ -167,24 +167,21 @@ jobs:
         if: "matrix.build_package == 'py-runtime-pkg'"
         shell: bash
         run: |
-          package_dir="./iree-install/python_packages/iree_runtime"
-          export CIBW_BEFORE_BUILD="python ./main_checkout/build_tools/github_actions/build_dist.py py-runtime-pkg"
-          # TODO: cibuildwheel sanity checks this, but our setup.py is the
-          # *output* of the build :( Make source packages.
-          mkdir -p $package_dir && touch $package_dir/setup.py
-          python -m cibuildwheel --output-dir bindist $package_dir
+          export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
+          python -m cibuildwheel --output-dir bindist ./runtime/
 
       - name: Build runtime wheels (instrumented)
         if: "matrix.build_package == 'instrumented-py-runtime-pkg'"
         shell: bash
         run: |
-          package_dir="./iree-install/python_packages/iree_runtime"
           export CIBW_BEFORE_ALL_LINUX="./main_checkout/build_tools/github_actions/install_tracy_cli_deps_manylinux2014.sh"
-          export CIBW_BEFORE_BUILD="python ./main_checkout/build_tools/github_actions/build_dist.py instrumented-py-runtime-pkg"
-          # TODO: cibuildwheel sanity checks this, but our setup.py is the
-          # *output* of the build :( Make source packages.
-          mkdir -p $package_dir && touch $package_dir/setup.py
-          python -m cibuildwheel --output-dir bindist $package_dir
+          export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
+          # I really dislike how cibuildwheel and GitHub actions make tunneling
+          # environment variables so obtuse. Burn this all with fire.
+          export CIBW_ENVIRONMENT_LINUX="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
+          export CIBW_ENVIRONMENT_MACOS="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
+          export CIBW_ENVIRONMENT_WINDOWS="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
+          python -m cibuildwheel --output-dir bindist ./runtime/
 
       # Experimental iree.compiler package.
       - name: Build compiler wheels

CMakeLists.txt

@@ -654,12 +654,11 @@ endif()
 
 # Order constraint: The python bindings install tools targets from iree/tools
 # and tracy, and must come after it.
-if(${IREE_BUILD_PYTHON_BINDINGS})
-  add_subdirectory(bindings/python)
+if(IREE_BUILD_PYTHON_BINDINGS)
   # Write out a .env file to make IDEs and developers happy.
   # Yes, we are writing this to the source dir. It is only for IDEs and if
   # it gets clobbered, it is fine.
-  set(_pythonpath_env "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/bindings/python\n")
+  set(_pythonpath_env "PYTHONPATH=$<SHELL_PATH:${CMAKE_CURRENT_BINARY_DIR}/iree/compiler/python;${CMAKE_CURRENT_BINARY_DIR}/runtime/bindings/python>\n")
   file(GENERATE OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/.env"
     CONTENT "${_pythonpath_env}"
   )
@@ -699,3 +698,13 @@ set(IREE_PUBLIC_INCLUDE_DIRS "${IREE_COMMON_INCLUDE_DIRS}"
 #-------------------------------------------------------------------------------
 
 add_subdirectory(build_tools/benchmarks)
+
+#-------------------------------------------------------------------------------
+# Transitional directories.
+#-------------------------------------------------------------------------------
+
+# The runtime uses tools and targets from elsewhere in the tree in a way
+# that is not supported by CMake, so we include it last (it aggregates
+# many things into top-level APIs). This should resolve once we finish
+# relayering everything.
+add_subdirectory(runtime)

build_tools/cmake/iree_python.cmake

@@ -39,8 +39,6 @@ endfunction()
 #     scoped)
 #   - Be installed under python_packages/PACKAGE_NAME
 #   - Have a local path of MODULE_PATH (i.e. namespace package path)
-#   - Process a setup.py.in from the current directory (if NOT AUGMENT_EXISTING_PACKAGE)
-#   - Process a version.py.in from the current directory (if NOT AUGMENT_EXISTING_PACKAGE)
 # Will set parent scope variables:
 #   - PY_INSTALL_COMPONENT: Install component. Echoed back from the argument
 #     for easier addition after this call.
@@ -73,7 +71,7 @@ function(iree_py_install_package)
     "DEPS;ADDL_PACKAGE_FILES;FILES_MATCHING"
     ${ARGN})
   set(_install_component ${ARG_COMPONENT})
-  set(_install_packages_dir "${CMAKE_INSTALL_PREFIX}/python_packages/${ARG_PACKAGE_NAME}")
+  set(_install_packages_dir "python_packages/${ARG_PACKAGE_NAME}")
   set(_install_module_dir "${_install_packages_dir}/${ARG_MODULE_PATH}")
   set(_target_name install-${_install_component})
 
@@ -83,44 +81,6 @@ function(iree_py_install_package)
     set(_files_matching ${ARG_FILES_MATCHING})
   endif()
 
-  if(NOT ARG_AUGMENT_EXISTING_PACKAGE)
-    configure_file(setup.py.in setup.py)
-    install(
-      FILES
-        ${CMAKE_CURRENT_BINARY_DIR}/setup.py
-        ${ARG_ADDL_PACKAGE_FILES}
-      COMPONENT ${_install_component}
-      DESTINATION "${_install_packages_dir}"
-    )
-    configure_file(version.py.in version.py)
-    install(
-      FILES
-        ${CMAKE_CURRENT_BINARY_DIR}/version.py
-      COMPONENT ${_install_component}
-      DESTINATION "${_install_module_dir}"
-    )
-
-    set(_component_option -DCMAKE_INSTALL_COMPONENT="${ARG_COMPONENT}")
-    add_custom_target(${_target_name}
-      COMMAND "${CMAKE_COMMAND}"
-              ${_component_option}
-              -P "${CMAKE_BINARY_DIR}/cmake_install.cmake"
-      USES_TERMINAL)
-    add_custom_target(${_target_name}-stripped
-      COMMAND "${CMAKE_COMMAND}"
-              ${_component_option}
-              -DCMAKE_INSTALL_DO_STRIP=1
-              -P "${CMAKE_BINARY_DIR}/cmake_install.cmake"
-      USES_TERMINAL)
-  endif()
-
-  # Explicit add dependencies in case if we are just extending a package
-  # vs adding the targets.
-  if(ARG_DEPS)
-    add_dependencies(${_target_name} ${ARG_DEPS})
-    add_dependencies(${_target_name}-stripped ${ARG_DEPS})
-  endif()
-
   install(
     DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
     COMPONENT ${_install_component}
@@ -336,7 +296,7 @@ function(iree_py_test)
 
   set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
   set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT
-      "PYTHONPATH=${IREE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH}"
+      "PYTHONPATH=${IREE_BINARY_DIR}/iree/compiler/python:${IREE_BINARY_DIR}/runtime/bindings/python:$ENV{PYTHONPATH}"
       "TEST_TMPDIR=${IREE_BINARY_DIR}/tmp/${_NAME}_test_tmpdir"
   )
   iree_add_test_environment_properties(${_NAME_PATH})

build_tools/github_actions/README.md

@@ -2,18 +2,3 @@
 
 ## Debugging releases cookbook
 
-### Build in the same Linux container as the release
-
-```
-docker run --rm -it -v $(pwd):/work stellaraccident/manylinux2014_x86_64-bazel-4.2.2:latest /bin/bash
-
-# From within docker.
-export PATH=/opt/python/cp38-cp38/bin:$PATH
-python -m pip install -r bindings/python/build_requirements.txt
-cd /work
-python ./build_tools/gitub_actions/cmake_ci.py -B../iree-build \
-  -DCMAKE_INSTALL_PREFIX=../iree-install -DCMAKE_BUILD_TYPE=Release \
-  -DIREE_BUILD_SAMPLES=OFF -DIREE_BUILD_PYTHON_BINDINGS=ON \
-  -DIREE_BUILD_TENSORFLOW_COMPILER=OFF -DIREE_BUILD_TFLITE_COMPILER=OFF -DIREE_BUILD_XLA_COMPILER=OFF
-python ./build_tools/github_actions/cmake_ci.py --build ../iree-build
-```

build_tools/github_actions/build_dist.py

@@ -38,7 +38,6 @@
   source .venv/bin/activate
 
   python ./main_checkout/build_tools/github_actions/build_dist.py main-dist
-  python ./main_checkout/build_tools/github_actions/build_dist.py py-runtime-pkg
   python ./main_checkout/build_tools/github_actions/build_dist.py py-xla-compiler-tools-pkg
   python ./main_checkout/build_tools/github_actions/build_dist.py py-tflite-compiler-tools-pkg
   python ./main_checkout/build_tools/github_actions/build_dist.py py-tf-compiler-tools-pkg
@@ -178,67 +177,6 @@ def build_main_dist():
       tf.add(os.path.join(INSTALL_DIR, entry), arcname=entry, recursive=True)
 
 
-def build_py_runtime_pkg(instrumented: bool = False):
-  """Builds the iree-install/python_packages/iree_runtime package.
-
-  This includes native, python-version dependent code and is designed to
-  be built multiple times.
-
-  Note that an instrumented build may require additional dependencies.
-  See: install_tracy_cli_deps_manylinux2014.sh for how to set up on that
-  container.
-  """
-  install_python_requirements()
-
-  # Clean up install and build trees.
-  shutil.rmtree(INSTALL_DIR, ignore_errors=True)
-  remove_cmake_cache()
-  extra_cmake_flags = []
-
-  # Extra options for instrumentation.
-  if instrumented:
-    print("*** Enabling options for instrumented build ***")
-    extra_cmake_flags.extend([
-        f"-DIREE_ENABLE_RUNTIME_TRACING=ON",
-        f"-DIREE_BUILD_TRACY=ON",
-    ])
-
-  # Enable CUDA if on platforms where we expect to have the deps and produce
-  # such binaries.
-  if platform.system() == "Linux":
-    print("*** Enabling CUDA runtime ***")
-    extra_cmake_flags.extend([
-        "-DIREE_HAL_DRIVER_CUDA=ON",
-    ])
-
-  # CMake configure.
-  print("*** Configuring ***")
-  subprocess.run([
-      sys.executable,
-      CMAKE_CI_SCRIPT,
-      f"-B{BUILD_DIR}",
-      "--log-level=VERBOSE",
-      f"-DCMAKE_INSTALL_PREFIX={INSTALL_DIR}",
-      f"-DCMAKE_BUILD_TYPE=Release",
-      f"-DIREE_BUILD_COMPILER=OFF",
-      f"-DIREE_BUILD_PYTHON_BINDINGS=ON",
-      f"-DIREE_BUILD_SAMPLES=OFF",
-      f"-DIREE_BUILD_TESTS=OFF",
-  ] + extra_cmake_flags,
-                 check=True)
-
-  print("*** Building ***")
-  subprocess.run([
-      sys.executable,
-      CMAKE_CI_SCRIPT,
-      "--build",
-      BUILD_DIR,
-      "--target",
-      "install-IreePythonPackage-runtime-stripped",
-  ],
-                 check=True)
-
-
 def build_py_tf_compiler_tools_pkg():
   """Builds the iree-install/python_packages/iree_tools_tf package."""
   install_python_requirements()
@@ -279,10 +217,6 @@ def build_py_tf_compiler_tools_pkg():
 command = sys.argv[1]
 if command == "main-dist":
   build_main_dist()
-elif command == "py-runtime-pkg":
-  build_py_runtime_pkg()
-elif command == "instrumented-py-runtime-pkg":
-  build_py_runtime_pkg(instrumented=True)
 elif command == "py-tf-compiler-tools-pkg":
   build_py_tf_compiler_tools_pkg()
 else:

build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-swiftshader/build.sh

@@ -76,7 +76,7 @@ tests_passed=true
 
 echo "***** Testing with CTest *****"
 if ! ctest --timeout 900 --output-on-failure \
-   --tests-regex "^integrations/tensorflow/|^bindings/python/" \
+   --tests-regex "^integrations/tensorflow/|^runtime/bindings/python/" \
    --label-exclude "^nokokoro$|^vulkan_uses_vk_khr_shader_float16_int8$"
 then
    tests_passed=false

build_tools/kokoro/gcp_ubuntu/cmake-bazel/linux/x86-turing/build.sh

@@ -83,7 +83,7 @@ tests_passed=true
 # as well.
 echo "***** Testing with CTest *****"
 if ! ctest --timeout 900 --output-on-failure \
-   --tests-regex "^integrations/tensorflow/|^bindings/python/" \
+   --tests-regex "^integrations/tensorflow/|^runtime/bindings/python/" \
    --label-regex "^driver=vulkan$|^driver=cuda$" \
    --label-exclude "^nokokoro$"
 then