Move C runtime source directories into runtime/src/iree. (iree-org#8950)

* Pries apart the global include directory situation on both the Bazel and CMake side. On Bazel, we use a new iree_runtime_cc_(library|binary) macro that adds an implicit dep for include propagation and (in the future) can set copts. 
* On the CMake side, we use a path-based implicit dep to similar effect. I tried a couple of other ways and this was the least intrusive.
* Reworks bazel_to_cmake target rewriting to account for the new split root.
* Removes the CMake DATA include::this:file.png style of data includes (used in one place) in favor of a path, since package names are no longer reversible to a location. This seems to be the only place we made that assumption.
* Will do a couple more followups to completely retire the iree/iree directory (in favor of top-level compiler/ and tools/ directories).

Progress on iree-org#8955

CMakeLists.txt

@@ -614,17 +614,8 @@ add_custom_target(iree-test-deps COMMENT "Building IREE test deps")
 
 add_subdirectory(build_tools/embed_data/)
 
-add_subdirectory(iree/base)
-add_subdirectory(iree/builtins)
-add_subdirectory(iree/hal)
-add_subdirectory(iree/modules)
-add_subdirectory(iree/runtime)
-add_subdirectory(iree/schemas)
-add_subdirectory(iree/task)
-add_subdirectory(iree/testing)
 # Note: Test deps are not built as part of all (use the iree-test-deps target).
 add_subdirectory(iree/test EXCLUDE_FROM_ALL)
-add_subdirectory(iree/vm)
 
 if(${IREE_BUILD_BENCHMARKS})
   find_program(IREE_IMPORT_TFLITE_PATH iree-import-tflite)
@@ -645,6 +636,10 @@ if(${IREE_BUILD_COMPILER})
   add_subdirectory(iree/compiler)
 endif()
 
+if(IREE_BUILD_SAMPLES)
+  add_subdirectory(iree/samples)
+endif()
+
 add_subdirectory(iree/tools)
 
 if(IREE_BUILD_TRACY)
@@ -676,10 +671,6 @@ if(${IREE_BUILD_BINDINGS_TFLITE})
   add_subdirectory(bindings/tflite)
 endif()
 
-if(${IREE_BUILD_SAMPLES})
-  add_subdirectory(iree/samples)
-endif()
-
 if(${IREE_BUILD_EXPERIMENTAL_REMOTING})
   # NOTE: Currently liburing is only used by the experimental remoting
   # support, so keeping it scoped here. If this broadens, then include along

build_tools/bazel/build_core.sh

@@ -83,7 +83,7 @@ bazel \
   --bazelrc=build_tools/bazel/iree.bazelrc \
   query \
     --config=non_darwin \
-    //iree/... + //build_tools/... + \
+    //iree/... + //runtime/... + //build_tools/... + \
     //llvm-external-projects/iree-dialects/... | \
       xargs --max-args 1000000 --max-chars 1000000 --exit \
         bazel \

build_tools/bazel_to_cmake/bazel_to_cmake.py

@@ -71,15 +71,17 @@ def parse_arguments():
       " whether the directory is skipped."
       " 2: Also output when conversion was successful.")
 
-  # Specify only one of these (defaults to --root_dir=iree).
+  # Specify only one of these (defaults to --root_dir=<main source dirs>).
   group = parser.add_mutually_exclusive_group()
   group.add_argument("--dir",
                      help="Converts the BUILD file in the given directory",
                      default=None)
   group.add_argument(
       "--root_dir",
-      help="Converts all BUILD files under a root directory (defaults to iree/)",
-      default="iree")
+      nargs="+",
+      help=
+      "Converts all BUILD files under a root directory (defaults to iree, runtime)",
+      default=["iree", "runtime"])
 
   args = parser.parse_args()
 
@@ -240,12 +242,14 @@ def main(args):
   write_files = not args.preview
 
   if args.root_dir:
-    root_directory_path = os.path.join(repo_root, args.root_dir)
-    log(f"Converting directory tree rooted at: {root_directory_path}")
-    convert_directories((root for root, _, _ in os.walk(root_directory_path)),
-                        write_files=write_files,
-                        allow_partial_conversion=args.allow_partial_conversion,
-                        verbosity=args.verbosity)
+    for root_dir in args.root_dir:
+      root_directory_path = os.path.join(repo_root, root_dir)
+      log(f"Converting directory tree rooted at: {root_directory_path}")
+      convert_directories(
+          (root for root, _, _ in os.walk(root_directory_path)),
+          write_files=write_files,
+          allow_partial_conversion=args.allow_partial_conversion,
+          verbosity=args.verbosity)
   elif args.dir:
     convert_directories([os.path.join(repo_root, args.dir)],
                         write_files=write_files,

build_tools/bazel_to_cmake/bazel_to_cmake_converter.py

@@ -69,11 +69,19 @@ def _convert_option_block(option, option_value):
 def _convert_target_block(name, target):
   if target is None:
     return ""
-  # Bazel target name to cmake binary name
-  # Bazel `//iree/custom:custom-translate` -> CMake `iree_custom_custom-translate`
-  target = target.replace("//iree", "iree")  # iree/custom:custom-translate
-  target = target.replace(":", "_")  # iree/custom_custom-translate
-  target = target.replace("/", "_")  # iree_custom_custom-translate
+
+  # Targets in this context can't be aliases, because CMake. So we first convert
+  # it in the usual way and then syntactically change it back from the pretty
+  # alias name to the underscored variant. Example:
+  #   //iree/tools:iree-translate
+  #   iree::tools::iree-translate
+  #   iree_tools_iree-translate
+  cmake_aliases = bazel_to_cmake_targets.convert_target(target)
+  if len(cmake_aliases) != 1:
+    raise ValueError(
+        f"Expected a CMake alias from {target}. Got {cmake_aliases}")
+  target = cmake_aliases[0]
+  target = target.replace("::", "_")
   return _convert_string_arg_block(name, target, quote=False)
 
 
@@ -325,6 +333,9 @@ def cc_library(self,
                             f"{testonly_block}"
                             f"  PUBLIC\n)\n\n")
 
+  def iree_runtime_cc_library(self, deps=[], **kwargs):
+    self.cc_library(deps=deps + ["//runtime/src:runtime_defines"], **kwargs)
+
   def cc_test(self,
               name,
               hdrs=None,
@@ -358,6 +369,9 @@ def cc_test(self,
                             f"{labels_block}"
                             f")\n\n")
 
+  def iree_runtime_cc_test(self, deps=[], **kwargs):
+    self.cc_test(deps=deps + ["//runtime/src:runtime_defines"], **kwargs)
+
   def cc_binary(self,
                 name,
                 srcs=None,
@@ -400,6 +414,7 @@ def c_embed_data(self,
                    strip_prefix=None,
                    flatten=None,
                    identifier=None,
+                   deps=None,
                    **kwargs):
     name_block = _convert_string_arg_block("NAME", name, quote=False)
     srcs_block = _convert_srcs_block(srcs)
@@ -410,10 +425,12 @@ def c_embed_data(self,
     testonly_block = _convert_option_block("TESTONLY", testonly)
     identifier_block = _convert_string_arg_block("IDENTIFIER", identifier)
     flatten_block = _convert_option_block("FLATTEN", flatten)
+    deps_block = _convert_target_list_block("DEPS", deps)
 
     self.converter.body += (f"iree_c_embed_data(\n"
                             f"{name_block}"
                             f"{srcs_block}"
+                            f"{deps_block}"
                             f"{c_file_output_block}"
                             f"{h_file_output_block}"
                             f"{identifier_block}"

build_tools/bazel_to_cmake/bazel_to_cmake_targets.py

@@ -5,12 +5,15 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import re
+
 # Bazel to CMake target name conversions used by bazel_to_cmake.py.
 
 EXPLICIT_TARGET_MAPPING = {
     # Internal utilities to emulate various binary/library options.
     "//build_tools:default_linkopts": [],
     "//build_tools:dl": ["${CMAKE_DL_LIBS}"],
+    "//runtime/src:runtime_defines": [],
 
     # IREE llvm-external-projects
     "//llvm-external-projects/iree-dialects:IREEPyDMTransforms": [
@@ -192,6 +195,17 @@ def _convert_iree_dialects_target(target):
   return [target.rsplit(":")[-1]]
 
 
+def _convert_bazel_path(bazel_path: str) -> str:
+  # Bazel `:api`            -> CMake `::api`
+  # Bazel `//iree/base`     -> CMake `iree::base`
+  # Bazel `//iree/base:foo` -> CMake `iree::base::foo`
+  if bazel_path.startswith("//"):
+    bazel_path = bazel_path[len("//"):]
+  bazel_path = bazel_path.replace(":", "::")  # iree/base::foo or ::foo
+  bazel_path = bazel_path.replace("/", "::")  # iree::base
+  return bazel_path
+
+
 def convert_target(target):
   """Converts a Bazel target to a list of CMake targets.
 
@@ -216,14 +230,19 @@ def convert_target(target):
     return _convert_mlir_target(target)
   if target.startswith("//llvm-external-projects/iree-dialects"):
     return _convert_iree_dialects_target(target)
+
+  # Map //runtime/src/iree/(.*) -> iree::\1
+  m = re.match("^//runtime/src/iree/(.+)", target)
+  if m:
+    return ["iree::" + _convert_bazel_path(m.group(1))]
+
+  # Map //runtime/bindings/(.*) -> iree::bindings\1
+  m = re.match("^//runtime/bindings/(.+)", target)
+  if m:
+    return ["iree::bindings::" + _convert_bazel_path(m.group(1))]
+
+  # Default (legacy) rewrite.
   if not target.startswith("@"):
-    # Bazel `:api`            -> CMake `::api`
-    # Bazel `//iree/base`     -> CMake `iree::base`
-    # Bazel `//iree/base:foo` -> CMake `iree::base::foo`
-    if target.startswith("//"):
-      target = target[len("//"):]
-    target = target.replace(":", "::")  # iree/base::foo or ::foo
-    target = target.replace("/", "::")  # iree::base
-    return [target]
+    return [_convert_bazel_path(target)]
 
   raise KeyError(f"No conversion found for target '{target}'")

build_tools/cmake/flatbuffer_c_library.cmake

@@ -117,8 +117,6 @@ function(flatbuffer_c_library)
   add_dependencies(${_NAME} ${_GEN_TARGET})
   target_include_directories(${_NAME} SYSTEM
     INTERFACE
-      "$<BUILD_INTERFACE:${IREE_SOURCE_DIR}>"
-      "$<BUILD_INTERFACE:${IREE_BINARY_DIR}>"
       ${CMAKE_CURRENT_BINARY_DIR}
     )
   target_link_libraries(${_NAME}

build_tools/cmake/iree_c_embed_data.cmake

@@ -32,7 +32,7 @@ function(iree_c_embed_data)
     _RULE
     "PUBLIC;TESTONLY;FLATTEN"
     "NAME;IDENTIFIER;STRIP_PREFIX;C_FILE_OUTPUT;H_FILE_OUTPUT"
-    "SRCS;GENERATED_SRCS"
+    "DEPS;SRCS;GENERATED_SRCS"
     ${ARGN}
   )
 
@@ -87,6 +87,7 @@ function(iree_c_embed_data)
     NAME ${_RULE_NAME}
     HDRS "${_RULE_H_FILE_OUTPUT}"
     SRCS "${_RULE_C_FILE_OUTPUT}"
+    DEPS "${_RULE_DEPS}"
     "${_PUBLIC_ARG}"
     "${_TESTONLY_ARG}"
   )

build_tools/cmake/iree_c_module.cmake

@@ -68,10 +68,14 @@ function(iree_c_module)
   iree_cc_library(
     NAME ${_RULE_NAME}
     HDRS "${_RULE_H_FILE_OUTPUT}"
-    SRCS "${IREE_SOURCE_DIR}/iree/vm/module_impl_emitc.c"
+    SRCS "${IREE_SOURCE_DIR}/runtime/src/iree/vm/module_impl_emitc.c"
     INCLUDES "${CMAKE_CURRENT_BINARY_DIR}"
-    COPTS "-DEMITC_IMPLEMENTATION=\"${_RULE_H_FILE_OUTPUT}\""
-    "${_TESTONLY_ARG}"
+    COPTS 
+      "-DEMITC_IMPLEMENTATION=\"${_RULE_H_FILE_OUTPUT}\""
+      "${_TESTONLY_ARG}"
+    DEPS
+      # Include paths and options for the runtime sources.
+      iree_defs_runtime
   )
 
   set(_GEN_TARGET "${_NAME}_gen")

build_tools/cmake/iree_cc_binary.cmake

@@ -122,6 +122,11 @@ function(iree_cc_binary)
   # Replace dependencies passed by ::name with iree::package::name
   list(TRANSFORM _RULE_DEPS REPLACE "^::" "${_PACKAGE_NS}::")
 
+  # Implicit deps.
+  if(IREE_IMPLICIT_DEFS_CC_DEPS)
+    list(APPEND _RULE_DEPS ${IREE_IMPLICIT_DEFS_CC_DEPS})
+  endif()
+
   target_link_libraries(${_NAME}
     PUBLIC
       ${_RULE_DEPS}

build_tools/cmake/iree_cc_library.cmake

@@ -95,6 +95,11 @@ function(iree_cc_library)
     set(_RULE_IS_INTERFACE 0)
   endif()
 
+  # Implicit deps.
+  if(IREE_IMPLICIT_DEFS_CC_DEPS)
+    list(APPEND _RULE_DEPS ${IREE_IMPLICIT_DEFS_CC_DEPS})
+  endif()
+
   if(NOT _RULE_IS_INTERFACE)
     add_library(${_OBJECTS_NAME} OBJECT)
     if(_RULE_SHARED)

build_tools/cmake/iree_cc_test.cmake

@@ -127,6 +127,11 @@ function(iree_cc_test)
 
   list(APPEND _RULE_DEPS "gmock")
 
+  # Implicit deps.
+  if(IREE_IMPLICIT_DEFS_CC_DEPS)
+    list(APPEND _RULE_DEPS ${IREE_IMPLICIT_DEFS_CC_DEPS})
+  endif()
+
   string(REPLACE "::" "/" _PACKAGE_PATH ${_PACKAGE_NS})
   set(_NAME_PATH "${_PACKAGE_PATH}/${_RULE_NAME}")
 

build_tools/cmake/iree_hal_cts_test_suite.cmake

@@ -113,7 +113,7 @@ function(iree_hal_cts_test_suite)
           MODULE_FILE_NAME
             "${_RULE_COMPILER_TARGET_BACKEND}_${_FILE_NAME}.bin"
           SRC
-            "${IREE_ROOT_DIR}/iree/hal/cts/testdata/${_FILE_NAME}.mlir"
+            "${IREE_ROOT_DIR}/runtime/src/iree/hal/cts/testdata/${_FILE_NAME}.mlir"
           FLAGS
             ${_TRANSLATE_FLAGS}
           TRANSLATE_TOOL
@@ -191,14 +191,14 @@ function(iree_hal_cts_test_suite)
 
     # Generate the source file for this [test x driver] pair.
     # TODO(scotttodd): Move to build time instead of configure time?
-    set(IREE_CTS_TEST_FILE_PATH "iree/hal/cts/${_TEST_NAME}_test.h")
+    set(IREE_CTS_TEST_FILE_PATH "runtime/src/iree/hal/cts/${_TEST_NAME}_test.h")
     set(IREE_CTS_DRIVER_REGISTRATION_HDR "${_RULE_DRIVER_REGISTRATION_HDR}")
     set(IREE_CTS_DRIVER_REGISTRATION_FN "${_RULE_DRIVER_REGISTRATION_FN}")
     set(IREE_CTS_TEST_CLASS_NAME "${_TEST_NAME}_test")
     set(IREE_CTS_DRIVER_NAME "${_RULE_DRIVER_NAME}")
 
     configure_file(
-      "${IREE_ROOT_DIR}/iree/hal/cts/cts_test_template.cc.in"
+      "${IREE_ROOT_DIR}/runtime/src/iree/hal/cts/cts_test_template.cc.in"
       ${_TEST_SOURCE_NAME}
     )
 

build_tools/cmake/iree_macros.cmake

@@ -56,8 +56,21 @@ endfunction()
 # Example when called from iree/base/CMakeLists.txt:
 #   iree::base
 function(iree_package_ns PACKAGE_NS)
-  string(REPLACE ${IREE_ROOT_DIR} "" _PACKAGE ${CMAKE_CURRENT_LIST_DIR})
-  string(SUBSTRING ${_PACKAGE} 1 -1 _PACKAGE)
+  # Get the relative path of the current dir (i.e. runtime/src/iree/vm).
+  string(REPLACE ${IREE_ROOT_DIR} "" _RELATIVE_PATH ${CMAKE_CURRENT_LIST_DIR})
+  string(SUBSTRING ${_RELATIVE_PATH} 1 -1 _RELATIVE_PATH)
+
+  # Some sub-trees form their own roots for package purposes. Rewrite them.
+  if(_RELATIVE_PATH MATCHES "^runtime/src/(.*)")
+    # runtime/src/iree/base -> iree/base
+    set(_PACKAGE "${CMAKE_MATCH_1}")
+  else()
+    # Default to pass-through. Examples:
+    #   iree/compiler/API
+    #   iree/tools
+    set(_PACKAGE "${_RELATIVE_PATH}")
+  endif()
+
   string(REPLACE "/" "::" _PACKAGE_NS ${_PACKAGE})
   set(${PACKAGE_NS} ${_PACKAGE_NS} PARENT_SCOPE)
 endfunction()
@@ -189,8 +202,8 @@ endfunction()
 #
 # Parameters:
 # NAME: name of the target to add data dependencies to
-# DATA: List of targets and/or files in the source tree. Files should use the
-#       same format as targets (i.e. iree::package::subpackage::file.txt)
+# DATA: List of targets and/or files in the source tree (relative to the 
+# project root).
 function(iree_add_data_dependencies)
   cmake_parse_arguments(
     _RULE
@@ -209,11 +222,12 @@ function(iree_add_data_dependencies)
       add_dependencies(${_RULE_NAME} ${_DATA_LABEL})
     else()
       # Not a target, assume to be a file instead.
-      string(REPLACE "::" "/" _FILE_PATH ${_DATA_LABEL})
+      set(_FILE_PATH ${_DATA_LABEL})
 
       # Create a target which copies the data file into the build directory.
       # If this file is included in multiple rules, only create the target once.
       string(REPLACE "::" "_" _DATA_TARGET ${_DATA_LABEL})
+      string(REPLACE "/" "_" _DATA_TARGET ${_DATA_TARGET})
       if(NOT TARGET ${_DATA_TARGET})
         set(_INPUT_PATH "${PROJECT_SOURCE_DIR}/${_FILE_PATH}")
         set(_OUTPUT_PATH "${PROJECT_BINARY_DIR}/${_FILE_PATH}")

build_tools/embed_data/BUILD

@@ -47,7 +47,7 @@ cc_test(
     deps = [
         ":testembed1",
         ":testembed2",
-        "//iree/testing:gtest",
-        "//iree/testing:gtest_main",
+        "//runtime/src/iree/testing:gtest",
+        "//runtime/src/iree/testing:gtest_main",
     ],
 )

build_tools/kokoro/gcp_ubuntu/cmake/baremetal/riscv32/test.sh

@@ -16,7 +16,7 @@ export PS4='[$(date -u "+%T %Z")] '
 
 # Run the embedded_library module loader and simple_embedding under QEMU.
 echo "Test elf_module_test_binary"
-pushd "${BUILD_RISCV_DIR?}/iree/hal/local/elf" > /dev/null
+pushd "${BUILD_RISCV_DIR?}/runtime/src/iree/hal/local/elf" > /dev/null
 "${QEMU_RV32_BIN?}" -cpu rv32,x-v=true,x-k=true,vlen=256,elen=64,vext_spec=v1.0 \
 elf_module_test_binary
 popd > /dev/null