python_configure.bzl

# Adapted with modifications from tensorflow/third_party/py/
"""Repository rule for Python autoconfiguration.

`python_configure` depends on the following environment variables:

  * `PYTHON3_BIN_PATH`: location of python binary.
  * `PYTHON3_LIB_PATH`: Location of python libraries.
"""

_BAZEL_SH = "BAZEL_SH"
_PYTHON3_BIN_PATH = "PYTHON3_BIN_PATH"
_PYTHON3_LIB_PATH = "PYTHON3_LIB_PATH"

_HEADERS_HELP = (
    "Are Python headers installed? Try installing " +
    "python3-dev on Debian-based systems. Try python3-devel " +
    "on Redhat-based systems."
)

def _tpl(repository_ctx, tpl, substitutions = {}, out = None):
    if not out:
        out = tpl
    repository_ctx.template(
        out,
        Label("//third_party/py:%s.tpl" % tpl),
        substitutions,
    )

def _fail(msg):
    """Output failure message when auto configuration fails."""
    red = "\033[0;31m"
    no_color = "\033[0m"
    fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg))

def _is_windows(repository_ctx):
    """Returns true if the host operating system is windows."""
    os_name = repository_ctx.os.name.lower()
    return os_name.find("windows") != -1

def _execute(
        repository_ctx,
        cmdline,
        error_msg = None,
        error_details = None,
        empty_stdout_fine = False):
    """Executes an arbitrary shell command.

    Args:
        repository_ctx: the repository_ctx object
        cmdline: list of strings, the command to execute
        error_msg: string, a summary of the error if the command fails
        error_details: string, details about the error or steps to fix it
        empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise
        it's an error
    Return:
        the result of repository_ctx.execute(cmdline)
  """
    result = repository_ctx.execute(cmdline)
    if result.stderr or not (empty_stdout_fine or result.stdout):
        _fail("\n".join([
            error_msg.strip() if error_msg else "Repository command failed",
            result.stderr.strip(),
            error_details if error_details else "",
        ]))
    else:
        return result

def _read_dir(repository_ctx, src_dir):
    """Returns a string with all files in a directory.

  Finds all files inside a directory, traversing subfolders and following
  symlinks. The returned string contains the full path of all files
  separated by line breaks.
  """
    if _is_windows(repository_ctx):
        src_dir = src_dir.replace("/", "\\")
        find_result = _execute(
            repository_ctx,
            ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
            empty_stdout_fine = True,
        )

        # src_files will be used in genrule.outs where the paths must
        # use forward slashes.
        return find_result.stdout.replace("\\", "/")
    else:
        find_result = _execute(
            repository_ctx,
            ["find", src_dir, "-follow", "-type", "f"],
            empty_stdout_fine = True,
        )
        return find_result.stdout

def _genrule(src_dir, genrule_name, command, outs):
    """Returns a string with a genrule.

  Genrule executes the given command and produces the given outputs.
  """
    return ("genrule(\n" + '    name = "' + genrule_name + '",\n' +
            "    outs = [\n" + outs + "\n    ],\n" + '    cmd = """\n' +
            command + '\n   """,\n' + ")\n")

def _normalize_path(path):
    """Returns a path with '/' and remove the trailing slash."""
    path = path.replace("\\", "/")
    if path[-1] == "/":
        path = path[:-1]
    return path

def _symlink_genrule_for_dir(
        repository_ctx,
        src_dir,
        dest_dir,
        genrule_name,
        src_files = [],
        dest_files = []):
    """Returns a genrule to symlink(or copy if on Windows) a set of files.

  If src_dir is passed, files will be read from the given directory; otherwise
  we assume files are in src_files and dest_files
  """
    if src_dir != None:
        src_dir = _normalize_path(src_dir)
        dest_dir = _normalize_path(dest_dir)
        files = "\n".join(
            sorted(_read_dir(repository_ctx, src_dir).splitlines()),
        )

        # Create a list with the src_dir stripped to use for outputs.
        dest_files = files.replace(src_dir, "").splitlines()
        src_files = files.splitlines()
    command = []
    outs = []
    for i in range(len(dest_files)):
        if dest_files[i] != "":
            # If we have only one file to link we do not want to use the dest_dir, as
            # $(@D) will include the full path to the file.
            dest = "$(@D)/" + dest_dir + dest_files[i] if len(
                dest_files,
            ) != 1 else "$(@D)/" + dest_files[i]

            # On Windows, symlink is not supported, so we just copy all the files.
            cmd = "cp -f" if _is_windows(repository_ctx) else "ln -s"
            command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
            outs.append('        "' + dest_dir + dest_files[i] + '",')
    return _genrule(
        src_dir,
        genrule_name,
        " && ".join(command),
        "\n".join(outs),
    )

def _get_python_bin(repository_ctx, bin_path_key, default_bin_path, allow_absent):
    """Gets the python bin path."""
    python_bin = repository_ctx.os.environ.get(bin_path_key, default_bin_path)
    if not repository_ctx.path(python_bin).exists:
        # It's a command, use 'which' to find its path.
        python_bin_path = repository_ctx.which(python_bin)
    else:
        # It's a path, use it as it is.
        python_bin_path = python_bin
    if python_bin_path != None:
        return str(python_bin_path)
    if not allow_absent:
        _fail("Cannot find python in PATH, please make sure " +
              "python is installed and add its directory in PATH, or --define " +
              "%s='/something/else'.\nPATH=%s" %
              (bin_path_key, repository_ctx.os.environ.get("PATH", "")))
    else:
        return None

def _get_bash_bin(repository_ctx):
    """Gets the bash bin path."""
    bash_bin = repository_ctx.os.environ.get(_BAZEL_SH)
    if bash_bin != None:
        return bash_bin
    else:
        bash_bin_path = repository_ctx.which("bash")
        if bash_bin_path != None:
            return str(bash_bin_path)
        else:
            _fail(
                "Cannot find bash in PATH, please make sure " +
                "bash is installed and add its directory in PATH, or --define " +
                "%s='/path/to/bash'.\nPATH=%s" %
                (_BAZEL_SH, repository_ctx.os.environ.get("PATH", "")),
            )

def _get_python_lib(repository_ctx, python_bin, lib_path_key):
    """Gets the python lib path."""
    python_lib = repository_ctx.os.environ.get(lib_path_key)
    if python_lib != None:
        return python_lib
    print_lib = (
        "<<END\n" + "from __future__ import print_function\n" +
        "import site\n" + "import os\n" + "\n" + "try:\n" +
        "  input = raw_input\n" + "except NameError:\n" + "  pass\n" + "\n" +
        "python_paths = []\n" + "if os.getenv('PYTHONPATH') is not None:\n" +
        "  python_paths = os.getenv('PYTHONPATH').split(':')\n" + "try:\n" +
        "  library_paths = site.getsitepackages()\n" +
        "except AttributeError:\n" +
        " import sysconfig\n" +
        " library_paths = [sysconfig.get_path('purelib')]\n" +
        "all_paths = set(python_paths + library_paths)\n" + "paths = []\n" +
        "for path in all_paths:\n" + "  if os.path.isdir(path):\n" +
        "    paths.append(path)\n" + "if len(paths) >=1:\n" +
        "  print(paths[0])\n" + "END"
    )
    cmd = '"%s" - %s' % (python_bin, print_lib)
    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
    return result.stdout.strip("\n")

def _check_python_lib(repository_ctx, python_lib):
    """Checks the python lib path."""
    cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
    if result.return_code == 1:
        _fail("Invalid python library path: %s" % python_lib)

def _check_python_bin(repository_ctx, python_bin, bin_path_key, allow_absent):
    """Checks the python bin path."""
    cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
    result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
    if result.return_code == 1:
        if not allow_absent:
            _fail("--define %s='%s' is not executable. Is it the python binary?" %
                  (bin_path_key, python_bin))
        else:
            return None
    return True

def _get_python_include(repository_ctx, python_bin):
    """Gets the python include path."""
    result = _execute(
        repository_ctx,
        [
            python_bin,
            "-c",
            "from __future__ import print_function;" +
            "import sysconfig;" +
            "print(sysconfig.get_path('include'))",
        ],
        error_msg = "Problem getting python include path for {}.".format(python_bin),
        error_details = (
            "Is the Python binary path set up right? " + "(See ./configure or " +
            python_bin + ".) " + _HEADERS_HELP
        ),
    )
    include_path = result.stdout.splitlines()[0]
    _execute(
        repository_ctx,
        [
            python_bin,
            "-c",
            "import os;" +
            "main_header = os.path.join(r'{}', 'Python.h');".format(include_path) +
            "assert os.path.exists(main_header), main_header + ' does not exist.'",
        ],
        error_msg = "Unable to find Python headers for {}".format(python_bin),
        error_details = _HEADERS_HELP,
        empty_stdout_fine = True,
    )
    return include_path

def _get_python_import_lib_name(repository_ctx, python_bin, bin_path_key):
    """Get Python import library name (pythonXY.lib) on Windows."""
    result = _execute(
        repository_ctx,
        [
            python_bin,
            "-c",
            "import sys;" + 'print("python" + str(sys.version_info[0]) + ' +
            '      str(sys.version_info[1]) + ".lib")',
        ],
        error_msg = "Problem getting python import library.",
        error_details = ("Is the Python binary path set up right? " +
                         "(See ./configure or " + bin_path_key + ".) "),
    )
    return result.stdout.splitlines()[0]

def _create_single_version_package(
        repository_ctx,
        variety_name,
        bin_path_key,
        default_bin_path,
        lib_path_key,
        allow_absent):
    """Creates the repository containing files set up to build with Python."""
    empty_include_rule = "filegroup(\n  name=\"{}_include\",\n  srcs=[],\n)".format(variety_name)

    python_bin = _get_python_bin(repository_ctx, bin_path_key, default_bin_path, allow_absent)
    if (python_bin == None or
        _check_python_bin(repository_ctx,
                          python_bin,
                          bin_path_key,
                          allow_absent) == None) and allow_absent:
            python_include_rule = empty_include_rule
    else:
        python_lib = _get_python_lib(repository_ctx, python_bin, lib_path_key)
        _check_python_lib(repository_ctx, python_lib)
        python_include = _get_python_include(repository_ctx, python_bin)
        python_include_rule = _symlink_genrule_for_dir(
            repository_ctx,
            python_include,
            "{}_include".format(variety_name),
            "{}_include".format(variety_name),
        )
    python_import_lib_genrule = ""

    # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
    # See https://docs.python.org/3/extending/windows.html
    if _is_windows(repository_ctx):
        python_include = _normalize_path(python_include)
        python_import_lib_name = _get_python_import_lib_name(
            repository_ctx,
            python_bin,
            bin_path_key,
        )
        python_import_lib_src = python_include.rsplit(
            "/",
            1,
        )[0] + "/libs/" + python_import_lib_name
        python_import_lib_genrule = _symlink_genrule_for_dir(
            repository_ctx,
            None,
            "",
            "{}_import_lib".format(variety_name),
            [python_import_lib_src],
            [python_import_lib_name],
        )
    _tpl(
        repository_ctx,
        "variety",
        {
            "%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
            "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
            "%{VARIETY_NAME}": variety_name,
        },
        out = "{}/BUILD".format(variety_name),
    )

def _python_autoconf_impl(repository_ctx):
    """Implementation of the python_autoconf repository rule."""
    _create_single_version_package(
        repository_ctx,
        "_python3",
        _PYTHON3_BIN_PATH,
        "python3" if not _is_windows(repository_ctx) else "python.exe",
        _PYTHON3_LIB_PATH,
        False
    )
    _tpl(repository_ctx, "BUILD")

python_configure = repository_rule(
    implementation = _python_autoconf_impl,
    environ = [
        _BAZEL_SH,
        _PYTHON3_BIN_PATH,
        _PYTHON3_LIB_PATH,
    ],
    attrs = {
        "_build_tpl": attr.label(
            default = Label("//third_party/py:BUILD.tpl"),
            allow_single_file = True,
        ),
        "_variety_tpl": attr.label(
            default = Label("//third_party/py:variety.tpl"),
            allow_single_file = True,
        ),
    },
)
"""Detects and configures the local Python.

It expects the system have a working Python 3 installation.

Add the following to your WORKSPACE FILE:

```python
python_configure(name = "local_config_python")
```

Args:
  name: A unique name for this workspace rule.
"""