|
"""Repository rule for Python autoconfiguration. |
|
|
|
`python_configure` depends on the following environment variables: |
|
|
|
* `PYTHON_BIN_PATH`: location of python binary. |
|
* `PYTHON_LIB_PATH`: Location of python libraries. |
|
""" |
|
|
|
load( |
|
"//third_party/remote_config:common.bzl", |
|
"BAZEL_SH", |
|
"PYTHON_BIN_PATH", |
|
"PYTHON_LIB_PATH", |
|
"TF_PYTHON_CONFIG_REPO", |
|
"auto_config_fail", |
|
"config_repo_label", |
|
"execute", |
|
"get_bash_bin", |
|
"get_host_environ", |
|
"get_python_bin", |
|
"is_windows", |
|
"raw_exec", |
|
"read_dir", |
|
) |
|
|
|
def _genrule(src_dir, genrule_name, command, outs): |
|
"""Returns a string with a genrule. |
|
|
|
Genrule executes the given command and produces the given outputs. |
|
""" |
|
return ( |
|
"genrule(\n" + |
|
' name = "' + |
|
genrule_name + '",\n' + |
|
" outs = [\n" + |
|
outs + |
|
"\n ],\n" + |
|
' cmd = """\n' + |
|
command + |
|
'\n """,\n' + |
|
")\n" |
|
) |
|
|
|
def _norm_path(path): |
|
"""Returns a path with '/' and remove the trailing slash.""" |
|
path = path.replace("\\", "/") |
|
if path[-1] == "/": |
|
path = path[:-1] |
|
return path |
|
|
|
def _symlink_genrule_for_dir( |
|
repository_ctx, |
|
src_dir, |
|
dest_dir, |
|
genrule_name, |
|
src_files = [], |
|
dest_files = []): |
|
"""Returns a genrule to symlink(or copy if on Windows) a set of files. |
|
|
|
If src_dir is passed, files will be read from the given directory; otherwise |
|
we assume files are in src_files and dest_files |
|
""" |
|
if src_dir != None: |
|
src_dir = _norm_path(src_dir) |
|
dest_dir = _norm_path(dest_dir) |
|
files = "\n".join(read_dir(repository_ctx, src_dir)) |
|
|
|
# Create a list with the src_dir stripped to use for outputs. |
|
dest_files = files.replace(src_dir, "").splitlines() |
|
src_files = files.splitlines() |
|
command = [] |
|
outs = [] |
|
for i in range(len(dest_files)): |
|
if dest_files[i] != "": |
|
# If we have only one file to link we do not want to use the dest_dir, as |
|
# $(@D) will include the full path to the file. |
|
dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i] |
|
|
|
# Copy the headers to create a sandboxable setup. |
|
cmd = "cp -f" |
|
command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) |
|
outs.append(' "' + dest_dir + dest_files[i] + '",') |
|
genrule = _genrule( |
|
src_dir, |
|
genrule_name, |
|
" && ".join(command), |
|
"\n".join(outs), |
|
) |
|
return genrule |
|
|
|
def _get_python_lib(repository_ctx, python_bin): |
|
"""Gets the python lib path.""" |
|
python_lib = get_host_environ(repository_ctx, PYTHON_LIB_PATH) |
|
if python_lib != None: |
|
return python_lib |
|
|
|
# The interesting program to execute. |
|
print_lib = [ |
|
"from __future__ import print_function", |
|
"import site", |
|
"import os", |
|
"python_paths = []", |
|
"if os.getenv('PYTHONPATH') is not None:", |
|
" python_paths = os.getenv('PYTHONPATH').split(':')", |
|
"try:", |
|
" library_paths = site.getsitepackages()", |
|
"except AttributeError:", |
|
" from distutils.sysconfig import get_python_lib", |
|
" library_paths = [get_python_lib()]", |
|
"all_paths = set(python_paths + library_paths)", |
|
"paths = []", |
|
"for path in all_paths:", |
|
" if os.path.isdir(path):", |
|
" paths.append(path)", |
|
"if len(paths) >=1:", |
|
" print(paths[0])", |
|
] |
|
|
|
# The below script writes the above program to a file |
|
# and executes it. This is to work around the limitation |
|
# of not being able to upload files as part of execute. |
|
cmd = "from os import linesep;" |
|
cmd += "f = open('script.py', 'w');" |
|
for line in print_lib: |
|
cmd += "f.write(\"%s\" + linesep);" % line |
|
cmd += "f.close();" |
|
cmd += "from os import system;" |
|
cmd += "system(r\"%s script.py\");" % python_bin |
|
|
|
result = execute(repository_ctx, [python_bin, "-c", cmd]) |
|
return result.stdout.strip() |
|
|
|
def _check_python_lib(repository_ctx, python_lib): |
|
"""Checks the python lib path.""" |
|
cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) |
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd]) |
|
if result.return_code == 1: |
|
auto_config_fail("Invalid python library path: %s" % python_lib) |
|
|
|
def _check_python_bin(repository_ctx, python_bin): |
|
"""Checks the python bin path.""" |
|
cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) |
|
result = raw_exec(repository_ctx, [get_bash_bin(repository_ctx), "-c", cmd]) |
|
if result.return_code == 1: |
|
auto_config_fail("--define %s='%s' is not executable. Is it the python binary?" % ( |
|
PYTHON_BIN_PATH, |
|
python_bin, |
|
)) |
|
|
|
def _get_python_include(repository_ctx, python_bin): |
|
"""Gets the python include path.""" |
|
result = execute( |
|
repository_ctx, |
|
[ |
|
python_bin, |
|
"-W ignore", |
|
"-c", |
|
"from __future__ import print_function;" + |
|
"from distutils import sysconfig;" + |
|
"print(sysconfig.get_python_inc())", |
|
], |
|
error_msg = "Problem getting python include path.", |
|
error_details = ("Is the Python binary path set up right? " + |
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) " + |
|
"Is distutils installed?"), |
|
) |
|
return result.stdout.splitlines()[0] |
|
|
|
def _get_python_import_lib_name(repository_ctx, python_bin): |
|
"""Get Python import library name (pythonXY.lib) on Windows.""" |
|
result = execute( |
|
repository_ctx, |
|
[ |
|
python_bin, |
|
"-c", |
|
"import sys;" + |
|
'print("python" + str(sys.version_info[0]) + ' + |
|
' str(sys.version_info[1]) + ".lib")', |
|
], |
|
error_msg = "Problem getting python import library.", |
|
error_details = ("Is the Python binary path set up right? " + |
|
"(See ./configure or " + PYTHON_BIN_PATH + ".) "), |
|
) |
|
return result.stdout.splitlines()[0] |
|
|
|
def _get_numpy_include(repository_ctx, python_bin): |
|
"""Gets the numpy include path.""" |
|
return execute( |
|
repository_ctx, |
|
[ |
|
python_bin, |
|
"-W ignore", |
|
"-c", |
|
"from __future__ import print_function;" + |
|
"import numpy;" + |
|
" print(numpy.get_include());", |
|
], |
|
error_msg = "Problem getting numpy include path.", |
|
error_details = "Is numpy installed?", |
|
).stdout.splitlines()[0] |
|
|
|
def _create_local_python_repository(repository_ctx): |
|
"""Creates the repository containing files set up to build with Python.""" |
|
|
|
# Resolve all labels before doing any real work. Resolving causes the |
|
# function to be restarted with all previous state being lost. This |
|
# can easily lead to a O(n^2) runtime in the number of labels. |
|
build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl")) |
|
|
|
python_bin = get_python_bin(repository_ctx) |
|
_check_python_bin(repository_ctx, python_bin) |
|
python_lib = _get_python_lib(repository_ctx, python_bin) |
|
_check_python_lib(repository_ctx, python_lib) |
|
python_include = _get_python_include(repository_ctx, python_bin) |
|
numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy" |
|
python_include_rule = _symlink_genrule_for_dir( |
|
repository_ctx, |
|
python_include, |
|
"python_include", |
|
"python_include", |
|
) |
|
python_import_lib_genrule = "" |
|
|
|
# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib |
|
# See https://docs.python.org/3/extending/windows.html |
|
if is_windows(repository_ctx): |
|
python_include = _norm_path(python_include) |
|
python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin) |
|
python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name |
|
python_import_lib_genrule = _symlink_genrule_for_dir( |
|
repository_ctx, |
|
None, |
|
"", |
|
"python_import_lib", |
|
[python_import_lib_src], |
|
[python_import_lib_name], |
|
) |
|
numpy_include_rule = _symlink_genrule_for_dir( |
|
repository_ctx, |
|
numpy_include, |
|
"numpy_include/numpy", |
|
"numpy_include", |
|
) |
|
|
|
platform_constraint = "" |
|
if repository_ctx.attr.platform_constraint: |
|
platform_constraint = "\"%s\"" % repository_ctx.attr.platform_constraint |
|
repository_ctx.template("BUILD", build_tpl, { |
|
"%{PYTHON_BIN_PATH}": python_bin, |
|
"%{PYTHON_INCLUDE_GENRULE}": python_include_rule, |
|
"%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, |
|
"%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, |
|
"%{PLATFORM_CONSTRAINT}": platform_constraint, |
|
}) |
|
|
|
def _create_remote_python_repository(repository_ctx, remote_config_repo): |
|
"""Creates pointers to a remotely configured repo set up to build with Python. |
|
""" |
|
repository_ctx.template("BUILD", config_repo_label(remote_config_repo, ":BUILD"), {}) |
|
|
|
def _python_autoconf_impl(repository_ctx): |
|
"""Implementation of the python_autoconf repository rule.""" |
|
if get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO) != None: |
|
_create_remote_python_repository( |
|
repository_ctx, |
|
get_host_environ(repository_ctx, TF_PYTHON_CONFIG_REPO), |
|
) |
|
else: |
|
_create_local_python_repository(repository_ctx) |
|
|
|
_ENVIRONS = [ |
|
BAZEL_SH, |
|
PYTHON_BIN_PATH, |
|
PYTHON_LIB_PATH, |
|
] |
|
|
|
local_python_configure = repository_rule( |
|
implementation = _create_local_python_repository, |
|
environ = _ENVIRONS, |
|
attrs = { |
|
"environ": attr.string_dict(), |
|
"platform_constraint": attr.string(), |
|
}, |
|
) |
|
|
|
remote_python_configure = repository_rule( |
|
implementation = _create_local_python_repository, |
|
environ = _ENVIRONS, |
|
remotable = True, |
|
attrs = { |
|
"environ": attr.string_dict(), |
|
"platform_constraint": attr.string(), |
|
}, |
|
) |
|
|
|
python_configure = repository_rule( |
|
implementation = _python_autoconf_impl, |
|
environ = _ENVIRONS + [TF_PYTHON_CONFIG_REPO], |
|
attrs = { |
|
"platform_constraint": attr.string(), |
|
}, |
|
) |
|
"""Detects and configures the local Python. |
|
|
|
Add the following to your WORKSPACE FILE: |
|
|
|
```python |
|
python_configure(name = "local_config_python") |
|
``` |
|
|
|
Args: |
|
name: A unique name for this workspace rule. |
|
""" |
|
|