diff options
Diffstat (limited to 'python/pip_install')
26 files changed, 1881 insertions, 1160 deletions
diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index e8e8633..4bcd5b8 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -1,16 +1,95 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = ["//:__subpackages__"], +) + +bzl_library( + name = "pip_repository_bzl", + srcs = ["pip_repository.bzl"], + deps = [ + ":repositories_bzl", + ":requirements_parser_bzl", + "//python:repositories_bzl", + "//python:versions_bzl", + "//python/pip_install/private:generate_group_library_build_bazel_bzl", + "//python/pip_install/private:generate_whl_library_build_bazel_bzl", + "//python/pip_install/private:srcs_bzl", + "//python/private:bzlmod_enabled_bzl", + "//python/private:normalize_name_bzl", + "//python/private:parse_whl_name_bzl", + "//python/private:patch_whl_bzl", + "//python/private:render_pkg_aliases_bzl", + "//python/private:toolchains_repo_bzl", + "//python/private:which_bzl", + "//python/private:whl_target_platforms_bzl", + "@bazel_skylib//lib:sets", + ], +) + +bzl_library( + name = "requirements_bzl", + srcs = ["requirements.bzl"], + deps = [ + ":repositories_bzl", + "//python:defs_bzl", + ], +) + +bzl_library( + name = "requirements_parser_bzl", + srcs = ["requirements_parser.bzl"], +) + +bzl_library( + name = "repositories_bzl", + srcs = ["repositories.bzl"], + deps = [ + "//:version_bzl", + "//python/private:bazel_tools_bzl", + "@bazel_skylib//lib:versions", + ], +) + filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ "BUILD.bazel", + "pip_repository_requirements.bzl.tmpl", "//python/pip_install/private:distribution", "//python/pip_install/tools/dependency_resolver:distribution", - "//python/pip_install/tools/lib:distribution", "//python/pip_install/tools/wheel_installer:distribution", ], visibility = ["//:__pkg__"], ) filegroup( + name = "repositories", + srcs = ["repositories.bzl"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + +filegroup( + name = "requirements_txt", + srcs = ["tools/requirements.txt"], + visibility = ["//tools/private/update_deps:__pkg__"], +) + +filegroup( name = "bzl", srcs = glob(["*.bzl"]) + [ "//python/pip_install/private:bzl_srcs", @@ -22,8 +101,9 @@ filegroup( name = "py_srcs", srcs = [ "//python/pip_install/tools/dependency_resolver:py_srcs", - "//python/pip_install/tools/lib:py_srcs", "//python/pip_install/tools/wheel_installer:py_srcs", + "//python/private:repack_whl.py", + "//tools:wheelmaker.py", ], visibility = ["//python/pip_install/private:__pkg__"], ) diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 4a3d512..0000000 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,35 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. - -This file is different from the other bzlmod template -because we do not support entry_point yet. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - # TODO: https://github.com/bazelbuild/rules_python/issues/1262 - print("not implemented") diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 99d1fb0..3e4878b 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,19 +14,29 @@ "" -load("//python:repositories.bzl", "get_interpreter_dirname", "is_standalone_interpreter") +load("@bazel_skylib//lib:sets.bzl", "sets") +load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") +load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_whl_name.bzl", "parse_whl_name") +load("//python/private:patch_whl.bzl", "patch_whl") +load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases") load("//python/private:toolchains_repo.bzl", "get_host_os_arch") +load("//python/private:which.bzl", "which_with_fail") +load("//python/private:whl_target_platforms.bzl", "whl_target_platforms") CPPFLAGS = "CPPFLAGS" COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" +_WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" + def _construct_pypath(rctx): """Helper function to construct a PYTHONPATH. @@ -35,18 +45,15 @@ def _construct_pypath(rctx): Args: rctx: Handle to the repository_context. + Returns: String of the PYTHONPATH. """ - # Get the root directory of these rules - rules_root = rctx.path(Label("//:BUILD.bazel")).dirname - thirdparty_roots = [ - # Includes all the external dependencies from repositories.bzl - rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname - for repo in all_requirements - ] separator = ":" if not "windows" in rctx.os.name.lower() else ";" - pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots]) + pypath = separator.join([ + str(rctx.path(entry).dirname) + for entry in rctx.attr._python_path_entries + ]) return pypath def _get_python_interpreter_attr(rctx): @@ -71,7 +78,9 @@ def _resolve_python_interpreter(rctx): Args: rctx: Handle to the rule repository context. - Returns: Python interpreter path. + + Returns: + `path` object, for the resolved path to the Python interpreter. """ python_interpreter = _get_python_interpreter_attr(rctx) @@ -86,10 +95,13 @@ def _resolve_python_interpreter(rctx): if os == WINDOWS_NAME: python_interpreter = python_interpreter.realpath elif "/" not in python_interpreter: + # It's a plain command, e.g. "python3", to look up in the environment. found_python_interpreter = rctx.which(python_interpreter) if not found_python_interpreter: fail("python interpreter `{}` not found in PATH".format(python_interpreter)) python_interpreter = found_python_interpreter + else: + python_interpreter = rctx.path(python_interpreter) return python_interpreter def _get_xcode_location_cflags(rctx): @@ -104,10 +116,7 @@ def _get_xcode_location_cflags(rctx): if not rctx.os.name.lower().startswith("mac os"): return [] - # Locate xcode-select - xcode_select = rctx.which("xcode-select") - - xcode_sdk_location = rctx.execute([xcode_select, "--print-path"]) + xcode_sdk_location = rctx.execute([which_with_fail("xcode-select", rctx), "--print-path"]) if xcode_sdk_location.return_code != 0: return [] @@ -121,7 +130,7 @@ def _get_xcode_location_cflags(rctx): "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), ] -def _get_toolchain_unix_cflags(rctx): +def _get_toolchain_unix_cflags(rctx, python_interpreter): """Gather cflags from a standalone toolchain for unix systems. Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg @@ -133,11 +142,11 @@ def _get_toolchain_unix_cflags(rctx): return [] # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, rctx.attr.python_interpreter_target): + if not is_standalone_interpreter(rctx, python_interpreter): return [] er = rctx.execute([ - rctx.path(rctx.attr.python_interpreter_target).realpath, + python_interpreter, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", ]) @@ -145,7 +154,7 @@ def _get_toolchain_unix_cflags(rctx): fail("could not get python version from interpreter (status {}): {}".format(er.return_code, er.stderr)) _python_version = er.stdout include_path = "{}/include/python{}".format( - get_interpreter_dirname(rctx, rctx.attr.python_interpreter_target), + python_interpreter.dirname, _python_version, ) @@ -216,11 +225,12 @@ def _parse_optional_attrs(rctx, args): return args -def _create_repository_execution_environment(rctx): +def _create_repository_execution_environment(rctx, python_interpreter): """Create a environment dictionary for processes we spawn with rctx.execute. Args: - rctx: The repository context. + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. Returns: Dictionary of environment variable suitable to pass to rctx.execute. """ @@ -228,7 +238,7 @@ def _create_repository_execution_environment(rctx): # Gather any available CPPFLAGS values cppflags = [] cppflags.extend(_get_xcode_location_cflags(rctx)) - cppflags.extend(_get_toolchain_unix_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) env = { "PYTHONPATH": _construct_pypath(rctx), @@ -268,163 +278,49 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _pkg_aliases(rctx, repo_name, bzl_packages): - """Create alias declarations for each python dependency. - - The aliases should be appended to the pip_repository BUILD.bazel file. These aliases - allow users to use requirement() without needed a corresponding `use_repo()` for each dep - when using bzlmod. - - Args: - rctx: the repository context. - repo_name: the repository name of the parent that is visible to the users. - bzl_packages: the list of packages to setup. - """ - for name in bzl_packages: - build_content = """package(default_visibility = ["//visibility:public"]) - -alias( - name = "{name}", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "pkg", - actual = "@{repo_name}_{dep}//:pkg", -) - -alias( - name = "whl", - actual = "@{repo_name}_{dep}//:whl", -) - -alias( - name = "data", - actual = "@{repo_name}_{dep}//:data", -) - -alias( - name = "dist_info", - actual = "@{repo_name}_{dep}//:dist_info", -) -""".format( - name = name, - repo_name = repo_name, - dep = name, - ) - rctx.file("{}/BUILD.bazel".format(name), build_content) - -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - _pkg_aliases(rctx, repo_name, bzl_packages) - - # NOTE: we are using the canonical name with the double '@' in order to - # always uniquely identify a repository, as the labels are being passed as - # a string and the resolution of the label happens at the call-site of the - # `requirement`, et al. macros. - macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - - rctx.file("BUILD.bazel", build_contents) - rctx.template("requirements.bzl", rctx.attr._template, substitutions = { - "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "data") - for p in bzl_packages - ]), - "%%ALL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, p) - for p in bzl_packages - ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - macro_tmpl.format(p, "whl") - for p in bzl_packages - ]), - "%%MACRO_TMPL%%": macro_tmpl, - "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, - }) - -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - -pip_hub_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", - ), - "whl_library_alias_names": attr.string_list( - mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", - ), - "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_hub_repository_bzlmod = repository_rule( - attrs = pip_hub_repository_bzlmod_attrs, - doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", - implementation = _pip_hub_repository_bzlmod_impl, -) - -def _pip_repository_bzlmod_impl(rctx): +def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) parsed_requirements_txt = parse_requirements(content) packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - -def _pip_repository_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) + bzl_packages = sorted([normalize_name(name) for name, _ in parsed_requirements_txt.requirements]) - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + # Normalize cycles first + requirement_cycles = { + name: sorted(sets.to_list(sets.make(deps))) + for name, deps in rctx.attr.experimental_requirement_cycles.items() + } - bzl_packages = sorted([name for name, _ in packages]) + # Check for conflicts between cycles _before_ we normalize package names so + # that reported errors use the names the user specified + for i in range(len(requirement_cycles)): + left_group = requirement_cycles.keys()[i] + left_deps = requirement_cycles.values()[i] + for j in range(len(requirement_cycles) - (i + 1)): + right_deps = requirement_cycles.values()[1 + i + j] + right_group = requirement_cycles.keys()[1 + i + j] + for d in left_deps: + if d in right_deps: + fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) + + # And normalize the names as used in the cycle specs + # + # NOTE: We must check that a listed dependency is actually in the actual + # requirements set for the current platform so that we can support cycles in + # platform-conditional requirements. Otherwise we'll blindly generate a + # label referencing a package which may not be installed on the current + # platform. + requirement_cycles = { + normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + for name, group in requirement_cycles.items() + } imports = [ - 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library")', + # NOTE: Maintain the order consistent with `buildifier` + 'load("@rules_python//python:pip.bzl", "pip_utils")', + 'load("@rules_python//python/pip_install:pip_repository.bzl", "group_library", "whl_library")', ] annotations = {} @@ -456,28 +352,37 @@ def _pip_repository_impl(rctx): if rctx.attr.python_interpreter_target: config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.experimental_target_platforms: + config["experimental_target_platforms"] = rctx.attr.experimental_target_platforms if rctx.attr.incompatible_generate_aliases: - _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + macro_tmpl = "@%s//{}:{}" % rctx.attr.name + aliases = render_pkg_aliases(repo_name = rctx.attr.name, bzl_packages = bzl_packages) + for path, contents in aliases.items(): + rctx.file(path, contents) + else: + macro_tmpl = "@%s_{}//:{}" % rctx.attr.name rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}:data".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:data".format(rctx.attr.name, p) + macro_tmpl.format(p, "data") for p in bzl_packages ]), "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) + macro_tmpl.format(p, "pkg") for p in bzl_packages ]), - "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}//{}:whl".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) + "%%ALL_REQUIREMENT_GROUPS%%": _format_dict(_repr_dict(requirement_cycles)), + "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": _format_dict(_repr_dict({ + p: macro_tmpl.format(p, "whl") for p in bzl_packages - ]), + })), "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), "%%CONFIG%%": _format_dict(_repr_dict(config)), "%%EXTRA_PIP_ARGS%%": json.encode(options), - "%%IMPORTS%%": "\n".join(sorted(imports)), + "%%IMPORTS%%": "\n".join(imports), + "%%MACRO_TMPL%%": macro_tmpl, "%%NAME%%": rctx.attr.name, "%%PACKAGES%%": _format_repr_list( [ @@ -522,6 +427,86 @@ can be passed. """, default = {}, ), + "experimental_requirement_cycles": attr.string_list_dict( + default = {}, + doc = """\ +A mapping of dependency cycle names to a list of requirements which form that cycle. + +Requirements which form cycles will be installed together and taken as +dependencies together in order to ensure that the cycle is always satisified. + +Example: + `sphinx` depends on `sphinxcontrib-serializinghtml` + When listing both as requirements, ala + + ``` + py_binary( + name = "doctool", + ... + deps = [ + "@pypi//sphinx:pkg", + "@pypi//sphinxcontrib_serializinghtml", + ] + ) + ``` + + Will produce a Bazel error such as + + ``` + ERROR: .../external/pypi_sphinxcontrib_serializinghtml/BUILD.bazel:44:6: in alias rule @pypi_sphinxcontrib_serializinghtml//:pkg: cycle in dependency graph: + //:doctool (...) + @pypi//sphinxcontrib_serializinghtml:pkg (...) + .-> @pypi_sphinxcontrib_serializinghtml//:pkg (...) + | @pypi_sphinxcontrib_serializinghtml//:_pkg (...) + | @pypi_sphinx//:pkg (...) + | @pypi_sphinx//:_pkg (...) + `-- @pypi_sphinxcontrib_serializinghtml//:pkg (...) + ``` + + Which we can resolve by configuring these two requirements to be installed together as a cycle + + ``` + pip_parse( + ... + experimental_requirement_cycles = { + "sphinx": [ + "sphinx", + "sphinxcontrib-serializinghtml", + ] + }, + ) + ``` + +Warning: + If a dependency participates in multiple cycles, all of those cycles must be + collapsed down to one. For instance `a <-> b` and `a <-> c` cannot be listed + as two separate cycles. +""", + ), + "experimental_target_platforms": attr.string_list( + default = [], + doc = """\ +A list of platforms that we will generate the conditional dependency graph for +cross platform wheels by parsing the wheel metadata. This will generate the +correct dependencies for packages like `sphinx` or `pylint`, which include +`colorama` when installed and used on Windows platforms. + +An empty list means falling back to the legacy behaviour where the host +platform is the target platform. + +WARNING: It may not work as expected in cases where the python interpreter +implementation that is being used at runtime is different between different platforms. +This has been tested for CPython only. + +Special values: `all` (for generating deps for all platforms), `host` (for +generating deps for the host platform only). `linux_*` and other `<os>_*` values. +In the future we plan to set `all` as the default to this attribute. + +For specific target platforms use values of the form `<os>_<arch>` where `<os>` +is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`, +`aarch64`, `s390x` and `ppc64le`. +""", + ), "extra_pip_args": attr.string_list( doc = "Extra arguments to pass on to pip. Must not contain spaces.", ), @@ -580,8 +565,21 @@ pip_repository_attrs = { doc = "Optional annotations to apply to packages", ), "incompatible_generate_aliases": attr.bool( - default = False, - doc = "Allow generating aliases '@pip//<pkg>' -> '@pip_<pkg>//:pkg'.", + default = True, + doc = """\ +If true, extra aliases will be created in the main `hub` repo - i.e. the repo +where the `requirements.bzl` is located. This means that for a Python package +`PyYAML` initialized within a `pip` `hub_repo` there will be the following +aliases generated: +- `@pip//pyyaml` will point to `@pip_pyyaml//:pkg` +- `@pip//pyyaml:data` will point to `@pip_pyyaml//:data` +- `@pip//pyyaml:dist_info` will point to `@pip_pyyaml//:dist_info` +- `@pip//pyyaml:pkg` will point to `@pip_pyyaml//:pkg` +- `@pip//pyyaml:whl` will point to `@pip_pyyaml//:whl` + +This is to keep the dependencies coming from PyPI to have more ergonomic label +names and support smooth transition to `bzlmod`. +""", ), "requirements_darwin": attr.label( allow_single_file = True, @@ -593,10 +591,14 @@ pip_repository_attrs = { ), "requirements_lock": attr.label( allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. + doc = """\ +A fully resolved 'requirements.txt' pip requirement file containing the +transitive set of your dependencies. If this file is passed instead of +'requirements' no resolve will take place and pip_repository will create +individual repositories for each of your dependencies so that wheels are +fetched/built only for the targets specified by 'build/run/test'. Note that if +your lockfile is platform-dependent, you can use the `requirements_[platform]` +attributes. """, ), "requirements_windows": attr.label( @@ -612,22 +614,31 @@ pip_repository_attrs.update(**common_attrs) pip_repository = repository_rule( attrs = pip_repository_attrs, - doc = """A rule for importing `requirements.txt` dependencies into Bazel. + doc = """Accepts a locked/compiled requirements file and installs the dependencies listed within. + +Those dependencies become available in a generated `requirements.bzl` file. +You can instead check this `requirements.bzl` file into your repo, see the "vendoring" section below. + +In your WORKSPACE file: -This rule imports a `requirements.txt` file and generates a new -`requirements.bzl` file. This is used via the `WORKSPACE` pattern: +```starlark +load("@rules_python//python:pip.bzl", "pip_parse") -```python -pip_repository( - name = "foo", - requirements = ":requirements.txt", +pip_parse( + name = "pip_deps", + requirements_lock = ":requirements.txt", ) + +load("@pip_deps//:requirements.bzl", "install_deps") + +install_deps() ``` -You can then reference imported dependencies from your `BUILD` file with: +You can then reference installed dependencies from a `BUILD` file with: + +```starlark +load("@pip_deps//:requirements.bzl", "requirement") -```python -load("@foo//:requirements.bzl", "requirement") py_library( name = "bar", ... @@ -639,17 +650,52 @@ py_library( ) ``` -Or alternatively: -```python -load("@foo//:requirements.bzl", "all_requirements") -py_binary( - name = "baz", - ... - deps = [ - ":foo", - ] + all_requirements, +In addition to the `requirement` macro, which is used to access the generated `py_library` +target generated from a package's wheel, The generated `requirements.bzl` file contains +functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. + +[whl_ep]: https://packaging.python.org/specifications/entry-points/ + +```starlark +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "pip-compile", + actual = entry_point( + pkg = "pip-tools", + script = "pip-compile", + ), ) ``` + +Note that for packages whose name and script are the same, only the name of the package +is needed when calling the `entry_point` macro. + +```starlark +load("@pip_deps//:requirements.bzl", "entry_point") + +alias( + name = "flake8", + actual = entry_point("flake8"), +) +``` + +### Vendoring the requirements.bzl file + +In some cases you may not want to generate the requirements.bzl file as a repository rule +while Bazel is fetching dependencies. For example, if you produce a reusable Bazel module +such as a ruleset, you may want to include the requirements.bzl file rather than make your users +install the WORKSPACE setup to generate it. +See https://github.com/bazelbuild/rules_python/issues/608 + +This is the same workflow as Gazelle, which creates `go_repository` rules with +[`update-repos`](https://github.com/bazelbuild/bazel-gazelle#update-repos) + +To do this, use the "write to source file" pattern documented in +https://blog.aspect.dev/bazel-can-write-to-the-source-folder +to put a copy of the generated requirements.bzl into your project. +Then load the requirements.bzl file directly rather than from the generated repository. +See the example in rules_python/examples/pip_parse_vendored. """, implementation = _pip_repository_impl, environ = common_env, @@ -663,23 +709,59 @@ def _whl_library_impl(rctx): "python.pip_install.tools.wheel_installer.wheel_installer", "--requirement", rctx.attr.requirement, - "--repo", - rctx.attr.repo, - "--repo-prefix", - rctx.attr.repo_prefix, ] - if rctx.attr.annotation: - args.extend([ - "--annotation", - rctx.path(rctx.attr.annotation), - ]) args = _parse_optional_attrs(rctx, args) + # Manually construct the PYTHONPATH since we cannot use the toolchain here + environment = _create_repository_execution_environment(rctx, python_interpreter) + result = rctx.execute( args, - # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx), + environment = environment, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + if result.return_code: + fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code)) + + whl_path = rctx.path(json.decode(rctx.read("whl_file.json"))["whl_file"]) + if not rctx.delete("whl_file.json"): + fail("failed to delete the whl_file.json file") + + if rctx.attr.whl_patches: + patches = {} + for patch_file, json_args in rctx.attr.whl_patches.items(): + patch_dst = struct(**json.decode(json_args)) + if whl_path.basename in patch_dst.whls: + patches[patch_file] = patch_dst.patch_strip + + whl_path = patch_whl( + rctx, + python_interpreter = python_interpreter, + whl_path = whl_path, + patches = patches, + quiet = rctx.attr.quiet, + timeout = rctx.attr.timeout, + ) + + target_platforms = rctx.attr.experimental_target_platforms + if target_platforms: + parsed_whl = parse_whl_name(whl_path.basename) + if parsed_whl.platform_tag != "any": + # NOTE @aignas 2023-12-04: if the wheel is a platform specific + # wheel, we only include deps for that target platform + target_platforms = [ + "{}_{}".format(p.os, p.cpu) + for p in whl_target_platforms(parsed_whl.platform_tag) + ] + + result = rctx.execute( + args + [ + "--whl-file", + whl_path, + ] + ["--platform={}".format(p) for p in target_platforms], + environment = environment, quiet = rctx.attr.quiet, timeout = rctx.attr.timeout, ) @@ -687,8 +769,76 @@ def _whl_library_impl(rctx): if result.return_code: fail("whl_library %s failed: %s (%s) error code: '%s'" % (rctx.attr.name, result.stdout, result.stderr, result.return_code)) + metadata = json.decode(rctx.read("metadata.json")) + rctx.delete("metadata.json") + + entry_points = {} + for item in metadata["entry_points"]: + name = item["name"] + module = item["module"] + attribute = item["attribute"] + + # There is an extreme edge-case with entry_points that end with `.py` + # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 + entry_point_without_py = name[:-3] + "_py" if name.endswith(".py") else name + entry_point_target_name = ( + _WHEEL_ENTRY_POINT_PREFIX + "_" + entry_point_without_py + ) + entry_point_script_name = entry_point_target_name + ".py" + + rctx.file( + entry_point_script_name, + _generate_entry_point_contents(module, attribute), + ) + entry_points[entry_point_without_py] = entry_point_script_name + + build_file_contents = generate_whl_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + whl_name = whl_path.basename, + dependencies = metadata["deps"], + dependencies_by_platform = metadata["deps_by_platform"], + group_name = rctx.attr.group_name, + group_deps = rctx.attr.group_deps, + data_exclude = rctx.attr.pip_data_exclude, + tags = [ + "pypi_name=" + metadata["name"], + "pypi_version=" + metadata["version"], + ], + entry_points = entry_points, + annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))), + ) + rctx.file("BUILD.bazel", build_file_contents) + return +def _generate_entry_point_contents( + module, + attribute, + shebang = "#!/usr/bin/env python3"): + """Generate the contents of an entry point script. + + Args: + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. + shebang (str, optional): The shebang to use for the entry point python + file. + + Returns: + str: A string of python code. + """ + contents = """\ +{shebang} +import sys +from {module} import {attribute} +if __name__ == "__main__": + sys.exit({attribute}()) +""".format( + shebang = shebang, + module = module, + attribute = attribute, + ) + return contents + whl_library_attrs = { "annotation": attr.label( doc = ( @@ -697,6 +847,13 @@ whl_library_attrs = { ), allow_files = True, ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), "repo": attr.string( mandatory = True, doc = "Pointer to parent repo name. Used to make these rules rerun if the parent repo changes.", @@ -705,6 +862,26 @@ whl_library_attrs = { mandatory = True, doc = "Python requirement string describing the package to make available", ), + "whl_patches": attr.label_keyed_string_dict( + doc = """a label-keyed-string dict that has + json.encode(struct([whl_file], patch_strip]) as values. This + is to maintain flexibility and correct bzlmod extension interface + until we have a better way to define whl_library and move whl + patching to a separate place. INTERNAL USE ONLY.""", + ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_requirements + ], + ), } whl_library_attrs.update(**common_attrs) @@ -752,6 +929,29 @@ def package_annotation( srcs_exclude_glob = srcs_exclude_glob, )) +def _group_library_impl(rctx): + build_file_contents = generate_group_library_build_bazel( + repo_prefix = rctx.attr.repo_prefix, + groups = rctx.attr.groups, + ) + rctx.file("BUILD.bazel", build_file_contents) + +group_library = repository_rule( + attrs = { + "groups": attr.string_list_dict( + doc = "A mapping of group names to requirements within that group.", + ), + "repo_prefix": attr.string( + doc = "Prefix used for the whl_library created components of each group", + ), + }, + implementation = _group_library_impl, + doc = """ +Create a package containing only wrapper py_library and whl_library rules for implementing dependency groups. +This is an implementation detail of dependency groups and should not be used alone. + """, +) + # pip_repository implementation def _format_list(items): diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl index 411f334..2b88f5c 100644 --- a/python/pip_install/pip_repository_requirements.bzl.tmpl +++ b/python/pip_install/pip_repository_requirements.bzl.tmpl @@ -8,7 +8,9 @@ from %%REQUIREMENTS_LOCK%% all_requirements = %%ALL_REQUIREMENTS%% -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% +all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% + +all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% @@ -16,25 +18,22 @@ _packages = %%PACKAGES%% _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - def requirement(name): - return "@%%NAME%%_" + _clean_name(name) + "//:pkg" + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") def whl_requirement(name): - return "@%%NAME%%_" + _clean_name(name) + "//:whl" + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") def data_requirement(name): - return "@%%NAME%%_" + _clean_name(name) + "//:data" + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data") def dist_info_requirement(name): - return "@%%NAME%%_" + _clean_name(name) + "//:dist_info" + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") def entry_point(pkg, script = None): if not script: script = pkg - return "@%%NAME%%_" + _clean_name(pkg) + "//:rules_python_wheel_entry_point_" + script + return "@%%NAME%%_" + pip_utils.normalize_name(pkg) + "//:rules_python_wheel_entry_point_" + script def _get_annotation(requirement): # This expects to parse `setuptools==58.2.0 --hash=sha256:2551203ae6955b9876741a26ab3e767bb3242dafe86a32a749ea0d78b6792f11` @@ -43,12 +42,42 @@ def _get_annotation(requirement): return _annotations.get(name) def install_deps(**whl_library_kwargs): + """Repository rule macro. Install dependencies from `pip_parse`. + + Args: + **whl_library_kwargs: Additional arguments which will flow to underlying + `whl_library` calls. See pip_repository.bzl for details. + """ + + # Set up the requirement groups + all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% + + requirement_group_mapping = { + requirement: group_name + for group_name, group_requirements in all_requirement_groups.items() + for requirement in group_requirements + } + + group_repo = "%%NAME%%__groups" + group_library( + name = group_repo, + repo_prefix = "%%NAME%%_", + groups = all_requirement_groups, + ) + + # Install wheels which may be participants in a group whl_config = dict(_config) whl_config.update(whl_library_kwargs) + for name, requirement in _packages: + group_name = requirement_group_mapping.get(name.replace("%%NAME%%_", "")) + group_deps = all_requirement_groups.get(group_name, []) + whl_library( name = name, requirement = requirement, + group_name = group_name, + group_deps = group_deps, annotation = _get_annotation(requirement), **whl_config ) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 2df60b0..0000000 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,33 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - if not script: - script = pkg - return "@@%%NAME%%_{}//:rules_python_wheel_entry_point_{}".format(_clean_name(pkg), script) diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel index 86b4b3d..887d2d3 100644 --- a/python/pip_install/private/BUILD.bazel +++ b/python/pip_install/private/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load(":pip_install_utils.bzl", "srcs_module") package(default_visibility = ["//:__subpackages__"]) @@ -22,3 +23,26 @@ srcs_module( srcs = "//python/pip_install:py_srcs", dest = ":srcs.bzl", ) + +bzl_library( + name = "generate_whl_library_build_bazel_bzl", + srcs = ["generate_whl_library_build_bazel.bzl"], + deps = [ + "//python/private:labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "generate_group_library_build_bazel_bzl", + srcs = ["generate_group_library_build_bazel.bzl"], + deps = [ + "//python/private:labels_bzl", + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "srcs_bzl", + srcs = ["srcs.bzl"], +) diff --git a/python/pip_install/private/generate_group_library_build_bazel.bzl b/python/pip_install/private/generate_group_library_build_bazel.bzl new file mode 100644 index 0000000..c122b04 --- /dev/null +++ b/python/pip_install/private/generate_group_library_build_bazel.bzl @@ -0,0 +1,105 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the BUILD.bazel contents for a repo defined by a group_library.""" + +load( + "//python/private:labels.bzl", + "PY_LIBRARY_IMPL_LABEL", + "PY_LIBRARY_PUBLIC_LABEL", + "WHEEL_FILE_IMPL_LABEL", + "WHEEL_FILE_PUBLIC_LABEL", +) +load("//python/private:normalize_name.bzl", "normalize_name") + +_PRELUDE = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +""" + +_GROUP_TEMPLATE = """\ +## Group {name} + +filegroup( + name = "{name}_{whl_public_label}", + srcs = [], + data = {whl_deps}, + visibility = {visibility}, +) + +py_library( + name = "{name}_{lib_public_label}", + srcs = [], + deps = {lib_deps}, + visibility = {visibility}, +) +""" + +def _generate_group_libraries(repo_prefix, group_name, group_members): + """Generate the component libraries implementing a group. + + A group consists of two underlying composite libraries, one `filegroup` + which wraps all the whls of the members and one `py_library` which wraps the + pkgs of the members. + + Implementation detail of `generate_group_library_build_bazel` which uses + this to construct a BUILD.bazel. + + Args: + repo_prefix: str; the pip_parse repo prefix. + group_name: str; the name which the user provided for the dep group. + group_members: list[str]; the names of the _packages_ (not repositories) + which make up the group. + """ + + lib_dependencies = [ + "@%s%s//:%s" % (repo_prefix, normalize_name(d), PY_LIBRARY_IMPL_LABEL) + for d in group_members + ] + whl_file_deps = [ + "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) + for d in group_members + ] + visibility = [ + "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) + for d in group_members + ] + + return _GROUP_TEMPLATE.format( + name = normalize_name(group_name), + whl_public_label = WHEEL_FILE_PUBLIC_LABEL, + whl_deps = repr(whl_file_deps), + lib_public_label = PY_LIBRARY_PUBLIC_LABEL, + lib_deps = repr(lib_dependencies), + visibility = repr(visibility), + ) + +def generate_group_library_build_bazel( + repo_prefix, + groups): + """Generate a BUILD file for a repository of group implementations + + Args: + repo_prefix: the repo prefix that should be used for dependency lists. + groups: a mapping of group names to lists of names of component packages. + + Returns: + A complete BUILD file as a string + """ + + content = [_PRELUDE] + + for group_name, group_members in groups.items(): + content.append(_generate_group_libraries(repo_prefix, group_name, group_members)) + + return "\n\n".join(content) diff --git a/python/pip_install/private/generate_whl_library_build_bazel.bzl b/python/pip_install/private/generate_whl_library_build_bazel.bzl new file mode 100644 index 0000000..568b00e --- /dev/null +++ b/python/pip_install/private/generate_whl_library_build_bazel.bzl @@ -0,0 +1,344 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generate the BUILD.bazel contents for a repo defined by a whl_library.""" + +load( + "//python/private:labels.bzl", + "DATA_LABEL", + "DIST_INFO_LABEL", + "PY_LIBRARY_IMPL_LABEL", + "PY_LIBRARY_PUBLIC_LABEL", + "WHEEL_ENTRY_POINT_PREFIX", + "WHEEL_FILE_IMPL_LABEL", + "WHEEL_FILE_PUBLIC_LABEL", +) +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:text_util.bzl", "render") + +_COPY_FILE_TEMPLATE = """\ +copy_file( + name = "{dest}.copy", + src = "{src}", + out = "{dest}", + is_executable = {is_executable}, +) +""" + +_ENTRY_POINT_RULE_TEMPLATE = """\ +py_binary( + name = "{name}", + srcs = ["{src}"], + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["."], + deps = ["{pkg}"], +) +""" + +_BUILD_TEMPLATE = """\ +load("@rules_python//python:defs.bzl", "py_library", "py_binary") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "{dist_info_label}", + srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), +) + +filegroup( + name = "{data_label}", + srcs = glob(["data/**"], allow_empty = True), +) + +filegroup( + name = "{whl_file_impl_label}", + srcs = ["{whl_name}"], + data = {whl_file_deps}, + visibility = {impl_vis}, +) + +py_library( + name = "{py_library_impl_label}", + srcs = glob( + ["site-packages/**/*.py"], + exclude={srcs_exclude}, + # Empty sources are allowed to support wheels that don't have any + # pure-Python code, e.g. pymssql, which is written in Cython. + allow_empty = True, + ), + data = {data} + glob( + ["site-packages/**/*"], + exclude={data_exclude}, + ), + # This makes this directory a top-level in the python import + # search path for anything that depends on this. + imports = ["site-packages"], + deps = {dependencies}, + tags = {tags}, + visibility = {impl_vis}, +) + +alias( + name = "{py_library_public_label}", + actual = "{py_library_actual_label}", +) + +alias( + name = "{whl_file_public_label}", + actual = "{whl_file_actual_label}", +) +""" + +def _render_list_and_select(deps, deps_by_platform, tmpl): + deps = render.list([tmpl.format(d) for d in deps]) + + if not deps_by_platform: + return deps + + deps_by_platform = { + p if p.startswith("@") else ":is_" + p: [ + tmpl.format(d) + for d in deps + ] + for p, deps in deps_by_platform.items() + } + + # Add the default, which means that we will be just using the dependencies in + # `deps` for platforms that are not handled in a special way by the packages + deps_by_platform["//conditions:default"] = [] + deps_by_platform = render.select(deps_by_platform, value_repr = render.list) + + if deps == "[]": + return deps_by_platform + else: + return "{} + {}".format(deps, deps_by_platform) + +def generate_whl_library_build_bazel( + *, + repo_prefix, + whl_name, + dependencies, + dependencies_by_platform, + data_exclude, + tags, + entry_points, + annotation = None, + group_name = None, + group_deps = []): + """Generate a BUILD file for an unzipped Wheel + + Args: + repo_prefix: the repo prefix that should be used for dependency lists. + whl_name: the whl_name that this is generated for. + dependencies: a list of PyPI packages that are dependencies to the py_library. + dependencies_by_platform: a dict[str, list] of PyPI packages that may vary by platform. + data_exclude: more patterns to exclude from the data attribute of generated py_library rules. + tags: list of tags to apply to generated py_library rules. + entry_points: A dict of entry points to add py_binary rules for. + annotation: The annotation for the build file. + group_name: Optional[str]; name of the dependency group (if any) which contains this library. + If set, this library will behave as a shim to group implementation rules which will provide + simultaneously installed dependencies which would otherwise form a cycle. + group_deps: List[str]; names of fellow members of the group (if any). These will be excluded + from generated deps lists so as to avoid direct cycles. These dependencies will be provided + at runtime by the group rules which wrap this library and its fellows together. + + Returns: + A complete BUILD file as a string + """ + + additional_content = [] + data = [] + srcs_exclude = [] + data_exclude = [] + data_exclude + dependencies = sorted([normalize_name(d) for d in dependencies]) + dependencies_by_platform = { + platform: sorted([normalize_name(d) for d in deps]) + for platform, deps in dependencies_by_platform.items() + } + tags = sorted(tags) + + for entry_point, entry_point_script_name in entry_points.items(): + additional_content.append( + _generate_entry_point_rule( + name = "{}_{}".format(WHEEL_ENTRY_POINT_PREFIX, entry_point), + script = entry_point_script_name, + pkg = ":" + PY_LIBRARY_PUBLIC_LABEL, + ), + ) + + if annotation: + for src, dest in annotation.copy_files.items(): + data.append(dest) + additional_content.append(_generate_copy_commands(src, dest)) + for src, dest in annotation.copy_executables.items(): + data.append(dest) + additional_content.append( + _generate_copy_commands(src, dest, is_executable = True), + ) + data.extend(annotation.data) + data_exclude.extend(annotation.data_exclude_glob) + srcs_exclude.extend(annotation.srcs_exclude_glob) + if annotation.additive_build_content: + additional_content.append(annotation.additive_build_content) + + _data_exclude = [ + "**/* *", + "**/*.py", + "**/*.pyc", + "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created + # RECORD is known to contain sha256 checksums of files which might include the checksums + # of generated files produced when wheels are installed. The file is ignored to avoid + # Bazel caching issues. + "**/*.dist-info/RECORD", + ] + for item in data_exclude: + if item not in _data_exclude: + _data_exclude.append(item) + + # Ensure this list is normalized + # Note: mapping used as set + group_deps = { + normalize_name(d): True + for d in group_deps + } + + dependencies = [ + d + for d in dependencies + if d not in group_deps + ] + dependencies_by_platform = { + p: deps + for p, deps in dependencies_by_platform.items() + for deps in [[d for d in deps if d not in group_deps]] + if deps + } + + for p in dependencies_by_platform: + if p.startswith("@"): + continue + + os, _, cpu = p.partition("_") + + additional_content.append( + """\ +config_setting( + name = "is_{os}_{cpu}", + constraint_values = [ + "@platforms//cpu:{cpu}", + "@platforms//os:{os}", + ], + visibility = ["//visibility:private"], +) +""".format(os = os, cpu = cpu), + ) + + lib_dependencies = _render_list_and_select( + deps = dependencies, + deps_by_platform = dependencies_by_platform, + tmpl = "@{}{{}}//:{}".format(repo_prefix, PY_LIBRARY_PUBLIC_LABEL), + ) + + whl_file_deps = _render_list_and_select( + deps = dependencies, + deps_by_platform = dependencies_by_platform, + tmpl = "@{}{{}}//:{}".format(repo_prefix, WHEEL_FILE_PUBLIC_LABEL), + ) + + # If this library is a member of a group, its public label aliases need to + # point to the group implementation rule not the implementation rules. We + # also need to mark the implementation rules as visible to the group + # implementation. + if group_name: + group_repo = repo_prefix + "_groups" + library_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), PY_LIBRARY_PUBLIC_LABEL) + whl_impl_label = "@%s//:%s_%s" % (group_repo, normalize_name(group_name), WHEEL_FILE_PUBLIC_LABEL) + impl_vis = "@%s//:__pkg__" % (group_repo,) + + else: + library_impl_label = PY_LIBRARY_IMPL_LABEL + whl_impl_label = WHEEL_FILE_IMPL_LABEL + impl_vis = "//visibility:private" + + contents = "\n".join( + [ + _BUILD_TEMPLATE.format( + py_library_public_label = PY_LIBRARY_PUBLIC_LABEL, + py_library_impl_label = PY_LIBRARY_IMPL_LABEL, + py_library_actual_label = library_impl_label, + dependencies = render.indent(lib_dependencies, " " * 4).lstrip(), + whl_file_deps = render.indent(whl_file_deps, " " * 4).lstrip(), + data_exclude = repr(_data_exclude), + whl_name = whl_name, + whl_file_public_label = WHEEL_FILE_PUBLIC_LABEL, + whl_file_impl_label = WHEEL_FILE_IMPL_LABEL, + whl_file_actual_label = whl_impl_label, + tags = repr(tags), + data_label = DATA_LABEL, + dist_info_label = DIST_INFO_LABEL, + entry_point_prefix = WHEEL_ENTRY_POINT_PREFIX, + srcs_exclude = repr(srcs_exclude), + data = repr(data), + impl_vis = repr([impl_vis]), + ), + ] + additional_content, + ) + + # NOTE: Ensure that we terminate with a new line + return contents.rstrip() + "\n" + +def _generate_copy_commands(src, dest, is_executable = False): + """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target + + [cf]: https://github.com/bazelbuild/bazel-skylib/blob/1.1.1/docs/copy_file_doc.md + + Args: + src (str): The label for the `src` attribute of [copy_file][cf] + dest (str): The label for the `out` attribute of [copy_file][cf] + is_executable (bool, optional): Whether or not the file being copied is executable. + sets `is_executable` for [copy_file][cf] + + Returns: + str: A `copy_file` instantiation. + """ + return _COPY_FILE_TEMPLATE.format( + src = src, + dest = dest, + is_executable = is_executable, + ) + +def _generate_entry_point_rule(*, name, script, pkg): + """Generate a Bazel `py_binary` rule for an entry point script. + + Note that the script is used to determine the name of the target. The name of + entry point targets should be uniuqe to avoid conflicts with existing sources or + directories within a wheel. + + Args: + name (str): The name of the generated py_binary. + script (str): The path to the entry point's python file. + pkg (str): The package owning the entry point. This is expected to + match up with the `py_library` defined for each repository. + + Returns: + str: A `py_binary` instantiation. + """ + return _ENTRY_POINT_RULE_TEMPLATE.format( + name = name, + src = script.replace("\\", "/"), + pkg = pkg, + ) diff --git a/python/pip_install/private/srcs.bzl b/python/pip_install/private/srcs.bzl index f3064a3..e92e49f 100644 --- a/python/pip_install/private/srcs.bzl +++ b/python/pip_install/private/srcs.bzl @@ -9,11 +9,10 @@ This file is auto-generated from the `@rules_python//python/pip_install/private: PIP_INSTALL_PY_SRCS = [ "@rules_python//python/pip_install/tools/dependency_resolver:__init__.py", "@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py", - "@rules_python//python/pip_install/tools/lib:__init__.py", - "@rules_python//python/pip_install/tools/lib:annotation.py", - "@rules_python//python/pip_install/tools/lib:arguments.py", - "@rules_python//python/pip_install/tools/lib:bazel.py", + "@rules_python//python/pip_install/tools/wheel_installer:arguments.py", "@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel.py", "@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py", + "@rules_python//python/private:repack_whl.py", + "@rules_python//tools:wheelmaker.py", ] diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index efe3bc7..91bdd4b 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -14,21 +14,20 @@ "" -load("@bazel_skylib//lib:versions.bzl", "versions") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") -load("//:version.bzl", "MINIMUM_BAZEL_VERSION") _RULE_DEPS = [ + # START: maintained by 'bazel run //tools/private:update_pip_deps' ( "pypi__build", - "https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl", - "38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69", + "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", + "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", ), ( "pypi__click", - "https://files.pythonhosted.org/packages/76/0a/b6c5f311e32aeb3b406e03c079ade51e905ea630fc19d1262a46249c1c86/click-8.0.1-py3-none-any.whl", - "fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6", + "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", ), ( "pypi__colorama", @@ -36,14 +35,24 @@ _RULE_DEPS = [ "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", ), ( + "pypi__importlib_metadata", + "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", + "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", + ), + ( "pypi__installer", "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", ), ( + "pypi__more_itertools", + "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", + "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", + ), + ( "pypi__packaging", - "https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl", - "957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3", + "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", + "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", ), ( "pypi__pep517", @@ -52,18 +61,23 @@ _RULE_DEPS = [ ), ( "pypi__pip", - "https://files.pythonhosted.org/packages/09/bd/2410905c76ee14c62baf69e3f4aa780226c1bbfc9485731ad018e35b0cb5/pip-22.3.1-py3-none-any.whl", - "908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077", + "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", + "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", ), ( "pypi__pip_tools", - "https://files.pythonhosted.org/packages/5e/e8/f6d7d1847c7351048da870417724ace5c4506e816b38db02f4d7c675c189/pip_tools-6.12.1-py3-none-any.whl", - "f0c0c0ec57b58250afce458e2e6058b1f30a4263db895b7d72fd6311bf1dc6f7", + "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", + "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", + ), + ( + "pypi__pyproject_hooks", + "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", - "782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96", + "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", + "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", ), ( "pypi__tomli", @@ -72,24 +86,15 @@ _RULE_DEPS = [ ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/bd/7c/d38a0b30ce22fc26ed7dbc087c6d00851fb3395e9d0dac40bec1f905030c/wheel-0.38.4-py3-none-any.whl", - "b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8", - ), - ( - "pypi__importlib_metadata", - "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl", - "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", + "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", ), ( "pypi__zipp", - "https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl", - "8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - ), - ( - "pypi__more_itertools", - "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl", - "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb", + "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", + "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", ), + # END: maintained by 'bazel run //tools/private:update_pip_deps' ] _GENERIC_WHEEL = """\ @@ -126,17 +131,8 @@ def requirement(pkg): def pip_install_dependencies(): """ - Fetch dependencies these rules depend on. Workspaces that use the pip_install rule can call this. - - (However we call it from pip_install, making it optional for users to do so.) + Fetch dependencies these rules depend on. Workspaces that use the pip_parse rule can call this. """ - - # We only support Bazel LTS and rolling releases. - # Give the user an obvious error to upgrade rather than some obscure missing symbol later. - # It's not guaranteed that users call this function, but it's used by all the pip fetch - # repository rules so it's likely that most users get the right error. - versions.check(MINIMUM_BAZEL_VERSION) - for (name, url, sha256) in _RULE_DEPS: maybe( http_archive, diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 84ee203..5caf762 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -19,6 +19,7 @@ load("//python/pip_install:repositories.bzl", "requirement") def compile_pip_requirements( name, + src = None, extra_args = [], extra_deps = [], generate_hashes = True, @@ -48,12 +49,17 @@ def compile_pip_requirements( Args: name: base name for generated targets, typically "requirements". + src: file containing inputs to dependency resolution. If not specified, + defaults to `pyproject.toml`. Supported formats are: + * a requirements text file, usually named `requirements.in` + * A `.toml` file, where the `project.dependencies` list is used as per + [PEP621](https://peps.python.org/pep-0621/). extra_args: passed to pip-compile. extra_deps: extra dependencies passed to pip-compile. generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. py_test: the py_test rule to be used. - requirements_in: file expressing desired dependencies. + requirements_in: file expressing desired dependencies. Deprecated, use src instead. requirements_txt: result of "compiling" the requirements.in file. requirements_linux: File of linux specific resolve output to check validate if requirement.in has changes. requirements_darwin: File of darwin specific resolve output to check validate if requirement.in has changes. @@ -62,7 +68,11 @@ def compile_pip_requirements( visibility: passed to both the _test and .update rules. **kwargs: other bazel attributes passed to the "_test" rule. """ - requirements_in = name + ".in" if requirements_in == None else requirements_in + if requirements_in and src: + fail("Only one of 'src' and 'requirements_in' attributes can be used") + else: + src = requirements_in or src or "pyproject.toml" + requirements_txt = name + ".txt" if requirements_txt == None else requirements_txt # "Default" target produced by this macro @@ -74,7 +84,7 @@ def compile_pip_requirements( visibility = visibility, ) - data = [name, requirements_in, requirements_txt] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] + data = [name, requirements_txt, src] + [f for f in (requirements_linux, requirements_darwin, requirements_windows) if f != None] # Use the Label constructor so this is expanded in the context of the file # where it appears, which is to say, in @rules_python @@ -83,27 +93,36 @@ def compile_pip_requirements( loc = "$(rlocationpath {})" args = [ - loc.format(requirements_in), + loc.format(src), loc.format(requirements_txt), - # String None is a placeholder for argv ordering. - loc.format(requirements_linux) if requirements_linux else "None", - loc.format(requirements_darwin) if requirements_darwin else "None", - loc.format(requirements_windows) if requirements_windows else "None", "//%s:%s.update" % (native.package_name(), name), - ] + (["--generate-hashes"] if generate_hashes else []) + extra_args + "--resolver=backtracking", + "--allow-unsafe", + ] + if generate_hashes: + args.append("--generate-hashes") + if requirements_linux: + args.append("--requirements-linux={}".format(loc.format(requirements_linux))) + if requirements_darwin: + args.append("--requirements-darwin={}".format(loc.format(requirements_darwin))) + if requirements_windows: + args.append("--requirements-windows={}".format(loc.format(requirements_windows))) + args.extend(extra_args) deps = [ requirement("build"), requirement("click"), requirement("colorama"), + requirement("importlib_metadata"), + requirement("more_itertools"), + requirement("packaging"), requirement("pep517"), requirement("pip"), requirement("pip_tools"), + requirement("pyproject_hooks"), requirement("setuptools"), requirement("tomli"), - requirement("importlib_metadata"), requirement("zipp"), - requirement("more_itertools"), Label("//python/runfiles:runfiles"), ] + extra_deps diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index e277cf9..5e914bc 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -19,7 +19,9 @@ import os import shutil import sys from pathlib import Path +from typing import Optional, Tuple +import click import piptools.writer as piptools_writer from piptools.scripts.compile import cli @@ -77,24 +79,25 @@ def _locate(bazel_runfiles, file): return bazel_runfiles.Rlocation(file) -if __name__ == "__main__": - if len(sys.argv) < 4: - print( - "Expected at least two arguments: requirements_in requirements_out", - file=sys.stderr, - ) - sys.exit(1) - - parse_str_none = lambda s: None if s == "None" else s +@click.command(context_settings={"ignore_unknown_options": True}) +@click.argument("requirements_in") +@click.argument("requirements_txt") +@click.argument("update_target_label") +@click.option("--requirements-linux") +@click.option("--requirements-darwin") +@click.option("--requirements-windows") +@click.argument("extra_args", nargs=-1, type=click.UNPROCESSED) +def main( + requirements_in: str, + requirements_txt: str, + update_target_label: str, + requirements_linux: Optional[str], + requirements_darwin: Optional[str], + requirements_windows: Optional[str], + extra_args: Tuple[str, ...], +) -> None: bazel_runfiles = runfiles.Create() - requirements_in = sys.argv.pop(1) - requirements_txt = sys.argv.pop(1) - requirements_linux = parse_str_none(sys.argv.pop(1)) - requirements_darwin = parse_str_none(sys.argv.pop(1)) - requirements_windows = parse_str_none(sys.argv.pop(1)) - update_target_label = sys.argv.pop(1) - requirements_file = _select_golden_requirements_file( requirements_txt=requirements_txt, requirements_linux=requirements_linux, requirements_darwin=requirements_darwin, requirements_windows=requirements_windows @@ -128,6 +131,8 @@ if __name__ == "__main__": os.environ["LC_ALL"] = "C.UTF-8" os.environ["LANG"] = "C.UTF-8" + argv = [] + UPDATE = True # Detect if we are running under `bazel test`. if "TEST_TMPDIR" in os.environ: @@ -136,8 +141,7 @@ if __name__ == "__main__": # to the real user cache, Bazel sandboxing makes the file read-only # and we fail. # In theory this makes the test more hermetic as well. - sys.argv.append("--cache-dir") - sys.argv.append(os.environ["TEST_TMPDIR"]) + argv.append(f"--cache-dir={os.environ['TEST_TMPDIR']}") # Make a copy for pip-compile to read and mutate. requirements_out = os.path.join( os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out" @@ -153,14 +157,13 @@ if __name__ == "__main__": os.environ["CUSTOM_COMPILE_COMMAND"] = update_command os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull - sys.argv.append("--output-file") - sys.argv.append(requirements_file_relative if UPDATE else requirements_out) - sys.argv.append( + argv.append(f"--output-file={requirements_file_relative if UPDATE else requirements_out}") + argv.append( requirements_in_relative if Path(requirements_in_relative).exists() else resolved_requirements_in ) - print(sys.argv) + argv.extend(extra_args) if UPDATE: print("Updating " + requirements_file_relative) @@ -176,7 +179,7 @@ if __name__ == "__main__": resolved_requirements_file, requirements_file_tree ) ) - cli() + cli(argv) requirements_file_relative_path = Path(requirements_file_relative) content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") @@ -185,7 +188,7 @@ if __name__ == "__main__": # cli will exit(0) on success try: print("Checking " + requirements_file) - cli() + cli(argv) print("cli() should exit", file=sys.stderr) sys.exit(1) except SystemExit as e: @@ -219,3 +222,7 @@ if __name__ == "__main__": file=sys.stderr, ) sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/python/pip_install/tools/lib/BUILD.bazel b/python/pip_install/tools/lib/BUILD.bazel deleted file mode 100644 index 37a8b09..0000000 --- a/python/pip_install/tools/lib/BUILD.bazel +++ /dev/null @@ -1,82 +0,0 @@ -load("//python:defs.bzl", "py_library", "py_test") -load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file") - -py_library( - name = "lib", - srcs = [ - "annotation.py", - "arguments.py", - "bazel.py", - ], - visibility = ["//python/pip_install:__subpackages__"], -) - -package_annotations_file( - name = "mock_annotations", - annotations = { - "pkg_a": package_annotation(), - "pkg_b": package_annotation( - data_exclude_glob = [ - "*.foo", - "*.bar", - ], - ), - "pkg_c": package_annotation( - # The `join` and `strip` here accounts for potential differences - # in new lines between unix and windows hosts. - additive_build_content = "\n".join([line.strip() for line in """\ -cc_library( - name = "my_target", - hdrs = glob(["**/*.h"]), - srcs = glob(["**/*.cc"]), -) -""".splitlines()]), - data = [":my_target"], - ), - "pkg_d": package_annotation( - srcs_exclude_glob = ["pkg_d/tests/**"], - ), - }, - tags = ["manual"], -) - -py_test( - name = "annotations_test", - size = "small", - srcs = ["annotations_test.py"], - data = [":mock_annotations"], - env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"}, - deps = [ - ":lib", - "//python/runfiles", - ], -) - -py_test( - name = "arguments_test", - size = "small", - srcs = [ - "arguments_test.py", - ], - deps = [ - ":lib", - ], -) - -filegroup( - name = "distribution", - srcs = glob( - ["*"], - exclude = ["*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) - -filegroup( - name = "py_srcs", - srcs = glob( - include = ["**/*.py"], - exclude = ["**/*_test.py"], - ), - visibility = ["//python/pip_install:__subpackages__"], -) diff --git a/python/pip_install/tools/lib/__init__.py b/python/pip_install/tools/lib/__init__.py deleted file mode 100644 index bbdfb4c..0000000 --- a/python/pip_install/tools/lib/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/python/pip_install/tools/lib/annotation.py b/python/pip_install/tools/lib/annotation.py deleted file mode 100644 index c980080..0000000 --- a/python/pip_install/tools/lib/annotation.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import logging -from collections import OrderedDict -from pathlib import Path -from typing import Any, Dict, List - - -class Annotation(OrderedDict): - """A python representation of `@rules_python//python:pip.bzl%package_annotation`""" - - def __init__(self, content: Dict[str, Any]) -> None: - - missing = [] - ordered_content = OrderedDict() - for field in ( - "additive_build_content", - "copy_executables", - "copy_files", - "data", - "data_exclude_glob", - "srcs_exclude_glob", - ): - if field not in content: - missing.append(field) - continue - ordered_content.update({field: content.pop(field)}) - - if missing: - raise ValueError("Data missing from initial annotation: {}".format(missing)) - - if content: - raise ValueError( - "Unexpected data passed to annotations: {}".format( - sorted(list(content.keys())) - ) - ) - - return OrderedDict.__init__(self, ordered_content) - - @property - def additive_build_content(self) -> str: - return self["additive_build_content"] - - @property - def copy_executables(self) -> Dict[str, str]: - return self["copy_executables"] - - @property - def copy_files(self) -> Dict[str, str]: - return self["copy_files"] - - @property - def data(self) -> List[str]: - return self["data"] - - @property - def data_exclude_glob(self) -> List[str]: - return self["data_exclude_glob"] - - @property - def srcs_exclude_glob(self) -> List[str]: - return self["srcs_exclude_glob"] - - -class AnnotationsMap: - """A mapping of python package names to [Annotation]""" - - def __init__(self, json_file: Path): - content = json.loads(json_file.read_text()) - - self._annotations = {pkg: Annotation(data) for (pkg, data) in content.items()} - - @property - def annotations(self) -> Dict[str, Annotation]: - return self._annotations - - def collect(self, requirements: List[str]) -> Dict[str, Annotation]: - unused = self.annotations - collection = {} - for pkg in requirements: - if pkg in unused: - collection.update({pkg: unused.pop(pkg)}) - - if unused: - logging.warning( - "Unused annotations: {}".format(sorted(list(unused.keys()))) - ) - - return collection - - -def annotation_from_str_path(path: str) -> Annotation: - """Load an annotation from a json encoded file - - Args: - path (str): The path to a json encoded file - - Returns: - Annotation: The deserialized annotations - """ - json_file = Path(path) - content = json.loads(json_file.read_text()) - return Annotation(content) - - -def annotations_map_from_str_path(path: str) -> AnnotationsMap: - """Load an annotations map from a json encoded file - - Args: - path (str): The path to a json encoded file - - Returns: - AnnotationsMap: The deserialized annotations map - """ - return AnnotationsMap(Path(path)) diff --git a/python/pip_install/tools/lib/annotations_test.py b/python/pip_install/tools/lib/annotations_test.py deleted file mode 100644 index f7c360f..0000000 --- a/python/pip_install/tools/lib/annotations_test.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os -import textwrap -import unittest -from pathlib import Path - -from python.pip_install.tools.lib.annotation import Annotation, AnnotationsMap -from python.runfiles import runfiles - - -class AnnotationsTestCase(unittest.TestCase): - - maxDiff = None - - def test_annotations_constructor(self) -> None: - annotations_env = os.environ.get("MOCK_ANNOTATIONS") - self.assertIsNotNone(annotations_env) - - r = runfiles.Create() - - annotations_path = Path(r.Rlocation("rules_python/{}".format(annotations_env))) - self.assertTrue(annotations_path.exists()) - - annotations_map = AnnotationsMap(annotations_path) - self.assertListEqual( - list(annotations_map.annotations.keys()), - ["pkg_a", "pkg_b", "pkg_c", "pkg_d"], - ) - - collection = annotations_map.collect(["pkg_a", "pkg_b", "pkg_c", "pkg_d"]) - - self.assertEqual( - collection["pkg_a"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": [], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_b"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": ["*.foo", "*.bar"], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_c"], - Annotation( - { - # The `join` and `strip` here accounts for potential - # differences in new lines between unix and windows - # hosts. - "additive_build_content": "\n".join( - [ - line.strip() - for line in textwrap.dedent( - """\ - cc_library( - name = "my_target", - hdrs = glob(["**/*.h"]), - srcs = glob(["**/*.cc"]), - ) - """ - ).splitlines() - ] - ), - "copy_executables": {}, - "copy_files": {}, - "data": [":my_target"], - "data_exclude_glob": [], - "srcs_exclude_glob": [], - } - ), - ) - - self.assertEqual( - collection["pkg_d"], - Annotation( - { - "additive_build_content": None, - "copy_executables": {}, - "copy_files": {}, - "data": [], - "data_exclude_glob": [], - "srcs_exclude_glob": ["pkg_d/tests/**"], - } - ), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/pip_install/tools/lib/annotations_test_helpers.bzl b/python/pip_install/tools/lib/annotations_test_helpers.bzl deleted file mode 100644 index 4f56bb7..0000000 --- a/python/pip_install/tools/lib/annotations_test_helpers.bzl +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Helper macros and rules for testing the `annotations` module of `tools`""" - -load("//python:pip.bzl", _package_annotation = "package_annotation") - -package_annotation = _package_annotation - -def _package_annotations_file_impl(ctx): - output = ctx.actions.declare_file(ctx.label.name + ".annotations.json") - - annotations = {package: json.decode(data) for (package, data) in ctx.attr.annotations.items()} - ctx.actions.write( - output = output, - content = json.encode_indent(annotations, indent = " " * 4), - ) - - return DefaultInfo( - files = depset([output]), - runfiles = ctx.runfiles(files = [output]), - ) - -package_annotations_file = rule( - implementation = _package_annotations_file_impl, - doc = ( - "Consumes `package_annotation` definitions in the same way " + - "`pip_repository` rules do to produce an annotations file." - ), - attrs = { - "annotations": attr.string_dict( - doc = "See `@rules_python//python:pip.bzl%package_annotation", - mandatory = True, - ), - }, -) diff --git a/python/pip_install/tools/lib/bazel.py b/python/pip_install/tools/lib/bazel.py deleted file mode 100644 index 81119e9..0000000 --- a/python/pip_install/tools/lib/bazel.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -WHEEL_FILE_LABEL = "whl" -PY_LIBRARY_LABEL = "pkg" -DATA_LABEL = "data" -DIST_INFO_LABEL = "dist_info" -WHEEL_ENTRY_POINT_PREFIX = "rules_python_wheel_entry_point" - - -def sanitise_name(name: str, prefix: str) -> str: - """Sanitises the name to be compatible with Bazel labels. - - See the doc in ../../../private/normalize_name.bzl. - """ - return prefix + re.sub(r"[-_.]+", "_", name).lower() - - -def _whl_name_to_repo_root(whl_name: str, repo_prefix: str) -> str: - return "@{}//".format(sanitise_name(whl_name, prefix=repo_prefix)) - - -def sanitised_repo_library_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), PY_LIBRARY_LABEL - ) - - -def sanitised_repo_file_label(whl_name: str, repo_prefix: str) -> str: - return '"{}:{}"'.format( - _whl_name_to_repo_root(whl_name, repo_prefix), WHEEL_FILE_LABEL - ) diff --git a/python/pip_install/tools/requirements.txt b/python/pip_install/tools/requirements.txt new file mode 100755 index 0000000..bf9fe46 --- /dev/null +++ b/python/pip_install/tools/requirements.txt @@ -0,0 +1,14 @@ +build +click +colorama +importlib_metadata +installer +more_itertools +packaging +pep517 +pip +pip_tools +setuptools +tomli +wheel +zipp diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel index 54bbc46..a396488 100644 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ b/python/pip_install/tools/wheel_installer/BUILD.bazel @@ -4,14 +4,16 @@ load("//python/pip_install:repositories.bzl", "requirement") py_library( name = "lib", srcs = [ + "arguments.py", "namespace_pkgs.py", "wheel.py", "wheel_installer.py", ], + visibility = ["//third_party/rules_pycross/pycross/private:__subpackages__"], deps = [ - "//python/pip_install/tools/lib", requirement("installer"), requirement("pip"), + requirement("packaging"), requirement("setuptools"), ], ) @@ -25,6 +27,17 @@ py_binary( ) py_test( + name = "arguments_test", + size = "small", + srcs = [ + "arguments_test.py", + ], + deps = [ + ":lib", + ], +) + +py_test( name = "namespace_pkgs_test", size = "small", srcs = [ @@ -36,6 +49,18 @@ py_test( ) py_test( + name = "wheel_test", + size = "small", + srcs = [ + "wheel_test.py", + ], + data = ["//examples/wheel:minimal_with_py_package"], + deps = [ + ":lib", + ], +) + +py_test( name = "wheel_installer_test", size = "small", srcs = [ diff --git a/python/pip_install/tools/lib/arguments.py b/python/pip_install/tools/wheel_installer/arguments.py index 974f03c..71133c2 100644 --- a/python/pip_install/tools/lib/arguments.py +++ b/python/pip_install/tools/wheel_installer/arguments.py @@ -12,16 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import json -from argparse import ArgumentParser +import pathlib +from typing import Any, Dict, Set +from python.pip_install.tools.wheel_installer import wheel -def parse_common_args(parser: ArgumentParser) -> ArgumentParser: + +def parser(**kwargs: Any) -> argparse.ArgumentParser: + """Create a parser for the wheel_installer tool.""" + parser = argparse.ArgumentParser( + **kwargs, + ) parser.add_argument( - "--repo", + "--requirement", action="store", required=True, - help="The external repo name to install dependencies. In the format '@{REPO_NAME}'", + help="A single PEP508 requirement specifier string.", ) parser.add_argument( "--isolated", @@ -34,6 +42,12 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser: help="Extra arguments to pass down to pip.", ) parser.add_argument( + "--platform", + action="extend", + type=wheel.Platform.from_string, + help="Platforms to target dependencies. Can be used multiple times.", + ) + parser.add_argument( "--pip_data_exclude", action="store", help="Additional data exclusion parameters to add to the pip packages BUILD file.", @@ -49,21 +63,22 @@ def parse_common_args(parser: ArgumentParser) -> ArgumentParser: help="Extra environment variables to set on the pip environment.", ) parser.add_argument( - "--repo-prefix", - required=True, - help="Prefix to prepend to packages", - ) - parser.add_argument( "--download_only", action="store_true", help="Use 'pip download' instead of 'pip wheel'. Disables building wheels from source, but allows use of " "--platform, --python-version, --implementation, and --abi in --extra_pip_args.", ) + parser.add_argument( + "--whl-file", + type=pathlib.Path, + help="Extract a whl file to be used within Bazel.", + ) return parser -def deserialize_structured_args(args): +def deserialize_structured_args(args: Dict[str, str]) -> Dict: """Deserialize structured arguments passed from the starlark rules. + Args: args: dict of parsed command line arguments """ @@ -74,3 +89,18 @@ def deserialize_structured_args(args): else: args[arg_name] = [] return args + + +def get_platforms(args: argparse.Namespace) -> Set: + """Aggregate platforms into a single set. + + Args: + args: dict of parsed command line arguments + """ + platforms = set() + if args.platform is None: + return platforms + + platforms.update(args.platform) + + return platforms diff --git a/python/pip_install/tools/lib/arguments_test.py b/python/pip_install/tools/wheel_installer/arguments_test.py index dfa96a8..840c2fa 100644 --- a/python/pip_install/tools/lib/arguments_test.py +++ b/python/pip_install/tools/wheel_installer/arguments_test.py @@ -16,35 +16,30 @@ import argparse import json import unittest -from python.pip_install.tools.lib import arguments +from python.pip_install.tools.wheel_installer import arguments, wheel class ArgumentsTestCase(unittest.TestCase): def test_arguments(self) -> None: - parser = argparse.ArgumentParser() - parser = arguments.parse_common_args(parser) + parser = arguments.parser() repo_name = "foo" repo_prefix = "pypi_" index_url = "--index_url=pypi.org/simple" extra_pip_args = [index_url] + requirement = "foo==1.0.0 --hash=sha256:deadbeef" args_dict = vars( parser.parse_args( args=[ - "--repo", - repo_name, + f'--requirement="{requirement}"', f"--extra_pip_args={json.dumps({'arg': extra_pip_args})}", - "--repo-prefix", - repo_prefix, ] ) ) args_dict = arguments.deserialize_structured_args(args_dict) - self.assertIn("repo", args_dict) + self.assertIn("requirement", args_dict) self.assertIn("extra_pip_args", args_dict) self.assertEqual(args_dict["pip_data_exclude"], []) self.assertEqual(args_dict["enable_implicit_namespace_pkgs"], False) - self.assertEqual(args_dict["repo"], repo_name) - self.assertEqual(args_dict["repo_prefix"], repo_prefix) self.assertEqual(args_dict["extra_pip_args"], extra_pip_args) def test_deserialize_structured_args(self) -> None: @@ -57,6 +52,18 @@ class ArgumentsTestCase(unittest.TestCase): self.assertEqual(args["environment"], {"PIP_DO_SOMETHING": "True"}) self.assertEqual(args["extra_pip_args"], []) + def test_platform_aggregation(self) -> None: + parser = arguments.parser() + args = parser.parse_args( + args=[ + "--platform=host", + "--platform=linux_*", + "--platform=all", + "--requirement=foo", + ] + ) + self.assertEqual(set(wheel.Platform.all()), arguments.get_platforms(args)) + if __name__ == "__main__": unittest.main() diff --git a/python/pip_install/tools/wheel_installer/wheel.py b/python/pip_install/tools/wheel_installer/wheel.py index 84af04c..efd916d 100644 --- a/python/pip_install/tools/wheel_installer/wheel.py +++ b/python/pip_install/tools/wheel_installer/wheel.py @@ -13,18 +13,372 @@ # limitations under the License. """Utility class to inspect an extracted wheel directory""" + import email -from typing import Dict, Optional, Set, Tuple +import platform +import re +import sys +from collections import defaultdict +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple, Union import installer -import pkg_resources +from packaging.requirements import Requirement from pip._vendor.packaging.utils import canonicalize_name +class OS(Enum): + linux = 1 + osx = 2 + windows = 3 + darwin = osx + win32 = windows + + +class Arch(Enum): + x86_64 = 1 + x86_32 = 2 + aarch64 = 3 + ppc = 4 + s390x = 5 + amd64 = x86_64 + arm64 = aarch64 + i386 = x86_32 + i686 = x86_32 + x86 = x86_32 + ppc64le = ppc + + +@dataclass(frozen=True) +class Platform: + os: OS + arch: Optional[Arch] = None + + @classmethod + def all(cls, want_os: Optional[OS] = None) -> List["Platform"]: + return sorted( + [ + cls(os=os, arch=arch) + for os in OS + for arch in Arch + if not want_os or want_os == os + ] + ) + + @classmethod + def host(cls) -> List["Platform"]: + """Use the Python interpreter to detect the platform. + + We extract `os` from sys.platform and `arch` from platform.machine + + Returns: + A list of parsed values which makes the signature the same as + `Platform.all` and `Platform.from_string`. + """ + return [ + cls( + os=OS[sys.platform.lower()], + # FIXME @aignas 2023-12-13: Hermetic toolchain on Windows 3.11.6 + # is returning an empty string here, so lets default to x86_64 + arch=Arch[platform.machine().lower() or "x86_64"], + ) + ] + + def __lt__(self, other: Any) -> bool: + """Add a comparison method, so that `sorted` returns the most specialized platforms first.""" + if not isinstance(other, Platform) or other is None: + raise ValueError(f"cannot compare {other} with Platform") + + if self.arch is None and other.arch is not None: + return True + + if self.arch is not None and other.arch is None: + return True + + # Here we ensure that we sort by OS before sorting by arch + + if self.arch is None and other.arch is None: + return self.os.value < other.os.value + + if self.os.value < other.os.value: + return True + + if self.os.value == other.os.value: + return self.arch.value < other.arch.value + + return False + + def __str__(self) -> str: + if self.arch is None: + return f"@platforms//os:{self.os.name.lower()}" + + return self.os.name.lower() + "_" + self.arch.name.lower() + + @classmethod + def from_string(cls, platform: Union[str, List[str]]) -> List["Platform"]: + """Parse a string and return a list of platforms""" + platform = [platform] if isinstance(platform, str) else list(platform) + ret = set() + for p in platform: + if p == "host": + ret.update(cls.host()) + elif p == "all": + ret.update(cls.all()) + elif p.endswith("*"): + os, _, _ = p.partition("_") + ret.update(cls.all(OS[os])) + else: + os, _, arch = p.partition("_") + ret.add(cls(os=OS[os], arch=Arch[arch])) + + return sorted(ret) + + # NOTE @aignas 2023-12-05: below is the minimum number of accessors that are defined in + # https://peps.python.org/pep-0496/ to make rules_python generate dependencies. + # + # WARNING: It may not work in cases where the python implementation is different between + # different platforms. + + # derived from OS + @property + def os_name(self) -> str: + if self.os == OS.linux or self.os == OS.osx: + return "posix" + elif self.os == OS.windows: + return "nt" + else: + return "" + + @property + def sys_platform(self) -> str: + if self.os == OS.linux: + return "linux" + elif self.os == OS.osx: + return "darwin" + elif self.os == OS.windows: + return "win32" + else: + return "" + + @property + def platform_system(self) -> str: + if self.os == OS.linux: + return "Linux" + elif self.os == OS.osx: + return "Darwin" + elif self.os == OS.windows: + return "Windows" + + # derived from OS and Arch + @property + def platform_machine(self) -> str: + """Guess the target 'platform_machine' marker. + + NOTE @aignas 2023-12-05: this may not work on really new systems, like + Windows if they define the platform markers in a different way. + """ + if self.arch == Arch.x86_64: + return "x86_64" + elif self.arch == Arch.x86_32 and self.os != OS.osx: + return "i386" + elif self.arch == Arch.x86_32: + return "" + elif self.arch == Arch.aarch64 and self.os == OS.linux: + return "aarch64" + elif self.arch == Arch.aarch64: + # Assuming that OSX and Windows use this one since the precedent is set here: + # https://github.com/cgohlke/win_arm64-wheels + return "arm64" + elif self.os != OS.linux: + return "" + elif self.arch == Arch.ppc64le: + return "ppc64le" + elif self.arch == Arch.s390x: + return "s390x" + else: + return "" + + def env_markers(self, extra: str) -> Dict[str, str]: + return { + "extra": extra, + "os_name": self.os_name, + "sys_platform": self.sys_platform, + "platform_machine": self.platform_machine, + "platform_system": self.platform_system, + "platform_release": "", # unset + "platform_version": "", # unset + # we assume that the following are the same as the interpreter used to setup the deps: + # "implementation_version": "X.Y.Z", + # "implementation_name": "cpython" + # "python_version": "X.Y", + # "python_full_version": "X.Y.Z", + # "platform_python_implementation: "CPython", + } + + +@dataclass(frozen=True) +class FrozenDeps: + deps: List[str] + deps_select: Dict[str, List[str]] + + +class Deps: + def __init__( + self, + name: str, + extras: Optional[Set[str]] = None, + platforms: Optional[Set[Platform]] = None, + ): + self.name: str = Deps._normalize(name) + self._deps: Set[str] = set() + self._select: Dict[Platform, Set[str]] = defaultdict(set) + self._want_extras: Set[str] = extras or {""} # empty strings means no extras + self._platforms: Set[Platform] = platforms or set() + + def _add(self, dep: str, platform: Optional[Platform]): + dep = Deps._normalize(dep) + + # Packages may create dependency cycles when specifying optional-dependencies / 'extras'. + # Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32. + if dep == self.name: + return + + if platform: + self._select[platform].add(dep) + else: + self._deps.add(dep) + + @staticmethod + def _normalize(name: str) -> str: + return re.sub(r"[-_.]+", "_", name).lower() + + def add(self, *wheel_reqs: str) -> None: + reqs = [Requirement(wheel_req) for wheel_req in wheel_reqs] + + # Resolve any extra extras due to self-edges + self._want_extras = self._resolve_extras(reqs) + + # process self-edges first to resolve the extras used + for req in reqs: + self._add_req(req) + + def _resolve_extras(self, reqs: List[Requirement]) -> Set[str]: + """Resolve extras which are due to depending on self[some_other_extra]. + + Some packages may have cyclic dependencies resulting from extras being used, one example is + `elint`, where we have one set of extras as aliases for other extras + and we have an extra called 'all' that includes all other extras. + + When the `requirements.txt` is generated by `pip-tools`, then it is likely that + this step is not needed, but for other `requirements.txt` files this may be useful. + + NOTE @aignas 2023-12-08: the extra resolution is not platform dependent, but + in order for it to become platform dependent we would have to have separate targets for each extra in + self._want_extras. + """ + extras = self._want_extras + + self_reqs = [] + for req in reqs: + if Deps._normalize(req.name) != self.name: + continue + + if req.marker is None: + # I am pretty sure we cannot reach this code as it does not + # make sense to specify packages in this way, but since it is + # easy to handle, lets do it. + # + # TODO @aignas 2023-12-08: add a test + extras = extras | req.extras + else: + # process these in a separate loop + self_reqs.append(req) + + # A double loop is not strictly optimal, but always correct without recursion + for req in self_reqs: + if any(req.marker.evaluate({"extra": extra}) for extra in extras): + extras = extras | req.extras + else: + continue + + # Iterate through all packages to ensure that we include all of the extras from previously + # visited packages. + for req_ in self_reqs: + if any(req_.marker.evaluate({"extra": extra}) for extra in extras): + extras = extras | req_.extras + + return extras + + def _add_req(self, req: Requirement) -> None: + extras = self._want_extras + + if req.marker is None: + self._add(req.name, None) + return + + marker_str = str(req.marker) + + # NOTE @aignas 2023-12-08: in order to have reasonable select statements + # we do have to have some parsing of the markers, so it begs the question + # if packaging should be reimplemented in Starlark to have the best solution + # for now we will implement it in Python and see what the best parsing result + # can be before making this decision. + if not self._platforms or not any( + tag in marker_str + for tag in [ + "os_name", + "sys_platform", + "platform_machine", + "platform_system", + ] + ): + if any(req.marker.evaluate({"extra": extra}) for extra in extras): + self._add(req.name, None) + return + + for plat in self._platforms: + if not any( + req.marker.evaluate(plat.env_markers(extra)) for extra in extras + ): + continue + + if "platform_machine" in marker_str: + self._add(req.name, plat) + else: + self._add(req.name, Platform(plat.os)) + + def build(self) -> FrozenDeps: + if not self._select: + return FrozenDeps( + deps=sorted(self._deps), + deps_select={}, + ) + + # Get all of the OS-specific dependencies applicable to all architectures + select = { + p: deps for p, deps in self._select.items() if deps and p.arch is None + } + # Now add them to all arch specific dependencies + select.update( + { + p: deps | select.get(Platform(p.os), set()) + for p, deps in self._select.items() + if deps and p.arch is not None + } + ) + + return FrozenDeps( + deps=sorted(self._deps), + deps_select={str(p): sorted(deps) for p, deps in sorted(select.items())}, + ) + + class Wheel: """Representation of the compressed .whl file""" - def __init__(self, path: str): + def __init__(self, path: Path): self._path = path @property @@ -70,19 +424,20 @@ class Wheel: return entry_points_mapping - def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: - dependency_set = set() - + def dependencies( + self, + extras_requested: Set[str] = None, + platforms: Optional[Set[Platform]] = None, + ) -> FrozenDeps: + dependency_set = Deps( + self.name, + extras=extras_requested, + platforms=platforms, + ) for wheel_req in self.metadata.get_all("Requires-Dist", []): - req = pkg_resources.Requirement(wheel_req) # type: ignore - - if req.marker is None or any( - req.marker.evaluate({"extra": extra}) - for extra in extras_requested or [""] - ): - dependency_set.add(req.name) # type: ignore + dependency_set.add(wheel_req) - return dependency_set + return dependency_set.build() def unzip(self, directory: str) -> None: installation_schemes = { diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 9b363c3..801ef95 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -12,24 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse +"""Build and/or fetch a single wheel based on the requirement passed in""" + import errno import glob import json import os import re -import shutil import subprocess import sys -import textwrap from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Dict, Iterable, List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple from pip._vendor.packaging.utils import canonicalize_name -from python.pip_install.tools.lib import annotation, arguments, bazel -from python.pip_install.tools.wheel_installer import namespace_pkgs, wheel +from python.pip_install.tools.wheel_installer import arguments, namespace_pkgs, wheel def _configure_reproducible_wheels() -> None: @@ -103,201 +101,12 @@ def _setup_namespace_pkg_compatibility(wheel_dir: str) -> None: namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) -def _generate_entry_point_contents( - module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" -) -> str: - """Generate the contents of an entry point script. - - Args: - module (str): The name of the module to use. - attribute (str): The name of the attribute to call. - shebang (str, optional): The shebang to use for the entry point python - file. - - Returns: - str: A string of python code. - """ - return textwrap.dedent( - """\ - {shebang} - import sys - from {module} import {attribute} - if __name__ == "__main__": - sys.exit({attribute}()) - """.format( - shebang=shebang, module=module, attribute=attribute - ) - ) - - -def _generate_entry_point_rule(name: str, script: str, pkg: str) -> str: - """Generate a Bazel `py_binary` rule for an entry point script. - - Note that the script is used to determine the name of the target. The name of - entry point targets should be uniuqe to avoid conflicts with existing sources or - directories within a wheel. - - Args: - name (str): The name of the generated py_binary. - script (str): The path to the entry point's python file. - pkg (str): The package owning the entry point. This is expected to - match up with the `py_library` defined for each repository. - - - Returns: - str: A `py_binary` instantiation. - """ - return textwrap.dedent( - """\ - py_binary( - name = "{name}", - srcs = ["{src}"], - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["."], - deps = ["{pkg}"], - ) - """.format( - name=name, src=str(script).replace("\\", "/"), pkg=pkg - ) - ) - - -def _generate_copy_commands(src, dest, is_executable=False) -> str: - """Generate a [@bazel_skylib//rules:copy_file.bzl%copy_file][cf] target - - [cf]: https://github.com/bazelbuild/bazel-skylib/blob/1.1.1/docs/copy_file_doc.md - - Args: - src (str): The label for the `src` attribute of [copy_file][cf] - dest (str): The label for the `out` attribute of [copy_file][cf] - is_executable (bool, optional): Whether or not the file being copied is executable. - sets `is_executable` for [copy_file][cf] - - Returns: - str: A `copy_file` instantiation. - """ - return textwrap.dedent( - """\ - copy_file( - name = "{dest}.copy", - src = "{src}", - out = "{dest}", - is_executable = {is_executable}, - ) - """.format( - src=src, - dest=dest, - is_executable=is_executable, - ) - ) - - -def _generate_build_file_contents( - name: str, - dependencies: List[str], - whl_file_deps: List[str], - data_exclude: List[str], - tags: List[str], - srcs_exclude: List[str] = [], - data: List[str] = [], - additional_content: List[str] = [], -) -> str: - """Generate a BUILD file for an unzipped Wheel - - Args: - name: the target name of the py_library - dependencies: a list of Bazel labels pointing to dependencies of the library - whl_file_deps: a list of Bazel labels pointing to wheel file dependencies of this wheel. - data_exclude: more patterns to exclude from the data attribute of generated py_library rules. - tags: list of tags to apply to generated py_library rules. - additional_content: A list of additional content to append to the BUILD file. - - Returns: - A complete BUILD file as a string - - We allow for empty Python sources as for Wheels containing only compiled C code - there may be no Python sources whatsoever (e.g. packages written in Cython: like `pymssql`). - """ - - data_exclude = list( - set( - [ - "**/* *", - "**/*.py", - "**/*.pyc", - "**/*.pyc.*", # During pyc creation, temp files named *.pyc.NNNN are created - # RECORD is known to contain sha256 checksums of files which might include the checksums - # of generated files produced when wheels are installed. The file is ignored to avoid - # Bazel caching issues. - "**/*.dist-info/RECORD", - ] - + data_exclude - ) - ) - - return "\n".join( - [ - textwrap.dedent( - """\ - load("@rules_python//python:defs.bzl", "py_library", "py_binary") - load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - - package(default_visibility = ["//visibility:public"]) - - filegroup( - name = "{dist_info_label}", - srcs = glob(["site-packages/*.dist-info/**"], allow_empty = True), - ) - - filegroup( - name = "{data_label}", - srcs = glob(["data/**"], allow_empty = True), - ) - - filegroup( - name = "{whl_file_label}", - srcs = glob(["*.whl"], allow_empty = True), - data = [{whl_file_deps}], - ) - - py_library( - name = "{name}", - srcs = glob(["site-packages/**/*.py"], exclude={srcs_exclude}, allow_empty = True), - data = {data} + glob(["site-packages/**/*"], exclude={data_exclude}), - # This makes this directory a top-level in the python import - # search path for anything that depends on this. - imports = ["site-packages"], - deps = [{dependencies}], - tags = [{tags}], - ) - """.format( - name=name, - dependencies=",".join(sorted(dependencies)), - data_exclude=json.dumps(sorted(data_exclude)), - whl_file_label=bazel.WHEEL_FILE_LABEL, - whl_file_deps=",".join(sorted(whl_file_deps)), - tags=",".join(sorted(['"%s"' % t for t in tags])), - data_label=bazel.DATA_LABEL, - dist_info_label=bazel.DIST_INFO_LABEL, - entry_point_prefix=bazel.WHEEL_ENTRY_POINT_PREFIX, - srcs_exclude=json.dumps(sorted(srcs_exclude)), - data=json.dumps(sorted(data)), - ) - ) - ] - + additional_content - ) - - def _extract_wheel( wheel_file: str, extras: Dict[str, Set[str]], - pip_data_exclude: List[str], enable_implicit_namespace_pkgs: bool, - repo_prefix: str, + platforms: List[wheel.Platform], installation_dir: Path = Path("."), - annotation: Optional[annotation.Annotation] = None, ) -> None: """Extracts wheel into given directory and creates py_library and filegroup targets. @@ -305,9 +114,7 @@ def _extract_wheel( wheel_file: the filepath of the .whl installation_dir: the destination directory for installation of the wheel. extras: a list of extras to add as dependencies for the installed wheel - pip_data_exclude: list of file patterns to exclude from the generated data section of the py_library enable_implicit_namespace_pkgs: if true, disables conversion of implicit namespace packages and will unzip as-is - annotation: An optional set of annotations to apply to the BUILD contents of the wheel. """ whl = wheel.Wheel(wheel_file) @@ -317,93 +124,47 @@ def _extract_wheel( _setup_namespace_pkg_compatibility(installation_dir) extras_requested = extras[whl.name] if whl.name in extras else set() - # Packages may create dependency cycles when specifying optional-dependencies / 'extras'. - # Example: github.com/google/etils/blob/a0b71032095db14acf6b33516bca6d885fe09e35/pyproject.toml#L32. - self_edge_dep = set([whl.name]) - whl_deps = sorted(whl.dependencies(extras_requested) - self_edge_dep) - - sanitised_dependencies = [ - bazel.sanitised_repo_library_label(d, repo_prefix=repo_prefix) for d in whl_deps - ] - sanitised_wheel_file_dependencies = [ - bazel.sanitised_repo_file_label(d, repo_prefix=repo_prefix) for d in whl_deps - ] - - entry_points = [] - for name, (module, attribute) in sorted(whl.entry_points().items()): - # There is an extreme edge-case with entry_points that end with `.py` - # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 - entry_point_without_py = f"{name[:-3]}_py" if name.endswith(".py") else name - entry_point_target_name = ( - f"{bazel.WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}" - ) - entry_point_script_name = f"{entry_point_target_name}.py" - (installation_dir / entry_point_script_name).write_text( - _generate_entry_point_contents(module, attribute) - ) - entry_points.append( - _generate_entry_point_rule( - entry_point_target_name, - entry_point_script_name, - bazel.PY_LIBRARY_LABEL, - ) - ) - with open(os.path.join(installation_dir, "BUILD.bazel"), "w") as build_file: - additional_content = entry_points - data = [] - data_exclude = pip_data_exclude - srcs_exclude = [] - if annotation: - for src, dest in annotation.copy_files.items(): - data.append(dest) - additional_content.append(_generate_copy_commands(src, dest)) - for src, dest in annotation.copy_executables.items(): - data.append(dest) - additional_content.append( - _generate_copy_commands(src, dest, is_executable=True) - ) - data.extend(annotation.data) - data_exclude.extend(annotation.data_exclude_glob) - srcs_exclude.extend(annotation.srcs_exclude_glob) - if annotation.additive_build_content: - additional_content.append(annotation.additive_build_content) - - contents = _generate_build_file_contents( - name=bazel.PY_LIBRARY_LABEL, - dependencies=sanitised_dependencies, - whl_file_deps=sanitised_wheel_file_dependencies, - data_exclude=data_exclude, - data=data, - srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], - additional_content=additional_content, - ) - build_file.write(contents) + dependencies = whl.dependencies(extras_requested, platforms) + + with open(os.path.join(installation_dir, "metadata.json"), "w") as f: + metadata = { + "name": whl.name, + "version": whl.version, + "deps": dependencies.deps, + "deps_by_platform": dependencies.deps_select, + "entry_points": [ + { + "name": name, + "module": module, + "attribute": attribute, + } + for name, (module, attribute) in sorted(whl.entry_points().items()) + ], + } + json.dump(metadata, f) def main() -> None: - parser = argparse.ArgumentParser( - description="Build and/or fetch a single wheel based on the requirement passed in" - ) - parser.add_argument( - "--requirement", - action="store", - required=True, - help="A single PEP508 requirement specifier string.", - ) - parser.add_argument( - "--annotation", - type=annotation.annotation_from_str_path, - help="A json encoded file containing annotations for rendered packages.", - ) - arguments.parse_common_args(parser) - args = parser.parse_args() + args = arguments.parser(description=__doc__).parse_args() deserialized_args = dict(vars(args)) arguments.deserialize_structured_args(deserialized_args) _configure_reproducible_wheels() + if args.whl_file: + whl = Path(args.whl_file) + + name, extras_for_pkg = _parse_requirement_for_extra(args.requirement) + extras = {name: extras_for_pkg} if extras_for_pkg and name else dict() + _extract_wheel( + wheel_file=whl, + extras=extras, + enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, + platforms=arguments.get_platforms(args), + ) + return + pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) @@ -434,18 +195,10 @@ def main() -> None: if e.errno != errno.ENOENT: raise - name, extras_for_pkg = _parse_requirement_for_extra(args.requirement) - extras = {name: extras_for_pkg} if extras_for_pkg and name else dict() - - whl = next(iter(glob.glob("*.whl"))) - _extract_wheel( - wheel_file=whl, - extras=extras, - pip_data_exclude=deserialized_args["pip_data_exclude"], - enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs, - repo_prefix=args.repo_prefix, - annotation=args.annotation, - ) + whl = Path(next(iter(glob.glob("*.whl")))) + + with open("whl_file.json", "w") as f: + json.dump({"whl_file": f"{whl.resolve()}"}, f) if __name__ == "__main__": diff --git a/python/pip_install/tools/wheel_installer/wheel_installer_test.py b/python/pip_install/tools/wheel_installer/wheel_installer_test.py index 8758b67..6eacd1f 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer_test.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer_test.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import shutil import tempfile import unittest from pathlib import Path -from python.pip_install.tools.wheel_installer import wheel_installer +from python.pip_install.tools.wheel_installer import wheel, wheel_installer class TestRequirementExtrasParsing(unittest.TestCase): @@ -54,30 +55,6 @@ class TestRequirementExtrasParsing(unittest.TestCase): ) -class BazelTestCase(unittest.TestCase): - def test_generate_entry_point_contents(self): - got = wheel_installer._generate_entry_point_contents("sphinx.cmd.build", "main") - want = """#!/usr/bin/env python3 -import sys -from sphinx.cmd.build import main -if __name__ == "__main__": - sys.exit(main()) -""" - self.assertEqual(got, want) - - def test_generate_entry_point_contents_with_shebang(self): - got = wheel_installer._generate_entry_point_contents( - "sphinx.cmd.build", "main", shebang="#!/usr/bin/python" - ) - want = """#!/usr/bin/python -import sys -from sphinx.cmd.build import main -if __name__ == "__main__": - sys.exit(main()) -""" - self.assertEqual(got, want) - - class TestWhlFilegroup(unittest.TestCase): def setUp(self) -> None: self.wheel_name = "example_minimal_package-0.0.1-py3-none-any.whl" @@ -90,18 +67,61 @@ class TestWhlFilegroup(unittest.TestCase): def test_wheel_exists(self) -> None: wheel_installer._extract_wheel( - self.wheel_path, + Path(self.wheel_path), installation_dir=Path(self.wheel_dir), extras={}, - pip_data_exclude=[], enable_implicit_namespace_pkgs=False, - repo_prefix="prefix_", + platforms=[], + ) + + want_files = [ + "metadata.json", + "site-packages", + self.wheel_name, + ] + self.assertEqual( + sorted(want_files), + sorted( + [ + str(p.relative_to(self.wheel_dir)) + for p in Path(self.wheel_dir).glob("*") + ] + ), + ) + with open("{}/metadata.json".format(self.wheel_dir)) as metadata_file: + metadata_file_content = json.load(metadata_file) + + want = dict( + version="0.0.1", + name="example-minimal-package", + deps=[], + deps_by_platform={}, + entry_points=[], + ) + self.assertEqual(want, metadata_file_content) + + +class TestWheelPlatform(unittest.TestCase): + def test_wheel_os_alias(self): + self.assertEqual("OS.osx", str(wheel.OS.osx)) + self.assertEqual(str(wheel.OS.darwin), str(wheel.OS.osx)) + + def test_wheel_arch_alias(self): + self.assertEqual("Arch.x86_64", str(wheel.Arch.x86_64)) + self.assertEqual(str(wheel.Arch.amd64), str(wheel.Arch.x86_64)) + + def test_wheel_platform_alias(self): + give = wheel.Platform( + os=wheel.OS.darwin, + arch=wheel.Arch.amd64, + ) + alias = wheel.Platform( + os=wheel.OS.osx, + arch=wheel.Arch.x86_64, ) - self.assertIn(self.wheel_name, os.listdir(self.wheel_dir)) - with open("{}/BUILD.bazel".format(self.wheel_dir)) as build_file: - build_file_content = build_file.read() - self.assertIn("filegroup", build_file_content) + self.assertEqual("osx_x86_64", str(give)) + self.assertEqual(str(alias), str(give)) if __name__ == "__main__": diff --git a/python/pip_install/tools/wheel_installer/wheel_test.py b/python/pip_install/tools/wheel_installer/wheel_test.py new file mode 100644 index 0000000..721b710 --- /dev/null +++ b/python/pip_install/tools/wheel_installer/wheel_test.py @@ -0,0 +1,220 @@ +import unittest + +from python.pip_install.tools.wheel_installer import wheel + + +class DepsTest(unittest.TestCase): + def test_simple(self): + deps = wheel.Deps("foo") + deps.add("bar") + + got = deps.build() + + self.assertIsInstance(got, wheel.FrozenDeps) + self.assertEqual(["bar"], got.deps) + self.assertEqual({}, got.deps_select) + + def test_can_add_os_specific_deps(self): + platforms = { + "linux_x86_64", + "osx_x86_64", + "windows_x86_64", + } + deps = wheel.Deps("foo", platforms=set(wheel.Platform.from_string(platforms))) + deps.add( + "bar", + "posix_dep; os_name=='posix'", + "win_dep; os_name=='nt'", + ) + + got = deps.build() + + self.assertEqual(["bar"], got.deps) + self.assertEqual( + { + "@platforms//os:linux": ["posix_dep"], + "@platforms//os:osx": ["posix_dep"], + "@platforms//os:windows": ["win_dep"], + }, + got.deps_select, + ) + + def test_can_add_platform_specific_deps(self): + platforms = { + "linux_x86_64", + "osx_x86_64", + "osx_aarch64", + "windows_x86_64", + } + deps = wheel.Deps("foo", platforms=set(wheel.Platform.from_string(platforms))) + deps.add( + "bar", + "posix_dep; os_name=='posix'", + "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", + "win_dep; os_name=='nt'", + ) + + got = deps.build() + + self.assertEqual(["bar"], got.deps) + self.assertEqual( + { + "osx_aarch64": ["m1_dep", "posix_dep"], + "@platforms//os:linux": ["posix_dep"], + "@platforms//os:osx": ["posix_dep"], + "@platforms//os:windows": ["win_dep"], + }, + got.deps_select, + ) + + def test_non_platform_markers_are_added_to_common_deps(self): + platforms = { + "linux_x86_64", + "osx_x86_64", + "osx_aarch64", + "windows_x86_64", + } + deps = wheel.Deps("foo", platforms=set(wheel.Platform.from_string(platforms))) + deps.add( + "bar", + "baz; implementation_name=='cpython'", + "m1_dep; sys_platform=='darwin' and platform_machine=='arm64'", + ) + + got = deps.build() + + self.assertEqual(["bar", "baz"], got.deps) + self.assertEqual( + { + "osx_aarch64": ["m1_dep"], + }, + got.deps_select, + ) + + def test_self_is_ignored(self): + deps = wheel.Deps("foo", extras={"ssl"}) + deps.add( + "bar", + "req_dep; extra == 'requests'", + "foo[requests]; extra == 'ssl'", + "ssl_lib; extra == 'ssl'", + ) + + got = deps.build() + + self.assertEqual(["bar", "req_dep", "ssl_lib"], got.deps) + self.assertEqual({}, got.deps_select) + + def test_handle_etils(self): + deps = wheel.Deps("etils", extras={"all"}) + requires = """ +etils[array-types] ; extra == "all" +etils[eapp] ; extra == "all" +etils[ecolab] ; extra == "all" +etils[edc] ; extra == "all" +etils[enp] ; extra == "all" +etils[epath] ; extra == "all" +etils[epath-gcs] ; extra == "all" +etils[epath-s3] ; extra == "all" +etils[epy] ; extra == "all" +etils[etqdm] ; extra == "all" +etils[etree] ; extra == "all" +etils[etree-dm] ; extra == "all" +etils[etree-jax] ; extra == "all" +etils[etree-tf] ; extra == "all" +etils[enp] ; extra == "array-types" +pytest ; extra == "dev" +pytest-subtests ; extra == "dev" +pytest-xdist ; extra == "dev" +pyink ; extra == "dev" +pylint>=2.6.0 ; extra == "dev" +chex ; extra == "dev" +torch ; extra == "dev" +optree ; extra == "dev" +dataclass_array ; extra == "dev" +sphinx-apitree[ext] ; extra == "docs" +etils[dev,all] ; extra == "docs" +absl-py ; extra == "eapp" +simple_parsing ; extra == "eapp" +etils[epy] ; extra == "eapp" +jupyter ; extra == "ecolab" +numpy ; extra == "ecolab" +mediapy ; extra == "ecolab" +packaging ; extra == "ecolab" +etils[enp] ; extra == "ecolab" +etils[epy] ; extra == "ecolab" +etils[epy] ; extra == "edc" +numpy ; extra == "enp" +etils[epy] ; extra == "enp" +fsspec ; extra == "epath" +importlib_resources ; extra == "epath" +typing_extensions ; extra == "epath" +zipp ; extra == "epath" +etils[epy] ; extra == "epath" +gcsfs ; extra == "epath-gcs" +etils[epath] ; extra == "epath-gcs" +s3fs ; extra == "epath-s3" +etils[epath] ; extra == "epath-s3" +typing_extensions ; extra == "epy" +absl-py ; extra == "etqdm" +tqdm ; extra == "etqdm" +etils[epy] ; extra == "etqdm" +etils[array_types] ; extra == "etree" +etils[epy] ; extra == "etree" +etils[enp] ; extra == "etree" +etils[etqdm] ; extra == "etree" +dm-tree ; extra == "etree-dm" +etils[etree] ; extra == "etree-dm" +jax[cpu] ; extra == "etree-jax" +etils[etree] ; extra == "etree-jax" +tensorflow ; extra == "etree-tf" +etils[etree] ; extra == "etree-tf" +etils[ecolab] ; extra == "lazy-imports" +""" + + deps.add(*requires.strip().split("\n")) + + got = deps.build() + want = [ + "absl_py", + "dm_tree", + "fsspec", + "gcsfs", + "importlib_resources", + "jax", + "jupyter", + "mediapy", + "numpy", + "packaging", + "s3fs", + "simple_parsing", + "tensorflow", + "tqdm", + "typing_extensions", + "zipp", + ] + + self.assertEqual(want, got.deps) + self.assertEqual({}, got.deps_select) + + +class PlatformTest(unittest.TestCase): + def test_can_get_host(self): + host = wheel.Platform.host() + self.assertIsNotNone(host) + self.assertEqual(1, len(wheel.Platform.from_string("host"))) + self.assertEqual(host, wheel.Platform.from_string("host")) + + def test_can_get_all(self): + all_platforms = wheel.Platform.all() + self.assertEqual(15, len(all_platforms)) + self.assertEqual(all_platforms, wheel.Platform.from_string("all")) + + def test_can_get_all_for_os(self): + linuxes = wheel.Platform.all(wheel.OS.linux) + self.assertEqual(5, len(linuxes)) + self.assertEqual(linuxes, wheel.Platform.from_string("linux_*")) + + +if __name__ == "__main__": + unittest.main() |