From 60c61e51646f610e7886d6bc1eaddb154f80ad44 Mon Sep 17 00:00:00 2001 From: Chris Love <335402+chrislovecnm@users.noreply.github.com> Date: Fri, 26 May 2023 14:20:35 -0600 Subject: [PATCH 01/16] feat(bzlmod): Allowing multiple python.toolchain extension calls (#1230) We do this work for two reasons. First, we must support Module dependencies and sub-modules using `python.toolchain`. There are already two known instances of sub-modules setting up a Python toolchain and colliding with another module (nanobind and rules_testing both run into this). Second, the upcoming multi-version support is going to work by having each `python.toolchain()` call register its particular version with the extra toolchain constraint. This also helps unify the version-aware and non-version-aware code paths (the non-version aware paths are just version-aware with a single version registered as the default) This commit implements various business logic in the toolchain class. Toolchains in Sub Modules It will create a toolchain in a sub-module if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and sub-modules. You cannot configure more than one toolchain as the default toolchain. Toolchain set as the default version. This extension will not create a toolchain in a sub-module if the sub-module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain, it is set as the default toolchain. See #1229 for more information --- examples/bzlmod/.bazelrc | 7 + examples/bzlmod/BUILD.bazel | 42 +++++- examples/bzlmod/MODULE.bazel | 75 +++++++++-- examples/bzlmod/__main__.py | 2 + examples/bzlmod/other_module/MODULE.bazel | 52 ++++++++ .../other_module/other_module/pkg/BUILD.bazel | 12 ++ examples/bzlmod/runfiles/BUILD.bazel | 1 - .../bzlmod_build_file_generation/BUILD.bazel | 2 +- .../bzlmod_build_file_generation/MODULE.bazel | 21 ++- python/extensions/private/interpreter_hub.bzl | 17 ++- python/extensions/python.bzl | 120 +++++++++++++++--- python/repositories.bzl | 9 +- 12 files changed, 320 insertions(+), 40 deletions(-) diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc index b8c233f98c..6f557e67b9 100644 --- a/examples/bzlmod/.bazelrc +++ b/examples/bzlmod/.bazelrc @@ -1,3 +1,10 @@ common --experimental_enable_bzlmod coverage --java_runtime_version=remotejdk_11 + +test --test_output=errors --enable_runfiles + +# Windows requires these for multi-python support: +build --enable_runfiles + +startup --windows_enable_symlinks diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index e1f5790631..86498226f9 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,9 +1,21 @@ +# Load various rules so that we can have bazel download +# various rulesets and dependencies. +# The `load` statement imports the symbol for the rule, in the defined +# ruleset. When the symbol is loaded you can use the rule. + +# The names @pip and @python_39 are values that are repository +# names. Those names are defined in the MODULES.bazel file. load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement") -load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") +load("@python_39//:defs.bzl", py_test_with_transition = "py_test") + +# This is not working yet till the toolchain hub registration is working +# load("@python_310//:defs.bzl", py_binary_310 = "py_binary") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") +# This stanza calls a rule that generates targets for managing pip dependencies +# with pip-compile. compile_pip_requirements( name = "requirements", extra_args = ["--allow-unsafe"], @@ -12,6 +24,11 @@ compile_pip_requirements( requirements_windows = "requirements_windows.txt", ) +# The rules below are language specific rules defined in +# rules_python. See +# https://bazel.build/reference/be/python + +# see https://bazel.build/reference/be/python#py_library py_library( name = "lib", srcs = ["lib.py"], @@ -22,6 +39,7 @@ py_library( ], ) +# see https://bazel.build/reference/be/python#py_binary py_binary( name = "bzlmod", srcs = ["__main__.py"], @@ -32,6 +50,23 @@ py_binary( ], ) +# This is still WIP. Not working till we have the toolchain +# registration functioning. + +# This is used for testing mulitple versions of Python. This is +# used only when you need to support multiple versions of Python +# in the same project. +# py_binary_310( +# name = "main_310", +# srcs = ["__main__.py"], +# main = "__main__.py", +# visibility = ["//:__subpackages__"], +# deps = [ +# ":lib", +# ], +# ) + +# see https://bazel.build/reference/be/python#py_test py_test( name = "test", srcs = ["test.py"], @@ -46,6 +81,11 @@ py_test_with_transition( deps = [":lib"], ) +# This example is also used for integration tests within +# rules_python. We are using +# https://github.com/bazelbuild/bazel-skylib +# to run some of the tests. +# See: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/build_test_doc.md build_test( name = "all_wheels", targets = all_whl_requirements, diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 145cebd276..bb4183bde2 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,32 +11,89 @@ local_path_override( path = "../..", ) +# This name is passed into python.toolchain and it's use_repo statement. +# We also use the same value in the python.host_python_interpreter call. +PYTHON_NAME_39 = "python_39" + +PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" + +INTERPRETER_NAME_39 = "interpreter_39" + +PYTHON_NAME_310 = "python_310" + +PYTHON_310_TOOLCHAINS = PYTHON_NAME_310 + "_toolchains" + +INTERPRETER_NAME_310 = "interpreter_310" + +# We next initialize the python toolchain using the extension. +# You can set different Python versions in this block. python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python3_9", + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_39, configure_coverage_tool = True, + # Only set when you have mulitple toolchain versions. + is_default = True, python_version = "3.9", ) -use_repo(python, "python3_9") -use_repo(python, "python3_9_toolchains") +# We are also using a second version of Python in this project. +# Typically you will only need a single version of Python, but +# If you need a different vesion we support more than one. +# Note: we do not supporting using multiple pip extensions, this is +# work in progress. +python.toolchain( + name = PYTHON_NAME_310, + configure_coverage_tool = True, + python_version = "3.10", +) + +# use_repo imports one or more repos generated by the given module extension +# into the scope of the current module. We are importing the various repos +# created by the above python.toolchain calls. +use_repo( + python, + PYTHON_NAME_39, + PYTHON_39_TOOLCHAINS, + PYTHON_NAME_310, + PYTHON_310_TOOLCHAINS, +) + +# This call registers the Python toolchains. +# Note: there is work under way to move this code to within +# rules_python, and the user won't have to make this call, +# unless they are registering custom toolchains. register_toolchains( - "@python3_9_toolchains//:all", + "@{}//:all".format(PYTHON_39_TOOLCHAINS), + "@{}//:all".format(PYTHON_310_TOOLCHAINS), ) +# The interpreter extension discovers the platform specific Python binary. +# It creates a symlink to the binary, and we pass the label to the following +# pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3_9", - python_name = "python3_9", + name = INTERPRETER_NAME_39, + python_name = PYTHON_NAME_39, +) + +# This second call is only needed if you are using mulitple different +# Python versions/interpreters. +interpreter.install( + name = INTERPRETER_NAME_310, + python_name = PYTHON_NAME_310, ) -use_repo(interpreter, "interpreter_python3_9") +use_repo(interpreter, INTERPRETER_NAME_39, INTERPRETER_NAME_310) pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( name = "pip", - # Intentionally set it false because the "true" case is already covered by examples/bzlmod_build_file_generation + # Intentionally set it false because the "true" case is already + # covered by examples/bzlmod_build_file_generation incompatible_generate_aliases = False, - python_interpreter_target = "@interpreter_python3_9//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME_39), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/examples/bzlmod/__main__.py b/examples/bzlmod/__main__.py index 099493b3c8..daf17495c2 100644 --- a/examples/bzlmod/__main__.py +++ b/examples/bzlmod/__main__.py @@ -13,6 +13,8 @@ # limitations under the License. from lib import main +import sys if __name__ == "__main__": print(main([["A", 1], ["B", 2]])) + print(sys.version) diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index 992e120760..eebfbcaa58 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -2,4 +2,56 @@ module( name = "other_module", ) +# This module is using the same version of rules_python +# that the parent module uses. bazel_dep(name = "rules_python", version = "") + +# It is not best practice to use a python.toolchian in +# a submodule. This code only exists to test that +# we support doing this. This code is only for rules_python +# testing purposes. +PYTHON_NAME_39 = "python_39" + +PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" + +PYTHON_NAME_311 = "python_311" + +PYTHON_311_TOOLCHAINS = PYTHON_NAME_311 + "_toolchains" + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_39, + configure_coverage_tool = True, + python_version = "3.9", +) +python.toolchain( + # This name is used in the various use_repo statements + # below, and in the local extension that is in this + # example. + name = PYTHON_NAME_311, + configure_coverage_tool = True, + # In a submodule this is ignored + is_default = True, + python_version = "3.11", +) + +# created by the above python.toolchain calls. +use_repo( + python, + PYTHON_NAME_39, + PYTHON_39_TOOLCHAINS, + PYTHON_NAME_311, + PYTHON_311_TOOLCHAINS, +) + +# This call registers the Python toolchains. +# Note: there is work under way to move this code to within +# rules_python, and the user won't have to make this call, +# unless they are registering custom toolchains. +register_toolchains( + "@{}//:all".format(PYTHON_39_TOOLCHAINS), + "@{}//:all".format(PYTHON_311_TOOLCHAINS), +) diff --git a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel index 9a130e3554..952a674d48 100644 --- a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel +++ b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel @@ -1,3 +1,4 @@ +load("@python_311//:defs.bzl", py_binary_311 = "py_binary") load("@rules_python//python:defs.bzl", "py_library") py_library( @@ -8,4 +9,15 @@ py_library( deps = ["@rules_python//python/runfiles"], ) +# This is used for testing mulitple versions of Python. This is +# used only when you need to support multiple versions of Python +# in the same project. +py_binary_311( + name = "lib_311", + srcs = ["lib.py"], + data = ["data/data.txt"], + visibility = ["//visibility:public"], + deps = ["@rules_python//python/runfiles"], +) + exports_files(["data/data.txt"]) diff --git a/examples/bzlmod/runfiles/BUILD.bazel b/examples/bzlmod/runfiles/BUILD.bazel index 3503ac3017..add56b3bd0 100644 --- a/examples/bzlmod/runfiles/BUILD.bazel +++ b/examples/bzlmod/runfiles/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_python//python:defs.bzl", "py_test") -# gazelle:ignore py_test( name = "runfiles_test", srcs = ["runfiles_test.py"], diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c667f1e49b..05a15cce28 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -7,7 +7,7 @@ # requirements. load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") -load("@python3//:defs.bzl", py_test_with_transition = "py_test") +load("@python//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index 179fe1bdea..45a1318ac8 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -46,7 +46,11 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") # This name is passed into python.toolchain and it's use_repo statement. # We also use the same name for python.host_python_interpreter. -PYTHON_NAME = "python3" +PYTHON_NAME = "python" + +PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains" + +INTERPRETER_NAME = "interpreter" # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. @@ -56,6 +60,7 @@ python.toolchain( # example. name = PYTHON_NAME, configure_coverage_tool = True, + is_default = True, python_version = "3.9", ) @@ -63,12 +68,16 @@ python.toolchain( # into the scope of the current module. # All of the python3 repositories use the PYTHON_NAME as there prefix. They # are not catenated for ease of reading. -use_repo(python, PYTHON_NAME, "python3_toolchains") +use_repo( + python, + PYTHON_NAME, + PYTHON_TOOLCHAINS, +) # Register an already-defined toolchain so that Bazel can use it during # toolchain resolution. register_toolchains( - "@python3_toolchains//:all", + "@{}//:all".format(PYTHON_TOOLCHAINS), ) # The interpreter extension discovers the platform specific Python binary. @@ -76,10 +85,10 @@ register_toolchains( # pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3", + name = INTERPRETER_NAME, python_name = PYTHON_NAME, ) -use_repo(interpreter, "interpreter_python3") +use_repo(interpreter, INTERPRETER_NAME) # Use the extension, pip.parse, to call the `pip_repository` rule that invokes # `pip`, with `incremental` set. The pip call accepts a locked/compiled @@ -102,7 +111,7 @@ pip.parse( # is used for both resolving dependencies and running tests/binaries. # If this isn't specified, then you'll get whatever is locally installed # on your system. - python_interpreter_target = "@interpreter_python3//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/python/extensions/private/interpreter_hub.bzl b/python/extensions/private/interpreter_hub.bzl index f1ca670cf2..82fcbf698f 100644 --- a/python/extensions/private/interpreter_hub.bzl +++ b/python/extensions/private/interpreter_hub.bzl @@ -19,8 +19,9 @@ load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platf _build_file_for_hub_template = """ INTERPRETER_LABELS = {{ -{lines} +{interpreter_labels} }} +DEFAULT_TOOLCHAIN_NAME = "{default}" """ _line_for_hub_template = """\ @@ -35,13 +36,19 @@ def _hub_repo_impl(rctx): is_windows = (os == WINDOWS_NAME) path = "python.exe" if is_windows else "bin/python3" - lines = "\n".join([_line_for_hub_template.format( + interpreter_labels = "\n".join([_line_for_hub_template.format( name = name, platform = platform, path = path, ) for name in rctx.attr.toolchains]) - rctx.file("interpreters.bzl", _build_file_for_hub_template.format(lines = lines)) + rctx.file( + "interpreters.bzl", + _build_file_for_hub_template.format( + interpreter_labels = interpreter_labels, + default = rctx.attr.default_toolchain, + ), + ) hub_repo = repository_rule( doc = """\ @@ -50,6 +57,10 @@ and the labels to said interpreters. This map is used to by the interpreter hub """, implementation = _hub_repo_impl, attrs = { + "default_toolchain": attr.string( + doc = "Name of the default toolchain", + mandatory = True, + ), "toolchains": attr.string_list( doc = "List of the base names the toolchain repo defines.", mandatory = True, diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 9a3d9ed959..cae1988e8a 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -17,35 +17,117 @@ load("@rules_python//python:repositories.bzl", "python_register_toolchains") load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo") +# Printing a warning msg not debugging, so we have to disable +# the buildifier check. +# buildifier: disable=print +def _print_warn(msg): + print("WARNING:", msg) + +def _python_register_toolchains(toolchain_attr, version_constraint): + python_register_toolchains( + name = toolchain_attr.name, + python_version = toolchain_attr.python_version, + register_coverage_tool = toolchain_attr.configure_coverage_tool, + ignore_root_user_error = toolchain_attr.ignore_root_user_error, + set_python_version_constraint = version_constraint, + ) + def _python_impl(module_ctx): - toolchains = [] + # We collect all of the toolchain names to create + # the INTERPRETER_LABELS map. This is used + # by interpreter_extensions.bzl via the hub_repo call below. + toolchain_names = [] + + # Used to store the default toolchain name so we can pass it to the hub + default_toolchain_name = None + + # Used to store toolchains that are in sub modules. + sub_toolchains_map = {} + for mod in module_ctx.modules: for toolchain_attr in mod.tags.toolchain: - python_register_toolchains( - name = toolchain_attr.name, - python_version = toolchain_attr.python_version, - bzlmod = True, - # Toolchain registration in bzlmod is done in MODULE file - register_toolchains = False, - register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = toolchain_attr.ignore_root_user_error, - ) - - # We collect all of the toolchain names to create - # the INTERPRETER_LABELS map. This is used - # by interpreter_extensions.bzl - toolchains.append(toolchain_attr.name) + # If we are in the root module we always register the toolchain. + # We wait to register the default toolchain till the end. + if mod.is_root: + toolchain_names.append(toolchain_attr.name) + + # If we have the default version or we only have one toolchain + # in the root module we set the toolchain as the default toolchain. + if toolchain_attr.is_default or len(mod.tags.toolchain) == 1: + # We have already found one default toolchain, and we can + # only have one. + if default_toolchain_name != None: + fail("""We found more than one toolchain that is marked +as the default version. Only set one toolchain with is_default set as +True. The toolchain is named: {}""".format(toolchain_attr.name)) + + # We register the default toolchain. + _python_register_toolchains(toolchain_attr, False) + default_toolchain_name = toolchain_attr.name + else: + # Always register toolchains that are in the root module. + _python_register_toolchains(toolchain_attr, version_constraint = True) + else: + # We add the toolchain to a map, and we later create the + # toolchain if the root module does not have a toolchain with + # the same name. We have to loop through all of the modules to + # ensure that we get a full list of the root toolchains. + sub_toolchains_map[toolchain_attr.name] = toolchain_attr + + # We did not find a default toolchain so we fail. + if default_toolchain_name == None: + fail("""Unable to find a default toolchain in the root module. +Please define a toolchain that has is_version set to True.""") + # Create the toolchains in the submodule(s). + for name, toolchain_attr in sub_toolchains_map.items(): + # We cannot have a toolchain in a sub module that has the same name of + # a toolchain in the root module. This will cause name clashing. + if name in toolchain_names: + _print_warn("""Not creating the toolchain from sub module, with the name {}. The root + module has a toolchain of the same name.""".format(toolchain_attr.name)) + continue + toolchain_names.append(name) + _python_register_toolchains(toolchain_attr, True) + + # Create the hub for the interpreters and the + # the default toolchain. hub_repo( name = "pythons_hub", - toolchains = toolchains, + toolchains = toolchain_names, + default_toolchain = default_toolchain_name, ) python = module_extension( - doc = "Bzlmod extension that is used to register a Python toolchain.", + doc = """Bzlmod extension that is used to register Python toolchains. +""", implementation = _python_impl, tag_classes = { "toolchain": tag_class( + doc = """Tag class used to register Python toolchains. +Use this tag class to register one or more Python toolchains. This class +is also potentially called by sub modules. The following covers different +business rules and use cases. + +Toolchains in the Root Module + +This class registers all toolchains in the root module. + +Toolchains in Sub Modules + +It will create a toolchain that is in a sub module, if the toolchain +of the same name does not exist in the root module. The extension stops name +clashing between toolchains in the root module and toolchains in sub modules. +You cannot configure more than one toolchain as the default toolchain. + +Toolchain set as the default version + +This extension will not create a toolchain that exists in a sub module, +if the sub module toolchain is marked as the default version. If you have +more than one toolchain in your root module, you need to set one of the +toolchains as the default version. If there is only one toolchain it +is set as the default toolchain. +""", attrs = { "configure_coverage_tool": attr.bool( mandatory = False, @@ -56,6 +138,10 @@ python = module_extension( doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", mandatory = False, ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), "name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True), }, diff --git a/python/repositories.bzl b/python/repositories.bzl index 358df4341b..4f36b12a14 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -463,7 +463,6 @@ def python_register_toolchains( register_coverage_tool = False, set_python_version_constraint = False, tool_versions = TOOL_VERSIONS, - bzlmod = False, **kwargs): """Convenience macro for users which does typical setup. @@ -486,9 +485,15 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. - bzlmod: Whether this rule is being run under a bzlmod module extension. **kwargs: passed to each python_repositories call. """ + + # If we have @@ we have bzlmod + bzlmod = str(Label("//:unused")).startswith("@@") + if bzlmod: + # you cannot used native.register_toolchains when using bzlmod. + register_toolchains = False + base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) if python_version in MINOR_MAPPING: From 62e95a46fec4421d2ae8060c02ea45f800f5ce57 Mon Sep 17 00:00:00 2001 From: Zhongpeng Lin Date: Sat, 27 May 2023 18:35:51 -0700 Subject: [PATCH 02/16] build: Upgrade Gazelle to v0.31.0 (#1240) Gazelle v0.31.0 comes with a lifecycle manager for extension, allowing the Python extension to properly shut down external Python processes without relying on timer. Upgrading Gazelle in this PR. Using the lifecycle manager will come next. --- gazelle/MODULE.bazel | 2 +- gazelle/README.md | 10 +++++----- gazelle/WORKSPACE | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index bd634020f3..ae94a5f863 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "rules_python", version = "0.18.0") bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go") -bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") +bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") diff --git a/gazelle/README.md b/gazelle/README.md index e36f3a303a..ba8520d36b 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -35,14 +35,14 @@ Here is a snippet of a `MODULE.bazel` file. ```starlark # The following stanza defines the dependency rules_python. -bazel_dep(name = "rules_python", version = "0.20.0") +bazel_dep(name = "rules_python", version = "0.22.0") -# The following stanza defines the dependency rules_python. +# The following stanza defines the dependency rules_python_gazelle_plugin. # For typical setups you set the version. -bazel_dep(name = "rules_python_gazelle_plugin", version = "0.20.0") +bazel_dep(name = "rules_python_gazelle_plugin", version = "0.22.0") -# The following stanza defines the dependency rules_python. -bazel_dep(name = "gazelle", version = "0.30.0", repo_name = "bazel_gazelle") +# The following stanza defines the dependency gazelle. +bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") # Import the python repositories generated by the given module extension into the scope of the current module. use_repo(python, "python3_9") diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index 55cf1b0d40..eef16e924d 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -13,10 +13,10 @@ http_archive( http_archive( name = "bazel_gazelle", - sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97", + sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", ], ) From 18a7bb5b506538835c75d46d4245da08fa695df3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 30 May 2023 14:00:58 -0700 Subject: [PATCH 03/16] fix: make `import python.runfiles` work with `--experimental_python_import_all_repositories=false` (#1243) Because `--experimental_python_import_all_repositories` defaults to true, every repository's directory is added to `sys.path`, which makes `import python.runfiles` work. However, we shouldn't rely on that behavior for a couple reasons: * We recommend disabling it for fewer sys.path entries (even non-Python related repos get added to the path). * Some users _must_ disable it because the resulting PYTHONPATH is too long. To fix, set the `imports` attribute on `//python/runfiles:runfiles` so that `import python.runfiles` works. The net result is the `rules_python` repo directory is added to sys.path even if `--experimental_python_import_all_repositories=false`. Fixes #1241 --- python/runfiles/BUILD.bazel | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel index 3a93d40f32..c6cfc2fa94 100644 --- a/python/runfiles/BUILD.bazel +++ b/python/runfiles/BUILD.bazel @@ -27,6 +27,11 @@ py_library( "__init__.py", "runfiles.py", ], + imports = [ + # Add the repo root so `import python.runfiles.runfiles` works. This makes it agnostic + # to the --experimental_python_import_all_repositories setting. + "../..", + ], visibility = ["//visibility:public"], ) From 148622aa92fdd5afcaf9f153d7bb6afce713e553 Mon Sep 17 00:00:00 2001 From: Chris Love <335402+chrislovecnm@users.noreply.github.com> Date: Wed, 31 May 2023 19:48:28 -0600 Subject: [PATCH 04/16] feat(bzlmod): Moving register.toolchains internal (#1238) This commit moves the register.toolchains bzlmod call to inside of rules_python. Instead of a user having to call register.toolchains in their MODULE.bazel, rules_python/MODULE.bazel calls it on the internal hub. This is a breaking change if you are using register.toolchains inside of submodules. Using register.toolchains inside of submodules is not recommended anyways. This is now broken because we are not creating a repo for every Python version toolchain. All of the toochain calls exist now in the hub's repo BUILD.bazel file. --- .bazelrc | 4 +- BZLMOD_SUPPORT.md | 2 +- MODULE.bazel | 5 + README.md | 37 +++-- examples/bzlmod/BUILD.bazel | 19 --- examples/bzlmod/MODULE.bazel | 26 +--- examples/bzlmod/other_module/MODULE.bazel | 15 -- examples/bzlmod/tests/BUILD.bazel | 142 ++++++++++++++++++ examples/bzlmod/tests/cross_version_test.py | 39 +++++ examples/bzlmod/tests/version.py | 17 +++ examples/bzlmod/tests/version_test.py | 23 +++ examples/bzlmod/tests/version_test.sh | 24 +++ .../bzlmod_build_file_generation/BUILD.bazel | 8 - .../bzlmod_build_file_generation/MODULE.bazel | 21 --- examples/py_proto_library/MODULE.bazel | 6 +- python/extensions/private/interpreter_hub.bzl | 69 --------- python/extensions/private/pythons_hub.bzl | 136 +++++++++++++++++ python/extensions/python.bzl | 120 ++++++++++++--- python/private/toolchains_repo.bzl | 81 +++++++--- python/repositories.bzl | 14 +- 20 files changed, 579 insertions(+), 229 deletions(-) create mode 100644 examples/bzlmod/tests/BUILD.bazel create mode 100644 examples/bzlmod/tests/cross_version_test.py create mode 100644 examples/bzlmod/tests/version.py create mode 100644 examples/bzlmod/tests/version_test.py create mode 100755 examples/bzlmod/tests/version_test.sh delete mode 100644 python/extensions/private/interpreter_hub.bzl create mode 100644 python/extensions/private/pythons_hub.bzl diff --git a/.bazelrc b/.bazelrc index fe542b3722..3c317412ce 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/BZLMOD_SUPPORT.md b/BZLMOD_SUPPORT.md index cf95d12a0e..8efd0df6d7 100644 --- a/BZLMOD_SUPPORT.md +++ b/BZLMOD_SUPPORT.md @@ -31,7 +31,7 @@ A second example, in [examples/bzlmod_build_file_generation](examples/bzlmod_bui This rule set does not have full feature partity with the older `WORKSPACE` type configuration: -1. Multiple python versions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example. +1. Multiple pip extensions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example. 2. Gazelle does not support finding deps in sub-modules. For instance we can have a dep like ` "@our_other_module//other_module/pkg:lib",` in a `py_test` definition. Check ["issues"](/bazelbuild/rules_python/issues) for an up to date list. diff --git a/MODULE.bazel b/MODULE.bazel index ddd946c78a..b45c2ff03d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -47,5 +47,10 @@ use_repo( "pypi__coverage_cp39_x86_64-unknown-linux-gnu", ) +# We need to do another use_extension call to expose the "pythons_hub" +# repo. python = use_extension("@rules_python//python/extensions:python.bzl", "python") use_repo(python, "pythons_hub") + +# This call registers the Python toolchains. +register_toolchains("@pythons_hub//:all") diff --git a/README.md b/README.md index a3f18869e6..6893a1da28 100644 --- a/README.md +++ b/README.md @@ -53,32 +53,39 @@ To import rules_python in your project, you first need to add it to your To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `MODULE.bazel` file: -```python +```starlark # Find the latest version number here: https://github.com/bazelbuild/rules_python/releases # and change the version number if needed in the line below. -bazel_dep(name = "rules_python", version = "0.20.0") +bazel_dep(name = "rules_python", version = "0.21.0") + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + name = "python", + configure_coverage_tool = True, + is_default = True, + python_version = "3.9", +) -# You do not have to use pip for the toolchain, but most people -# will use it for the dependency management. -pip = use_extension("@rules_python//python:extensions.bzl", "pip") +interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") +interpreter.install( + name = "interpreter", + python_name = "python", +) +use_repo(interpreter, "interpreter") +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( name = "pip", + incompatible_generate_aliases = True, + python_interpreter_target = "@interpreter//:python", requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", ) - use_repo(pip, "pip") - -# Register a specific python toolchain instead of using the host version -python = use_extension("@rules_python//python:extensions.bzl", "python") - -use_repo(python, "python3_10_toolchains") - -register_toolchains( - "@python3_10_toolchains//:all", -) ``` +For more documentation see the bzlmod examples under the [examples](examples) folder. + ### Using a WORKSPACE file To import rules_python in your project, you first need to add it to your diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 86498226f9..0a068ce640 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -8,9 +8,6 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement") load("@python_39//:defs.bzl", py_test_with_transition = "py_test") - -# This is not working yet till the toolchain hub registration is working -# load("@python_310//:defs.bzl", py_binary_310 = "py_binary") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -50,22 +47,6 @@ py_binary( ], ) -# This is still WIP. Not working till we have the toolchain -# registration functioning. - -# This is used for testing mulitple versions of Python. This is -# used only when you need to support multiple versions of Python -# in the same project. -# py_binary_310( -# name = "main_310", -# srcs = ["__main__.py"], -# main = "__main__.py", -# visibility = ["//:__subpackages__"], -# deps = [ -# ":lib", -# ], -# ) - # see https://bazel.build/reference/be/python#py_test py_test( name = "test", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index bb4183bde2..24bb4581f4 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -15,14 +15,10 @@ local_path_override( # We also use the same value in the python.host_python_interpreter call. PYTHON_NAME_39 = "python_39" -PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" - INTERPRETER_NAME_39 = "interpreter_39" PYTHON_NAME_310 = "python_310" -PYTHON_310_TOOLCHAINS = PYTHON_NAME_310 + "_toolchains" - INTERPRETER_NAME_310 = "interpreter_310" # We next initialize the python toolchain using the extension. @@ -50,25 +46,9 @@ python.toolchain( python_version = "3.10", ) -# use_repo imports one or more repos generated by the given module extension -# into the scope of the current module. We are importing the various repos -# created by the above python.toolchain calls. -use_repo( - python, - PYTHON_NAME_39, - PYTHON_39_TOOLCHAINS, - PYTHON_NAME_310, - PYTHON_310_TOOLCHAINS, -) - -# This call registers the Python toolchains. -# Note: there is work under way to move this code to within -# rules_python, and the user won't have to make this call, -# unless they are registering custom toolchains. -register_toolchains( - "@{}//:all".format(PYTHON_39_TOOLCHAINS), - "@{}//:all".format(PYTHON_310_TOOLCHAINS), -) +# You only need to load this repositories if you are using muiltple Python versions. +# See the tests folder for various examples. +use_repo(python, PYTHON_NAME_39, "python_aliases") # The interpreter extension discovers the platform specific Python binary. # It creates a symlink to the binary, and we pass the label to the following diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index eebfbcaa58..5fb745266f 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -12,12 +12,8 @@ bazel_dep(name = "rules_python", version = "") # testing purposes. PYTHON_NAME_39 = "python_39" -PYTHON_39_TOOLCHAINS = PYTHON_NAME_39 + "_toolchains" - PYTHON_NAME_311 = "python_311" -PYTHON_311_TOOLCHAINS = PYTHON_NAME_311 + "_toolchains" - python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( # This name is used in the various use_repo statements @@ -42,16 +38,5 @@ python.toolchain( use_repo( python, PYTHON_NAME_39, - PYTHON_39_TOOLCHAINS, PYTHON_NAME_311, - PYTHON_311_TOOLCHAINS, -) - -# This call registers the Python toolchains. -# Note: there is work under way to move this code to within -# rules_python, and the user won't have to make this call, -# unless they are registering custom toolchains. -register_toolchains( - "@{}//:all".format(PYTHON_39_TOOLCHAINS), - "@{}//:all".format(PYTHON_311_TOOLCHAINS), ) diff --git a/examples/bzlmod/tests/BUILD.bazel b/examples/bzlmod/tests/BUILD.bazel new file mode 100644 index 0000000000..5331f4ab96 --- /dev/null +++ b/examples/bzlmod/tests/BUILD.bazel @@ -0,0 +1,142 @@ +load("@python_aliases//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test") +load("@python_aliases//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test") +load("@python_aliases//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test") +load("@rules_python//python:defs.bzl", "py_binary", "py_test") + +py_binary( + name = "version_default", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_9( + name = "version_3_9", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_10( + name = "version_3_10", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_11( + name = "version_3_11", + srcs = ["version.py"], + main = "version.py", +) + +# This is a work in progress and the commented +# tests will not work until we can support +# multiple pips with bzlmod. + +#py_test( +# name = "my_lib_default_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_9( +# name = "my_lib_3_9_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_10( +# name = "my_lib_3_10_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_11( +# name = "my_lib_3_11_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +py_test( + name = "version_default_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.9"}, # The default defined in the WORKSPACE. + main = "version_test.py", +) + +py_test_3_9( + name = "version_3_9_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.9"}, + main = "version_test.py", +) + +py_test_3_10( + name = "version_3_10_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.10"}, + main = "version_test.py", +) + +py_test_3_11( + name = "version_3_11_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.11"}, + main = "version_test.py", +) + +py_test( + name = "version_default_takes_3_10_subprocess_test", + srcs = ["cross_version_test.py"], + data = [":version_3_10"], + env = { + "SUBPROCESS_VERSION_CHECK": "3.10", + "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_10)", + "VERSION_CHECK": "3.9", + }, + main = "cross_version_test.py", +) + +py_test_3_10( + name = "version_3_10_takes_3_9_subprocess_test", + srcs = ["cross_version_test.py"], + data = [":version_3_9"], + env = { + "SUBPROCESS_VERSION_CHECK": "3.9", + "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_9)", + "VERSION_CHECK": "3.10", + }, + main = "cross_version_test.py", +) + +sh_test( + name = "version_test_binary_default", + srcs = ["version_test.sh"], + data = [":version_default"], + env = { + "VERSION_CHECK": "3.9", # The default defined in the WORKSPACE. + "VERSION_PY_BINARY": "$(rootpath :version_default)", + }, +) + +sh_test( + name = "version_test_binary_3_9", + srcs = ["version_test.sh"], + data = [":version_3_9"], + env = { + "VERSION_CHECK": "3.9", + "VERSION_PY_BINARY": "$(rootpath :version_3_9)", + }, +) + +sh_test( + name = "version_test_binary_3_10", + srcs = ["version_test.sh"], + data = [":version_3_10"], + env = { + "VERSION_CHECK": "3.10", + "VERSION_PY_BINARY": "$(rootpath :version_3_10)", + }, +) diff --git a/examples/bzlmod/tests/cross_version_test.py b/examples/bzlmod/tests/cross_version_test.py new file mode 100644 index 0000000000..437be2ed5a --- /dev/null +++ b/examples/bzlmod/tests/cross_version_test.py @@ -0,0 +1,39 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +process = subprocess.run( + [os.getenv("SUBPROCESS_VERSION_PY_BINARY")], + stdout=subprocess.PIPE, + universal_newlines=True, +) + +subprocess_current = process.stdout.strip() +subprocess_expected = os.getenv("SUBPROCESS_VERSION_CHECK") + +if subprocess_current != subprocess_expected: + print( + f"expected subprocess version '{subprocess_expected}' is different than returned '{subprocess_current}'" + ) + sys.exit(1) + +expected = os.getenv("VERSION_CHECK") +current = f"{sys.version_info.major}.{sys.version_info.minor}" + +if current != expected: + print(f"expected version '{expected}' is different than returned '{current}'") + sys.exit(1) diff --git a/examples/bzlmod/tests/version.py b/examples/bzlmod/tests/version.py new file mode 100644 index 0000000000..2d293c1571 --- /dev/null +++ b/examples/bzlmod/tests/version.py @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +print(f"{sys.version_info.major}.{sys.version_info.minor}") diff --git a/examples/bzlmod/tests/version_test.py b/examples/bzlmod/tests/version_test.py new file mode 100644 index 0000000000..444f5e4321 --- /dev/null +++ b/examples/bzlmod/tests/version_test.py @@ -0,0 +1,23 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +expected = os.getenv("VERSION_CHECK") +current = f"{sys.version_info.major}.{sys.version_info.minor}" + +if current != expected: + print(f"expected version '{expected}' is different than returned '{current}'") + sys.exit(1) diff --git a/examples/bzlmod/tests/version_test.sh b/examples/bzlmod/tests/version_test.sh new file mode 100755 index 0000000000..3bedb95ef9 --- /dev/null +++ b/examples/bzlmod/tests/version_test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit -o nounset -o pipefail + +version_py_binary=$("${VERSION_PY_BINARY}") + +if [[ "${version_py_binary}" != "${VERSION_CHECK}" ]]; then + echo >&2 "expected version '${VERSION_CHECK}' is different than returned '${version_py_binary}'" + exit 1 +fi diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index 05a15cce28..498969ba3a 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -7,7 +7,6 @@ # requirements. load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") -load("@python//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") @@ -72,13 +71,6 @@ gazelle( gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", ) -py_test_with_transition( - name = "test_with_transition", - srcs = ["__test__.py"], - main = "__test__.py", - deps = [":bzlmod_build_file_generation"], -) - # The following targets are created and maintained by gazelle py_library( name = "bzlmod_build_file_generation", diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index 45a1318ac8..d69dd7da48 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -48,38 +48,17 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") # We also use the same name for python.host_python_interpreter. PYTHON_NAME = "python" -PYTHON_TOOLCHAINS = PYTHON_NAME + "_toolchains" - INTERPRETER_NAME = "interpreter" # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. python.toolchain( - # This name is used in the various use_repo statements - # below, and in the local extension that is in this - # example. name = PYTHON_NAME, configure_coverage_tool = True, is_default = True, python_version = "3.9", ) -# Import the python repositories generated by the given module extension -# into the scope of the current module. -# All of the python3 repositories use the PYTHON_NAME as there prefix. They -# are not catenated for ease of reading. -use_repo( - python, - PYTHON_NAME, - PYTHON_TOOLCHAINS, -) - -# Register an already-defined toolchain so that Bazel can use it during -# toolchain resolution. -register_toolchains( - "@{}//:all".format(PYTHON_TOOLCHAINS), -) - # The interpreter extension discovers the platform specific Python binary. # It creates a symlink to the binary, and we pass the label to the following # pip.parse call. diff --git a/examples/py_proto_library/MODULE.bazel b/examples/py_proto_library/MODULE.bazel index 6fb1a05548..3116c40b2d 100644 --- a/examples/py_proto_library/MODULE.bazel +++ b/examples/py_proto_library/MODULE.bazel @@ -18,11 +18,7 @@ python.toolchain( configure_coverage_tool = True, python_version = "3.9", ) -use_repo(python, "python3_9_toolchains") - -register_toolchains( - "@python3_9_toolchains//:all", -) +use_repo(python, "python3_9") # We are using rules_proto to define rules_proto targets to be consumed by py_proto_library. bazel_dep(name = "rules_proto", version = "5.3.0-21.7") diff --git a/python/extensions/private/interpreter_hub.bzl b/python/extensions/private/interpreter_hub.bzl deleted file mode 100644 index 82fcbf698f..0000000000 --- a/python/extensions/private/interpreter_hub.bzl +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" - -load("//python:versions.bzl", "WINDOWS_NAME") -load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platform") - -_build_file_for_hub_template = """ -INTERPRETER_LABELS = {{ -{interpreter_labels} -}} -DEFAULT_TOOLCHAIN_NAME = "{default}" -""" - -_line_for_hub_template = """\ - "{name}": Label("@{name}_{platform}//:{path}"), -""" - -def _hub_repo_impl(rctx): - (os, arch) = get_host_os_arch(rctx) - platform = get_host_platform(os, arch) - - rctx.file("BUILD.bazel", "") - is_windows = (os == WINDOWS_NAME) - path = "python.exe" if is_windows else "bin/python3" - - interpreter_labels = "\n".join([_line_for_hub_template.format( - name = name, - platform = platform, - path = path, - ) for name in rctx.attr.toolchains]) - - rctx.file( - "interpreters.bzl", - _build_file_for_hub_template.format( - interpreter_labels = interpreter_labels, - default = rctx.attr.default_toolchain, - ), - ) - -hub_repo = repository_rule( - doc = """\ -This private rule create a repo with a BUILD file that contains a map of interpreter names -and the labels to said interpreters. This map is used to by the interpreter hub extension. -""", - implementation = _hub_repo_impl, - attrs = { - "default_toolchain": attr.string( - doc = "Name of the default toolchain", - mandatory = True, - ), - "toolchains": attr.string_list( - doc = "List of the base names the toolchain repo defines.", - mandatory = True, - ), - }, -) diff --git a/python/extensions/private/pythons_hub.bzl b/python/extensions/private/pythons_hub.bzl new file mode 100644 index 0000000000..5baaef96fd --- /dev/null +++ b/python/extensions/private/pythons_hub.bzl @@ -0,0 +1,136 @@ +# Copyright 2023 The Bazel Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" + +load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME") +load( + "//python/private:toolchains_repo.bzl", + "get_host_os_arch", + "get_host_platform", + "get_repository_name", + "python_toolchain_build_file_content", +) + +def _have_same_length(*lists): + if not lists: + fail("expected at least one list") + return len({len(length): None for length in lists}) == 1 + +def _get_version(python_version): + # we need to get the MINOR_MAPPING or use the full version + if python_version in MINOR_MAPPING: + python_version = MINOR_MAPPING[python_version] + return python_version + +def _python_toolchain_build_file_content( + prefixes, + python_versions, + set_python_version_constraints, + user_repository_names, + workspace_location): + """This macro iterates over each of the lists and returns the toolchain content. + + python_toolchain_build_file_content is called to generate each of the toolchain + definitions. + """ + + if not _have_same_length(python_versions, set_python_version_constraints, user_repository_names): + fail("all lists must have the same length") + + rules_python = get_repository_name(workspace_location) + + # Iterate over the length of python_versions and call + # build the toolchain content by calling python_toolchain_build_file_content + return "\n".join([python_toolchain_build_file_content( + prefix = prefixes[i], + python_version = _get_version(python_versions[i]), + set_python_version_constraint = set_python_version_constraints[i], + user_repository_name = user_repository_names[i], + rules_python = rules_python, + ) for i in range(len(python_versions))]) + +_build_file_for_hub_template = """ +INTERPRETER_LABELS = {{ +{interpreter_labels} +}} +""" + +_line_for_hub_template = """\ + "{name}": Label("@{name}_{platform}//:{path}"), +""" + +def _hub_repo_impl(rctx): + # Create the various toolchain definitions and + # write them to the BUILD file. + rctx.file( + "BUILD.bazel", + _python_toolchain_build_file_content( + rctx.attr.toolchain_prefixes, + rctx.attr.toolchain_python_versions, + rctx.attr.toolchain_set_python_version_constraints, + rctx.attr.toolchain_user_repository_names, + rctx.attr._rules_python_workspace, + ), + executable = False, + ) + + (os, arch) = get_host_os_arch(rctx) + platform = get_host_platform(os, arch) + is_windows = (os == WINDOWS_NAME) + path = "python.exe" if is_windows else "bin/python3" + + # Create a dict that is later used to create + # a symlink to a interpreter. + interpreter_labels = "".join([_line_for_hub_template.format( + name = name, + platform = platform, + path = path, + ) for name in rctx.attr.toolchain_user_repository_names]) + + rctx.file( + "interpreters.bzl", + _build_file_for_hub_template.format( + interpreter_labels = interpreter_labels, + ), + executable = False, + ) + +hub_repo = repository_rule( + doc = """\ +This private rule create a repo with a BUILD file that contains a map of interpreter names +and the labels to said interpreters. This map is used to by the interpreter hub extension. +This rule also writes out the various toolchains for the different Python versions. +""", + implementation = _hub_repo_impl, + attrs = { + "toolchain_prefixes": attr.string_list( + doc = "List prefixed for the toolchains", + mandatory = True, + ), + "toolchain_python_versions": attr.string_list( + doc = "List of Python versions for the toolchains", + mandatory = True, + ), + "toolchain_set_python_version_constraints": attr.string_list( + doc = "List of version contraints for the toolchains", + mandatory = True, + ), + "toolchain_user_repository_names": attr.string_list( + doc = "List of the user repo names for the toolchains", + mandatory = True, + ), + "_rules_python_workspace": attr.label(default = Label("//:does_not_matter_what_this_name_is")), + }, +) diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index cae1988e8a..4732cfb078 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -14,8 +14,28 @@ "Python toolchain module extensions for use with bzlmod" -load("@rules_python//python:repositories.bzl", "python_register_toolchains") -load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo") +load("//python:repositories.bzl", "python_register_toolchains") +load("//python/extensions/private:pythons_hub.bzl", "hub_repo") +load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases") + +# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all +# targets using any of these toolchains due to the changed repository name. +_MAX_NUM_TOOLCHAINS = 9999 +_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) + +def _toolchain_prefix(index, name): + """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting. + + Examples: + _toolchain_prefix( 2, "foo") == "_0002_foo_" + _toolchain_prefix(2000, "foo") == "_2000_foo_" + """ + return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name) + +def _left_pad_zero(index, length): + if index < 0: + fail("index must be non-negative") + return ("0" * length + str(index))[-length:] # Printing a warning msg not debugging, so we have to disable # the buildifier check. @@ -24,6 +44,8 @@ def _print_warn(msg): print("WARNING:", msg) def _python_register_toolchains(toolchain_attr, version_constraint): + """Calls python_register_toolchains and returns a struct used to collect the toolchains. + """ python_register_toolchains( name = toolchain_attr.name, python_version = toolchain_attr.python_version, @@ -31,24 +53,33 @@ def _python_register_toolchains(toolchain_attr, version_constraint): ignore_root_user_error = toolchain_attr.ignore_root_user_error, set_python_version_constraint = version_constraint, ) + return struct( + python_version = toolchain_attr.python_version, + set_python_version_constraint = str(version_constraint), + name = toolchain_attr.name, + ) def _python_impl(module_ctx): - # We collect all of the toolchain names to create - # the INTERPRETER_LABELS map. This is used - # by interpreter_extensions.bzl via the hub_repo call below. - toolchain_names = [] + # Use to store all of the toolchains + toolchains = [] - # Used to store the default toolchain name so we can pass it to the hub - default_toolchain_name = None + # Used to check if toolchains already exist + toolchain_names = [] # Used to store toolchains that are in sub modules. sub_toolchains_map = {} + default_toolchain = None + python_versions = {} for mod in module_ctx.modules: for toolchain_attr in mod.tags.toolchain: # If we are in the root module we always register the toolchain. # We wait to register the default toolchain till the end. if mod.is_root: + if toolchain_attr.name in toolchain_names: + fail("""We found more than one toolchain that is named: {}. +All toolchains must have an unique name.""".format(toolchain_attr.name)) + toolchain_names.append(toolchain_attr.name) # If we have the default version or we only have one toolchain @@ -56,17 +87,27 @@ def _python_impl(module_ctx): if toolchain_attr.is_default or len(mod.tags.toolchain) == 1: # We have already found one default toolchain, and we can # only have one. - if default_toolchain_name != None: + if default_toolchain != None: fail("""We found more than one toolchain that is marked as the default version. Only set one toolchain with is_default set as True. The toolchain is named: {}""".format(toolchain_attr.name)) - # We register the default toolchain. - _python_register_toolchains(toolchain_attr, False) - default_toolchain_name = toolchain_attr.name - else: - # Always register toolchains that are in the root module. - _python_register_toolchains(toolchain_attr, version_constraint = True) + # We store the default toolchain to have it + # as the last toolchain added to toolchains + default_toolchain = _python_register_toolchains( + toolchain_attr, + version_constraint = False, + ) + python_versions[toolchain_attr.python_version] = toolchain_attr.name + continue + + toolchains.append( + _python_register_toolchains( + toolchain_attr, + version_constraint = True, + ), + ) + python_versions[toolchain_attr.python_version] = toolchain_attr.name else: # We add the toolchain to a map, and we later create the # toolchain if the root module does not have a toolchain with @@ -75,7 +116,7 @@ True. The toolchain is named: {}""".format(toolchain_attr.name)) sub_toolchains_map[toolchain_attr.name] = toolchain_attr # We did not find a default toolchain so we fail. - if default_toolchain_name == None: + if default_toolchain == None: fail("""Unable to find a default toolchain in the root module. Please define a toolchain that has is_version set to True.""") @@ -88,14 +129,39 @@ Please define a toolchain that has is_version set to True.""") module has a toolchain of the same name.""".format(toolchain_attr.name)) continue toolchain_names.append(name) - _python_register_toolchains(toolchain_attr, True) - - # Create the hub for the interpreters and the - # the default toolchain. + toolchains.append( + _python_register_toolchains( + toolchain_attr, + version_constraint = True, + ), + ) + python_versions[toolchain_attr.python_version] = toolchain_attr.name + + # The last toolchain in the BUILD file is set as the default + # toolchain. We need the default last. + toolchains.append(default_toolchain) + + if len(toolchains) > _MAX_NUM_TOOLCHAINS: + fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) + + # Create the pythons_hub repo for the interpreter meta data and the + # the various toolchains. hub_repo( name = "pythons_hub", - toolchains = toolchain_names, - default_toolchain = default_toolchain_name, + toolchain_prefixes = [ + _toolchain_prefix(index, toolchain.name) + for index, toolchain in enumerate(toolchains) + ], + toolchain_python_versions = [t.python_version for t in toolchains], + toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains], + toolchain_user_repository_names = [t.name for t in toolchains], + ) + + # This is require in order to support multiple version py_test + # and py_binary + multi_toolchain_aliases( + name = "python_aliases", + python_versions = python_versions, ) python = module_extension( @@ -142,8 +208,14 @@ is set as the default toolchain. mandatory = False, doc = "Whether the toolchain is the default version", ), - "name": attr.string(mandatory = True), - "python_version": attr.string(mandatory = True), + "name": attr.string( + mandatory = True, + doc = "Name of the toolchain", + ), + "python_version": attr.string( + mandatory = True, + doc = "The Python version that we are creating the toolchain for.", + ), }, ), }, diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 9bed73e55c..b5ac81a491 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -35,28 +35,37 @@ def get_repository_name(repository_workspace): dummy_label = "//:_" return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@" -def _toolchains_repo_impl(rctx): - python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( - rules_python = get_repository_name(rctx.attr._rules_python_workspace), - python_version = rctx.attr.python_version, - ) +def python_toolchain_build_file_content( + prefix, + python_version, + set_python_version_constraint, + user_repository_name, + rules_python): + """Creates the content for toolchain definitions for a build file. - build_content = """\ -# Generated by python/private/toolchains_repo.bzl -# -# These can be registered in the workspace file or passed to --extra_toolchains -# flag. By default all these toolchains are registered by the -# python_register_toolchains macro so you don't normally need to interact with -# these targets. + Args: + prefix: Python toolchain name prefixes + python_version: Python versions for the toolchains + set_python_version_constraint: string "True" or "False" + user_repository_name: names for the user repos + rules_python: rules_python label -""" + Returns: + build_content: Text containing toolchain definitions + """ + + python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( + rules_python = rules_python, + python_version = python_version, + ) - for [platform, meta] in PLATFORMS.items(): - build_content += """\ -# Bazel selects this toolchain to get a Python interpreter -# for executing build actions. + # We create a list of toolchain content from iterating over + # the enumeration of PLATFORMS. We enumerate PLATFORMS in + # order to get us an index to increment the increment. + return "".join([ + """ toolchain( - name = "{platform}_toolchain", + name = "{prefix}{platform}_toolchain", target_compatible_with = {compatible_with}, target_settings = ["{python_version_constraint}"] if {set_python_version_constraint} else [], toolchain = "@{user_repository_name}_{platform}//:python_runtimes", @@ -64,14 +73,42 @@ toolchain( ) """.format( compatible_with = meta.compatible_with, - name = rctx.attr.name, platform = platform, python_version_constraint = python_version_constraint, - set_python_version_constraint = rctx.attr.set_python_version_constraint, - user_repository_name = rctx.attr.user_repository_name, + # We have to use a String value here because bzlmod is passing in a + # string as we cannot have list of bools in build rule attribues. + # This if statement does not appear to work unless it is in the + # toolchain file. + set_python_version_constraint = True if set_python_version_constraint == "True" else False, + user_repository_name = user_repository_name, + prefix = prefix, ) + for platform, meta in PLATFORMS.items() + ]) + +def _toolchains_repo_impl(rctx): + build_content = """\ +# Generated by python/private/toolchains_repo.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains +# flag. By default all these toolchains are registered by the +# python_register_toolchains macro so you don't normally need to interact with +# these targets. + +""" + + # Get the repository name + rules_python = get_repository_name(rctx.attr._rules_python_workspace) + + toolchains = python_toolchain_build_file_content( + prefix = "", + python_version = rctx.attr.python_version, + set_python_version_constraint = str(rctx.attr.set_python_version_constraint), + user_repository_name = rctx.attr.user_repository_name, + rules_python = rules_python, + ) - rctx.file("BUILD.bazel", build_content) + rctx.file("BUILD.bazel", build_content + toolchains) toolchains_repo = repository_rule( _toolchains_repo_impl, diff --git a/python/repositories.bzl b/python/repositories.bzl index 4f36b12a14..e841e2888c 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -564,16 +564,20 @@ def python_register_toolchains( platform = platform, )) - toolchains_repo( - name = toolchain_repo_name, + toolchain_aliases( + name = name, python_version = python_version, - set_python_version_constraint = set_python_version_constraint, user_repository_name = name, ) - toolchain_aliases( - name = name, + # in bzlmod we write out our own toolchain repos + if bzlmod: + return + + toolchains_repo( + name = toolchain_repo_name, python_version = python_version, + set_python_version_constraint = set_python_version_constraint, user_repository_name = name, ) From 7f6de725d99aa3dec9e2b20cfc038bdc4704cc1c Mon Sep 17 00:00:00 2001 From: Aiden Grossman <39388941+boomanaiden154@users.noreply.github.com> Date: Fri, 2 Jun 2023 08:26:35 -0700 Subject: [PATCH 05/16] docs(compile_pip_requirements): Add note on requirements.txt VC (#1245) The documentation is currently ambiguous on whether or not to check requirements.txt into version control. This has raised some confusion in other projects (e.g., https://github.com/google/gematria/pull/3). This makes it clear that requirements.txt files produced by this rule should be checked into version control in an easy to find place to avoid confusion. --- docs/pip.md | 3 +++ python/pip_install/requirements.bzl | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docs/pip.md b/docs/pip.md index e4c3f21b79..8ad5b6903a 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -45,6 +45,9 @@ It also generates two targets for running pip-compile: - validate with `bazel test [name]_test` - update with `bazel run [name].update` +If you are using a version control system, the requirements.txt generated by this rule should +be checked into it to ensure that all developers/users have the same dependency versions. + **PARAMETERS** diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 7594471897..86fd408647 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -42,6 +42,9 @@ def compile_pip_requirements( - validate with `bazel test [name]_test` - update with `bazel run [name].update` + If you are using a version control system, the requirements.txt generated by this rule should + be checked into it to ensure that all developers/users have the same dependency versions. + Args: name: base name for generated targets, typically "requirements". extra_args: passed to pip-compile. From 4c365e71d1146f8f8f7c43ad19129c72ac49f4bf Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 2 Jun 2023 15:13:18 -0700 Subject: [PATCH 06/16] cleanup: Set toolchain target_setting directly instead of via inline ternary (#1246) The generated toolchain BUILD file is confusing to read because it relies on a ternary expression in the BUILD file to set the `target_settings` attribute. This makes debugging harder because, upon first look, all the toolchains appear to have the version constraint set. It's only upon closer inspection that you can see the 1-character difference of "False" vs "True" embedded into the middle of a line amongst other similar looking lines. Also: * Adds a bit of validation logic for the `set_python_version_constraint` argument because it's conceptually a boolean, but is passed as a string, so is prone to having an incorrect value passed. * Documents the `set_python_version_constraint` arg, since it has a particular range of values it accepts. --- python/private/toolchains_repo.bzl | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index b5ac81a491..f47ea8f064 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -46,18 +46,30 @@ def python_toolchain_build_file_content( Args: prefix: Python toolchain name prefixes python_version: Python versions for the toolchains - set_python_version_constraint: string "True" or "False" + set_python_version_constraint: string, "True" if the toolchain should + have the Python version constraint added as a requirement for + matching the toolchain, "False" if not. user_repository_name: names for the user repos rules_python: rules_python label Returns: build_content: Text containing toolchain definitions """ - - python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( - rules_python = rules_python, - python_version = python_version, - ) + if set_python_version_constraint == "True": + constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( + rules_python = rules_python, + python_version = python_version, + ) + target_settings = '["{}"]'.format(constraint) + elif set_python_version_constraint == "False": + target_settings = "[]" + else: + fail(("Invalid set_python_version_constraint value: got {} {}, wanted " + + "either the string 'True' or the string 'False'; " + + "(did you convert bool to string?)").format( + type(set_python_version_constraint), + repr(set_python_version_constraint), + )) # We create a list of toolchain content from iterating over # the enumeration of PLATFORMS. We enumerate PLATFORMS in @@ -67,19 +79,18 @@ def python_toolchain_build_file_content( toolchain( name = "{prefix}{platform}_toolchain", target_compatible_with = {compatible_with}, - target_settings = ["{python_version_constraint}"] if {set_python_version_constraint} else [], + target_settings = {target_settings}, toolchain = "@{user_repository_name}_{platform}//:python_runtimes", toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) """.format( compatible_with = meta.compatible_with, platform = platform, - python_version_constraint = python_version_constraint, # We have to use a String value here because bzlmod is passing in a # string as we cannot have list of bools in build rule attribues. # This if statement does not appear to work unless it is in the # toolchain file. - set_python_version_constraint = True if set_python_version_constraint == "True" else False, + target_settings = target_settings, user_repository_name = user_repository_name, prefix = prefix, ) From afdbedd3a58cb5d65e659e1dcc77108cfd22715e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 3 Jun 2023 11:55:34 -0700 Subject: [PATCH 07/16] fix(bzlmod): give precedence to the first seen versioned toolchain (#1244) This fixes an issue where the last submodule, instead of the first, was given precedence when creating versioned toolchains. To fix, when creating the map of versioned toolchains, if a version is already defined, then a subsequent usage is ignored. A warning is emitted when this occurs. This also fixes a similar problem that can occur to the root module. If the root module uses a particular version marked as the default, and is using the versioned rules, and a submodule also uses that same version, then the submodule's toolchain would be used. This happened because the root module's toolchain would be moved last, so its versioned rules would match the submodule's versioned toolchain. This also does some cleanup and refactoring to: * Compute the toolchains in one loop iteration * Give more informative error messages * Reject duplicate names within a module, even for the non-root module. * Reject duplicate versions within a module. --- python/extensions/python.bzl | 203 +++++++++++++++++++++++------------ 1 file changed, 133 insertions(+), 70 deletions(-) diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 4732cfb078..a604df6ca3 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -60,82 +60,97 @@ def _python_register_toolchains(toolchain_attr, version_constraint): ) def _python_impl(module_ctx): - # Use to store all of the toolchains + # The toolchain info structs to register, in the order to register them in. toolchains = [] - # Used to check if toolchains already exist - toolchain_names = [] - - # Used to store toolchains that are in sub modules. - sub_toolchains_map = {} + # We store the default toolchain separately to ensure it is the last + # toolchain added to toolchains. default_toolchain = None - python_versions = {} + + # Map of toolchain name to registering module + global_toolchain_names = {} + + # Map of string Major.Minor to the toolchain name and module name + global_toolchain_versions = {} for mod in module_ctx.modules: + module_toolchain_names = [] + module_toolchain_versions = [] + for toolchain_attr in mod.tags.toolchain: - # If we are in the root module we always register the toolchain. - # We wait to register the default toolchain till the end. + toolchain_name = toolchain_attr.name + + # Duplicate names within a module indicate a misconfigured module. + if toolchain_name in module_toolchain_names: + _fail_duplicate_module_toolchain_name(mod.name, toolchain_name) + module_toolchain_names.append(toolchain_name) + + # Ignore name collisions in the global scope because there isn't + # much else that can be done. Modules don't know and can't control + # what other modules do, so the first in the dependency graph wins. + if toolchain_name in global_toolchain_names: + _warn_duplicate_global_toolchain_name( + toolchain_name, + first_module = global_toolchain_names[toolchain_name], + second_module = mod.name, + ) + continue + global_toolchain_names[toolchain_name] = mod.name + + # Duplicate versions within a module indicate a misconfigured module. + toolchain_version = toolchain_attr.python_version + if toolchain_version in module_toolchain_versions: + _fail_duplicate_module_toolchain_version(toolchain_version, mod.name) + module_toolchain_versions.append(toolchain_version) + + # Ignore version collisions in the global scope because there isn't + # much else that can be done. Modules don't know and can't control + # what other modules do, so the first in the dependency graph wins. + if toolchain_version in global_toolchain_versions: + _warn_duplicate_global_toolchain_version( + toolchain_version, + first = global_toolchain_versions[toolchain_version], + second_toolchain_name = toolchain_name, + second_module_name = mod.name, + ) + continue + global_toolchain_versions[toolchain_version] = struct( + toolchain_name = toolchain_name, + module_name = mod.name, + ) + + # Only the root module is allowed to set the default toolchain + # to prevent submodules from clobbering each other. + # A single toolchain in the root module is treated as the default + # because it's unambigiuous. if mod.is_root: - if toolchain_attr.name in toolchain_names: - fail("""We found more than one toolchain that is named: {}. -All toolchains must have an unique name.""".format(toolchain_attr.name)) - - toolchain_names.append(toolchain_attr.name) - - # If we have the default version or we only have one toolchain - # in the root module we set the toolchain as the default toolchain. - if toolchain_attr.is_default or len(mod.tags.toolchain) == 1: - # We have already found one default toolchain, and we can - # only have one. - if default_toolchain != None: - fail("""We found more than one toolchain that is marked -as the default version. Only set one toolchain with is_default set as -True. The toolchain is named: {}""".format(toolchain_attr.name)) - - # We store the default toolchain to have it - # as the last toolchain added to toolchains - default_toolchain = _python_register_toolchains( - toolchain_attr, - version_constraint = False, - ) - python_versions[toolchain_attr.python_version] = toolchain_attr.name - continue - - toolchains.append( - _python_register_toolchains( - toolchain_attr, - version_constraint = True, - ), + is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1 + else: + is_default = False + + # We have already found one default toolchain, and we can only have + # one. + if is_default and default_toolchain != None: + _fail_multiple_default_toolchains( + first = default_toolchain.name, + second = toolchain_name, ) - python_versions[toolchain_attr.python_version] = toolchain_attr.name + + toolchain_info = _python_register_toolchains( + toolchain_attr, + version_constraint = not is_default, + ) + + if is_default: + default_toolchain = toolchain_info else: - # We add the toolchain to a map, and we later create the - # toolchain if the root module does not have a toolchain with - # the same name. We have to loop through all of the modules to - # ensure that we get a full list of the root toolchains. - sub_toolchains_map[toolchain_attr.name] = toolchain_attr + toolchains.append(toolchain_info) - # We did not find a default toolchain so we fail. + # A default toolchain is required so that the non-version-specific rules + # are able to match a toolchain. if default_toolchain == None: - fail("""Unable to find a default toolchain in the root module. -Please define a toolchain that has is_version set to True.""") - - # Create the toolchains in the submodule(s). - for name, toolchain_attr in sub_toolchains_map.items(): - # We cannot have a toolchain in a sub module that has the same name of - # a toolchain in the root module. This will cause name clashing. - if name in toolchain_names: - _print_warn("""Not creating the toolchain from sub module, with the name {}. The root - module has a toolchain of the same name.""".format(toolchain_attr.name)) - continue - toolchain_names.append(name) - toolchains.append( - _python_register_toolchains( - toolchain_attr, - version_constraint = True, - ), - ) - python_versions[toolchain_attr.python_version] = toolchain_attr.name + fail("No default toolchain found: exactly one toolchain must have " + + "is_default=True set") # The last toolchain in the BUILD file is set as the default # toolchain. We need the default last. @@ -161,9 +176,57 @@ Please define a toolchain that has is_version set to True.""") # and py_binary multi_toolchain_aliases( name = "python_aliases", - python_versions = python_versions, + python_versions = { + version: entry.toolchain_name + for version, entry in global_toolchain_versions.items() + }, ) +def _fail_duplicate_module_toolchain_name(module_name, toolchain_name): + fail(("Duplicate module toolchain name: module '{module}' attempted " + + "to use the name '{toolchain}' multiple times in itself").format( + toolchain = toolchain_name, + module = module_name, + )) + +def _warn_duplicate_global_toolchain_name(name, first_module, second_module): + _print_warn(( + "Ignoring toolchain '{name}' from module '{second_module}': " + + "Toolchain with the same name from module '{first_module}' has precedence" + ).format( + name = name, + first_module = first_module, + second_module = second_module, + )) + +def _fail_duplicate_module_toolchain_version(version, module): + fail(("Duplicate module toolchain version: module '{module}' attempted " + + "to use version '{version}' multiple times in itself").format( + version = version, + module = module, + )) + +def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name): + _print_warn(( + "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + + "Toolchain '{first_toolchain}' from module '{first_module}' " + + "already registered Python version {version} and has precedence" + ).format( + first_toolchain = first.toolchain_name, + first_module = first.module_name, + second_module = second_module_name, + second_toolchain = second_toolchain_name, + version = version, + )) + +def _fail_multiple_default_toolchains(first, second): + fail(("Multiple default toolchains: only one toolchain " + + "can have is_default=True. First default " + + "was toolchain '{first}'. Second was '{second}'").format( + first = first, + second = second, + )) + python = module_extension( doc = """Bzlmod extension that is used to register Python toolchains. """, @@ -184,15 +247,15 @@ Toolchains in Sub Modules It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and toolchains in sub modules. -You cannot configure more than one toolchain as the default toolchain. +You cannot configure more than one toolchain as the default toolchain. Toolchain set as the default version -This extension will not create a toolchain that exists in a sub module, +This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the -toolchains as the default version. If there is only one toolchain it -is set as the default toolchain. +toolchains as the default version. If there is only one toolchain it +is set as the default toolchain. """, attrs = { "configure_coverage_tool": attr.bool( From d573c60788c213c5ff1b07c42d202623cba988c9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 6 Jun 2023 01:19:30 +0900 Subject: [PATCH 08/16] chore: add a pre-commit hook to maintain deleted packages (#1208) Currently the users need to run the script manually and this PR adds a pre-commit hook which should facilitate the maintenance of the deleted packages within the `rules_python .bazelrc`. --- .bazelrc | 4 ++-- .pre-commit-config.yaml | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 3c317412ce..2cf2a8a3a9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be5f47fc45..e5010539ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,3 +37,11 @@ repos: rev: 23.1.0 hooks: - id: black + - repo: local + hooks: + - id: update-deleted-packages + name: Update deleted packages + language: script + entry: ./tools/bazel_integration_test/update_deleted_packages.sh + files: ^((examples|tests)/*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc|tools/bazel_integration_test/update_deleted_packages.sh)$ + pass_filenames: false From 3912266f15207eeeeecd0cf6d7a8e75db89e7b75 Mon Sep 17 00:00:00 2001 From: Derek Cormier Date: Mon, 5 Jun 2023 10:52:27 -0700 Subject: [PATCH 09/16] chore: auto-publish gazelle module to BCR (#1247) --- .bcr/config.yml | 1 + .bcr/gazelle/metadata.template.json | 20 ++++++++++++++++++++ .bcr/gazelle/presubmit.yml | 27 +++++++++++++++++++++++++++ .bcr/gazelle/source.template.json | 5 +++++ 4 files changed, 53 insertions(+) create mode 100644 .bcr/gazelle/metadata.template.json create mode 100644 .bcr/gazelle/presubmit.yml create mode 100644 .bcr/gazelle/source.template.json diff --git a/.bcr/config.yml b/.bcr/config.yml index 024e524293..7bdd70fbaf 100644 --- a/.bcr/config.yml +++ b/.bcr/config.yml @@ -15,3 +15,4 @@ fixedReleaser: login: f0rmiga email: thulio@aspect.dev +moduleRoots: [".", "gazelle"] diff --git a/.bcr/gazelle/metadata.template.json b/.bcr/gazelle/metadata.template.json new file mode 100644 index 0000000000..9cd4291e5a --- /dev/null +++ b/.bcr/gazelle/metadata.template.json @@ -0,0 +1,20 @@ +{ + "homepage": "https://github.com/bazelbuild/rules_python", + "maintainers": [ + { + "name": "Richard Levasseur", + "email": "rlevasseur@google.com", + "github": "rickeylev" + }, + { + "name": "Thulio Ferraz Assis", + "email": "thulio@aspect.dev", + "github": "f0rmiga" + } + ], + "repository": [ + "github:bazelbuild/rules_python" + ], + "versions": [], + "yanked_versions": {} +} diff --git a/.bcr/gazelle/presubmit.yml b/.bcr/gazelle/presubmit.yml new file mode 100644 index 0000000000..be948ad0f2 --- /dev/null +++ b/.bcr/gazelle/presubmit.yml @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bcr_test_module: + module_path: "../examples/build_file_generation" + matrix: + platform: ["debian11", "macos", "ubuntu2004", "windows"] + tasks: + run_tests: + name: "Run test module" + platform: ${{ platform }} + build_targets: + - "//..." + - ":modules_map" + test_targets: + - "//..." diff --git a/.bcr/gazelle/source.template.json b/.bcr/gazelle/source.template.json new file mode 100644 index 0000000000..cf06458e50 --- /dev/null +++ b/.bcr/gazelle/source.template.json @@ -0,0 +1,5 @@ +{ + "integrity": "", + "strip_prefix": "{REPO}-{VERSION}/gazelle", + "url": "https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/rules_python-{TAG}.tar.gz" +} From 28e15c2092a9512064e4b2a97a0fa3d1a83bc5ae Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 6 Jun 2023 02:59:27 +0900 Subject: [PATCH 10/16] fix(coverage): bump to latest coverage.py and fix import shadowing (#1249) Fixes #1196. Currently the `coverage.py` module does not work if updated to the latest version with the following error: ``` ImportError: cannot import name 'MappingProxyType' from partially initialized module 'types' (most likely due to a circular import) ...~pypi__coverage_cp39_x86_64-unknown-linux-gnu/coverage/types.py) ``` Where the `MappingProxyType` actually exists in Python's std lib. To fix, modify sys.path before the first import of coverage. Summary: - chore(coverage): bump coverage.py to 7.2.7 - fix(coverage): patch sys.path before importing anything from coverage - test(coverage): add extra assertions about the module names --- MODULE.bazel | 1 + examples/bzlmod/test.py | 12 ++++++ python/private/coverage.patch | 26 +++++++------ python/private/coverage_deps.bzl | 64 +++++++++++++++++--------------- 4 files changed, 61 insertions(+), 42 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b45c2ff03d..5381ba1cc4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,6 +34,7 @@ use_repo( "pypi__coverage_cp310_aarch64-unknown-linux-gnu", "pypi__coverage_cp310_x86_64-apple-darwin", "pypi__coverage_cp310_x86_64-unknown-linux-gnu", + "pypi__coverage_cp311_aarch64-apple-darwin", "pypi__coverage_cp311_aarch64-unknown-linux-gnu", "pypi__coverage_cp311_x86_64-apple-darwin", "pypi__coverage_cp311_x86_64-unknown-linux-gnu", diff --git a/examples/bzlmod/test.py b/examples/bzlmod/test.py index a36c19dc63..0486916e1b 100644 --- a/examples/bzlmod/test.py +++ b/examples/bzlmod/test.py @@ -30,6 +30,18 @@ def test_coverage_doesnt_shadow_stdlib(self): except ImportError: self.skipTest("not running under coverage, skipping") + self.assertEqual( + "html", + f"{html_stdlib.__name__}", + "'html' from stdlib was not loaded correctly", + ) + + self.assertEqual( + "coverage.html", + f"{html_coverage.__name__}", + "'coverage.html' was not loaded correctly", + ) + self.assertNotEqual( html_stdlib, html_coverage, diff --git a/python/private/coverage.patch b/python/private/coverage.patch index 4cab60cddc..cb4402e19c 100644 --- a/python/private/coverage.patch +++ b/python/private/coverage.patch @@ -1,15 +1,17 @@ # Because of how coverage is run, the current directory is the first in # sys.path. This is a problem for the tests, because they may import a module of # the same name as a module in the current directory. -diff --git a/coverage/cmdline.py b/coverage/cmdline.py -index dbf66e0a..780505ac 100644 ---- a/coverage/cmdline.py -+++ b/coverage/cmdline.py -@@ -937,6 +937,7 @@ def main(argv=None): - This is installed as the script entry point. - - """ -+ sys.path.append(sys.path.pop(0)) - if argv is None: - argv = sys.argv[1:] - try: +# +# NOTE @aignas 2023-06-05: we have to do this before anything from coverage gets +# imported. +diff --git a/coverage/__main__.py b/coverage/__main__.py +index 79aa4e2b..291fcff8 100644 +--- a/coverage/__main__.py ++++ b/coverage/__main__.py +@@ -4,5 +4,6 @@ + """Coverage.py's main entry point.""" + + import sys ++sys.path.append(sys.path.pop(0)) + from coverage.cmdline import main + sys.exit(main()) diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl index 377dfb7494..a4801cad37 100644 --- a/python/private/coverage_deps.bzl +++ b/python/private/coverage_deps.bzl @@ -28,70 +28,74 @@ load( _coverage_deps = { "cp310": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/89/a2/cbf599e50bb4be416e0408c4cf523c354c51d7da39935461a9687e039481/coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", - "784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660", + "https://files.pythonhosted.org/packages/3d/80/7060a445e1d2c9744b683dc935248613355657809d6c6b2716cdf4ca4766/coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", + "6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/15/b0/3639d84ee8a900da0cf6450ab46e22517e4688b6cec0ba8ab6f8166103a2/coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4", + "https://files.pythonhosted.org/packages/b8/9d/926fce7e03dbfc653104c2d981c0fa71f0572a9ebd344d24c573bd6f7c4f/coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/c4/8d/5ec7d08f4601d2d792563fe31db5e9322c306848fec1e65ec8885927f739/coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", - "ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53", + "https://files.pythonhosted.org/packages/01/24/be01e62a7bce89bcffe04729c540382caa5a06bee45ae42136c93e2499f5/coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", + "d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/3c/7d/d5211ea782b193ab8064b06dc0cc042cf1a4ca9c93a530071459172c550f/coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0", + "https://files.pythonhosted.org/packages/b4/bd/1b2331e3a04f4cc9b7b332b1dd0f3a1261dfc4114f8479bebfcc2afee9e8/coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063", ), }, "cp311": { + "aarch64-apple-darwin": ( + "https://files.pythonhosted.org/packages/67/d7/cd8fe689b5743fffac516597a1222834c42b80686b99f5b44ef43ccc2a43/coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", + "5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe", + ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/36/f3/5cbd79cf4cd059c80b59104aca33b8d05af4ad5bf5b1547645ecee716378/coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75", + "https://files.pythonhosted.org/packages/8c/95/16eed713202406ca0a37f8ac259bbf144c9d24f9b8097a8e6ead61da2dbb/coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/50/cf/455930004231fa87efe8be06d13512f34e070ddfee8b8bf5a050cdc47ab3/coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", - "4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795", + "https://files.pythonhosted.org/packages/c6/fa/529f55c9a1029c840bcc9109d5a15ff00478b7ff550a1ae361f8745f8ad5/coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", + "06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/6a/63/8e82513b7e4a1b8d887b4e85c1c2b6c9b754a581b187c0b084f3330ac479/coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91", + "https://files.pythonhosted.org/packages/a7/cd/3ce94ad9d407a052dc2a74fbeb1c7947f442155b28264eb467ee78dea812/coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb", ), }, "cp38": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/07/82/79fa21ceca9a9b091eb3c67e27eb648dade27b2c9e1eb23af47232a2a365/coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", - "2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba", + "https://files.pythonhosted.org/packages/28/d7/9a8de57d87f4bbc6f9a6a5ded1eaac88a89bf71369bb935dac3c0cf2893e/coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", + "3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/40/3b/cd68cb278c4966df00158811ec1e357b9a7d132790c240fc65da57e10013/coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e", + "https://files.pythonhosted.org/packages/c8/e4/e6182e4697665fb594a7f4e4f27cb3a4dd00c2e3d35c5c706765de8c7866/coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/05/63/a789b462075395d34f8152229dccf92b25ca73eac05b3f6cd75fa5017095/coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", - "d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c", + "https://files.pythonhosted.org/packages/c6/fc/be19131010930a6cf271da48202c8cc1d3f971f68c02fb2d3a78247f43dc/coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", + "54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/bd/a0/e263b115808226fdb2658f1887808c06ac3f1b579ef5dda02309e0d54459/coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b", + "https://files.pythonhosted.org/packages/44/55/49f65ccdd4dfd6d5528e966b28c37caec64170c725af32ab312889d2f857/coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e", ), }, "cp39": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/63/e9/f23e8664ec4032d7802a1cf920853196bcbdce7b56408e3efe1b2da08f3c/coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", - "95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc", + "https://files.pythonhosted.org/packages/ca/0c/3dfeeb1006c44b911ee0ed915350db30325d01808525ae7cc8d57643a2ce/coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", + "06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/18/95/27f80dcd8273171b781a19d109aeaed7f13d78ef6d1e2f7134a5826fd1b4/coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe", + "https://files.pythonhosted.org/packages/61/af/5964b8d7d9a5c767785644d9a5a63cacba9a9c45cc42ba06d25895ec87be/coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/ea/52/c08080405329326a7ff16c0dfdb4feefaa8edd7446413df67386fe1bbfe0/coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", - "633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745", + "https://files.pythonhosted.org/packages/88/da/495944ebf0ad246235a6bd523810d9f81981f9b81c6059ba1f56e943abe0/coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", + "537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/6b/f2/919f0fdc93d3991ca074894402074d847be8ac1e1d78e7e9e1c371b69a6f/coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5", + "https://files.pythonhosted.org/packages/fe/57/e4f8ad64d84ca9e759d783a052795f62a9f9111585e46068845b1cb52c2b/coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1", ), }, } From 93740219ff790016894e683000ee2427fb260268 Mon Sep 17 00:00:00 2001 From: cnorman Date: Wed, 7 Jun 2023 18:23:53 -0500 Subject: [PATCH 11/16] feat: add ppc64le releases and update to 3.10.11, 3.11.3 for python-build-standalone (#1234) This is being added in order to once again be able to build envoyproxy on the `ppc64le` architecture. Little Endian Power support was added to release https://github.com/indygreg/python-build-standalone/releases/tag/20230507. Signed-off-by: Christy Norman --- python/versions.bzl | 48 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/python/versions.bzl b/python/versions.bzl index baf6e33a06..a88c982c76 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -142,13 +142,14 @@ TOOL_VERSIONS = { "strip_prefix": "python", }, "3.9.16": { - "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", "sha256": { - "aarch64-apple-darwin": "d732d212d42315ac27c6da3e0b69636737a8d72086c980daf844344c010cab80", - "aarch64-unknown-linux-gnu": "1ba520c0db431c84305677f56eb9a4254f5097430ed443e92fc8617f8fba973d", - "x86_64-apple-darwin": "3948384af5e8d4ee7e5ccc648322b99c1c5cf4979954ed5e6b3382c69d6db71e", - "x86_64-pc-windows-msvc": "5274afd6b7ff2bddbd8306385ffb2336764b0e58535db968daeac656246f59a8", - "x86_64-unknown-linux-gnu": "7ba397787932393e65fc2fb9fcfabf54f2bb6751d5da2b45913cb25b2d493758", + "aarch64-apple-darwin": "c1de1d854717a6245f45262ef1bb17b09e2c587590e7e3f406593c143ff875bd", + "aarch64-unknown-linux-gnu": "f629b75ebfcafe9ceee2e796b7e4df5cf8dbd14f3c021afca078d159ab797acf", + "ppc64le-unknown-linux-gnu": "ff3ac35c58f67839aff9b5185a976abd3d1abbe61af02089f7105e876c1fe284", + "x86_64-apple-darwin": "3abc4d5fbbc80f5f848f280927ac5d13de8dc03aabb6ae65d8247cbb68e6f6bf", + "x86_64-pc-windows-msvc": "cdabb47204e96ce7ea31fbd0b5ed586114dd7d8f8eddf60a509a7f70b48a1c5e", + "x86_64-unknown-linux-gnu": "2b6e146234a4ef2a8946081fc3fbfffe0765b80b690425a49ebe40b47c33445b", }, "strip_prefix": "python", }, @@ -207,6 +208,18 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.11": { + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "8348bc3c2311f94ec63751fb71bd0108174be1c4def002773cf519ee1506f96f", + "aarch64-unknown-linux-gnu": "c7573fdb00239f86b22ea0e8e926ca881d24fde5e5890851339911d76110bc35", + "ppc64le-unknown-linux-gnu": "73a9d4c89ed51be39dd2de4e235078281087283e9fdedef65bec02f503e906ee", + "x86_64-apple-darwin": "bd3fc6e4da6f4033ebf19d66704e73b0804c22641ddae10bbe347c48f82374ad", + "x86_64-pc-windows-msvc": "9c2d3604a06fcd422289df73015cd00e7271d90de28d2c910f0e2309a7f73a68", + "x86_64-unknown-linux-gnu": "c5bcaac91bc80bfc29cf510669ecad12d506035ecb3ad85ef213416d54aecd79", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -218,6 +231,18 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.3": { + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "09e412506a8d63edbb6901742b54da9aa7faf120b8dbdce56c57b303fc892c86", + "aarch64-unknown-linux-gnu": "8190accbbbbcf7620f1ff6d668e4dd090c639665d11188ce864b62554d40e5ab", + "ppc64le-unknown-linux-gnu": "767d24f3570b35fedb945f5ac66224c8983f2d556ab83c5cfaa5f3666e9c212c", + "x86_64-apple-darwin": "f710b8d60621308149c100d5175fec39274ed0b9c99645484fd93d1716ef4310", + "x86_64-pc-windows-msvc": "24741066da6f35a7ff67bee65ce82eae870d84e1181843e64a7076d1571e95af", + "x86_64-unknown-linux-gnu": "da50b87d1ec42b3cb577dfd22a3655e43a53150f4f98a4bfb40757c9d7839ab5", + }, + "strip_prefix": "python", + }, } # buildifier: disable=unsorted-dict-items @@ -250,6 +275,17 @@ PLATFORMS = { # repository_ctx.execute(["uname", "-m"]).stdout.strip() arch = "aarch64", ), + "ppc64le-unknown-linux-gnu": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:ppc", + ], + os_name = LINUX_NAME, + # Note: this string differs between OSX and Linux + # Matches the value returned from: + # repository_ctx.execute(["uname", "-m"]).stdout.strip() + arch = "ppc64le", + ), "x86_64-apple-darwin": struct( compatible_with = [ "@platforms//os:macos", From b228f6047671abcf4c78ea7318916218fb098831 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 8 Jun 2023 07:50:20 -0700 Subject: [PATCH 12/16] fix(bzlmod)!: Remove ability to specify toolchain repo name. (#1258) The main reasons this is removed is because if modules choose different names for the same toolchain, only one of the two toolchains (which are, hopefully, identical) will be used. Which toolchain is used depends on the module graph dependency ordering. Furthermore, as of #1238, only one repo per version is created; others are ignored. This means a module doesn't know if the name it chooses will result in a repo being created with that name. Instead, the toolchain repos are named by rules_python: `python_{major}_{minor}`. These repo names are currently part of the public API, since they end up referenced in MODULE config (to wire the toolchain interpreter to pip). BREAKING CHANGES This removes the `name` arg from `python.toolchain()`. Users will need to remove such usages from their `MODULE.bazel` and update their `use_repo()` statements. If keeping the custom repo name is necessary, then repo mappings can be used. See #1232 for additional migration steps, commands, and information. --- .bazelignore | 3 + examples/bzlmod/BUILD.bazel | 2 +- examples/bzlmod/MODULE.bazel | 13 ++-- examples/bzlmod/other_module/MODULE.bazel | 12 +--- .../other_module/other_module/pkg/BUILD.bazel | 2 +- .../bzlmod_build_file_generation/MODULE.bazel | 3 +- examples/py_proto_library/MODULE.bazel | 3 +- python/extensions/interpreter.bzl | 2 +- python/extensions/python.bzl | 61 ++++--------------- 9 files changed, 27 insertions(+), 74 deletions(-) diff --git a/.bazelignore b/.bazelignore index a603a7bd8a..135f709824 100644 --- a/.bazelignore +++ b/.bazelignore @@ -6,3 +6,6 @@ bazel-rules_python bazel-bin bazel-out bazel-testlogs +examples/bzlmod/bazel-bzlmod +examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation +examples/py_proto_library/bazel-py_proto_library diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 0a068ce640..3bff20836d 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -7,7 +7,7 @@ # names. Those names are defined in the MODULES.bazel file. load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement") -load("@python_39//:defs.bzl", py_test_with_transition = "py_test") +load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 24bb4581f4..40dfb6aed1 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,13 +11,13 @@ local_path_override( path = "../..", ) -# This name is passed into python.toolchain and it's use_repo statement. -# We also use the same value in the python.host_python_interpreter call. -PYTHON_NAME_39 = "python_39" +# This name is generated by python.toolchain(), and is later passed +# to use_repo() and interpreter.install(). +PYTHON_NAME_39 = "python_3_9" INTERPRETER_NAME_39 = "interpreter_39" -PYTHON_NAME_310 = "python_310" +PYTHON_NAME_310 = "python_3_10" INTERPRETER_NAME_310 = "interpreter_310" @@ -25,10 +25,6 @@ INTERPRETER_NAME_310 = "interpreter_310" # You can set different Python versions in this block. python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - # This name is used in the various use_repo statements - # below, and in the local extension that is in this - # example. - name = PYTHON_NAME_39, configure_coverage_tool = True, # Only set when you have mulitple toolchain versions. is_default = True, @@ -41,7 +37,6 @@ python.toolchain( # Note: we do not supporting using multiple pip extensions, this is # work in progress. python.toolchain( - name = PYTHON_NAME_310, configure_coverage_tool = True, python_version = "3.10", ) diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index 5fb745266f..cc23a51601 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -10,24 +10,16 @@ bazel_dep(name = "rules_python", version = "") # a submodule. This code only exists to test that # we support doing this. This code is only for rules_python # testing purposes. -PYTHON_NAME_39 = "python_39" +PYTHON_NAME_39 = "python_3_9" -PYTHON_NAME_311 = "python_311" +PYTHON_NAME_311 = "python_3_11" python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - # This name is used in the various use_repo statements - # below, and in the local extension that is in this - # example. - name = PYTHON_NAME_39, configure_coverage_tool = True, python_version = "3.9", ) python.toolchain( - # This name is used in the various use_repo statements - # below, and in the local extension that is in this - # example. - name = PYTHON_NAME_311, configure_coverage_tool = True, # In a submodule this is ignored is_default = True, diff --git a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel index 952a674d48..6e37df8233 100644 --- a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel +++ b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel @@ -1,4 +1,4 @@ -load("@python_311//:defs.bzl", py_binary_311 = "py_binary") +load("@python_3_11//:defs.bzl", py_binary_311 = "py_binary") load("@rules_python//python:defs.bzl", "py_library") py_library( diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index d69dd7da48..fab2a26a83 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -46,14 +46,13 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") # This name is passed into python.toolchain and it's use_repo statement. # We also use the same name for python.host_python_interpreter. -PYTHON_NAME = "python" +PYTHON_NAME = "python_3_9" INTERPRETER_NAME = "interpreter" # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. python.toolchain( - name = PYTHON_NAME, configure_coverage_tool = True, is_default = True, python_version = "3.9", diff --git a/examples/py_proto_library/MODULE.bazel b/examples/py_proto_library/MODULE.bazel index 3116c40b2d..feb938da5c 100644 --- a/examples/py_proto_library/MODULE.bazel +++ b/examples/py_proto_library/MODULE.bazel @@ -14,11 +14,10 @@ local_path_override( python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python3_9", configure_coverage_tool = True, python_version = "3.9", ) -use_repo(python, "python3_9") +use_repo(python, "python_3_9") # We are using rules_proto to define rules_proto targets to be consumed by py_proto_library. bazel_dep(name = "rules_proto", version = "5.3.0-21.7") diff --git a/python/extensions/interpreter.bzl b/python/extensions/interpreter.bzl index b9afe1abda..bbeadc26b8 100644 --- a/python/extensions/interpreter.bzl +++ b/python/extensions/interpreter.bzl @@ -53,7 +53,7 @@ def _interpreter_repo_impl(rctx): actual_interpreter_label = INTERPRETER_LABELS.get(rctx.attr.python_name) if actual_interpreter_label == None: - fail("Unable to find interpreter with name {}".format(rctx.attr.python_name)) + fail("Unable to find interpreter with name '{}'".format(rctx.attr.python_name)) rctx.symlink(actual_interpreter_label, "python") diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index a604df6ca3..bed62305de 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -43,11 +43,11 @@ def _left_pad_zero(index, length): def _print_warn(msg): print("WARNING:", msg) -def _python_register_toolchains(toolchain_attr, version_constraint): +def _python_register_toolchains(name, toolchain_attr, version_constraint): """Calls python_register_toolchains and returns a struct used to collect the toolchains. """ python_register_toolchains( - name = toolchain_attr.name, + name = name, python_version = toolchain_attr.python_version, register_coverage_tool = toolchain_attr.configure_coverage_tool, ignore_root_user_error = toolchain_attr.ignore_root_user_error, @@ -56,7 +56,7 @@ def _python_register_toolchains(toolchain_attr, version_constraint): return struct( python_version = toolchain_attr.python_version, set_python_version_constraint = str(version_constraint), - name = toolchain_attr.name, + name = name, ) def _python_impl(module_ctx): @@ -67,38 +67,17 @@ def _python_impl(module_ctx): # toolchain added to toolchains. default_toolchain = None - # Map of toolchain name to registering module - global_toolchain_names = {} - # Map of string Major.Minor to the toolchain name and module name global_toolchain_versions = {} for mod in module_ctx.modules: - module_toolchain_names = [] module_toolchain_versions = [] for toolchain_attr in mod.tags.toolchain: - toolchain_name = toolchain_attr.name - - # Duplicate names within a module indicate a misconfigured module. - if toolchain_name in module_toolchain_names: - _fail_duplicate_module_toolchain_name(mod.name, toolchain_name) - module_toolchain_names.append(toolchain_name) - - # Ignore name collisions in the global scope because there isn't - # much else that can be done. Modules don't know and can't control - # what other modules do, so the first in the dependency graph wins. - if toolchain_name in global_toolchain_names: - _warn_duplicate_global_toolchain_name( - toolchain_name, - first_module = global_toolchain_names[toolchain_name], - second_module = mod.name, - ) - continue - global_toolchain_names[toolchain_name] = mod.name + toolchain_version = toolchain_attr.python_version + toolchain_name = "python_" + toolchain_version.replace(".", "_") # Duplicate versions within a module indicate a misconfigured module. - toolchain_version = toolchain_attr.python_version if toolchain_version in module_toolchain_versions: _fail_duplicate_module_toolchain_version(toolchain_version, mod.name) module_toolchain_versions.append(toolchain_version) @@ -137,6 +116,7 @@ def _python_impl(module_ctx): ) toolchain_info = _python_register_toolchains( + toolchain_name, toolchain_attr, version_constraint = not is_default, ) @@ -182,23 +162,6 @@ def _python_impl(module_ctx): }, ) -def _fail_duplicate_module_toolchain_name(module_name, toolchain_name): - fail(("Duplicate module toolchain name: module '{module}' attempted " + - "to use the name '{toolchain}' multiple times in itself").format( - toolchain = toolchain_name, - module = module_name, - )) - -def _warn_duplicate_global_toolchain_name(name, first_module, second_module): - _print_warn(( - "Ignoring toolchain '{name}' from module '{second_module}': " + - "Toolchain with the same name from module '{first_module}' has precedence" - ).format( - name = name, - first_module = first_module, - second_module = second_module, - )) - def _fail_duplicate_module_toolchain_version(version, module): fail(("Duplicate module toolchain version: module '{module}' attempted " + "to use version '{version}' multiple times in itself").format( @@ -256,6 +219,12 @@ if the sub module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain it is set as the default toolchain. + +Toolchain repository name + +A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. +`python_3_10`. The `major` and `minor` components are +`major` and `minor` are the Python version from the `python_version` attribute. """, attrs = { "configure_coverage_tool": attr.bool( @@ -271,13 +240,9 @@ is set as the default toolchain. mandatory = False, doc = "Whether the toolchain is the default version", ), - "name": attr.string( - mandatory = True, - doc = "Name of the toolchain", - ), "python_version": attr.string( mandatory = True, - doc = "The Python version that we are creating the toolchain for.", + doc = "The Python version, in `major.minor` format, e.g '3.12', to create a toolchain for.", ), }, ), From 32b00530160c3a8894350fab7a195540df89819d Mon Sep 17 00:00:00 2001 From: Rasrack Date: Sat, 10 Jun 2023 00:52:13 +0200 Subject: [PATCH 13/16] fix: update correct requirements lock file when using os specific lock files (#1123) Currently the dependency_resolver.py ignores that you give requirement lock files for different os's, except when checking if the golden file needs updating. This causes dependecy_resolver.py to update the wrong lock i.e the non platform specific one if ran in "update mode". --- .bazelci/presubmit.yml | 24 +++++++++ .../dependency_resolver.py | 51 +++++++++---------- tests/compile_pip_requirements/BUILD.bazel | 30 +++++++++++ .../requirements_lock_darwin.txt | 10 ++++ .../requirements_lock_linux.txt | 10 ++++ .../requirements_lock_windows.txt | 10 ++++ 6 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 tests/compile_pip_requirements/requirements_lock_darwin.txt create mode 100644 tests/compile_pip_requirements/requirements_lock_linux.txt create mode 100644 tests/compile_pip_requirements/requirements_lock_windows.txt diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b468970fbb..ac24113d03 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -435,6 +435,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_debian: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Debian @@ -447,6 +453,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_macos: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on macOS @@ -459,6 +471,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_darwin.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Windows @@ -471,6 +489,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_windows.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_pip_repository_entry_points_ubuntu_min: <<: *minimum_supported_version diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 89e355806c..ceb20db7ef 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -95,25 +95,30 @@ def _locate(bazel_runfiles, file): requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) + requirements_file = _select_golden_requirements_file( + requirements_txt=requirements_txt, requirements_linux=requirements_linux, + requirements_darwin=requirements_darwin, requirements_windows=requirements_windows + ) + resolved_requirements_in = _locate(bazel_runfiles, requirements_in) - resolved_requirements_txt = _locate(bazel_runfiles, requirements_txt) + resolved_requirements_file = _locate(bazel_runfiles, requirements_file) # Files in the runfiles directory has the following naming schema: # Main repo: __main__/ # External repo: / # We want to strip both __main__ and from the absolute prefix # to keep the requirements lock file agnostic. - repository_prefix = requirements_txt[: requirements_txt.index("/") + 1] - absolute_path_prefix = resolved_requirements_txt[ - : -(len(requirements_txt) - len(repository_prefix)) + repository_prefix = requirements_file[: requirements_file.index("/") + 1] + absolute_path_prefix = resolved_requirements_file[ + : -(len(requirements_file) - len(repository_prefix)) ] # As requirements_in might contain references to generated files we want to # use the runfiles file first. Thus, we need to compute the relative path # from the execution root. # Note: Windows cannot reference generated files without runfiles support enabled. - requirements_in_relative = requirements_in[len(repository_prefix) :] - requirements_txt_relative = requirements_txt[len(repository_prefix) :] + requirements_in_relative = requirements_in[len(repository_prefix):] + requirements_file_relative = requirements_file[len(repository_prefix):] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -135,11 +140,11 @@ def _locate(bazel_runfiles, file): sys.argv.append(os.environ["TEST_TMPDIR"]) # Make a copy for pip-compile to read and mutate. requirements_out = os.path.join( - os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out" ) # Those two files won't necessarily be on the same filesystem, so we can't use os.replace # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. - shutil.copy(resolved_requirements_txt, requirements_out) + shutil.copy(resolved_requirements_file, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, @@ -150,7 +155,7 @@ def _locate(bazel_runfiles, file): sys.argv.append("--generate-hashes") sys.argv.append("--output-file") - sys.argv.append(requirements_txt_relative if UPDATE else requirements_out) + sys.argv.append(requirements_file_relative if UPDATE else requirements_out) sys.argv.append( requirements_in_relative if Path(requirements_in_relative).exists() @@ -159,28 +164,28 @@ def _locate(bazel_runfiles, file): print(sys.argv) if UPDATE: - print("Updating " + requirements_txt_relative) + print("Updating " + requirements_file_relative) if "BUILD_WORKSPACE_DIRECTORY" in os.environ: workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"] - requirements_txt_tree = os.path.join(workspace, requirements_txt_relative) - # In most cases, requirements_txt will be a symlink to the real file in the source tree. - # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy, + requirements_file_tree = os.path.join(workspace, requirements_file_relative) + # In most cases, requirements_file will be a symlink to the real file in the source tree. + # If symlinks are not enabled (e.g. on Windows), then requirements_file will be a copy, # and we should copy the updated requirements back to the source tree. - if not os.path.samefile(resolved_requirements_txt, requirements_txt_tree): + if not os.path.samefile(resolved_requirements_file, requirements_file_tree): atexit.register( lambda: shutil.copy( - resolved_requirements_txt, requirements_txt_tree + resolved_requirements_file, requirements_file_tree ) ) cli() - requirements_txt_relative_path = Path(requirements_txt_relative) - content = requirements_txt_relative_path.read_text() + requirements_file_relative_path = Path(requirements_file_relative) + content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") - requirements_txt_relative_path.write_text(content) + requirements_file_relative_path.write_text(content) else: # cli will exit(0) on success try: - print("Checking " + requirements_txt) + print("Checking " + requirements_file) cli() print("cli() should exit", file=sys.stderr) sys.exit(1) @@ -194,13 +199,7 @@ def _locate(bazel_runfiles, file): ) sys.exit(1) elif e.code == 0: - golden_filename = _select_golden_requirements_file( - requirements_txt, - requirements_linux, - requirements_darwin, - requirements_windows, - ) - golden = open(_locate(bazel_runfiles, golden_filename)).readlines() + golden = open(_locate(bazel_runfiles, requirements_file)).readlines() out = open(requirements_out).readlines() out = [line.replace(absolute_path_prefix, "") for line in out] if golden != out: diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index d6ac0086ab..87ffe706dd 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -32,3 +32,33 @@ compile_pip_requirements( requirements_in = "requirements.txt", requirements_txt = "requirements_lock.txt", ) + +genrule( + name = "generate_os_specific_requirements_in", + srcs = [], + outs = ["requirements_os_specific.in"], + cmd = """ +cat > $@ < Date: Fri, 9 Jun 2023 18:19:32 -0500 Subject: [PATCH 14/16] fix: use `only-binary` for `download_only` `pip download` (#1219) [`wheel_installer` assumes that if the pip command succeeded, there must be at least one `*.whl` file](https://github.com/lpulley/rules_python/blob/fdec44120aa45d748ab804f1d019002c6949b449/python/pip_install/tools/wheel_installer/wheel_installer.py#L439), but this is not currently true when `download_only` is enabled and the package provides no wheel; a `.tar.gz` will happily be downloaded, pip will succeed, and the `next(iter(glob.glob("*.whl")))` call will fail, producing a mysterious `StopIteration`: ``` Saved ./artifactory-0.1.17.tar.gz Successfully downloaded artifactory (Traceback (most recent call last): File "[redacted]/lib/python3.8/runpy.py", line 194, in _run_module_as_main return _run_code(code, main_globals, None, File "[redacted]/lib/python3.8/runpy.py", line 87, in _run_code exec(code, run_globals) File "[redacted]/external/rules_python/python/pip_install/tools/wheel_installer/wheel_installer.py", line 450, in main() File "[redacted]/external/rules_python/python/pip_install/tools/wheel_installer/wheel_installer.py", line 438, in main whl = next(iter(glob.glob("*.whl"))) StopIteration ) ``` By using `--only-binary=:all:` when using `pip download`, the pip command will fail if there is no suitable wheel to be downloaded. This should make the error much more obvious, since with `--only-binary=:all:` and no suitable wheel, pip fails and reports an error like this: ``` ERROR: Could not find a version that satisfies the requirement artifactory (from versions: none) ERROR: No matching distribution found for artifactory ``` --- python/pip_install/tools/wheel_installer/wheel_installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 77aa3a406c..5a6f49bb5b 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -406,7 +406,8 @@ def main() -> None: pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) - + ["download" if args.download_only else "wheel", "--no-deps"] + + (["download", "--only-binary=:all:"] if args.download_only else ["wheel"]) + + ["--no-deps"] + deserialized_args["extra_pip_args"] ) From 1f58f4c9fb5a5f17f35482b0ec7f8bc5448afb59 Mon Sep 17 00:00:00 2001 From: Tejaswi Konduri Date: Fri, 9 Jun 2023 17:32:54 -0700 Subject: [PATCH 15/16] feat: Adding variable support for distribution in py_wheel (#1251) This allows the `distribution` attribute to expand workspace status keys, just as the `version` attribute can. This allows, for example, the VCS's branch name (e.g test, release, etc) to be part of the distribution name without having to modify the BUILD file. Having distinct distribution names is necessary because tools like pip-compile, which determine version compatibility and replacements, and having the same distribution name would mean the different builds are seen as equivalent. Closes #1250 --- docs/packaging.md | 2 +- examples/wheel/BUILD.bazel | 2 +- examples/wheel/wheel_test.py | 12 +++++++++--- python/private/py_wheel.bzl | 6 ++++++ tools/wheelmaker.py | 37 +++++++++++++++++++++++------------- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/docs/packaging.md b/docs/packaging.md index b244b42767..16fa00c312 100755 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -82,7 +82,7 @@ in the way they expect. | console_scripts | Deprecated console_script entry points, e.g. {'main': 'examples.wheel.main:main'}.

Deprecated: prefer the entry_points attribute, which supports console_scripts as well as other entry points. | Dictionary: String -> String | optional | {} | | deps | Targets to be included in the distribution.

The targets to package are usually py_library rules or filesets (for packaging data files).

Note it's usually better to package py_library targets and use entry_points attribute to specify console_scripts than to package py_binary rules. py_binary targets would wrap a executable script that tries to locate .runfiles directory which is not packaged in the wheel. | List of labels | optional | [] | | description_file | A file containing text describing the package. | Label | optional | None | -| distribution | Name of the distribution.

This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. | String | required | | +| distribution | Name of the distribution.

This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies.

Workspace status keys are expanded using {NAME} format, for example: - distribution = "package.{CLASSIFIER}" - distribution = "{DISTRIBUTION}"

For the available keys, see https://bazel.build/docs/user-manual#workspace-status | String | required | | | entry_points | entry_points, e.g. {'console_scripts': ['main = examples.wheel.main:main']}. | Dictionary: String -> List of strings | optional | {} | | extra_distinfo_files | Extra files to add to distinfo directory in the archive. | Dictionary: Label -> String | optional | {} | | extra_requires | List of optional requirements for this package | Dictionary: String -> List of strings | optional | {} | diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index 61a43ae6cf..49a212b311 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -94,7 +94,7 @@ build_test( py_wheel( name = "minimal_with_py_library_with_stamp", # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" - distribution = "example_minimal_library", + distribution = "example_minimal_library{BUILD_USER}", python_tag = "py3", stamp = 1, version = "0.1.{BUILD_TIMESTAMP}", diff --git a/examples/wheel/wheel_test.py b/examples/wheel/wheel_test.py index c292c87132..591e0571de 100644 --- a/examples/wheel/wheel_test.py +++ b/examples/wheel/wheel_test.py @@ -374,30 +374,36 @@ def test_rule_creates_directory_and_is_included_in_wheel(self): ], ) - def test_rule_sets_stamped_version_in_wheel_metadata(self): + def test_rule_expands_workspace_status_keys_in_wheel_metadata(self): filename = os.path.join( os.environ["TEST_SRCDIR"], "rules_python", "examples", "wheel", - "example_minimal_library-0.1._BUILD_TIMESTAMP_-py3-none-any.whl", + "example_minimal_library_BUILD_USER_-0.1._BUILD_TIMESTAMP_-py3-none-any.whl", ) with zipfile.ZipFile(filename) as zf: metadata_file = None for f in zf.namelist(): self.assertNotIn("_BUILD_TIMESTAMP_", f) + self.assertNotIn("_BUILD_USER_", f) if os.path.basename(f) == "METADATA": metadata_file = f self.assertIsNotNone(metadata_file) version = None + name = None with zf.open(metadata_file) as fp: for line in fp: - if line.startswith(b'Version:'): + if line.startswith(b"Version:"): version = line.decode().split()[-1] + if line.startswith(b"Name:"): + name = line.decode().split()[-1] self.assertIsNotNone(version) + self.assertIsNotNone(name) self.assertNotIn("{BUILD_TIMESTAMP}", version) + self.assertNotIn("{BUILD_USER}", name) if __name__ == "__main__": diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index b6f2bfae56..84fdeace1d 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -40,6 +40,12 @@ Name of the distribution. This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. + +Workspace status keys are expanded using `{NAME}` format, for example: + - `distribution = "package.{CLASSIFIER}"` + - `distribution = "{DISTRIBUTION}"` + +For the available keys, see https://bazel.build/docs/user-manual#workspace-status """, ), "platform": attr.string( diff --git a/tools/wheelmaker.py b/tools/wheelmaker.py index 6138c934d5..edc25cc09f 100644 --- a/tools/wheelmaker.py +++ b/tools/wheelmaker.py @@ -167,12 +167,12 @@ def add_wheelfile(self): wheel_contents += "Tag: %s\n" % tag self.add_string(self.distinfo_path("WHEEL"), wheel_contents) - def add_metadata(self, metadata, description, version): + def add_metadata(self, metadata, name, description, version): """Write METADATA file to the distribution.""" # https://www.python.org/dev/peps/pep-0566/ # https://packaging.python.org/specifications/core-metadata/ - metadata += "Version: " + version - metadata += "\n\n" + metadata = re.sub("^Name: .*$", "Name: %s" % name, metadata, flags=re.MULTILINE) + metadata += "Version: %s\n\n" % version # setuptools seems to insert UNKNOWN as description when none is # provided. metadata += description if description else "UNKNOWN" @@ -207,18 +207,18 @@ def get_files_to_package(input_files): return files -def resolve_version_stamp( - version: str, volatile_status_stamp: Path, stable_status_stamp: Path +def resolve_argument_stamp( + argument: str, volatile_status_stamp: Path, stable_status_stamp: Path ) -> str: - """Resolve workspace status stamps format strings found in the version string + """Resolve workspace status stamps format strings found in the argument string Args: - version (str): The raw version represenation for the wheel (may include stamp variables) + argument (str): The raw argument represenation for the wheel (may include stamp variables) volatile_status_stamp (Path): The path to a volatile workspace status file stable_status_stamp (Path): The path to a stable workspace status file Returns: - str: A resolved version string + str: A resolved argument string """ lines = ( volatile_status_stamp.read_text().splitlines() @@ -229,9 +229,9 @@ def resolve_version_stamp( continue key, value = line.split(" ", maxsplit=1) stamp = "{" + key + "}" - version = version.replace(stamp, value) + argument = argument.replace(stamp, value) - return version + return argument def parse_args() -> argparse.Namespace: @@ -357,7 +357,16 @@ def main() -> None: strip_prefixes = [p for p in arguments.strip_path_prefix] if arguments.volatile_status_file and arguments.stable_status_file: - version = resolve_version_stamp( + name = resolve_argument_stamp( + arguments.name, + arguments.volatile_status_file, + arguments.stable_status_file, + ) + else: + name = arguments.name + + if arguments.volatile_status_file and arguments.stable_status_file: + version = resolve_argument_stamp( arguments.version, arguments.volatile_status_file, arguments.stable_status_file, @@ -366,7 +375,7 @@ def main() -> None: version = arguments.version with WheelMaker( - name=arguments.name, + name=name, version=version, build_tag=arguments.build_tag, python_tag=arguments.python_tag, @@ -398,7 +407,9 @@ def main() -> None: with open(arguments.metadata_file, "rt", encoding="utf-8") as metadata_file: metadata = metadata_file.read() - maker.add_metadata(metadata=metadata, description=description, version=version) + maker.add_metadata( + metadata=metadata, name=name, description=description, version=version + ) if arguments.entry_points_file: maker.add_file( From 2c28e61232cd77c25d1eb09cf63283f62b942558 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 12 Jun 2023 07:52:01 -0700 Subject: [PATCH 16/16] feat(bzlmod): Register a default toolchain (#1259) This makes rules_python always provide a default toolchain when using bzlmod. Note that, unlike workspace builds, the default is not the local system Python (`@bazel_tools//tools/python:autodetecting_toolchain`). Instead, the default is a hermetic runtime, but no guarantees are made about the particular version used. In practice, it will be the latest available Python version. Work towards #1233 --- .github/workflows/create_archive_and_notes.sh | 14 ------- BZLMOD_SUPPORT.md | 24 ++++++++++++ MODULE.bazel | 9 +++++ README.md | 37 +++++++++++++++---- python/extensions/python.bzl | 19 +++++++--- 5 files changed, 75 insertions(+), 28 deletions(-) diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index 0c0c4acf41..02279bcca1 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -40,20 +40,6 @@ pip.parse( ) use_repo(pip, "pip") - -# (Optional) Register a specific python toolchain instead of using the host version -python = use_extension("@rules_python//python:extensions.bzl", "python") - -python.toolchain( - name = "python3_9", - python_version = "3.9", -) - -use_repo(python, "python3_9_toolchains") - -register_toolchains( - "@python3_9_toolchains//:all", -) \`\`\` ## Using WORKSPACE diff --git a/BZLMOD_SUPPORT.md b/BZLMOD_SUPPORT.md index 8efd0df6d7..dbe5238dee 100644 --- a/BZLMOD_SUPPORT.md +++ b/BZLMOD_SUPPORT.md @@ -35,3 +35,27 @@ This rule set does not have full feature partity with the older `WORKSPACE` type 2. Gazelle does not support finding deps in sub-modules. For instance we can have a dep like ` "@our_other_module//other_module/pkg:lib",` in a `py_test` definition. Check ["issues"](/bazelbuild/rules_python/issues) for an up to date list. + +## Differences in behavior from WORKSPACE + +### Default toolchain is not the local system Python + +Under bzlmod, the default toolchain is no longer based on the locally installed +system Python. Instead, a recent Python version using the pre-built, +standalone runtimes are used. + +If you need the local system Python to be your toolchain, then it's suggested +that you setup and configure your own toolchain and register it. Note that using +the local system's Python is not advised because will vary between users and +platforms. + +If you want to use the same toolchain as what WORKSPACE used, then manually +register the builtin Bazel Python toolchain by doing +`register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain")`. +**IMPORTANT: this should only be done in a root module, and may intefere with +the toolchains rules_python registers**. + +NOTE: Regardless of your toolchain, due to +[#691](https://github.com/bazelbuild/rules_python/issues/691), `rules_python` +still relies on a local Python being available to bootstrap the program before +handing over execution to the toolchain Python. diff --git a/MODULE.bazel b/MODULE.bazel index 5381ba1cc4..6729d09c7a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -51,6 +51,15 @@ use_repo( # We need to do another use_extension call to expose the "pythons_hub" # repo. python = use_extension("@rules_python//python/extensions:python.bzl", "python") + +# The default toolchain to use if nobody configures a toolchain. +# NOTE: This is not a stable version. It is provided for convenience, but will +# change frequently to track the most recent Python version. +# NOTE: The root module can override this. +python.toolchain( + is_default = True, + python_version = "3.11", +) use_repo(python, "pythons_hub") # This call registers the Python toolchains. diff --git a/README.md b/README.md index 6893a1da28..cf4b04e224 100644 --- a/README.md +++ b/README.md @@ -45,31 +45,52 @@ the older way of configuring bazel with a `WORKSPACE` file. ### Using bzlmod +NOTE: bzlmod support is still experimental; APIs subject to change. + To import rules_python in your project, you first need to add it to your `MODULE.bazel` file, using the snippet provided in the [release you choose](https://github.com/bazelbuild/rules_python/releases). +Once the dependency is added, a Python toolchain will be automatically +registered and you'll be able to create runnable programs and tests. + + #### Toolchain registration with bzlmod -To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `MODULE.bazel` file: +NOTE: bzlmod support is still experimental; APIs subject to change. + +A default toolchain is automatically configured for by depending on +`rules_python`. Note, however, the version used tracks the most recent Python +release and will change often. + +If you want to register specific Python versions, then use +`python.toolchain()` for each version you need: ```starlark -# Find the latest version number here: https://github.com/bazelbuild/rules_python/releases -# and change the version number if needed in the line below. -bazel_dep(name = "rules_python", version = "0.21.0") +python = use_extension("@rules_python//python:extensions.bzl", "python") + +python.toolchain( + python_version = "3.9", +) +``` + +### Using pip with bzlmod +NOTE: bzlmod support is still experimental; APIs subject to change. + +To use dependencies from PyPI, the `pip.parse()` extension is used to +convert a requirements file into Bazel dependencies. + +```starlark python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python", - configure_coverage_tool = True, - is_default = True, python_version = "3.9", ) interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( name = "interpreter", - python_name = "python", + python_name = "python_3_9", ) use_repo(interpreter, "interpreter") diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index bed62305de..d7a466ae10 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -98,12 +98,20 @@ def _python_impl(module_ctx): module_name = mod.name, ) - # Only the root module is allowed to set the default toolchain - # to prevent submodules from clobbering each other. - # A single toolchain in the root module is treated as the default - # because it's unambigiuous. + # Only the root module and rules_python are allowed to specify the default + # toolchain for a couple reasons: + # * It prevents submodules from specifying different defaults and only + # one of them winning. + # * rules_python needs to set a soft default in case the root module doesn't, + # e.g. if the root module doesn't use Python itself. + # * The root module is allowed to override the rules_python default. if mod.is_root: + # A single toolchain is treated as the default because it's unambiguous. is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1 + elif mod.name == "rules_python" and not default_toolchain: + # We don't do the len() check because we want the default that rules_python + # sets to be clearly visible. + is_default = toolchain_attr.is_default else: is_default = False @@ -129,8 +137,7 @@ def _python_impl(module_ctx): # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: - fail("No default toolchain found: exactly one toolchain must have " + - "is_default=True set") + fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?") # The last toolchain in the BUILD file is set as the default # toolchain. We need the default last.