diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b468970fbb..ac24113d03 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -435,6 +435,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_debian: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Debian @@ -447,6 +453,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_linux.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_macos: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on macOS @@ -459,6 +471,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_darwin.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_compile_pip_requirements_windows: <<: *reusable_build_test_all name: compile_pip_requirements integration tests on Windows @@ -471,6 +489,12 @@ tasks: - "! git diff --exit-code" - "bazel run //:requirements.update" - "git diff --exit-code" + # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the + # right thing. + - "echo '' > requirements_lock_windows.txt" + - "! git diff --exit-code" + - "bazel run //:os_specific_requirements.update" + - "git diff --exit-code" integration_test_pip_repository_entry_points_ubuntu_min: <<: *minimum_supported_version diff --git a/.bazelignore b/.bazelignore index a603a7bd8a..135f709824 100644 --- a/.bazelignore +++ b/.bazelignore @@ -6,3 +6,6 @@ bazel-rules_python bazel-bin bazel-out bazel-testlogs +examples/bzlmod/bazel-bzlmod +examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation +examples/py_proto_library/bazel-py_proto_library diff --git a/.bazelrc b/.bazelrc index fe542b3722..2cf2a8a3a9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/.bcr/config.yml b/.bcr/config.yml index 024e524293..7bdd70fbaf 100644 --- a/.bcr/config.yml +++ b/.bcr/config.yml @@ -15,3 +15,4 @@ fixedReleaser: login: f0rmiga email: thulio@aspect.dev +moduleRoots: [".", "gazelle"] diff --git a/.bcr/gazelle/metadata.template.json b/.bcr/gazelle/metadata.template.json new file mode 100644 index 0000000000..9cd4291e5a --- /dev/null +++ b/.bcr/gazelle/metadata.template.json @@ -0,0 +1,20 @@ +{ + "homepage": "https://github.com/bazelbuild/rules_python", + "maintainers": [ + { + "name": "Richard Levasseur", + "email": "rlevasseur@google.com", + "github": "rickeylev" + }, + { + "name": "Thulio Ferraz Assis", + "email": "thulio@aspect.dev", + "github": "f0rmiga" + } + ], + "repository": [ + "github:bazelbuild/rules_python" + ], + "versions": [], + "yanked_versions": {} +} diff --git a/.bcr/gazelle/presubmit.yml b/.bcr/gazelle/presubmit.yml new file mode 100644 index 0000000000..be948ad0f2 --- /dev/null +++ b/.bcr/gazelle/presubmit.yml @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bcr_test_module: + module_path: "../examples/build_file_generation" + matrix: + platform: ["debian11", "macos", "ubuntu2004", "windows"] + tasks: + run_tests: + name: "Run test module" + platform: ${{ platform }} + build_targets: + - "//..." + - ":modules_map" + test_targets: + - "//..." diff --git a/.bcr/gazelle/source.template.json b/.bcr/gazelle/source.template.json new file mode 100644 index 0000000000..cf06458e50 --- /dev/null +++ b/.bcr/gazelle/source.template.json @@ -0,0 +1,5 @@ +{ + "integrity": "", + "strip_prefix": "{REPO}-{VERSION}/gazelle", + "url": "https://github.com/{OWNER}/{REPO}/releases/download/{TAG}/rules_python-{TAG}.tar.gz" +} diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index 0c0c4acf41..02279bcca1 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -40,20 +40,6 @@ pip.parse( ) use_repo(pip, "pip") - -# (Optional) Register a specific python toolchain instead of using the host version -python = use_extension("@rules_python//python:extensions.bzl", "python") - -python.toolchain( - name = "python3_9", - python_version = "3.9", -) - -use_repo(python, "python3_9_toolchains") - -register_toolchains( - "@python3_9_toolchains//:all", -) \`\`\` ## Using WORKSPACE diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be5f47fc45..e5010539ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,3 +37,11 @@ repos: rev: 23.1.0 hooks: - id: black + - repo: local + hooks: + - id: update-deleted-packages + name: Update deleted packages + language: script + entry: ./tools/bazel_integration_test/update_deleted_packages.sh + files: ^((examples|tests)/*/(MODULE.bazel|WORKSPACE|WORKSPACE.bzlmod|BUILD.bazel)|.bazelrc|tools/bazel_integration_test/update_deleted_packages.sh)$ + pass_filenames: false diff --git a/BZLMOD_SUPPORT.md b/BZLMOD_SUPPORT.md index cf95d12a0e..dbe5238dee 100644 --- a/BZLMOD_SUPPORT.md +++ b/BZLMOD_SUPPORT.md @@ -31,7 +31,31 @@ A second example, in [examples/bzlmod_build_file_generation](examples/bzlmod_bui This rule set does not have full feature partity with the older `WORKSPACE` type configuration: -1. Multiple python versions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example. +1. Multiple pip extensions are not yet supported, as demonstrated in [this](examples/multi_python_versions) example. 2. Gazelle does not support finding deps in sub-modules. For instance we can have a dep like ` "@our_other_module//other_module/pkg:lib",` in a `py_test` definition. Check ["issues"](/bazelbuild/rules_python/issues) for an up to date list. + +## Differences in behavior from WORKSPACE + +### Default toolchain is not the local system Python + +Under bzlmod, the default toolchain is no longer based on the locally installed +system Python. Instead, a recent Python version using the pre-built, +standalone runtimes are used. + +If you need the local system Python to be your toolchain, then it's suggested +that you setup and configure your own toolchain and register it. Note that using +the local system's Python is not advised because will vary between users and +platforms. + +If you want to use the same toolchain as what WORKSPACE used, then manually +register the builtin Bazel Python toolchain by doing +`register_toolchains("@bazel_tools//tools/python:autodetecting_toolchain")`. +**IMPORTANT: this should only be done in a root module, and may intefere with +the toolchains rules_python registers**. + +NOTE: Regardless of your toolchain, due to +[#691](https://github.com/bazelbuild/rules_python/issues/691), `rules_python` +still relies on a local Python being available to bootstrap the program before +handing over execution to the toolchain Python. diff --git a/MODULE.bazel b/MODULE.bazel index ddd946c78a..6729d09c7a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,6 +34,7 @@ use_repo( "pypi__coverage_cp310_aarch64-unknown-linux-gnu", "pypi__coverage_cp310_x86_64-apple-darwin", "pypi__coverage_cp310_x86_64-unknown-linux-gnu", + "pypi__coverage_cp311_aarch64-apple-darwin", "pypi__coverage_cp311_aarch64-unknown-linux-gnu", "pypi__coverage_cp311_x86_64-apple-darwin", "pypi__coverage_cp311_x86_64-unknown-linux-gnu", @@ -47,5 +48,19 @@ use_repo( "pypi__coverage_cp39_x86_64-unknown-linux-gnu", ) +# We need to do another use_extension call to expose the "pythons_hub" +# repo. python = use_extension("@rules_python//python/extensions:python.bzl", "python") + +# The default toolchain to use if nobody configures a toolchain. +# NOTE: This is not a stable version. It is provided for convenience, but will +# change frequently to track the most recent Python version. +# NOTE: The root module can override this. +python.toolchain( + is_default = True, + python_version = "3.11", +) use_repo(python, "pythons_hub") + +# This call registers the Python toolchains. +register_toolchains("@pythons_hub//:all") diff --git a/README.md b/README.md index a3f18869e6..cf4b04e224 100644 --- a/README.md +++ b/README.md @@ -45,40 +45,68 @@ the older way of configuring bazel with a `WORKSPACE` file. ### Using bzlmod +NOTE: bzlmod support is still experimental; APIs subject to change. + To import rules_python in your project, you first need to add it to your `MODULE.bazel` file, using the snippet provided in the [release you choose](https://github.com/bazelbuild/rules_python/releases). +Once the dependency is added, a Python toolchain will be automatically +registered and you'll be able to create runnable programs and tests. + + #### Toolchain registration with bzlmod -To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `MODULE.bazel` file: +NOTE: bzlmod support is still experimental; APIs subject to change. -```python -# Find the latest version number here: https://github.com/bazelbuild/rules_python/releases -# and change the version number if needed in the line below. -bazel_dep(name = "rules_python", version = "0.20.0") +A default toolchain is automatically configured for by depending on +`rules_python`. Note, however, the version used tracks the most recent Python +release and will change often. -# You do not have to use pip for the toolchain, but most people -# will use it for the dependency management. -pip = use_extension("@rules_python//python:extensions.bzl", "pip") +If you want to register specific Python versions, then use +`python.toolchain()` for each version you need: -pip.parse( - name = "pip", - requirements_lock = "//:requirements_lock.txt", +```starlark +python = use_extension("@rules_python//python:extensions.bzl", "python") + +python.toolchain( + python_version = "3.9", ) +``` -use_repo(pip, "pip") +### Using pip with bzlmod -# Register a specific python toolchain instead of using the host version -python = use_extension("@rules_python//python:extensions.bzl", "python") +NOTE: bzlmod support is still experimental; APIs subject to change. + +To use dependencies from PyPI, the `pip.parse()` extension is used to +convert a requirements file into Bazel dependencies. -use_repo(python, "python3_10_toolchains") +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + python_version = "3.9", +) -register_toolchains( - "@python3_10_toolchains//:all", +interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") +interpreter.install( + name = "interpreter", + python_name = "python_3_9", ) +use_repo(interpreter, "interpreter") + +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + name = "pip", + incompatible_generate_aliases = True, + python_interpreter_target = "@interpreter//:python", + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") ``` +For more documentation see the bzlmod examples under the [examples](examples) folder. + ### Using a WORKSPACE file To import rules_python in your project, you first need to add it to your diff --git a/docs/packaging.md b/docs/packaging.md index b244b42767..16fa00c312 100755 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -82,7 +82,7 @@ in the way they expect. | console_scripts | Deprecated console_script entry points, e.g. {'main': 'examples.wheel.main:main'}.

Deprecated: prefer the entry_points attribute, which supports console_scripts as well as other entry points. | Dictionary: String -> String | optional | {} | | deps | Targets to be included in the distribution.

The targets to package are usually py_library rules or filesets (for packaging data files).

Note it's usually better to package py_library targets and use entry_points attribute to specify console_scripts than to package py_binary rules. py_binary targets would wrap a executable script that tries to locate .runfiles directory which is not packaged in the wheel. | List of labels | optional | [] | | description_file | A file containing text describing the package. | Label | optional | None | -| distribution | Name of the distribution.

This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. | String | required | | +| distribution | Name of the distribution.

This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies.

Workspace status keys are expanded using {NAME} format, for example: - distribution = "package.{CLASSIFIER}" - distribution = "{DISTRIBUTION}"

For the available keys, see https://bazel.build/docs/user-manual#workspace-status | String | required | | | entry_points | entry_points, e.g. {'console_scripts': ['main = examples.wheel.main:main']}. | Dictionary: String -> List of strings | optional | {} | | extra_distinfo_files | Extra files to add to distinfo directory in the archive. | Dictionary: Label -> String | optional | {} | | extra_requires | List of optional requirements for this package | Dictionary: String -> List of strings | optional | {} | diff --git a/docs/pip.md b/docs/pip.md index e4c3f21b79..8ad5b6903a 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -45,6 +45,9 @@ It also generates two targets for running pip-compile: - validate with `bazel test [name]_test` - update with `bazel run [name].update` +If you are using a version control system, the requirements.txt generated by this rule should +be checked into it to ensure that all developers/users have the same dependency versions. + **PARAMETERS** diff --git a/examples/bzlmod/.bazelrc b/examples/bzlmod/.bazelrc index b8c233f98c..6f557e67b9 100644 --- a/examples/bzlmod/.bazelrc +++ b/examples/bzlmod/.bazelrc @@ -1,3 +1,10 @@ common --experimental_enable_bzlmod coverage --java_runtime_version=remotejdk_11 + +test --test_output=errors --enable_runfiles + +# Windows requires these for multi-python support: +build --enable_runfiles + +startup --windows_enable_symlinks diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index e1f5790631..3bff20836d 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,9 +1,18 @@ +# Load various rules so that we can have bazel download +# various rulesets and dependencies. +# The `load` statement imports the symbol for the rule, in the defined +# ruleset. When the symbol is loaded you can use the rule. + +# The names @pip and @python_39 are values that are repository +# names. Those names are defined in the MODULES.bazel file. load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@pip//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement") -load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") +load("@python_3_9//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") +# This stanza calls a rule that generates targets for managing pip dependencies +# with pip-compile. compile_pip_requirements( name = "requirements", extra_args = ["--allow-unsafe"], @@ -12,6 +21,11 @@ compile_pip_requirements( requirements_windows = "requirements_windows.txt", ) +# The rules below are language specific rules defined in +# rules_python. See +# https://bazel.build/reference/be/python + +# see https://bazel.build/reference/be/python#py_library py_library( name = "lib", srcs = ["lib.py"], @@ -22,6 +36,7 @@ py_library( ], ) +# see https://bazel.build/reference/be/python#py_binary py_binary( name = "bzlmod", srcs = ["__main__.py"], @@ -32,6 +47,7 @@ py_binary( ], ) +# see https://bazel.build/reference/be/python#py_test py_test( name = "test", srcs = ["test.py"], @@ -46,6 +62,11 @@ py_test_with_transition( deps = [":lib"], ) +# This example is also used for integration tests within +# rules_python. We are using +# https://github.com/bazelbuild/bazel-skylib +# to run some of the tests. +# See: https://github.com/bazelbuild/bazel-skylib/blob/main/docs/build_test_doc.md build_test( name = "all_wheels", targets = all_whl_requirements, diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 145cebd276..40dfb6aed1 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,32 +11,64 @@ local_path_override( path = "../..", ) +# This name is generated by python.toolchain(), and is later passed +# to use_repo() and interpreter.install(). +PYTHON_NAME_39 = "python_3_9" + +INTERPRETER_NAME_39 = "interpreter_39" + +PYTHON_NAME_310 = "python_3_10" + +INTERPRETER_NAME_310 = "interpreter_310" + +# We next initialize the python toolchain using the extension. +# You can set different Python versions in this block. python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python3_9", configure_coverage_tool = True, + # Only set when you have mulitple toolchain versions. + is_default = True, python_version = "3.9", ) -use_repo(python, "python3_9") -use_repo(python, "python3_9_toolchains") -register_toolchains( - "@python3_9_toolchains//:all", +# We are also using a second version of Python in this project. +# Typically you will only need a single version of Python, but +# If you need a different vesion we support more than one. +# Note: we do not supporting using multiple pip extensions, this is +# work in progress. +python.toolchain( + configure_coverage_tool = True, + python_version = "3.10", ) +# You only need to load this repositories if you are using muiltple Python versions. +# See the tests folder for various examples. +use_repo(python, PYTHON_NAME_39, "python_aliases") + +# The interpreter extension discovers the platform specific Python binary. +# It creates a symlink to the binary, and we pass the label to the following +# pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3_9", - python_name = "python3_9", + name = INTERPRETER_NAME_39, + python_name = PYTHON_NAME_39, +) + +# This second call is only needed if you are using mulitple different +# Python versions/interpreters. +interpreter.install( + name = INTERPRETER_NAME_310, + python_name = PYTHON_NAME_310, ) -use_repo(interpreter, "interpreter_python3_9") +use_repo(interpreter, INTERPRETER_NAME_39, INTERPRETER_NAME_310) pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( name = "pip", - # Intentionally set it false because the "true" case is already covered by examples/bzlmod_build_file_generation + # Intentionally set it false because the "true" case is already + # covered by examples/bzlmod_build_file_generation incompatible_generate_aliases = False, - python_interpreter_target = "@interpreter_python3_9//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME_39), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/examples/bzlmod/__main__.py b/examples/bzlmod/__main__.py index 099493b3c8..daf17495c2 100644 --- a/examples/bzlmod/__main__.py +++ b/examples/bzlmod/__main__.py @@ -13,6 +13,8 @@ # limitations under the License. from lib import main +import sys if __name__ == "__main__": print(main([["A", 1], ["B", 2]])) + print(sys.version) diff --git a/examples/bzlmod/other_module/MODULE.bazel b/examples/bzlmod/other_module/MODULE.bazel index 992e120760..cc23a51601 100644 --- a/examples/bzlmod/other_module/MODULE.bazel +++ b/examples/bzlmod/other_module/MODULE.bazel @@ -2,4 +2,33 @@ module( name = "other_module", ) +# This module is using the same version of rules_python +# that the parent module uses. bazel_dep(name = "rules_python", version = "") + +# It is not best practice to use a python.toolchian in +# a submodule. This code only exists to test that +# we support doing this. This code is only for rules_python +# testing purposes. +PYTHON_NAME_39 = "python_3_9" + +PYTHON_NAME_311 = "python_3_11" + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + configure_coverage_tool = True, + python_version = "3.9", +) +python.toolchain( + configure_coverage_tool = True, + # In a submodule this is ignored + is_default = True, + python_version = "3.11", +) + +# created by the above python.toolchain calls. +use_repo( + python, + PYTHON_NAME_39, + PYTHON_NAME_311, +) diff --git a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel index 9a130e3554..6e37df8233 100644 --- a/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel +++ b/examples/bzlmod/other_module/other_module/pkg/BUILD.bazel @@ -1,3 +1,4 @@ +load("@python_3_11//:defs.bzl", py_binary_311 = "py_binary") load("@rules_python//python:defs.bzl", "py_library") py_library( @@ -8,4 +9,15 @@ py_library( deps = ["@rules_python//python/runfiles"], ) +# This is used for testing mulitple versions of Python. This is +# used only when you need to support multiple versions of Python +# in the same project. +py_binary_311( + name = "lib_311", + srcs = ["lib.py"], + data = ["data/data.txt"], + visibility = ["//visibility:public"], + deps = ["@rules_python//python/runfiles"], +) + exports_files(["data/data.txt"]) diff --git a/examples/bzlmod/runfiles/BUILD.bazel b/examples/bzlmod/runfiles/BUILD.bazel index 3503ac3017..add56b3bd0 100644 --- a/examples/bzlmod/runfiles/BUILD.bazel +++ b/examples/bzlmod/runfiles/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_python//python:defs.bzl", "py_test") -# gazelle:ignore py_test( name = "runfiles_test", srcs = ["runfiles_test.py"], diff --git a/examples/bzlmod/test.py b/examples/bzlmod/test.py index a36c19dc63..0486916e1b 100644 --- a/examples/bzlmod/test.py +++ b/examples/bzlmod/test.py @@ -30,6 +30,18 @@ def test_coverage_doesnt_shadow_stdlib(self): except ImportError: self.skipTest("not running under coverage, skipping") + self.assertEqual( + "html", + f"{html_stdlib.__name__}", + "'html' from stdlib was not loaded correctly", + ) + + self.assertEqual( + "coverage.html", + f"{html_coverage.__name__}", + "'coverage.html' was not loaded correctly", + ) + self.assertNotEqual( html_stdlib, html_coverage, diff --git a/examples/bzlmod/tests/BUILD.bazel b/examples/bzlmod/tests/BUILD.bazel new file mode 100644 index 0000000000..5331f4ab96 --- /dev/null +++ b/examples/bzlmod/tests/BUILD.bazel @@ -0,0 +1,142 @@ +load("@python_aliases//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test") +load("@python_aliases//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test") +load("@python_aliases//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test") +load("@rules_python//python:defs.bzl", "py_binary", "py_test") + +py_binary( + name = "version_default", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_9( + name = "version_3_9", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_10( + name = "version_3_10", + srcs = ["version.py"], + main = "version.py", +) + +py_binary_3_11( + name = "version_3_11", + srcs = ["version.py"], + main = "version.py", +) + +# This is a work in progress and the commented +# tests will not work until we can support +# multiple pips with bzlmod. + +#py_test( +# name = "my_lib_default_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_9( +# name = "my_lib_3_9_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_10( +# name = "my_lib_3_10_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +#py_test_3_11( +# name = "my_lib_3_11_test", +# srcs = ["my_lib_test.py"], +# main = "my_lib_test.py", +# deps = ["//libs/my_lib"], +#) + +py_test( + name = "version_default_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.9"}, # The default defined in the WORKSPACE. + main = "version_test.py", +) + +py_test_3_9( + name = "version_3_9_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.9"}, + main = "version_test.py", +) + +py_test_3_10( + name = "version_3_10_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.10"}, + main = "version_test.py", +) + +py_test_3_11( + name = "version_3_11_test", + srcs = ["version_test.py"], + env = {"VERSION_CHECK": "3.11"}, + main = "version_test.py", +) + +py_test( + name = "version_default_takes_3_10_subprocess_test", + srcs = ["cross_version_test.py"], + data = [":version_3_10"], + env = { + "SUBPROCESS_VERSION_CHECK": "3.10", + "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_10)", + "VERSION_CHECK": "3.9", + }, + main = "cross_version_test.py", +) + +py_test_3_10( + name = "version_3_10_takes_3_9_subprocess_test", + srcs = ["cross_version_test.py"], + data = [":version_3_9"], + env = { + "SUBPROCESS_VERSION_CHECK": "3.9", + "SUBPROCESS_VERSION_PY_BINARY": "$(rootpath :version_3_9)", + "VERSION_CHECK": "3.10", + }, + main = "cross_version_test.py", +) + +sh_test( + name = "version_test_binary_default", + srcs = ["version_test.sh"], + data = [":version_default"], + env = { + "VERSION_CHECK": "3.9", # The default defined in the WORKSPACE. + "VERSION_PY_BINARY": "$(rootpath :version_default)", + }, +) + +sh_test( + name = "version_test_binary_3_9", + srcs = ["version_test.sh"], + data = [":version_3_9"], + env = { + "VERSION_CHECK": "3.9", + "VERSION_PY_BINARY": "$(rootpath :version_3_9)", + }, +) + +sh_test( + name = "version_test_binary_3_10", + srcs = ["version_test.sh"], + data = [":version_3_10"], + env = { + "VERSION_CHECK": "3.10", + "VERSION_PY_BINARY": "$(rootpath :version_3_10)", + }, +) diff --git a/examples/bzlmod/tests/cross_version_test.py b/examples/bzlmod/tests/cross_version_test.py new file mode 100644 index 0000000000..437be2ed5a --- /dev/null +++ b/examples/bzlmod/tests/cross_version_test.py @@ -0,0 +1,39 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +process = subprocess.run( + [os.getenv("SUBPROCESS_VERSION_PY_BINARY")], + stdout=subprocess.PIPE, + universal_newlines=True, +) + +subprocess_current = process.stdout.strip() +subprocess_expected = os.getenv("SUBPROCESS_VERSION_CHECK") + +if subprocess_current != subprocess_expected: + print( + f"expected subprocess version '{subprocess_expected}' is different than returned '{subprocess_current}'" + ) + sys.exit(1) + +expected = os.getenv("VERSION_CHECK") +current = f"{sys.version_info.major}.{sys.version_info.minor}" + +if current != expected: + print(f"expected version '{expected}' is different than returned '{current}'") + sys.exit(1) diff --git a/examples/bzlmod/tests/version.py b/examples/bzlmod/tests/version.py new file mode 100644 index 0000000000..2d293c1571 --- /dev/null +++ b/examples/bzlmod/tests/version.py @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +print(f"{sys.version_info.major}.{sys.version_info.minor}") diff --git a/examples/bzlmod/tests/version_test.py b/examples/bzlmod/tests/version_test.py new file mode 100644 index 0000000000..444f5e4321 --- /dev/null +++ b/examples/bzlmod/tests/version_test.py @@ -0,0 +1,23 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +expected = os.getenv("VERSION_CHECK") +current = f"{sys.version_info.major}.{sys.version_info.minor}" + +if current != expected: + print(f"expected version '{expected}' is different than returned '{current}'") + sys.exit(1) diff --git a/examples/bzlmod/tests/version_test.sh b/examples/bzlmod/tests/version_test.sh new file mode 100755 index 0000000000..3bedb95ef9 --- /dev/null +++ b/examples/bzlmod/tests/version_test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -o errexit -o nounset -o pipefail + +version_py_binary=$("${VERSION_PY_BINARY}") + +if [[ "${version_py_binary}" != "${VERSION_CHECK}" ]]; then + echo >&2 "expected version '${VERSION_CHECK}' is different than returned '${version_py_binary}'" + exit 1 +fi diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c667f1e49b..498969ba3a 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -7,7 +7,6 @@ # requirements. load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") -load("@python3//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") @@ -72,13 +71,6 @@ gazelle( gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", ) -py_test_with_transition( - name = "test_with_transition", - srcs = ["__test__.py"], - main = "__test__.py", - deps = [":bzlmod_build_file_generation"], -) - # The following targets are created and maintained by gazelle py_library( name = "bzlmod_build_file_generation", diff --git a/examples/bzlmod_build_file_generation/MODULE.bazel b/examples/bzlmod_build_file_generation/MODULE.bazel index 179fe1bdea..fab2a26a83 100644 --- a/examples/bzlmod_build_file_generation/MODULE.bazel +++ b/examples/bzlmod_build_file_generation/MODULE.bazel @@ -46,40 +46,27 @@ python = use_extension("@rules_python//python/extensions:python.bzl", "python") # This name is passed into python.toolchain and it's use_repo statement. # We also use the same name for python.host_python_interpreter. -PYTHON_NAME = "python3" +PYTHON_NAME = "python_3_9" + +INTERPRETER_NAME = "interpreter" # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. python.toolchain( - # This name is used in the various use_repo statements - # below, and in the local extension that is in this - # example. - name = PYTHON_NAME, configure_coverage_tool = True, + is_default = True, python_version = "3.9", ) -# Import the python repositories generated by the given module extension -# into the scope of the current module. -# All of the python3 repositories use the PYTHON_NAME as there prefix. They -# are not catenated for ease of reading. -use_repo(python, PYTHON_NAME, "python3_toolchains") - -# Register an already-defined toolchain so that Bazel can use it during -# toolchain resolution. -register_toolchains( - "@python3_toolchains//:all", -) - # The interpreter extension discovers the platform specific Python binary. # It creates a symlink to the binary, and we pass the label to the following # pip.parse call. interpreter = use_extension("@rules_python//python/extensions:interpreter.bzl", "interpreter") interpreter.install( - name = "interpreter_python3", + name = INTERPRETER_NAME, python_name = PYTHON_NAME, ) -use_repo(interpreter, "interpreter_python3") +use_repo(interpreter, INTERPRETER_NAME) # Use the extension, pip.parse, to call the `pip_repository` rule that invokes # `pip`, with `incremental` set. The pip call accepts a locked/compiled @@ -102,7 +89,7 @@ pip.parse( # is used for both resolving dependencies and running tests/binaries. # If this isn't specified, then you'll get whatever is locally installed # on your system. - python_interpreter_target = "@interpreter_python3//:python", + python_interpreter_target = "@{}//:python".format(INTERPRETER_NAME), requirements_lock = "//:requirements_lock.txt", requirements_windows = "//:requirements_windows.txt", ) diff --git a/examples/py_proto_library/MODULE.bazel b/examples/py_proto_library/MODULE.bazel index 6fb1a05548..feb938da5c 100644 --- a/examples/py_proto_library/MODULE.bazel +++ b/examples/py_proto_library/MODULE.bazel @@ -14,15 +14,10 @@ local_path_override( python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( - name = "python3_9", configure_coverage_tool = True, python_version = "3.9", ) -use_repo(python, "python3_9_toolchains") - -register_toolchains( - "@python3_9_toolchains//:all", -) +use_repo(python, "python_3_9") # We are using rules_proto to define rules_proto targets to be consumed by py_proto_library. bazel_dep(name = "rules_proto", version = "5.3.0-21.7") diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index 61a43ae6cf..49a212b311 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -94,7 +94,7 @@ build_test( py_wheel( name = "minimal_with_py_library_with_stamp", # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" - distribution = "example_minimal_library", + distribution = "example_minimal_library{BUILD_USER}", python_tag = "py3", stamp = 1, version = "0.1.{BUILD_TIMESTAMP}", diff --git a/examples/wheel/wheel_test.py b/examples/wheel/wheel_test.py index c292c87132..591e0571de 100644 --- a/examples/wheel/wheel_test.py +++ b/examples/wheel/wheel_test.py @@ -374,30 +374,36 @@ def test_rule_creates_directory_and_is_included_in_wheel(self): ], ) - def test_rule_sets_stamped_version_in_wheel_metadata(self): + def test_rule_expands_workspace_status_keys_in_wheel_metadata(self): filename = os.path.join( os.environ["TEST_SRCDIR"], "rules_python", "examples", "wheel", - "example_minimal_library-0.1._BUILD_TIMESTAMP_-py3-none-any.whl", + "example_minimal_library_BUILD_USER_-0.1._BUILD_TIMESTAMP_-py3-none-any.whl", ) with zipfile.ZipFile(filename) as zf: metadata_file = None for f in zf.namelist(): self.assertNotIn("_BUILD_TIMESTAMP_", f) + self.assertNotIn("_BUILD_USER_", f) if os.path.basename(f) == "METADATA": metadata_file = f self.assertIsNotNone(metadata_file) version = None + name = None with zf.open(metadata_file) as fp: for line in fp: - if line.startswith(b'Version:'): + if line.startswith(b"Version:"): version = line.decode().split()[-1] + if line.startswith(b"Name:"): + name = line.decode().split()[-1] self.assertIsNotNone(version) + self.assertIsNotNone(name) self.assertNotIn("{BUILD_TIMESTAMP}", version) + self.assertNotIn("{BUILD_USER}", name) if __name__ == "__main__": diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index bd634020f3..ae94a5f863 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "rules_python", version = "0.18.0") bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go") -bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") +bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") diff --git a/gazelle/README.md b/gazelle/README.md index e36f3a303a..ba8520d36b 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -35,14 +35,14 @@ Here is a snippet of a `MODULE.bazel` file. ```starlark # The following stanza defines the dependency rules_python. -bazel_dep(name = "rules_python", version = "0.20.0") +bazel_dep(name = "rules_python", version = "0.22.0") -# The following stanza defines the dependency rules_python. +# The following stanza defines the dependency rules_python_gazelle_plugin. # For typical setups you set the version. -bazel_dep(name = "rules_python_gazelle_plugin", version = "0.20.0") +bazel_dep(name = "rules_python_gazelle_plugin", version = "0.22.0") -# The following stanza defines the dependency rules_python. -bazel_dep(name = "gazelle", version = "0.30.0", repo_name = "bazel_gazelle") +# The following stanza defines the dependency gazelle. +bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") # Import the python repositories generated by the given module extension into the scope of the current module. use_repo(python, "python3_9") diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index 55cf1b0d40..eef16e924d 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -13,10 +13,10 @@ http_archive( http_archive( name = "bazel_gazelle", - sha256 = "448e37e0dbf61d6fa8f00aaa12d191745e14f07c31cabfa731f0c8e8a4f41b97", + sha256 = "29d5dafc2a5582995488c6735115d1d366fcd6a0fc2e2a153f02988706349825", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.28.0/bazel-gazelle-v0.28.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.31.0/bazel-gazelle-v0.31.0.tar.gz", ], ) diff --git a/python/extensions/interpreter.bzl b/python/extensions/interpreter.bzl index b9afe1abda..bbeadc26b8 100644 --- a/python/extensions/interpreter.bzl +++ b/python/extensions/interpreter.bzl @@ -53,7 +53,7 @@ def _interpreter_repo_impl(rctx): actual_interpreter_label = INTERPRETER_LABELS.get(rctx.attr.python_name) if actual_interpreter_label == None: - fail("Unable to find interpreter with name {}".format(rctx.attr.python_name)) + fail("Unable to find interpreter with name '{}'".format(rctx.attr.python_name)) rctx.symlink(actual_interpreter_label, "python") diff --git a/python/extensions/private/interpreter_hub.bzl b/python/extensions/private/interpreter_hub.bzl deleted file mode 100644 index f1ca670cf2..0000000000 --- a/python/extensions/private/interpreter_hub.bzl +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2023 The Bazel Authors. All rights reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" - -load("//python:versions.bzl", "WINDOWS_NAME") -load("//python/private:toolchains_repo.bzl", "get_host_os_arch", "get_host_platform") - -_build_file_for_hub_template = """ -INTERPRETER_LABELS = {{ -{lines} -}} -""" - -_line_for_hub_template = """\ - "{name}": Label("@{name}_{platform}//:{path}"), -""" - -def _hub_repo_impl(rctx): - (os, arch) = get_host_os_arch(rctx) - platform = get_host_platform(os, arch) - - rctx.file("BUILD.bazel", "") - is_windows = (os == WINDOWS_NAME) - path = "python.exe" if is_windows else "bin/python3" - - lines = "\n".join([_line_for_hub_template.format( - name = name, - platform = platform, - path = path, - ) for name in rctx.attr.toolchains]) - - rctx.file("interpreters.bzl", _build_file_for_hub_template.format(lines = lines)) - -hub_repo = repository_rule( - doc = """\ -This private rule create a repo with a BUILD file that contains a map of interpreter names -and the labels to said interpreters. This map is used to by the interpreter hub extension. -""", - implementation = _hub_repo_impl, - attrs = { - "toolchains": attr.string_list( - doc = "List of the base names the toolchain repo defines.", - mandatory = True, - ), - }, -) diff --git a/python/extensions/private/pythons_hub.bzl b/python/extensions/private/pythons_hub.bzl new file mode 100644 index 0000000000..5baaef96fd --- /dev/null +++ b/python/extensions/private/pythons_hub.bzl @@ -0,0 +1,136 @@ +# Copyright 2023 The Bazel Authors. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" + +load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME") +load( + "//python/private:toolchains_repo.bzl", + "get_host_os_arch", + "get_host_platform", + "get_repository_name", + "python_toolchain_build_file_content", +) + +def _have_same_length(*lists): + if not lists: + fail("expected at least one list") + return len({len(length): None for length in lists}) == 1 + +def _get_version(python_version): + # we need to get the MINOR_MAPPING or use the full version + if python_version in MINOR_MAPPING: + python_version = MINOR_MAPPING[python_version] + return python_version + +def _python_toolchain_build_file_content( + prefixes, + python_versions, + set_python_version_constraints, + user_repository_names, + workspace_location): + """This macro iterates over each of the lists and returns the toolchain content. + + python_toolchain_build_file_content is called to generate each of the toolchain + definitions. + """ + + if not _have_same_length(python_versions, set_python_version_constraints, user_repository_names): + fail("all lists must have the same length") + + rules_python = get_repository_name(workspace_location) + + # Iterate over the length of python_versions and call + # build the toolchain content by calling python_toolchain_build_file_content + return "\n".join([python_toolchain_build_file_content( + prefix = prefixes[i], + python_version = _get_version(python_versions[i]), + set_python_version_constraint = set_python_version_constraints[i], + user_repository_name = user_repository_names[i], + rules_python = rules_python, + ) for i in range(len(python_versions))]) + +_build_file_for_hub_template = """ +INTERPRETER_LABELS = {{ +{interpreter_labels} +}} +""" + +_line_for_hub_template = """\ + "{name}": Label("@{name}_{platform}//:{path}"), +""" + +def _hub_repo_impl(rctx): + # Create the various toolchain definitions and + # write them to the BUILD file. + rctx.file( + "BUILD.bazel", + _python_toolchain_build_file_content( + rctx.attr.toolchain_prefixes, + rctx.attr.toolchain_python_versions, + rctx.attr.toolchain_set_python_version_constraints, + rctx.attr.toolchain_user_repository_names, + rctx.attr._rules_python_workspace, + ), + executable = False, + ) + + (os, arch) = get_host_os_arch(rctx) + platform = get_host_platform(os, arch) + is_windows = (os == WINDOWS_NAME) + path = "python.exe" if is_windows else "bin/python3" + + # Create a dict that is later used to create + # a symlink to a interpreter. + interpreter_labels = "".join([_line_for_hub_template.format( + name = name, + platform = platform, + path = path, + ) for name in rctx.attr.toolchain_user_repository_names]) + + rctx.file( + "interpreters.bzl", + _build_file_for_hub_template.format( + interpreter_labels = interpreter_labels, + ), + executable = False, + ) + +hub_repo = repository_rule( + doc = """\ +This private rule create a repo with a BUILD file that contains a map of interpreter names +and the labels to said interpreters. This map is used to by the interpreter hub extension. +This rule also writes out the various toolchains for the different Python versions. +""", + implementation = _hub_repo_impl, + attrs = { + "toolchain_prefixes": attr.string_list( + doc = "List prefixed for the toolchains", + mandatory = True, + ), + "toolchain_python_versions": attr.string_list( + doc = "List of Python versions for the toolchains", + mandatory = True, + ), + "toolchain_set_python_version_constraints": attr.string_list( + doc = "List of version contraints for the toolchains", + mandatory = True, + ), + "toolchain_user_repository_names": attr.string_list( + doc = "List of the user repo names for the toolchains", + mandatory = True, + ), + "_rules_python_workspace": attr.label(default = Label("//:does_not_matter_what_this_name_is")), + }, +) diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 9a3d9ed959..d7a466ae10 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -14,38 +14,225 @@ "Python toolchain module extensions for use with bzlmod" -load("@rules_python//python:repositories.bzl", "python_register_toolchains") -load("@rules_python//python/extensions/private:interpreter_hub.bzl", "hub_repo") +load("//python:repositories.bzl", "python_register_toolchains") +load("//python/extensions/private:pythons_hub.bzl", "hub_repo") +load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases") + +# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all +# targets using any of these toolchains due to the changed repository name. +_MAX_NUM_TOOLCHAINS = 9999 +_TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) + +def _toolchain_prefix(index, name): + """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting. + + Examples: + _toolchain_prefix( 2, "foo") == "_0002_foo_" + _toolchain_prefix(2000, "foo") == "_2000_foo_" + """ + return "_{}_{}_".format(_left_pad_zero(index, _TOOLCHAIN_INDEX_PAD_LENGTH), name) + +def _left_pad_zero(index, length): + if index < 0: + fail("index must be non-negative") + return ("0" * length + str(index))[-length:] + +# Printing a warning msg not debugging, so we have to disable +# the buildifier check. +# buildifier: disable=print +def _print_warn(msg): + print("WARNING:", msg) + +def _python_register_toolchains(name, toolchain_attr, version_constraint): + """Calls python_register_toolchains and returns a struct used to collect the toolchains. + """ + python_register_toolchains( + name = name, + python_version = toolchain_attr.python_version, + register_coverage_tool = toolchain_attr.configure_coverage_tool, + ignore_root_user_error = toolchain_attr.ignore_root_user_error, + set_python_version_constraint = version_constraint, + ) + return struct( + python_version = toolchain_attr.python_version, + set_python_version_constraint = str(version_constraint), + name = name, + ) def _python_impl(module_ctx): + # The toolchain info structs to register, in the order to register them in. toolchains = [] + + # We store the default toolchain separately to ensure it is the last + # toolchain added to toolchains. + default_toolchain = None + + # Map of string Major.Minor to the toolchain name and module name + global_toolchain_versions = {} + for mod in module_ctx.modules: + module_toolchain_versions = [] + for toolchain_attr in mod.tags.toolchain: - python_register_toolchains( - name = toolchain_attr.name, - python_version = toolchain_attr.python_version, - bzlmod = True, - # Toolchain registration in bzlmod is done in MODULE file - register_toolchains = False, - register_coverage_tool = toolchain_attr.configure_coverage_tool, - ignore_root_user_error = toolchain_attr.ignore_root_user_error, + toolchain_version = toolchain_attr.python_version + toolchain_name = "python_" + toolchain_version.replace(".", "_") + + # Duplicate versions within a module indicate a misconfigured module. + if toolchain_version in module_toolchain_versions: + _fail_duplicate_module_toolchain_version(toolchain_version, mod.name) + module_toolchain_versions.append(toolchain_version) + + # Ignore version collisions in the global scope because there isn't + # much else that can be done. Modules don't know and can't control + # what other modules do, so the first in the dependency graph wins. + if toolchain_version in global_toolchain_versions: + _warn_duplicate_global_toolchain_version( + toolchain_version, + first = global_toolchain_versions[toolchain_version], + second_toolchain_name = toolchain_name, + second_module_name = mod.name, + ) + continue + global_toolchain_versions[toolchain_version] = struct( + toolchain_name = toolchain_name, + module_name = mod.name, ) - # We collect all of the toolchain names to create - # the INTERPRETER_LABELS map. This is used - # by interpreter_extensions.bzl - toolchains.append(toolchain_attr.name) + # Only the root module and rules_python are allowed to specify the default + # toolchain for a couple reasons: + # * It prevents submodules from specifying different defaults and only + # one of them winning. + # * rules_python needs to set a soft default in case the root module doesn't, + # e.g. if the root module doesn't use Python itself. + # * The root module is allowed to override the rules_python default. + if mod.is_root: + # A single toolchain is treated as the default because it's unambiguous. + is_default = toolchain_attr.is_default or len(mod.tags.toolchain) == 1 + elif mod.name == "rules_python" and not default_toolchain: + # We don't do the len() check because we want the default that rules_python + # sets to be clearly visible. + is_default = toolchain_attr.is_default + else: + is_default = False + + # We have already found one default toolchain, and we can only have + # one. + if is_default and default_toolchain != None: + _fail_multiple_default_toolchains( + first = default_toolchain.name, + second = toolchain_name, + ) + + toolchain_info = _python_register_toolchains( + toolchain_name, + toolchain_attr, + version_constraint = not is_default, + ) + + if is_default: + default_toolchain = toolchain_info + else: + toolchains.append(toolchain_info) + + # A default toolchain is required so that the non-version-specific rules + # are able to match a toolchain. + if default_toolchain == None: + fail("No default Python toolchain configured. Is rules_python missing `is_default=True`?") + # The last toolchain in the BUILD file is set as the default + # toolchain. We need the default last. + toolchains.append(default_toolchain) + + if len(toolchains) > _MAX_NUM_TOOLCHAINS: + fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) + + # Create the pythons_hub repo for the interpreter meta data and the + # the various toolchains. hub_repo( name = "pythons_hub", - toolchains = toolchains, + toolchain_prefixes = [ + _toolchain_prefix(index, toolchain.name) + for index, toolchain in enumerate(toolchains) + ], + toolchain_python_versions = [t.python_version for t in toolchains], + toolchain_set_python_version_constraints = [t.set_python_version_constraint for t in toolchains], + toolchain_user_repository_names = [t.name for t in toolchains], ) + # This is require in order to support multiple version py_test + # and py_binary + multi_toolchain_aliases( + name = "python_aliases", + python_versions = { + version: entry.toolchain_name + for version, entry in global_toolchain_versions.items() + }, + ) + +def _fail_duplicate_module_toolchain_version(version, module): + fail(("Duplicate module toolchain version: module '{module}' attempted " + + "to use version '{version}' multiple times in itself").format( + version = version, + module = module, + )) + +def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name): + _print_warn(( + "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + + "Toolchain '{first_toolchain}' from module '{first_module}' " + + "already registered Python version {version} and has precedence" + ).format( + first_toolchain = first.toolchain_name, + first_module = first.module_name, + second_module = second_module_name, + second_toolchain = second_toolchain_name, + version = version, + )) + +def _fail_multiple_default_toolchains(first, second): + fail(("Multiple default toolchains: only one toolchain " + + "can have is_default=True. First default " + + "was toolchain '{first}'. Second was '{second}'").format( + first = first, + second = second, + )) + python = module_extension( - doc = "Bzlmod extension that is used to register a Python toolchain.", + doc = """Bzlmod extension that is used to register Python toolchains. +""", implementation = _python_impl, tag_classes = { "toolchain": tag_class( + doc = """Tag class used to register Python toolchains. +Use this tag class to register one or more Python toolchains. This class +is also potentially called by sub modules. The following covers different +business rules and use cases. + +Toolchains in the Root Module + +This class registers all toolchains in the root module. + +Toolchains in Sub Modules + +It will create a toolchain that is in a sub module, if the toolchain +of the same name does not exist in the root module. The extension stops name +clashing between toolchains in the root module and toolchains in sub modules. +You cannot configure more than one toolchain as the default toolchain. + +Toolchain set as the default version + +This extension will not create a toolchain that exists in a sub module, +if the sub module toolchain is marked as the default version. If you have +more than one toolchain in your root module, you need to set one of the +toolchains as the default version. If there is only one toolchain it +is set as the default toolchain. + +Toolchain repository name + +A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. +`python_3_10`. The `major` and `minor` components are +`major` and `minor` are the Python version from the `python_version` attribute. +""", attrs = { "configure_coverage_tool": attr.bool( mandatory = False, @@ -56,8 +243,14 @@ python = module_extension( doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", mandatory = False, ), - "name": attr.string(mandatory = True), - "python_version": attr.string(mandatory = True), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), + "python_version": attr.string( + mandatory = True, + doc = "The Python version, in `major.minor` format, e.g '3.12', to create a toolchain for.", + ), }, ), }, diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 7594471897..86fd408647 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -42,6 +42,9 @@ def compile_pip_requirements( - validate with `bazel test [name]_test` - update with `bazel run [name].update` + If you are using a version control system, the requirements.txt generated by this rule should + be checked into it to ensure that all developers/users have the same dependency versions. + Args: name: base name for generated targets, typically "requirements". extra_args: passed to pip-compile. diff --git a/python/pip_install/tools/dependency_resolver/dependency_resolver.py b/python/pip_install/tools/dependency_resolver/dependency_resolver.py index 89e355806c..ceb20db7ef 100644 --- a/python/pip_install/tools/dependency_resolver/dependency_resolver.py +++ b/python/pip_install/tools/dependency_resolver/dependency_resolver.py @@ -95,25 +95,30 @@ def _locate(bazel_runfiles, file): requirements_windows = parse_str_none(sys.argv.pop(1)) update_target_label = sys.argv.pop(1) + requirements_file = _select_golden_requirements_file( + requirements_txt=requirements_txt, requirements_linux=requirements_linux, + requirements_darwin=requirements_darwin, requirements_windows=requirements_windows + ) + resolved_requirements_in = _locate(bazel_runfiles, requirements_in) - resolved_requirements_txt = _locate(bazel_runfiles, requirements_txt) + resolved_requirements_file = _locate(bazel_runfiles, requirements_file) # Files in the runfiles directory has the following naming schema: # Main repo: __main__/ # External repo: / # We want to strip both __main__ and from the absolute prefix # to keep the requirements lock file agnostic. - repository_prefix = requirements_txt[: requirements_txt.index("/") + 1] - absolute_path_prefix = resolved_requirements_txt[ - : -(len(requirements_txt) - len(repository_prefix)) + repository_prefix = requirements_file[: requirements_file.index("/") + 1] + absolute_path_prefix = resolved_requirements_file[ + : -(len(requirements_file) - len(repository_prefix)) ] # As requirements_in might contain references to generated files we want to # use the runfiles file first. Thus, we need to compute the relative path # from the execution root. # Note: Windows cannot reference generated files without runfiles support enabled. - requirements_in_relative = requirements_in[len(repository_prefix) :] - requirements_txt_relative = requirements_txt[len(repository_prefix) :] + requirements_in_relative = requirements_in[len(repository_prefix):] + requirements_file_relative = requirements_file[len(repository_prefix):] # Before loading click, set the locale for its parser. # If it leaks through to the system setting, it may fail: @@ -135,11 +140,11 @@ def _locate(bazel_runfiles, file): sys.argv.append(os.environ["TEST_TMPDIR"]) # Make a copy for pip-compile to read and mutate. requirements_out = os.path.join( - os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out" + os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out" ) # Those two files won't necessarily be on the same filesystem, so we can't use os.replace # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link. - shutil.copy(resolved_requirements_txt, requirements_out) + shutil.copy(resolved_requirements_file, requirements_out) update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % ( update_target_label, @@ -150,7 +155,7 @@ def _locate(bazel_runfiles, file): sys.argv.append("--generate-hashes") sys.argv.append("--output-file") - sys.argv.append(requirements_txt_relative if UPDATE else requirements_out) + sys.argv.append(requirements_file_relative if UPDATE else requirements_out) sys.argv.append( requirements_in_relative if Path(requirements_in_relative).exists() @@ -159,28 +164,28 @@ def _locate(bazel_runfiles, file): print(sys.argv) if UPDATE: - print("Updating " + requirements_txt_relative) + print("Updating " + requirements_file_relative) if "BUILD_WORKSPACE_DIRECTORY" in os.environ: workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"] - requirements_txt_tree = os.path.join(workspace, requirements_txt_relative) - # In most cases, requirements_txt will be a symlink to the real file in the source tree. - # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy, + requirements_file_tree = os.path.join(workspace, requirements_file_relative) + # In most cases, requirements_file will be a symlink to the real file in the source tree. + # If symlinks are not enabled (e.g. on Windows), then requirements_file will be a copy, # and we should copy the updated requirements back to the source tree. - if not os.path.samefile(resolved_requirements_txt, requirements_txt_tree): + if not os.path.samefile(resolved_requirements_file, requirements_file_tree): atexit.register( lambda: shutil.copy( - resolved_requirements_txt, requirements_txt_tree + resolved_requirements_file, requirements_file_tree ) ) cli() - requirements_txt_relative_path = Path(requirements_txt_relative) - content = requirements_txt_relative_path.read_text() + requirements_file_relative_path = Path(requirements_file_relative) + content = requirements_file_relative_path.read_text() content = content.replace(absolute_path_prefix, "") - requirements_txt_relative_path.write_text(content) + requirements_file_relative_path.write_text(content) else: # cli will exit(0) on success try: - print("Checking " + requirements_txt) + print("Checking " + requirements_file) cli() print("cli() should exit", file=sys.stderr) sys.exit(1) @@ -194,13 +199,7 @@ def _locate(bazel_runfiles, file): ) sys.exit(1) elif e.code == 0: - golden_filename = _select_golden_requirements_file( - requirements_txt, - requirements_linux, - requirements_darwin, - requirements_windows, - ) - golden = open(_locate(bazel_runfiles, golden_filename)).readlines() + golden = open(_locate(bazel_runfiles, requirements_file)).readlines() out = open(requirements_out).readlines() out = [line.replace(absolute_path_prefix, "") for line in out] if golden != out: diff --git a/python/pip_install/tools/wheel_installer/wheel_installer.py b/python/pip_install/tools/wheel_installer/wheel_installer.py index 77aa3a406c..5a6f49bb5b 100644 --- a/python/pip_install/tools/wheel_installer/wheel_installer.py +++ b/python/pip_install/tools/wheel_installer/wheel_installer.py @@ -406,7 +406,8 @@ def main() -> None: pip_args = ( [sys.executable, "-m", "pip"] + (["--isolated"] if args.isolated else []) - + ["download" if args.download_only else "wheel", "--no-deps"] + + (["download", "--only-binary=:all:"] if args.download_only else ["wheel"]) + + ["--no-deps"] + deserialized_args["extra_pip_args"] ) diff --git a/python/private/coverage.patch b/python/private/coverage.patch index 4cab60cddc..cb4402e19c 100644 --- a/python/private/coverage.patch +++ b/python/private/coverage.patch @@ -1,15 +1,17 @@ # Because of how coverage is run, the current directory is the first in # sys.path. This is a problem for the tests, because they may import a module of # the same name as a module in the current directory. -diff --git a/coverage/cmdline.py b/coverage/cmdline.py -index dbf66e0a..780505ac 100644 ---- a/coverage/cmdline.py -+++ b/coverage/cmdline.py -@@ -937,6 +937,7 @@ def main(argv=None): - This is installed as the script entry point. - - """ -+ sys.path.append(sys.path.pop(0)) - if argv is None: - argv = sys.argv[1:] - try: +# +# NOTE @aignas 2023-06-05: we have to do this before anything from coverage gets +# imported. +diff --git a/coverage/__main__.py b/coverage/__main__.py +index 79aa4e2b..291fcff8 100644 +--- a/coverage/__main__.py ++++ b/coverage/__main__.py +@@ -4,5 +4,6 @@ + """Coverage.py's main entry point.""" + + import sys ++sys.path.append(sys.path.pop(0)) + from coverage.cmdline import main + sys.exit(main()) diff --git a/python/private/coverage_deps.bzl b/python/private/coverage_deps.bzl index 377dfb7494..a4801cad37 100644 --- a/python/private/coverage_deps.bzl +++ b/python/private/coverage_deps.bzl @@ -28,70 +28,74 @@ load( _coverage_deps = { "cp310": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/89/a2/cbf599e50bb4be416e0408c4cf523c354c51d7da39935461a9687e039481/coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", - "784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660", + "https://files.pythonhosted.org/packages/3d/80/7060a445e1d2c9744b683dc935248613355657809d6c6b2716cdf4ca4766/coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", + "6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/15/b0/3639d84ee8a900da0cf6450ab46e22517e4688b6cec0ba8ab6f8166103a2/coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4", + "https://files.pythonhosted.org/packages/b8/9d/926fce7e03dbfc653104c2d981c0fa71f0572a9ebd344d24c573bd6f7c4f/coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/c4/8d/5ec7d08f4601d2d792563fe31db5e9322c306848fec1e65ec8885927f739/coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", - "ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53", + "https://files.pythonhosted.org/packages/01/24/be01e62a7bce89bcffe04729c540382caa5a06bee45ae42136c93e2499f5/coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", + "d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/3c/7d/d5211ea782b193ab8064b06dc0cc042cf1a4ca9c93a530071459172c550f/coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0", + "https://files.pythonhosted.org/packages/b4/bd/1b2331e3a04f4cc9b7b332b1dd0f3a1261dfc4114f8479bebfcc2afee9e8/coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063", ), }, "cp311": { + "aarch64-apple-darwin": ( + "https://files.pythonhosted.org/packages/67/d7/cd8fe689b5743fffac516597a1222834c42b80686b99f5b44ef43ccc2a43/coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", + "5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe", + ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/36/f3/5cbd79cf4cd059c80b59104aca33b8d05af4ad5bf5b1547645ecee716378/coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75", + "https://files.pythonhosted.org/packages/8c/95/16eed713202406ca0a37f8ac259bbf144c9d24f9b8097a8e6ead61da2dbb/coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/50/cf/455930004231fa87efe8be06d13512f34e070ddfee8b8bf5a050cdc47ab3/coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", - "4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795", + "https://files.pythonhosted.org/packages/c6/fa/529f55c9a1029c840bcc9109d5a15ff00478b7ff550a1ae361f8745f8ad5/coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", + "06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/6a/63/8e82513b7e4a1b8d887b4e85c1c2b6c9b754a581b187c0b084f3330ac479/coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91", + "https://files.pythonhosted.org/packages/a7/cd/3ce94ad9d407a052dc2a74fbeb1c7947f442155b28264eb467ee78dea812/coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb", ), }, "cp38": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/07/82/79fa21ceca9a9b091eb3c67e27eb648dade27b2c9e1eb23af47232a2a365/coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", - "2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba", + "https://files.pythonhosted.org/packages/28/d7/9a8de57d87f4bbc6f9a6a5ded1eaac88a89bf71369bb935dac3c0cf2893e/coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", + "3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/40/3b/cd68cb278c4966df00158811ec1e357b9a7d132790c240fc65da57e10013/coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e", + "https://files.pythonhosted.org/packages/c8/e4/e6182e4697665fb594a7f4e4f27cb3a4dd00c2e3d35c5c706765de8c7866/coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/05/63/a789b462075395d34f8152229dccf92b25ca73eac05b3f6cd75fa5017095/coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", - "d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c", + "https://files.pythonhosted.org/packages/c6/fc/be19131010930a6cf271da48202c8cc1d3f971f68c02fb2d3a78247f43dc/coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", + "54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/bd/a0/e263b115808226fdb2658f1887808c06ac3f1b579ef5dda02309e0d54459/coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b", + "https://files.pythonhosted.org/packages/44/55/49f65ccdd4dfd6d5528e966b28c37caec64170c725af32ab312889d2f857/coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e", ), }, "cp39": { "aarch64-apple-darwin": ( - "https://files.pythonhosted.org/packages/63/e9/f23e8664ec4032d7802a1cf920853196bcbdce7b56408e3efe1b2da08f3c/coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", - "95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc", + "https://files.pythonhosted.org/packages/ca/0c/3dfeeb1006c44b911ee0ed915350db30325d01808525ae7cc8d57643a2ce/coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", + "06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2", ), "aarch64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/18/95/27f80dcd8273171b781a19d109aeaed7f13d78ef6d1e2f7134a5826fd1b4/coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe", + "https://files.pythonhosted.org/packages/61/af/5964b8d7d9a5c767785644d9a5a63cacba9a9c45cc42ba06d25895ec87be/coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7", ), "x86_64-apple-darwin": ( - "https://files.pythonhosted.org/packages/ea/52/c08080405329326a7ff16c0dfdb4feefaa8edd7446413df67386fe1bbfe0/coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", - "633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745", + "https://files.pythonhosted.org/packages/88/da/495944ebf0ad246235a6bd523810d9f81981f9b81c6059ba1f56e943abe0/coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", + "537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9", ), "x86_64-unknown-linux-gnu": ( - "https://files.pythonhosted.org/packages/6b/f2/919f0fdc93d3991ca074894402074d847be8ac1e1d78e7e9e1c371b69a6f/coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5", + "https://files.pythonhosted.org/packages/fe/57/e4f8ad64d84ca9e759d783a052795f62a9f9111585e46068845b1cb52c2b/coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1", ), }, } diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index b6f2bfae56..84fdeace1d 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -40,6 +40,12 @@ Name of the distribution. This should match the project name onm PyPI. It's also the name that is used to refer to the package in other packages' dependencies. + +Workspace status keys are expanded using `{NAME}` format, for example: + - `distribution = "package.{CLASSIFIER}"` + - `distribution = "{DISTRIBUTION}"` + +For the available keys, see https://bazel.build/docs/user-manual#workspace-status """, ), "platform": attr.string( diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 9bed73e55c..f47ea8f064 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -35,43 +35,91 @@ def get_repository_name(repository_workspace): dummy_label = "//:_" return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@" -def _toolchains_repo_impl(rctx): - python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( - rules_python = get_repository_name(rctx.attr._rules_python_workspace), - python_version = rctx.attr.python_version, - ) +def python_toolchain_build_file_content( + prefix, + python_version, + set_python_version_constraint, + user_repository_name, + rules_python): + """Creates the content for toolchain definitions for a build file. - build_content = """\ -# Generated by python/private/toolchains_repo.bzl -# -# These can be registered in the workspace file or passed to --extra_toolchains -# flag. By default all these toolchains are registered by the -# python_register_toolchains macro so you don't normally need to interact with -# these targets. + Args: + prefix: Python toolchain name prefixes + python_version: Python versions for the toolchains + set_python_version_constraint: string, "True" if the toolchain should + have the Python version constraint added as a requirement for + matching the toolchain, "False" if not. + user_repository_name: names for the user repos + rules_python: rules_python label -""" + Returns: + build_content: Text containing toolchain definitions + """ + if set_python_version_constraint == "True": + constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( + rules_python = rules_python, + python_version = python_version, + ) + target_settings = '["{}"]'.format(constraint) + elif set_python_version_constraint == "False": + target_settings = "[]" + else: + fail(("Invalid set_python_version_constraint value: got {} {}, wanted " + + "either the string 'True' or the string 'False'; " + + "(did you convert bool to string?)").format( + type(set_python_version_constraint), + repr(set_python_version_constraint), + )) - for [platform, meta] in PLATFORMS.items(): - build_content += """\ -# Bazel selects this toolchain to get a Python interpreter -# for executing build actions. + # We create a list of toolchain content from iterating over + # the enumeration of PLATFORMS. We enumerate PLATFORMS in + # order to get us an index to increment the increment. + return "".join([ + """ toolchain( - name = "{platform}_toolchain", + name = "{prefix}{platform}_toolchain", target_compatible_with = {compatible_with}, - target_settings = ["{python_version_constraint}"] if {set_python_version_constraint} else [], + target_settings = {target_settings}, toolchain = "@{user_repository_name}_{platform}//:python_runtimes", toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) """.format( compatible_with = meta.compatible_with, - name = rctx.attr.name, platform = platform, - python_version_constraint = python_version_constraint, - set_python_version_constraint = rctx.attr.set_python_version_constraint, - user_repository_name = rctx.attr.user_repository_name, + # We have to use a String value here because bzlmod is passing in a + # string as we cannot have list of bools in build rule attribues. + # This if statement does not appear to work unless it is in the + # toolchain file. + target_settings = target_settings, + user_repository_name = user_repository_name, + prefix = prefix, ) + for platform, meta in PLATFORMS.items() + ]) + +def _toolchains_repo_impl(rctx): + build_content = """\ +# Generated by python/private/toolchains_repo.bzl +# +# These can be registered in the workspace file or passed to --extra_toolchains +# flag. By default all these toolchains are registered by the +# python_register_toolchains macro so you don't normally need to interact with +# these targets. + +""" + + # Get the repository name + rules_python = get_repository_name(rctx.attr._rules_python_workspace) + + toolchains = python_toolchain_build_file_content( + prefix = "", + python_version = rctx.attr.python_version, + set_python_version_constraint = str(rctx.attr.set_python_version_constraint), + user_repository_name = rctx.attr.user_repository_name, + rules_python = rules_python, + ) - rctx.file("BUILD.bazel", build_content) + rctx.file("BUILD.bazel", build_content + toolchains) toolchains_repo = repository_rule( _toolchains_repo_impl, diff --git a/python/repositories.bzl b/python/repositories.bzl index 358df4341b..e841e2888c 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -463,7 +463,6 @@ def python_register_toolchains( register_coverage_tool = False, set_python_version_constraint = False, tool_versions = TOOL_VERSIONS, - bzlmod = False, **kwargs): """Convenience macro for users which does typical setup. @@ -486,9 +485,15 @@ def python_register_toolchains( set_python_version_constraint: When set to true, target_compatible_with for the toolchains will include a version constraint. tool_versions: a dict containing a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. - bzlmod: Whether this rule is being run under a bzlmod module extension. **kwargs: passed to each python_repositories call. """ + + # If we have @@ we have bzlmod + bzlmod = str(Label("//:unused")).startswith("@@") + if bzlmod: + # you cannot used native.register_toolchains when using bzlmod. + register_toolchains = False + base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) if python_version in MINOR_MAPPING: @@ -559,16 +564,20 @@ def python_register_toolchains( platform = platform, )) - toolchains_repo( - name = toolchain_repo_name, + toolchain_aliases( + name = name, python_version = python_version, - set_python_version_constraint = set_python_version_constraint, user_repository_name = name, ) - toolchain_aliases( - name = name, + # in bzlmod we write out our own toolchain repos + if bzlmod: + return + + toolchains_repo( + name = toolchain_repo_name, python_version = python_version, + set_python_version_constraint = set_python_version_constraint, user_repository_name = name, ) diff --git a/python/runfiles/BUILD.bazel b/python/runfiles/BUILD.bazel index 3a93d40f32..c6cfc2fa94 100644 --- a/python/runfiles/BUILD.bazel +++ b/python/runfiles/BUILD.bazel @@ -27,6 +27,11 @@ py_library( "__init__.py", "runfiles.py", ], + imports = [ + # Add the repo root so `import python.runfiles.runfiles` works. This makes it agnostic + # to the --experimental_python_import_all_repositories setting. + "../..", + ], visibility = ["//visibility:public"], ) diff --git a/python/versions.bzl b/python/versions.bzl index baf6e33a06..a88c982c76 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -142,13 +142,14 @@ TOOL_VERSIONS = { "strip_prefix": "python", }, "3.9.16": { - "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", "sha256": { - "aarch64-apple-darwin": "d732d212d42315ac27c6da3e0b69636737a8d72086c980daf844344c010cab80", - "aarch64-unknown-linux-gnu": "1ba520c0db431c84305677f56eb9a4254f5097430ed443e92fc8617f8fba973d", - "x86_64-apple-darwin": "3948384af5e8d4ee7e5ccc648322b99c1c5cf4979954ed5e6b3382c69d6db71e", - "x86_64-pc-windows-msvc": "5274afd6b7ff2bddbd8306385ffb2336764b0e58535db968daeac656246f59a8", - "x86_64-unknown-linux-gnu": "7ba397787932393e65fc2fb9fcfabf54f2bb6751d5da2b45913cb25b2d493758", + "aarch64-apple-darwin": "c1de1d854717a6245f45262ef1bb17b09e2c587590e7e3f406593c143ff875bd", + "aarch64-unknown-linux-gnu": "f629b75ebfcafe9ceee2e796b7e4df5cf8dbd14f3c021afca078d159ab797acf", + "ppc64le-unknown-linux-gnu": "ff3ac35c58f67839aff9b5185a976abd3d1abbe61af02089f7105e876c1fe284", + "x86_64-apple-darwin": "3abc4d5fbbc80f5f848f280927ac5d13de8dc03aabb6ae65d8247cbb68e6f6bf", + "x86_64-pc-windows-msvc": "cdabb47204e96ce7ea31fbd0b5ed586114dd7d8f8eddf60a509a7f70b48a1c5e", + "x86_64-unknown-linux-gnu": "2b6e146234a4ef2a8946081fc3fbfffe0765b80b690425a49ebe40b47c33445b", }, "strip_prefix": "python", }, @@ -207,6 +208,18 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.11": { + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "8348bc3c2311f94ec63751fb71bd0108174be1c4def002773cf519ee1506f96f", + "aarch64-unknown-linux-gnu": "c7573fdb00239f86b22ea0e8e926ca881d24fde5e5890851339911d76110bc35", + "ppc64le-unknown-linux-gnu": "73a9d4c89ed51be39dd2de4e235078281087283e9fdedef65bec02f503e906ee", + "x86_64-apple-darwin": "bd3fc6e4da6f4033ebf19d66704e73b0804c22641ddae10bbe347c48f82374ad", + "x86_64-pc-windows-msvc": "9c2d3604a06fcd422289df73015cd00e7271d90de28d2c910f0e2309a7f73a68", + "x86_64-unknown-linux-gnu": "c5bcaac91bc80bfc29cf510669ecad12d506035ecb3ad85ef213416d54aecd79", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -218,6 +231,18 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.3": { + "url": "20230507/cpython-{python_version}+20230507-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "09e412506a8d63edbb6901742b54da9aa7faf120b8dbdce56c57b303fc892c86", + "aarch64-unknown-linux-gnu": "8190accbbbbcf7620f1ff6d668e4dd090c639665d11188ce864b62554d40e5ab", + "ppc64le-unknown-linux-gnu": "767d24f3570b35fedb945f5ac66224c8983f2d556ab83c5cfaa5f3666e9c212c", + "x86_64-apple-darwin": "f710b8d60621308149c100d5175fec39274ed0b9c99645484fd93d1716ef4310", + "x86_64-pc-windows-msvc": "24741066da6f35a7ff67bee65ce82eae870d84e1181843e64a7076d1571e95af", + "x86_64-unknown-linux-gnu": "da50b87d1ec42b3cb577dfd22a3655e43a53150f4f98a4bfb40757c9d7839ab5", + }, + "strip_prefix": "python", + }, } # buildifier: disable=unsorted-dict-items @@ -250,6 +275,17 @@ PLATFORMS = { # repository_ctx.execute(["uname", "-m"]).stdout.strip() arch = "aarch64", ), + "ppc64le-unknown-linux-gnu": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:ppc", + ], + os_name = LINUX_NAME, + # Note: this string differs between OSX and Linux + # Matches the value returned from: + # repository_ctx.execute(["uname", "-m"]).stdout.strip() + arch = "ppc64le", + ), "x86_64-apple-darwin": struct( compatible_with = [ "@platforms//os:macos", diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index d6ac0086ab..87ffe706dd 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -32,3 +32,33 @@ compile_pip_requirements( requirements_in = "requirements.txt", requirements_txt = "requirements_lock.txt", ) + +genrule( + name = "generate_os_specific_requirements_in", + srcs = [], + outs = ["requirements_os_specific.in"], + cmd = """ +cat > $@ < str: - """Resolve workspace status stamps format strings found in the version string + """Resolve workspace status stamps format strings found in the argument string Args: - version (str): The raw version represenation for the wheel (may include stamp variables) + argument (str): The raw argument represenation for the wheel (may include stamp variables) volatile_status_stamp (Path): The path to a volatile workspace status file stable_status_stamp (Path): The path to a stable workspace status file Returns: - str: A resolved version string + str: A resolved argument string """ lines = ( volatile_status_stamp.read_text().splitlines() @@ -229,9 +229,9 @@ def resolve_version_stamp( continue key, value = line.split(" ", maxsplit=1) stamp = "{" + key + "}" - version = version.replace(stamp, value) + argument = argument.replace(stamp, value) - return version + return argument def parse_args() -> argparse.Namespace: @@ -357,7 +357,16 @@ def main() -> None: strip_prefixes = [p for p in arguments.strip_path_prefix] if arguments.volatile_status_file and arguments.stable_status_file: - version = resolve_version_stamp( + name = resolve_argument_stamp( + arguments.name, + arguments.volatile_status_file, + arguments.stable_status_file, + ) + else: + name = arguments.name + + if arguments.volatile_status_file and arguments.stable_status_file: + version = resolve_argument_stamp( arguments.version, arguments.volatile_status_file, arguments.stable_status_file, @@ -366,7 +375,7 @@ def main() -> None: version = arguments.version with WheelMaker( - name=arguments.name, + name=name, version=version, build_tag=arguments.build_tag, python_tag=arguments.python_tag, @@ -398,7 +407,9 @@ def main() -> None: with open(arguments.metadata_file, "rt", encoding="utf-8") as metadata_file: metadata = metadata_file.read() - maker.add_metadata(metadata=metadata, description=description, version=version) + maker.add_metadata( + metadata=metadata, name=name, description=description, version=version + ) if arguments.entry_points_file: maker.add_file(