From 9e59206e806ca8e06cb9761358c90a0faadf7773 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 22 Aug 2023 14:48:11 -0700 Subject: [PATCH 01/41] doc: Note Python version changes in CHANGELOG (#1391) These patch level bumps were done in #1370 and are part of the 0.25.0 release. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 502545adec..bc86812707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,10 @@ A brief description of the categories of changes: ### Changed +* Python version patch level bumps: + * 3.9.16 -> 3.9.17 + * 3.10.9 -> 3.10.12 + * 3.11.1 -> 3.11.4 * (bzlmod) `pip.parse` can no longer automatically use the default Python version; this was an unreliable and unsafe behavior. The `python_version` arg must always be explicitly specified. From b4ab34edeb7156f30754f38f2aeb3ad832dcde57 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:23:54 -0700 Subject: [PATCH 02/41] fix: bcr releaser email (#1392) Fixes https://github.com/bazelbuild/bazel-central-registry/pull/863. The aspect email is no longer associated with the github user, so the CLA bot but doesn't think think the CLA is signed. To fix, change the email the BCR PRs are published under to an address that is associated with the github user (and thus the CLA). --- .bcr/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bcr/config.yml b/.bcr/config.yml index 7bdd70fbaf..7672aa554d 100644 --- a/.bcr/config.yml +++ b/.bcr/config.yml @@ -14,5 +14,5 @@ fixedReleaser: login: f0rmiga - email: thulio@aspect.dev + email: 3149049+f0rmiga@users.noreply.github.com moduleRoots: [".", "gazelle"] From e3449dc0ee233def5520c7b77ee292bbc0f3ff94 Mon Sep 17 00:00:00 2001 From: Zhongpeng Lin Date: Wed, 23 Aug 2023 21:45:39 +0800 Subject: [PATCH 03/41] Adding kwargs to gazelle_python_manifest (#1289) Adding kwargs to gazelle_python_manifest, so we can set the tags and size to the test. This is similar to: https://github.com/bazelbuild/rules_python/blob/5b8fa22a2f22501b18b4aea97c5dbfe3a6913a0c/python/pip_install/requirements.bzl#L20-L62 Alternatively, we can add individual `go_test` attributes as needed. Please advice which way is preferred. --- .../bzlmod_build_file_generation/BUILD.bazel | 1 + gazelle/manifest/defs.bzl | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index c5e27c2d49..bd2fc80933 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -53,6 +53,7 @@ gazelle_python_manifest( "//:requirements_lock.txt", "//:requirements_windows.txt", ], + tags = ["exclusive"], # NOTE: we can use this flag in order to make our setup compatible with # bzlmod. use_pip_repository_aliases = True, diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index f1266a0f46..d5afe7c143 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -25,7 +25,8 @@ def gazelle_python_manifest( pip_repository_name = "", pip_deps_repository_name = "", manifest = ":gazelle_python.yaml", - use_pip_repository_aliases = False): + use_pip_repository_aliases = False, + **kwargs): """A macro for defining the updating and testing targets for the Gazelle manifest file. Args: @@ -39,6 +40,8 @@ def gazelle_python_manifest( pip_deps_repository_name: deprecated - the old pip_install target name. modules_mapping: the target for the generated modules_mapping.json file. manifest: the target for the Gazelle manifest file. + **kwargs: other bazel attributes passed to the target target generated by + this macro. """ if pip_deps_repository_name != "": # buildifier: disable=print @@ -102,6 +105,14 @@ def gazelle_python_manifest( tags = ["manual"], ) + attrs = { + "env": { + "_TEST_MANIFEST": "$(rootpath {})".format(manifest), + "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), + "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), + }, + "size": "small", + } go_test( name = "{}.test".format(name), srcs = [Label("//manifest/test:test.go")], @@ -110,14 +121,10 @@ def gazelle_python_manifest( requirements, manifest_generator_hash, ], - env = { - "_TEST_MANIFEST": "$(rootpath {})".format(manifest), - "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), - "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), - }, rundir = ".", deps = [Label("//manifest")], - size = "small", + # kwargs could contain test-specific attributes like size or timeout + **dict(attrs, **kwargs) ) native.filegroup( From c32d2320d98f5b7633238bfee0c466eab5e703f3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 23 Aug 2023 08:25:12 -0700 Subject: [PATCH 04/41] docs: Use correct link to build badge image and build status page. (#1390) I'm not sure what happened, but the old image url doesn't work anymore. Also links to the canonical build status page; the old postsubmit url simply redirects to the canonical url. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 660e6e20af..10c7d0a4be 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python Rules for Bazel -[![Build status](https://badge.buildkite.com/1bcfe58b6f5741aacb09b12485969ba7a1205955a45b53e854.svg?branch=main)](https://buildkite.com/bazel/python-rules-python-postsubmit) +[![Build status](https://badge.buildkite.com/0bcfe58b6f5741aacb09b12485969ba7a1205955a45b53e854.svg?branch=main)](https://buildkite.com/bazel/rules-python-python) ## Overview From 9818a60e687956dca60b0e1884b217ef6a1d1821 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Sat, 26 Aug 2023 05:34:23 +0900 Subject: [PATCH 05/41] feat(py_console_script_binary)!: entry points with custom dependencies (#1363) Add `py_console_script_binary`, a macro/rule that allows better customization of how entry points are generated. Notable features of it are: * It allows passing in additional dependencies, which makes it easier for plugin dependencies to be added to tools such as pylint or sphinx. * The underlying `py_binary` rule can be passed in, allowing custom rules, such as the version-aware rules, to be used for the resulting binary. * Entry point generation is based upon a wheel's `entry_points.txt` file. This helps avoid loading external repositories unless they're actually used, allows entry points to have better version-aware support, and allows bzlmod to provide a supportable mechanism for entry points. Because the expected common use case is an entry point for our pip generated repos, there is special logic to make that easy and concisely do. Usage of `py_console_script_binary` is not tied to our pip code generation, though, and users can manually specify dependencies if they need to. BREAKING CHANGE: This is a breaking change, but only for bzlmod users. Note that bzlmod support is still beta. Bzlmod users will need to replace using `entry_point` from `requirements.bzl` with loading `py_console_script_binary` and defining the entry point locally: ``` load("@rules_python//python/entry_points:py_console_script_binary.bzl, "py_console_script_binary") py_console_script_binary(name="foo", pkg="@mypip//pylint") ``` For workspace users, this new macro is available to be used, but the old code is still present. Fixes #1362 Fixes #543 Fixes #979 Fixes #1262 Closes #980 Closes #1294 Closes #1055 --------- Co-authored-by: Richard Levasseur --- .bazelrc | 4 +- CHANGELOG.md | 14 ++ docs/BUILD.bazel | 11 + docs/py_console_script_binary.md | 87 ++++++++ examples/bzlmod/MODULE.bazel | 5 +- examples/bzlmod/entry_point/BUILD.bazel | 20 -- examples/bzlmod/entry_points/BUILD.bazel | 33 +++ .../bzlmod/entry_points/tests/BUILD.bazel | 63 ++++++ .../tests/file_with_pylint_errors.py | 6 + .../entry_points/tests/pylint_deps_test.py | 72 +++++++ .../bzlmod/entry_points/tests/pylint_test.py | 57 +++++ .../tests/yamllint_test.py} | 16 +- examples/bzlmod/requirements.in | 1 + examples/bzlmod/requirements_lock_3_10.txt | 6 + examples/bzlmod/requirements_lock_3_9.txt | 6 + examples/bzlmod/requirements_windows_3_10.txt | 6 + examples/bzlmod/requirements_windows_3_9.txt | 6 + examples/pip_parse_vendored/BUILD.bazel | 2 +- python/BUILD.bazel | 1 + python/config_settings/transition.bzl | 3 +- python/entry_points/BUILD.bazel | 39 ++++ .../entry_points/py_console_script_binary.bzl | 79 +++++++ ...ub_repository_requirements_bzlmod.bzl.tmpl | 6 - ...ip_repository_requirements_bzlmod.bzl.tmpl | 17 +- python/private/BUILD.bazel | 33 +++ python/private/py_console_script_binary.bzl | 87 ++++++++ python/private/py_console_script_gen.bzl | 93 +++++++++ python/private/py_console_script_gen.py | 180 ++++++++++++++++ python/private/toolchains_repo.bzl | 19 +- tests/BUILD.bazel | 1 + tests/entry_points/BUILD.bazel | 39 ++++ .../py_console_script_gen_test.py | 197 ++++++++++++++++++ tests/entry_points/simple_macro.bzl | 31 +++ 33 files changed, 1201 insertions(+), 39 deletions(-) create mode 100644 docs/py_console_script_binary.md delete mode 100644 examples/bzlmod/entry_point/BUILD.bazel create mode 100644 examples/bzlmod/entry_points/BUILD.bazel create mode 100644 examples/bzlmod/entry_points/tests/BUILD.bazel create mode 100644 examples/bzlmod/entry_points/tests/file_with_pylint_errors.py create mode 100644 examples/bzlmod/entry_points/tests/pylint_deps_test.py create mode 100644 examples/bzlmod/entry_points/tests/pylint_test.py rename examples/bzlmod/{entry_point/test_entry_point.py => entry_points/tests/yamllint_test.py} (64%) create mode 100644 python/entry_points/BUILD.bazel create mode 100644 python/entry_points/py_console_script_binary.bzl create mode 100644 python/private/py_console_script_binary.bzl create mode 100644 python/private/py_console_script_gen.bzl create mode 100644 python/private/py_console_script_gen.py create mode 100644 tests/entry_points/BUILD.bazel create mode 100644 tests/entry_points/py_console_script_gen_test.py create mode 100644 tests/entry_points/simple_macro.bzl diff --git a/.bazelrc b/.bazelrc index 3a5497a071..39b28d12e6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,8 +3,8 @@ # This lets us glob() up all the files inside the examples to make them inputs to tests # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_point,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,tests/compile_pip_requirements,tests/compile_pip_requirements_test_from_external_workspace,tests/ignore_root_user_error,tests/pip_repository_entry_points test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index bc86812707..9e7b6853f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,20 @@ A brief description of the categories of changes: * Particular sub-systems are identified using parentheses, e.g. `(bzlmod)` or `(docs)`. +## Unreleased + +### Added + +* (bzlmod, entry_point) Added + [`py_console_script_binary`](./docs/py_console_script_binary.md), which + allows adding custom dependencies to a package's entry points and customizing + the `py_binary` rule used to build it. + +### Removed + +* (bzlmod) The `entry_point` macro is no longer supported and has been removed + in favour of the `py_console_script_binary` macro for `bzlmod` users. + ## [0.25.0] - 2023-08-22 ### Changed diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 1fb4f81484..3a222ab8d2 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -27,6 +27,7 @@ _DOCS = { "pip_repository": "//docs:pip-repository", "py_cc_toolchain": "//docs:py_cc_toolchain-docs", "py_cc_toolchain_info": "//docs:py_cc_toolchain_info-docs", + "py_console_script_binary": "//docs:py-console-script-binary", "python": "//docs:core-docs", } @@ -128,6 +129,16 @@ stardoc( ], ) +stardoc( + name = "py-console-script-binary", + out = "py_console_script_binary.md_", + input = "//python/entry_points:py_console_script_binary.bzl", + target_compatible_with = _NOT_WINDOWS, + deps = [ + "//python/entry_points:py_console_script_binary_bzl", + ], +) + stardoc( name = "packaging-docs", out = "packaging.md_", diff --git a/docs/py_console_script_binary.md b/docs/py_console_script_binary.md new file mode 100644 index 0000000000..3d7b5e5bbd --- /dev/null +++ b/docs/py_console_script_binary.md @@ -0,0 +1,87 @@ + + + +Creates an executable (a non-test binary) for console_script entry points. + +Generate a `py_binary` target for a particular console_script `entry_point` +from a PyPI package, e.g. for creating an executable `pylint` target use: +```starlark +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "pylint", + pkg = "@pip//pylint", +) +``` + +Or for more advanced setups you can also specify extra dependencies and the +exact script name you want to call. It is useful for tools like flake8, pylint, +pytest, which have plugin discovery methods and discover dependencies from the +PyPI packages available in the PYTHONPATH. +```starlark +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "pylint_with_deps", + pkg = "@pip//pylint", + # Because `pylint` has multiple console_scripts available, we have to + # specify which we want if the name of the target name 'pylint_with_deps' + # cannot be used to guess the entry_point script. + script = "pylint", + deps = [ + # One can add extra dependencies to the entry point. + # This specifically allows us to add plugins to pylint. + "@pip//pylint_print", + ], +) +``` + +A specific Python version can be forced by using the generated version-aware +wrappers, e.g. to force Python 3.9: +```starlark +load("@python_versions//3.9:defs.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "yamllint", + pkg = "@pip//yamllint", +) +``` + +Alternatively, the the `py_console_script_binary.binary_rule` arg can be passed +the version-bound `py_binary` symbol, or any other `py_binary`-compatible rule +of your choosing: +```starlark +load("@python_versions//3.9:defs.bzl", "py_binary") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "yamllint", + pkg = "@pip//yamllint:pkg", + binary_rule = py_binary, +) +``` + + + + +## py_console_script_binary + +
+py_console_script_binary(name, pkg, entry_points_txt, script, binary_rule, kwargs)
+
+ +Generate a py_binary for a console_script entry_point. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| name | str, The name of the resulting target. | none | +| pkg | target, the package for which to generate the script. | none | +| entry_points_txt | optional target, the entry_points.txt file to parse for available console_script values. It may be a single file, or a group of files, but must contain a file named entry_points.txt. If not specified, defaults to the dist_info target in the same package as the pkg Label. | None | +| script | str, The console script name that the py_binary is going to be generated for. Defaults to the normalized name attribute. | None | +| binary_rule | callable, The rule/macro to use to instantiate the target. It's expected to behave like py_binary. Defaults to @rules_python//python:py_binary.bzl#py_binary. | <function py_binary> | +| kwargs | Extra parameters forwarded to binary_rule. | none | + + diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index be9466d883..0d1c7a736b 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -113,10 +113,7 @@ pip.parse( "@whl_mods_hub//:wheel.json": "wheel", }, ) - -# NOTE: The pip_39 repo is only used because the plain `@pip` repo doesn't -# yet support entry points; see https://github.com/bazelbuild/rules_python/issues/1262 -use_repo(pip, "pip", "pip_39") +use_repo(pip, "pip") bazel_dep(name = "other_module", version = "", repo_name = "our_other_module") local_path_override( diff --git a/examples/bzlmod/entry_point/BUILD.bazel b/examples/bzlmod/entry_point/BUILD.bazel deleted file mode 100644 index f68552c3ef..0000000000 --- a/examples/bzlmod/entry_point/BUILD.bazel +++ /dev/null @@ -1,20 +0,0 @@ -load("@pip_39//:requirements.bzl", "entry_point") -load("@rules_python//python:defs.bzl", "py_test") - -alias( - name = "yamllint", - actual = entry_point("yamllint"), -) - -py_test( - name = "entry_point_test", - srcs = ["test_entry_point.py"], - data = [ - ":yamllint", - ], - env = { - "YAMLLINT_ENTRY_POINT": "$(rlocationpath :yamllint)", - }, - main = "test_entry_point.py", - deps = ["@rules_python//python/runfiles"], -) diff --git a/examples/bzlmod/entry_points/BUILD.bazel b/examples/bzlmod/entry_points/BUILD.bazel new file mode 100644 index 0000000000..a0939cb65b --- /dev/null +++ b/examples/bzlmod/entry_points/BUILD.bazel @@ -0,0 +1,33 @@ +load("@python_versions//3.9:defs.bzl", py_console_script_binary_3_9 = "py_console_script_binary") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +# This is how you can define a `pylint` entrypoint which uses the default python version. +py_console_script_binary( + name = "pylint", + pkg = "@pip//pylint", + visibility = ["//entry_points:__subpackages__"], +) + +# We can also specify extra dependencies for the binary, which is useful for +# tools like flake8, pylint, pytest, which have plugin discovery methods. +py_console_script_binary( + name = "pylint_with_deps", + pkg = "@pip//pylint", + # Because `pylint` has multiple console_scripts available, we have to + # specify which we want if the name of the target name 'pylint_with_deps' + # cannot be used to guess the entry_point script. + script = "pylint", + visibility = ["//entry_points:__subpackages__"], + deps = [ + # One can add extra dependencies to the entry point. + "@pip//pylint_print", + ], +) + +# A specific Python version can be forced by using the generated version-aware +# wrappers, e.g. to force Python 3.9: +py_console_script_binary_3_9( + name = "yamllint", + pkg = "@pip//yamllint:pkg", + visibility = ["//entry_points:__subpackages__"], +) diff --git a/examples/bzlmod/entry_points/tests/BUILD.bazel b/examples/bzlmod/entry_points/tests/BUILD.bazel new file mode 100644 index 0000000000..5a65e9e1a3 --- /dev/null +++ b/examples/bzlmod/entry_points/tests/BUILD.bazel @@ -0,0 +1,63 @@ +load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@rules_python//python:defs.bzl", "py_test") + +# Below are targets for testing the `py_console_script_binary` feature and are +# not part of the example how to use the feature. + +# And a test that we can correctly run `pylint --version` +py_test( + name = "pylint_test", + srcs = ["pylint_test.py"], + data = ["//entry_points:pylint"], + env = { + "ENTRY_POINT": "$(rlocationpath //entry_points:pylint)", + }, + deps = ["@rules_python//python/runfiles"], +) + +# Next run pylint on the file to generate a report. +run_binary( + name = "pylint_report", + srcs = [ + ":file_with_pylint_errors.py", + ], + outs = ["pylint_report.txt"], + args = [ + "--output-format=text:$(location pylint_report.txt)", + "--load-plugins=pylint_print", + # The `exit-zero` ensures that `run_binary` is successful even though there are lint errors. + # We check the generated report in the test below. + "--exit-zero", + "$(location :file_with_pylint_errors.py)", + ], + env = { + # otherwise it may try to create ${HOME}/.cache/pylint + "PYLINTHOME": "./.pylint_home", + }, + tool = "//entry_points:pylint_with_deps", +) + +py_test( + name = "pylint_deps_test", + srcs = ["pylint_deps_test.py"], + data = [ + ":pylint_report", + "//entry_points:pylint_with_deps", + ], + env = { + "ENTRY_POINT": "$(rlocationpath //entry_points:pylint_with_deps)", + "PYLINT_REPORT": "$(rlocationpath :pylint_report)", + }, + deps = ["@rules_python//python/runfiles"], +) + +# And a test to check that yamllint works +py_test( + name = "yamllint_test", + srcs = ["yamllint_test.py"], + data = ["//entry_points:yamllint"], + env = { + "ENTRY_POINT": "$(rlocationpath //entry_points:yamllint)", + }, + deps = ["@rules_python//python/runfiles"], +) diff --git a/examples/bzlmod/entry_points/tests/file_with_pylint_errors.py b/examples/bzlmod/entry_points/tests/file_with_pylint_errors.py new file mode 100644 index 0000000000..bb3dbab660 --- /dev/null +++ b/examples/bzlmod/entry_points/tests/file_with_pylint_errors.py @@ -0,0 +1,6 @@ +""" +A file to demonstrate the pylint-print checker works. +""" + +if __name__ == "__main__": + print("Hello, World!") diff --git a/examples/bzlmod/entry_points/tests/pylint_deps_test.py b/examples/bzlmod/entry_points/tests/pylint_deps_test.py new file mode 100644 index 0000000000..f6743ce9b5 --- /dev/null +++ b/examples/bzlmod/entry_points/tests/pylint_deps_test.py @@ -0,0 +1,72 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import subprocess +import tempfile +import unittest + +from python.runfiles import runfiles + + +class ExampleTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + self.maxDiff = None + + super().__init__(*args, **kwargs) + + def test_pylint_entry_point(self): + rlocation_path = os.environ.get("ENTRY_POINT") + assert ( + rlocation_path is not None + ), "expected 'ENTRY_POINT' env variable to be set to rlocation of the tool" + + entry_point = pathlib.Path(runfiles.Create().Rlocation(rlocation_path)) + self.assertTrue(entry_point.exists(), f"'{entry_point}' does not exist") + + # Let's run the entrypoint and check the tool version. + # + # NOTE @aignas 2023-08-24: the Windows python launcher with Python 3.9 and bazel 6 is not happy if we start + # passing extra files via `subprocess.run` and it starts to fail with an error that the file which is the + # entry_point cannot be found. However, just calling `--version` seems to be fine. + proc = subprocess.run( + [str(entry_point), "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual( + "", + proc.stderr.decode("utf-8").strip(), + ) + self.assertRegex(proc.stdout.decode("utf-8").strip(), "^pylint 2\.15\.9") + + def test_pylint_report_has_expected_warnings(self): + rlocation_path = os.environ.get("PYLINT_REPORT") + assert ( + rlocation_path is not None + ), "expected 'PYLINT_REPORT' env variable to be set to rlocation of the report" + + pylint_report = pathlib.Path(runfiles.Create().Rlocation(rlocation_path)) + self.assertTrue(pylint_report.exists(), f"'{pylint_report}' does not exist") + + self.assertRegex( + pylint_report.read_text().strip(), + "W8201: Logging should be used instead of the print\(\) function\. \(print-function\)", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/bzlmod/entry_points/tests/pylint_test.py b/examples/bzlmod/entry_points/tests/pylint_test.py new file mode 100644 index 0000000000..c2532938d8 --- /dev/null +++ b/examples/bzlmod/entry_points/tests/pylint_test.py @@ -0,0 +1,57 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import subprocess +import unittest + +from python.runfiles import runfiles + + +class ExampleTest(unittest.TestCase): + def __init__(self, *args, **kwargs): + self.maxDiff = None + + super().__init__(*args, **kwargs) + + def test_pylint_entry_point(self): + rlocation_path = os.environ.get("ENTRY_POINT") + assert ( + rlocation_path is not None + ), "expected 'ENTRY_POINT' env variable to be set to rlocation of the tool" + + entry_point = pathlib.Path(runfiles.Create().Rlocation(rlocation_path)) + self.assertTrue(entry_point.exists(), f"'{entry_point}' does not exist") + + # Let's run the entrypoint and check the tool version. + # + # NOTE @aignas 2023-08-24: the Windows python launcher with Python 3.9 and bazel 6 is not happy if we start + # passing extra files via `subprocess.run` and it starts to fail with an error that the file which is the + # entry_point cannot be found. However, just calling `--version` seems to be fine. + proc = subprocess.run( + [str(entry_point), "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual( + "", + proc.stderr.decode("utf-8").strip(), + ) + self.assertRegex(proc.stdout.decode("utf-8").strip(), "^pylint 2\.15\.9") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/bzlmod/entry_point/test_entry_point.py b/examples/bzlmod/entry_points/tests/yamllint_test.py similarity index 64% rename from examples/bzlmod/entry_point/test_entry_point.py rename to examples/bzlmod/entry_points/tests/yamllint_test.py index 5a37458348..0a0235793b 100644 --- a/examples/bzlmod/entry_point/test_entry_point.py +++ b/examples/bzlmod/entry_points/tests/yamllint_test.py @@ -21,15 +21,25 @@ class ExampleTest(unittest.TestCase): - def test_entry_point(self): - rlocation_path = os.environ.get("YAMLLINT_ENTRY_POINT") + def __init__(self, *args, **kwargs): + self.maxDiff = None + + super().__init__(*args, **kwargs) + + def test_yamllint_entry_point(self): + rlocation_path = os.environ.get("ENTRY_POINT") assert ( rlocation_path is not None - ), "expected 'YAMLLINT_ENTRY_POINT' env variable to be set to rlocation of the tool" + ), "expected 'ENTRY_POINT' env variable to be set to rlocation of the tool" entry_point = pathlib.Path(runfiles.Create().Rlocation(rlocation_path)) self.assertTrue(entry_point.exists(), f"'{entry_point}' does not exist") + # Let's run the entrypoint and check the tool version. + # + # NOTE @aignas 2023-08-24: the Windows python launcher with Python 3.9 and bazel 6 is not happy if we start + # passing extra files via `subprocess.run` and it starts to fail with an error that the file which is the + # entry_point cannot be found. However, just calling `--version` seems to be fine. proc = subprocess.run( [str(entry_point), "--version"], check=True, diff --git a/examples/bzlmod/requirements.in b/examples/bzlmod/requirements.in index 47cdcf1ea8..702e15103d 100644 --- a/examples/bzlmod/requirements.in +++ b/examples/bzlmod/requirements.in @@ -7,4 +7,5 @@ s3cmd~=2.1.0 yamllint>=1.28.0 tabulate~=0.9.0 pylint~=2.15.5 +pylint-print python-dateutil>=2.8.2 diff --git a/examples/bzlmod/requirements_lock_3_10.txt b/examples/bzlmod/requirements_lock_3_10.txt index e3a185ac88..7f9bd3ac4a 100644 --- a/examples/bzlmod/requirements_lock_3_10.txt +++ b/examples/bzlmod/requirements_lock_3_10.txt @@ -83,6 +83,12 @@ platformdirs==3.5.1 \ pylint==2.15.10 \ --hash=sha256:9df0d07e8948a1c3ffa3b6e2d7e6e63d9fb457c5da5b961ed63106594780cc7e \ --hash=sha256:b3dc5ef7d33858f297ac0d06cc73862f01e4f2e74025ec3eff347ce0bc60baf5 + # via + # -r requirements.in + # pylint-print +pylint-print==1.0.1 \ + --hash=sha256:30aa207e9718ebf4ceb47fb87012092e6d8743aab932aa07aa14a73e750ad3d0 \ + --hash=sha256:a2b2599e7887b93e551db2624c523c1e6e9e58c3be8416cd98d41e4427e2669b # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ diff --git a/examples/bzlmod/requirements_lock_3_9.txt b/examples/bzlmod/requirements_lock_3_9.txt index ba1d4d7148..79c18127ec 100644 --- a/examples/bzlmod/requirements_lock_3_9.txt +++ b/examples/bzlmod/requirements_lock_3_9.txt @@ -66,6 +66,12 @@ platformdirs==2.6.0 \ pylint==2.15.9 \ --hash=sha256:18783cca3cfee5b83c6c5d10b3cdb66c6594520ffae61890858fe8d932e1c6b4 \ --hash=sha256:349c8cd36aede4d50a0754a8c0218b43323d13d5d88f4b2952ddfe3e169681eb + # via + # -r requirements.in + # pylint-print +pylint-print==1.0.1 \ + --hash=sha256:30aa207e9718ebf4ceb47fb87012092e6d8743aab932aa07aa14a73e750ad3d0 \ + --hash=sha256:a2b2599e7887b93e551db2624c523c1e6e9e58c3be8416cd98d41e4427e2669b # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ diff --git a/examples/bzlmod/requirements_windows_3_10.txt b/examples/bzlmod/requirements_windows_3_10.txt index 9a28ae8687..a8f05add6b 100644 --- a/examples/bzlmod/requirements_windows_3_10.txt +++ b/examples/bzlmod/requirements_windows_3_10.txt @@ -87,6 +87,12 @@ platformdirs==3.5.1 \ pylint==2.15.10 \ --hash=sha256:9df0d07e8948a1c3ffa3b6e2d7e6e63d9fb457c5da5b961ed63106594780cc7e \ --hash=sha256:b3dc5ef7d33858f297ac0d06cc73862f01e4f2e74025ec3eff347ce0bc60baf5 + # via + # -r requirements.in + # pylint-print +pylint-print==1.0.1 \ + --hash=sha256:30aa207e9718ebf4ceb47fb87012092e6d8743aab932aa07aa14a73e750ad3d0 \ + --hash=sha256:a2b2599e7887b93e551db2624c523c1e6e9e58c3be8416cd98d41e4427e2669b # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ diff --git a/examples/bzlmod/requirements_windows_3_9.txt b/examples/bzlmod/requirements_windows_3_9.txt index 08f0979d52..790e3d5d11 100644 --- a/examples/bzlmod/requirements_windows_3_9.txt +++ b/examples/bzlmod/requirements_windows_3_9.txt @@ -70,6 +70,12 @@ platformdirs==2.6.0 \ pylint==2.15.9 \ --hash=sha256:18783cca3cfee5b83c6c5d10b3cdb66c6594520ffae61890858fe8d932e1c6b4 \ --hash=sha256:349c8cd36aede4d50a0754a8c0218b43323d13d5d88f4b2952ddfe3e169681eb + # via + # -r requirements.in + # pylint-print +pylint-print==1.0.1 \ + --hash=sha256:30aa207e9718ebf4ceb47fb87012092e6d8743aab932aa07aa14a73e750ad3d0 \ + --hash=sha256:a2b2599e7887b93e551db2624c523c1e6e9e58c3be8416cd98d41e4427e2669b # via -r requirements.in python-dateutil==2.8.2 \ --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ diff --git a/examples/pip_parse_vendored/BUILD.bazel b/examples/pip_parse_vendored/BUILD.bazel index 56630e513d..b87b2aa812 100644 --- a/examples/pip_parse_vendored/BUILD.bazel +++ b/examples/pip_parse_vendored/BUILD.bazel @@ -16,7 +16,7 @@ genrule( cmd = " | ".join([ "cat $<", # Insert our load statement after the existing one so we don't produce a file with buildifier warnings - """sed -e '/^load.*/i\\'$$'\\n''load("@python39//:defs.bzl", "interpreter")'""", + """sed -e '/^load.*.whl_library/i\\'$$'\\n''load("@python39//:defs.bzl", "interpreter")'""", # Replace the bazel 6.0.0 specific comment with something that bazel 5.4.0 would produce. # This enables this example to be run as a test under bazel 5.4.0. """sed -e 's#@//#//#'""", diff --git a/python/BUILD.bazel b/python/BUILD.bazel index c5f25803c7..aa8c8bf3e4 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -36,6 +36,7 @@ filegroup( "//python/cc:distribution", "//python/config_settings:distribution", "//python/constraints:distribution", + "//python/entry_points:distribution", "//python/private:distribution", "//python/runfiles:distribution", ], diff --git a/python/config_settings/transition.bzl b/python/config_settings/transition.bzl index 20e03dc21d..f9f19f2940 100644 --- a/python/config_settings/transition.bzl +++ b/python/config_settings/transition.bzl @@ -17,7 +17,8 @@ them to the desired target platform. """ load("@bazel_skylib//lib:dicts.bzl", "dicts") -load("//python:defs.bzl", _py_binary = "py_binary", _py_test = "py_test") +load("//python:py_binary.bzl", _py_binary = "py_binary") +load("//python:py_test.bzl", _py_test = "py_test") load("//python/config_settings/private:py_args.bzl", "py_args") def _transition_python_version_impl(_, attr): diff --git a/python/entry_points/BUILD.bazel b/python/entry_points/BUILD.bazel new file mode 100644 index 0000000000..981a1ccf69 --- /dev/null +++ b/python/entry_points/BUILD.bazel @@ -0,0 +1,39 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +exports_files( + [ + "py_console_script_binary.bzl", + ], + visibility = ["//docs:__subpackages__"], +) + +bzl_library( + name = "py_console_script_binary_bzl", + srcs = [":py_console_script_binary.bzl"], + visibility = ["//visibility:public"], + deps = [ + "//python/private:py_console_script_binary_bzl", + ], +) + +filegroup( + name = "distribution", + srcs = glob([ + "*.bzl", + ]), + visibility = ["//python:__subpackages__"], +) diff --git a/python/entry_points/py_console_script_binary.bzl b/python/entry_points/py_console_script_binary.bzl new file mode 100644 index 0000000000..60e74f579d --- /dev/null +++ b/python/entry_points/py_console_script_binary.bzl @@ -0,0 +1,79 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Creates an executable (a non-test binary) for console_script entry points. + +Generate a `py_binary` target for a particular console_script `entry_point` +from a PyPI package, e.g. for creating an executable `pylint` target use: +```starlark +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "pylint", + pkg = "@pip//pylint", +) +``` + +Or for more advanced setups you can also specify extra dependencies and the +exact script name you want to call. It is useful for tools like flake8, pylint, +pytest, which have plugin discovery methods and discover dependencies from the +PyPI packages available in the PYTHONPATH. +```starlark +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "pylint_with_deps", + pkg = "@pip//pylint", + # Because `pylint` has multiple console_scripts available, we have to + # specify which we want if the name of the target name 'pylint_with_deps' + # cannot be used to guess the entry_point script. + script = "pylint", + deps = [ + # One can add extra dependencies to the entry point. + # This specifically allows us to add plugins to pylint. + "@pip//pylint_print", + ], +) +``` + +A specific Python version can be forced by using the generated version-aware +wrappers, e.g. to force Python 3.9: +```starlark +load("@python_versions//3.9:defs.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "yamllint", + pkg = "@pip//yamllint", +) +``` + +Alternatively, the the `py_console_script_binary.binary_rule` arg can be passed +the version-bound `py_binary` symbol, or any other `py_binary`-compatible rule +of your choosing: +```starlark +load("@python_versions//3.9:defs.bzl", "py_binary") +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "yamllint", + pkg = "@pip//yamllint:pkg", + binary_rule = py_binary, +) +``` +""" + +load("//python/private:py_console_script_binary.bzl", _py_console_script_binary = "py_console_script_binary") + +py_console_script_binary = _py_console_script_binary diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl index 4a3d512ae7..53d4ee99c4 100644 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl @@ -27,9 +27,3 @@ def data_requirement(name): def dist_info_requirement(name): return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") - -def entry_point(pkg, script = None): - """entry_point returns the target of the canonical label of the package entrypoints. - """ - # TODO: https://github.com/bazelbuild/rules_python/issues/1262 - print("not implemented") diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl index 2df60b0b52..00580f5593 100644 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl @@ -30,4 +30,19 @@ def entry_point(pkg, script = None): """ if not script: script = pkg - return "@@%%NAME%%_{}//:rules_python_wheel_entry_point_{}".format(_clean_name(pkg), script) + fail("""Please replace this instance of entry_point with the following: + +``` +load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +py_console_script_binary( + name = "{pkg}", + pkg = "@%%{pkg_label}", + script = "{script}", +) +``` +""".format( + pkg = _clean_name(pkg), + pkg_label = "%%MACRO_TMPL%%".format(_clean_name(pkg), "pkg"), + script = script, + )) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 29b5a6c885..48c3f8c73b 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -13,6 +13,8 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python:py_binary.bzl", "py_binary") +load("//python:py_library.bzl", "py_library") load("//python:versions.bzl", "print_toolchains_checksums") load(":stamp.bzl", "stamp_build_setting") @@ -88,6 +90,18 @@ bzl_library( visibility = ["//python/cc:__pkg__"], ) +bzl_library( + name = "py_console_script_binary_bzl", + srcs = [ + "py_console_script_binary.bzl", + "py_console_script_gen.bzl", + ], + visibility = ["//python/entry_points:__pkg__"], + deps = [ + "//python:py_binary_bzl", + ], +) + # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", @@ -119,3 +133,22 @@ exports_files( stamp_build_setting(name = "stamp") print_toolchains_checksums(name = "print_toolchains_checksums") + +# Used for py_console_script_gen rule +py_binary( + name = "py_console_script_gen_py", + srcs = ["py_console_script_gen.py"], + main = "py_console_script_gen.py", + visibility = [ + "//visibility:public", + ], +) + +py_library( + name = "py_console_script_gen_lib", + srcs = ["py_console_script_gen.py"], + imports = ["../.."], + visibility = [ + "//tests/entry_points:__pkg__", + ], +) diff --git a/python/private/py_console_script_binary.bzl b/python/private/py_console_script_binary.bzl new file mode 100644 index 0000000000..bd992a8f75 --- /dev/null +++ b/python/private/py_console_script_binary.bzl @@ -0,0 +1,87 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Implementation for the macro to generate a console_script py_binary from an 'entry_points.txt' config. +""" + +load("//python:py_binary.bzl", "py_binary") +load(":py_console_script_gen.bzl", "py_console_script_gen") + +def _dist_info(pkg): + """Return the first candidate for the dist_info target label. + + We cannot do `Label(pkg)` here because the string will be evaluated within + the context of the rules_python repo_mapping and it will fail because + rules_python does not know anything about the hub repos that the user has + available. + + NOTE: Works with `incompatible_generate_aliases` and without by assuming the + following formats: + * @pypi_pylint//:pkg + * @pypi//pylint + * @pypi//pylint:pkg + * Label("@pypi//pylint:pkg") + """ + + # str() is called to convert Label objects + return str(pkg).replace(":pkg", "") + ":dist_info" + +def py_console_script_binary( + *, + name, + pkg, + entry_points_txt = None, + script = None, + binary_rule = py_binary, + **kwargs): + """Generate a py_binary for a console_script entry_point. + + Args: + name: str, The name of the resulting target. + pkg: target, the package for which to generate the script. + entry_points_txt: optional target, the entry_points.txt file to parse + for available console_script values. It may be a single file, or a + group of files, but must contain a file named `entry_points.txt`. + If not specified, defaults to the `dist_info` target in the same + package as the `pkg` Label. + script: str, The console script name that the py_binary is going to be + generated for. Defaults to the normalized name attribute. + binary_rule: callable, The rule/macro to use to instantiate + the target. It's expected to behave like `py_binary`. + Defaults to @rules_python//python:py_binary.bzl#py_binary. + **kwargs: Extra parameters forwarded to binary_rule. + """ + main = "rules_python_entry_point_{}.py".format(name) + + if kwargs.pop("srcs", None): + fail("passing 'srcs' attribute to py_console_script_binary is unsupported") + + py_console_script_gen( + name = "_{}_gen".format(name), + # NOTE @aignas 2023-08-05: Works with `incompatible_generate_aliases` and without. + entry_points_txt = entry_points_txt or _dist_info(pkg), + out = main, + console_script = script, + console_script_guess = name, + visibility = ["//visibility:private"], + ) + + binary_rule( + name = name, + srcs = [main], + main = main, + deps = [pkg] + kwargs.pop("deps", []), + **kwargs + ) diff --git a/python/private/py_console_script_gen.bzl b/python/private/py_console_script_gen.bzl new file mode 100644 index 0000000000..7dd4dd2dad --- /dev/null +++ b/python/private/py_console_script_gen.bzl @@ -0,0 +1,93 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A private rule to generate an entry_point python file to be used in a py_binary. + +Right now it only supports console_scripts via the entry_points.txt file in the dist-info. + +NOTE @aignas 2023-08-07: This cannot be in pure starlark, because we need to +read a file and then create a `.py` file based on the contents of that file, +which cannot be done in pure starlark according to +https://github.com/bazelbuild/bazel/issues/14744 +""" + +_ENTRY_POINTS_TXT = "entry_points.txt" + +def _get_entry_points_txt(entry_points_txt): + """Get the entry_points.txt file + + TODO: use map_each to avoid flattening of the directories outside the execution phase. + """ + for file in entry_points_txt.files.to_list(): + if file.basename == _ENTRY_POINTS_TXT: + return file + + fail("{} does not contain {}".format(entry_points_txt, _ENTRY_POINTS_TXT)) + +def _py_console_script_gen_impl(ctx): + entry_points_txt = _get_entry_points_txt(ctx.attr.entry_points_txt) + + args = ctx.actions.args() + args.add("--console-script", ctx.attr.console_script) + args.add("--console-script-guess", ctx.attr.console_script_guess) + args.add(entry_points_txt) + args.add(ctx.outputs.out) + + ctx.actions.run( + inputs = [ + entry_points_txt, + ], + outputs = [ctx.outputs.out], + arguments = [args], + mnemonic = "PyConsoleScriptBinaryGen", + progress_message = "Generating py_console_script_binary main: %{label}", + executable = ctx.executable._tool, + ) + + return [DefaultInfo( + files = depset([ctx.outputs.out]), + )] + +py_console_script_gen = rule( + _py_console_script_gen_impl, + attrs = { + "console_script": attr.string( + doc = "The name of the console_script to create the .py file for. Optional if there is only a single entry-point available.", + default = "", + mandatory = False, + ), + "console_script_guess": attr.string( + doc = "The string used for guessing the console_script if it is not provided.", + default = "", + mandatory = False, + ), + "entry_points_txt": attr.label( + doc = "The filegroup to search for entry_points.txt.", + mandatory = True, + ), + "out": attr.output( + doc = "Output file location.", + mandatory = True, + ), + "_tool": attr.label( + default = ":py_console_script_gen_py", + executable = True, + cfg = "exec", + ), + }, + doc = """\ +Builds an entry_point script from an entry_points.txt file. +""", +) diff --git a/python/private/py_console_script_gen.py b/python/private/py_console_script_gen.py new file mode 100644 index 0000000000..30e93c2e5b --- /dev/null +++ b/python/private/py_console_script_gen.py @@ -0,0 +1,180 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +console_script generator from entry_points.txt contents. + +For Python versions earlier than 3.11 and for earlier bazel versions than 7.0 we need to workaround the issue of +sys.path[0] breaking out of the runfiles tree see the following for more context: +* https://github.com/bazelbuild/rules_python/issues/382 +* https://github.com/bazelbuild/bazel/pull/15701 + +In affected bazel and Python versions we see in programs such as `flake8`, `pylint` or `pytest` errors because the +first `sys.path` element is outside the `runfiles` directory and if the `name` of the `py_binary` is the same as +the program name, then the script (e.g. `flake8`) will start failing whilst trying to import its own internals from +the bazel entrypoint script. + +The mitigation strategy is to remove the first entry in the `sys.path` if it does not have `.runfiles` and it seems +to fix the behaviour of console_scripts under `bazel run`. + +This would not happen if we created an console_script binary in the root of an external repository, e.g. +`@pypi_pylint//` because the path for the external repository is already in the runfiles directory. +""" + +from __future__ import annotations + +import argparse +import configparser +import pathlib +import re +import sys +import textwrap + +_ENTRY_POINTS_TXT = "entry_points.txt" + +_TEMPLATE = """\ +import sys + +# See @rules_python//python/private:py_console_script_gen.py for explanation +if getattr(sys.flags, "safe_path", False): + # We are running on Python 3.11 and we don't need this workaround + pass +elif ".runfiles" not in sys.path[0]: + sys.path = sys.path[1:] + +try: + from {module} import {attr} +except ImportError: + entries = "\\n".join(sys.path) + print("Printing sys.path entries for easier debugging:", file=sys.stderr) + print(f"sys.path is:\\n{{entries}}", file=sys.stderr) + raise + +if __name__ == "__main__": + sys.exit({entry_point}()) +""" + + +class EntryPointsParser(configparser.ConfigParser): + """A class handling entry_points.txt + + See https://packaging.python.org/en/latest/specifications/entry-points/ + """ + + optionxform = staticmethod(str) + + +def _guess_entry_point(guess: str, console_scripts: dict[string, string]) -> str | None: + for key, candidate in console_scripts.items(): + if guess == key: + return candidate + + +def run( + *, + entry_points: pathlib.Path, + out: pathlib.Path, + console_script: str, + console_script_guess: str, +): + """Run the generator + + Args: + entry_points: The entry_points.txt file to be parsed. + out: The output file. + console_script: The console_script entry in the entry_points.txt file. + """ + config = EntryPointsParser() + config.read(entry_points) + + try: + console_scripts = dict(config["console_scripts"]) + except KeyError: + raise RuntimeError( + f"The package does not provide any console_scripts in it's {_ENTRY_POINTS_TXT}" + ) + + if console_script: + try: + entry_point = console_scripts[console_script] + except KeyError: + available = ", ".join(sorted(console_scripts.keys())) + raise RuntimeError( + f"The console_script '{console_script}' was not found, only the following are available: {available}" + ) from None + else: + # Get rid of the extension and the common prefix + entry_point = _guess_entry_point( + guess=console_script_guess, + console_scripts=console_scripts, + ) + + if not entry_point: + available = ", ".join(sorted(console_scripts.keys())) + raise RuntimeError( + f"Tried to guess that you wanted '{console_script_guess}', but could not find it. " + f"Please select one of the following console scripts: {available}" + ) from None + + module, _, entry_point = entry_point.rpartition(":") + attr, _, _ = entry_point.partition(".") + # TODO: handle 'extras' in entry_point generation + # See https://github.com/bazelbuild/rules_python/issues/1383 + # See https://packaging.python.org/en/latest/specifications/entry-points/ + + with open(out, "w") as f: + f.write( + _TEMPLATE.format( + module=module, + attr=attr, + entry_point=entry_point, + ), + ) + + +def main(): + parser = argparse.ArgumentParser(description="console_script generator") + parser.add_argument( + "--console-script", + help="The console_script to generate the entry_point template for.", + ) + parser.add_argument( + "--console-script-guess", + required=True, + help="The string used for guessing the console_script if it is not provided.", + ) + parser.add_argument( + "entry_points", + metavar="ENTRY_POINTS_TXT", + type=pathlib.Path, + help="The entry_points.txt within the dist-info of a PyPI wheel", + ) + parser.add_argument( + "out", + type=pathlib.Path, + metavar="OUT", + help="The output file.", + ) + args = parser.parse_args() + + run( + entry_points=args.entry_points, + out=args.out, + console_script=args.console_script, + console_script_guess=args.console_script_guess, + ) + + +if __name__ == "__main__": + main() diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index b2919c1041..20dc9763e0 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -182,7 +182,15 @@ alias(name = "pip", actual = select({{":" + item: "@{py_repository}_ rctx.file("defs.bzl", content = """\ # Generated by python/private/toolchains_repo.bzl -load("{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") +load( + "{rules_python}//python/config_settings:transition.bzl", + _py_binary = "py_binary", + _py_test = "py_test", +) +load( + "{rules_python}//python/entry_points:py_console_script_binary.bzl", + _py_console_script_binary = "py_console_script_binary", +) load("{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") host_platform = "{host_platform}" @@ -195,6 +203,13 @@ def py_binary(name, **kwargs): **kwargs ) +def py_console_script_binary(name, **kwargs): + return _py_console_script_binary( + name = name, + binary_rule = py_binary, + **kwargs + ) + def py_test(name, **kwargs): return _py_test( name = name, @@ -247,6 +262,7 @@ load( _host_platform = "host_platform", _interpreter = "interpreter", _py_binary = "py_binary", + _py_console_script_binary = "py_console_script_binary", _py_test = "py_test", ) @@ -254,6 +270,7 @@ compile_pip_requirements = _compile_pip_requirements host_platform = _host_platform interpreter = _interpreter py_binary = _py_binary +py_console_script_binary = _py_console_script_binary py_test = _py_test """.format( repository_name = repository_name, diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 2dd2282146..70dfa48857 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -30,5 +30,6 @@ build_test( "//python:py_test_bzl", "//python/cc:py_cc_toolchain_bzl", "//python/cc:py_cc_toolchain_info_bzl", + "//python/entry_points:py_console_script_binary_bzl", ], ) diff --git a/tests/entry_points/BUILD.bazel b/tests/entry_points/BUILD.bazel new file mode 100644 index 0000000000..7a22d3c92b --- /dev/null +++ b/tests/entry_points/BUILD.bazel @@ -0,0 +1,39 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("//python:py_test.bzl", "py_test") +load(":simple_macro.bzl", "py_console_script_binary_in_a_macro") + +py_test( + name = "py_console_script_gen_test", + srcs = ["py_console_script_gen_test.py"], + main = "py_console_script_gen_test.py", + visibility = ["//visibility:private"], + deps = [ + "//python/private:py_console_script_gen_lib", + ], +) + +py_console_script_binary_in_a_macro( + name = "twine", + pkg = "@publish_deps_twine//:pkg", +) + +build_test( + name = "build_entry_point", + targets = [ + ":twine", + ], +) diff --git a/tests/entry_points/py_console_script_gen_test.py b/tests/entry_points/py_console_script_gen_test.py new file mode 100644 index 0000000000..80b5f20bde --- /dev/null +++ b/tests/entry_points/py_console_script_gen_test.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +import tempfile +import textwrap +import unittest + +from python.private.py_console_script_gen import run + + +class RunTest(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + def test_no_console_scripts_error(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + outfile = tmpdir / "out.py" + given_contents = ( + textwrap.dedent( + """ + [non_console_scripts] + foo = foo.bar:fizz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + + with self.assertRaises(RuntimeError) as cm: + run( + entry_points=entry_points, + out=outfile, + console_script=None, + console_script_guess="", + ) + + self.assertEqual( + "The package does not provide any console_scripts in it's entry_points.txt", + cm.exception.args[0], + ) + + def test_no_entry_point_selected_error(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + outfile = tmpdir / "out.py" + given_contents = ( + textwrap.dedent( + """ + [console_scripts] + foo = foo.bar:fizz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + + with self.assertRaises(RuntimeError) as cm: + run( + entry_points=entry_points, + out=outfile, + console_script=None, + console_script_guess="bar-baz", + ) + + self.assertEqual( + "Tried to guess that you wanted 'bar-baz', but could not find it. Please select one of the following console scripts: foo", + cm.exception.args[0], + ) + + def test_incorrect_entry_point(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + outfile = tmpdir / "out.py" + given_contents = ( + textwrap.dedent( + """ + [console_scripts] + foo = foo.bar:fizz + bar = foo.bar:buzz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + + with self.assertRaises(RuntimeError) as cm: + run( + entry_points=entry_points, + out=outfile, + console_script="baz", + console_script_guess="", + ) + + self.assertEqual( + "The console_script 'baz' was not found, only the following are available: bar, foo", + cm.exception.args[0], + ) + + def test_a_single_entry_point(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + given_contents = ( + textwrap.dedent( + """ + [console_scripts] + foo = foo.bar:baz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + out = tmpdir / "foo.py" + + run( + entry_points=entry_points, + out=out, + console_script=None, + console_script_guess="foo", + ) + + got = out.read_text() + + want = textwrap.dedent( + """\ + import sys + + # See @rules_python//python/private:py_console_script_gen.py for explanation + if getattr(sys.flags, "safe_path", False): + # We are running on Python 3.11 and we don't need this workaround + pass + elif ".runfiles" not in sys.path[0]: + sys.path = sys.path[1:] + + try: + from foo.bar import baz + except ImportError: + entries = "\\n".join(sys.path) + print("Printing sys.path entries for easier debugging:", file=sys.stderr) + print(f"sys.path is:\\n{entries}", file=sys.stderr) + raise + + if __name__ == "__main__": + sys.exit(baz()) + """ + ) + self.assertEqual(want, got) + + def test_a_second_entry_point_class_method(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + given_contents = ( + textwrap.dedent( + """ + [console_scripts] + foo = foo.bar:Bar.baz + bar = foo.baz:Bar.baz + """ + ).strip() + + "\n" + ) + entry_points = tmpdir / "entry_points.txt" + entry_points.write_text(given_contents) + out = tmpdir / "out.py" + + run( + entry_points=entry_points, + out=out, + console_script="bar", + console_script_guess="", + ) + + got = out.read_text() + + self.assertRegex(got, "from foo\.baz import Bar") + self.assertRegex(got, "sys\.exit\(Bar\.baz\(\)\)") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/entry_points/simple_macro.bzl b/tests/entry_points/simple_macro.bzl new file mode 100644 index 0000000000..4764a3ff2e --- /dev/null +++ b/tests/entry_points/simple_macro.bzl @@ -0,0 +1,31 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A simple test macro. +""" + +load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary") + +def py_console_script_binary_in_a_macro(name, pkg): + """A simple macro to see that we can use our macro in a macro. + + Args: + name, str: the name of the target + pkg, str: the pkg target + """ + py_console_script_binary( + name = name, + pkg = Label(pkg), + ) From 6461a693a919ccbe3b9d2ae9b44b22fd6a98e289 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 4 Sep 2023 14:25:16 +0900 Subject: [PATCH 06/41] fix(whl_library): avoid unnecessary repository rule restarts (#1400) Put the `PYTHONPATH` entries used in wheel building as a default value to a private attribute of the `whl_library` repository rule and use resolved path of the interpreter target in creating execution environment to avoid repository rule restarts when fetching external dependencies. The extra private attribute on the `whl_library` removes all but one restart and the extra refactor removes the last restart observed when running, which also reduces the total execution time from around 50s to 43s on my machine: ```console $ cd examples/bzlmod $ bazel clean --expunge --async && bazel build //entry_points:yamllint ``` Fixes #1399 --- CHANGELOG.md | 6 ++++ python/pip_install/pip_repository.bzl | 44 ++++++++++++++++----------- python/repositories.bzl | 21 +++---------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e7b6853f5..def9aa0e5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,12 @@ A brief description of the categories of changes: * (bzlmod) The `entry_point` macro is no longer supported and has been removed in favour of the `py_console_script_binary` macro for `bzlmod` users. +### Fixed + +* (whl_library) No longer restarts repository rule when fetching external + dependencies improving initial build times involving external dependency + fetching. + ## [0.25.0] - 2023-08-22 ### Changed diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 87c7f6b77a..abe3ca787c 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -14,7 +14,7 @@ "" -load("//python:repositories.bzl", "get_interpreter_dirname", "is_standalone_interpreter") +load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") @@ -43,15 +43,11 @@ def _construct_pypath(rctx): Returns: String of the PYTHONPATH. """ - # Get the root directory of these rules - rules_root = rctx.path(Label("//:BUILD.bazel")).dirname - thirdparty_roots = [ - # Includes all the external dependencies from repositories.bzl - rctx.path(Label("@" + repo + "//:BUILD.bazel")).dirname - for repo in all_requirements - ] separator = ":" if not "windows" in rctx.os.name.lower() else ";" - pypath = separator.join([str(p) for p in [rules_root] + thirdparty_roots]) + pypath = separator.join([ + str(rctx.path(entry).dirname) + for entry in rctx.attr._python_path_entries + ]) return pypath def _get_python_interpreter_attr(rctx): @@ -123,7 +119,7 @@ def _get_xcode_location_cflags(rctx): "-isysroot {}/SDKs/MacOSX.sdk".format(xcode_root), ] -def _get_toolchain_unix_cflags(rctx): +def _get_toolchain_unix_cflags(rctx, python_interpreter): """Gather cflags from a standalone toolchain for unix systems. Pip won't be able to compile c extensions from sdists with the pre built python distributions from indygreg @@ -135,11 +131,11 @@ def _get_toolchain_unix_cflags(rctx): return [] # Only update the location when using a standalone toolchain. - if not is_standalone_interpreter(rctx, rctx.attr.python_interpreter_target): + if not is_standalone_interpreter(rctx, python_interpreter): return [] er = rctx.execute([ - rctx.path(rctx.attr.python_interpreter_target).realpath, + python_interpreter, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", ]) @@ -147,7 +143,7 @@ def _get_toolchain_unix_cflags(rctx): fail("could not get python version from interpreter (status {}): {}".format(er.return_code, er.stderr)) _python_version = er.stdout include_path = "{}/include/python{}".format( - get_interpreter_dirname(rctx, rctx.attr.python_interpreter_target), + python_interpreter.dirname, _python_version, ) @@ -218,11 +214,12 @@ def _parse_optional_attrs(rctx, args): return args -def _create_repository_execution_environment(rctx): +def _create_repository_execution_environment(rctx, python_interpreter): """Create a environment dictionary for processes we spawn with rctx.execute. Args: - rctx: The repository context. + rctx (repository_ctx): The repository context. + python_interpreter (path): The resolved python interpreter. Returns: Dictionary of environment variable suitable to pass to rctx.execute. """ @@ -230,7 +227,7 @@ def _create_repository_execution_environment(rctx): # Gather any available CPPFLAGS values cppflags = [] cppflags.extend(_get_xcode_location_cflags(rctx)) - cppflags.extend(_get_toolchain_unix_cflags(rctx)) + cppflags.extend(_get_toolchain_unix_cflags(rctx, python_interpreter)) env = { "PYTHONPATH": _construct_pypath(rctx), @@ -630,7 +627,7 @@ def _whl_library_impl(rctx): result = rctx.execute( args, # Manually construct the PYTHONPATH since we cannot use the toolchain here - environment = _create_repository_execution_environment(rctx), + environment = _create_repository_execution_environment(rctx, python_interpreter), quiet = rctx.attr.quiet, timeout = rctx.attr.timeout, ) @@ -720,6 +717,19 @@ whl_library_attrs = { mandatory = True, doc = "Python requirement string describing the package to make available", ), + "_python_path_entries": attr.label_list( + # Get the root directory of these rules and keep them as a default attribute + # in order to avoid unnecessary repository fetching restarts. + # + # This is very similar to what was done in https://github.com/bazelbuild/rules_go/pull/3478 + default = [ + Label("//:BUILD.bazel"), + ] + [ + # Includes all the external dependencies from repositories.bzl + Label("@" + repo + "//:BUILD.bazel") + for repo in all_requirements + ], + ), } whl_library_attrs.update(**common_attrs) diff --git a/python/repositories.bzl b/python/repositories.bzl index bd06f0b3d0..fbe23bc2e3 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -61,39 +61,26 @@ def py_repositories(): STANDALONE_INTERPRETER_FILENAME = "STANDALONE_INTERPRETER" -def get_interpreter_dirname(rctx, python_interpreter_target): - """Get a python interpreter target dirname. - - Args: - rctx (repository_ctx): The repository rule's context object. - python_interpreter_target (Target): A target representing a python interpreter. - - Returns: - str: The Python interpreter directory. - """ - - return rctx.path(Label("{}//:WORKSPACE".format(str(python_interpreter_target).split("//")[0]))).dirname - -def is_standalone_interpreter(rctx, python_interpreter_target): +def is_standalone_interpreter(rctx, python_interpreter_path): """Query a python interpreter target for whether or not it's a rules_rust provided toolchain Args: rctx (repository_ctx): The repository rule's context object. - python_interpreter_target (Target): A target representing a python interpreter. + python_interpreter_path (path): A path representing the interpreter. Returns: bool: Whether or not the target is from a rules_python generated toolchain. """ # Only update the location when using a hermetic toolchain. - if not python_interpreter_target: + if not python_interpreter_path: return False # This is a rules_python provided toolchain. return rctx.execute([ "ls", "{}/{}".format( - get_interpreter_dirname(rctx, python_interpreter_target), + python_interpreter_path.dirname, STANDALONE_INTERPRETER_FILENAME, ), ]).return_code == 0 From 37452ab462adc5199af20a349158354280ee2516 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 5 Sep 2023 08:34:56 -0700 Subject: [PATCH 07/41] refactor: add missing `//python/config_settings/private:distribution` target (#1402) I'm working on adding a new pip package download rule to the repo called `pypi_install`. For that, I set up a new `examples/pypi_install` directory. When I imported `rules_python` via a `local_repository`, however, the build complained about missing files. This patch aims to fix that issue by making the missing files available. I tried copying the other `distribution` targets when creating this one. --- python/config_settings/BUILD.bazel | 1 + python/config_settings/private/BUILD.bazel | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 272ba78f1f..ab4ee8d880 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -5,6 +5,7 @@ filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ "BUILD.bazel", + "//python/config_settings/private:distribution", ], visibility = ["//python:__pkg__"], ) diff --git a/python/config_settings/private/BUILD.bazel b/python/config_settings/private/BUILD.bazel index e69de29bb2..aa68c6508c 100644 --- a/python/config_settings/private/BUILD.bazel +++ b/python/config_settings/private/BUILD.bazel @@ -0,0 +1,7 @@ +filegroup( + name = "distribution", + srcs = glob(["*.bzl"]) + [ + "BUILD.bazel", + ], + visibility = ["//python/config_settings:__pkg__"], +) From 6d4fa3c72e292472c754c26d0ed22e48e09cc2fc Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Fri, 8 Sep 2023 08:56:38 -0700 Subject: [PATCH 08/41] Import pycross_wheel_library (#1403) This patch imports a few files from jvolkman/rules_pycross at 757033ff8afeb5f7090b1320759f6f03d9c4615c. I would like to re-use this rule for the `pypi_install` repo rule that I'm working on. This rule extracts a downloaded wheel and generates an appropriate `PyInfo` provider for it. All the .pyfiles are taken as-is without modification. I had to run buildifier on all the bazel-related files. As per bazelbuild/rules_python#1360, that meant that I had to add copyright headers. A followup patch will make tweaks so that the code can be used from within rules_python. References: #1360 --- third_party/rules_pycross/LICENSE | 201 ++++++++++++++++++ .../pycross/private/providers.bzl | 32 +++ .../pycross/private/tools/BUILD.bazel | 53 +++++ .../pycross/private/tools/namespace_pkgs.py | 109 ++++++++++ .../private/tools/namespace_pkgs_test.py | 179 ++++++++++++++++ .../pycross/private/tools/wheel_installer.py | 122 +++++++++++ .../pycross/private/wheel_library.bzl | 137 ++++++++++++ 7 files changed, 833 insertions(+) create mode 100644 third_party/rules_pycross/LICENSE create mode 100644 third_party/rules_pycross/pycross/private/providers.bzl create mode 100644 third_party/rules_pycross/pycross/private/tools/BUILD.bazel create mode 100644 third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py create mode 100644 third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py create mode 100644 third_party/rules_pycross/pycross/private/tools/wheel_installer.py create mode 100644 third_party/rules_pycross/pycross/private/wheel_library.bzl diff --git a/third_party/rules_pycross/LICENSE b/third_party/rules_pycross/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/third_party/rules_pycross/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rules_pycross/pycross/private/providers.bzl b/third_party/rules_pycross/pycross/private/providers.bzl new file mode 100644 index 0000000000..f55e98693a --- /dev/null +++ b/third_party/rules_pycross/pycross/private/providers.bzl @@ -0,0 +1,32 @@ +# Copyright 2023 Jeremy Volkman. All rights reserved. +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pycross providers.""" + +PycrossWheelInfo = provider( + doc = "Information about a Python wheel.", + fields = { + "name_file": "File: A file containing the canonical name of the wheel.", + "wheel_file": "File: The wheel file itself.", + }, +) + +PycrossTargetEnvironmentInfo = provider( + doc = "A target environment description.", + fields = { + "file": "The JSON file containing target environment information.", + "python_compatible_with": "A list of constraints used to select this platform.", + }, +) diff --git a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel new file mode 100644 index 0000000000..867b771aae --- /dev/null +++ b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel @@ -0,0 +1,53 @@ +# Copyright 2023 Jeremy Volkman. All rights reserved. +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +package(default_visibility = ["//visibility:private"]) + +py_library( + name = "namespace_pkgs", + srcs = [ + "namespace_pkgs.py", + ], +) + +py_test( + name = "namespace_pkgs_test", + size = "small", + srcs = [ + "namespace_pkgs_test.py", + ], + tags = [ + "unit", + # TODO(philsc): Make this work. + "manual", + ], + deps = [ + ":namespace_pkgs", + ], +) + +py_binary( + name = "wheel_installer", + srcs = ["wheel_installer.py"], + visibility = ["//visibility:public"], + deps = [ + ":namespace_pkgs", + # TODO(philsc): Make this work with what's available in rules_python. + #"@rules_pycross_pypi_deps_absl_py//:pkg", + #"@rules_pycross_pypi_deps_installer//:pkg", + ], +) diff --git a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py new file mode 100644 index 0000000000..59300ffcd1 --- /dev/null +++ b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py @@ -0,0 +1,109 @@ +"""Utility functions to discover python package types""" +import os +import textwrap +from pathlib import Path # supported in >= 3.4 +from typing import List +from typing import Optional +from typing import Set + + +def implicit_namespace_packages( + directory: str, ignored_dirnames: Optional[List[str]] = None +) -> Set[Path]: + """Discovers namespace packages implemented using the 'native namespace packages' method. + + AKA 'implicit namespace packages', which has been supported since Python 3.3. + See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages + + Args: + directory: The root directory to recursively find packages in. + ignored_dirnames: A list of directories to exclude from the search + + Returns: + The set of directories found under root to be packages using the native namespace method. + """ + namespace_pkg_dirs: Set[Path] = set() + standard_pkg_dirs: Set[Path] = set() + directory_path = Path(directory) + ignored_dirname_paths: List[Path] = [Path(p) for p in ignored_dirnames or ()] + # Traverse bottom-up because a directory can be a namespace pkg because its child contains module files. + for dirpath, dirnames, filenames in map( + lambda t: (Path(t[0]), *t[1:]), os.walk(directory_path, topdown=False) + ): + if "__init__.py" in filenames: + standard_pkg_dirs.add(dirpath) + continue + elif ignored_dirname_paths: + is_ignored_dir = dirpath in ignored_dirname_paths + child_of_ignored_dir = any( + d in dirpath.parents for d in ignored_dirname_paths + ) + if is_ignored_dir or child_of_ignored_dir: + continue + + dir_includes_py_modules = _includes_python_modules(filenames) + parent_of_namespace_pkg = any( + Path(dirpath, d) in namespace_pkg_dirs for d in dirnames + ) + parent_of_standard_pkg = any( + Path(dirpath, d) in standard_pkg_dirs for d in dirnames + ) + parent_of_pkg = parent_of_namespace_pkg or parent_of_standard_pkg + if ( + (dir_includes_py_modules or parent_of_pkg) + and + # The root of the directory should never be an implicit namespace + dirpath != directory_path + ): + namespace_pkg_dirs.add(dirpath) + return namespace_pkg_dirs + + +def add_pkgutil_style_namespace_pkg_init(dir_path: Path) -> None: + """Adds 'pkgutil-style namespace packages' init file to the given directory + + See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages + + Args: + dir_path: The directory to create an __init__.py for. + + Raises: + ValueError: If the directory already contains an __init__.py file + """ + ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py") + + if os.path.isfile(ns_pkg_init_filepath): + raise ValueError("%s already contains an __init__.py file." % dir_path) + + with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: + # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages + ns_pkg_init_f.write( + textwrap.dedent( + """\ + # __path__ manipulation added by bazelbuild/rules_python to support namespace pkgs. + __path__ = __import__('pkgutil').extend_path(__path__, __name__) + """ + ) + ) + + +def _includes_python_modules(files: List[str]) -> bool: + """ + In order to only transform directories that Python actually considers namespace pkgs + we need to detect if a directory includes Python modules. + + Which files are loadable as modules is extension based, and the particular set of extensions + varies by platform. + + See: + 1. https://github.com/python/cpython/blob/7d9d25dbedfffce61fc76bc7ccbfa9ae901bf56f/Lib/importlib/machinery.py#L19 + 2. PEP 420 -- Implicit Namespace Packages, Specification - https://www.python.org/dev/peps/pep-0420/#specification + 3. dynload_shlib.c and dynload_win.c in python/cpython. + """ + module_suffixes = { + ".py", # Source modules + ".pyc", # Compiled bytecode modules + ".so", # Unix extension modules + ".pyd", # https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll + } + return any(Path(f).suffix in module_suffixes for f in files) diff --git a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py new file mode 100644 index 0000000000..49945f9b8c --- /dev/null +++ b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py @@ -0,0 +1,179 @@ +import os +import pathlib +import shutil +import tempfile +import unittest +from typing import Optional +from typing import Set + +from pycross.private.tools import namespace_pkgs + + +class TempDir: + def __init__(self) -> None: + self.dir = tempfile.mkdtemp() + + def root(self) -> str: + return self.dir + + def add_dir(self, rel_path: str) -> None: + d = pathlib.Path(self.dir, rel_path) + d.mkdir(parents=True) + + def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: + f = pathlib.Path(self.dir, rel_path) + f.parent.mkdir(parents=True, exist_ok=True) + if contents: + with open(str(f), "w") as writeable_f: + writeable_f.write(contents) + else: + f.touch() + + def remove(self) -> None: + shutil.rmtree(self.dir) + + +class TestImplicitNamespacePackages(unittest.TestCase): + def assertPathsEqual(self, actual: Set[pathlib.Path], expected: Set[str]) -> None: + self.assertEqual(actual, {pathlib.Path(p) for p in expected}) + + def test_in_current_directory(self) -> None: + directory = TempDir() + directory.add_file("foo/bar/biz.py") + directory.add_file("foo/bee/boo.py") + directory.add_file("foo/buu/__init__.py") + directory.add_file("foo/buu/bii.py") + cwd = os.getcwd() + os.chdir(directory.root()) + expected = { + "foo", + "foo/bar", + "foo/bee", + } + try: + actual = namespace_pkgs.implicit_namespace_packages(".") + self.assertPathsEqual(actual, expected) + finally: + os.chdir(cwd) + directory.remove() + + def test_finds_correct_namespace_packages(self) -> None: + directory = TempDir() + directory.add_file("foo/bar/biz.py") + directory.add_file("foo/bee/boo.py") + directory.add_file("foo/buu/__init__.py") + directory.add_file("foo/buu/bii.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + directory.root() + "/foo/bee", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_ignores_empty_directories(self) -> None: + directory = TempDir() + directory.add_file("foo/bar/biz.py") + directory.add_dir("foo/cat") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_empty_case(self) -> None: + directory = TempDir() + directory.add_file("foo/__init__.py") + directory.add_file("foo/bar/__init__.py") + directory.add_file("foo/bar/biz.py") + + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertEqual(actual, set()) + + def test_ignores_non_module_files_in_directories(self) -> None: + directory = TempDir() + directory.add_file("foo/__init__.pyi") + directory.add_file("foo/py.typed") + + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertEqual(actual, set()) + + def test_parent_child_relationship_of_namespace_pkgs(self): + directory = TempDir() + directory.add_file("foo/bar/biff/my_module.py") + directory.add_file("foo/bar/biff/another_module.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + directory.root() + "/foo/bar/biff", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_parent_child_relationship_of_namespace_and_standard_pkgs(self): + directory = TempDir() + directory.add_file("foo/bar/biff/__init__.py") + directory.add_file("foo/bar/biff/another_module.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(self): + directory = TempDir() + directory.add_file("foo/bar/__init__.py") + directory.add_file("foo/bar/biff/another_module.py") + directory.add_file("foo/bar/biff/__init__.py") + directory.add_file("foo/bar/boof/big_module.py") + directory.add_file("foo/bar/boof/__init__.py") + directory.add_file("fim/in_a_ns_pkg.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/fim", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_recognized_all_nonstandard_module_types(self): + directory = TempDir() + directory.add_file("ayy/my_module.pyc") + directory.add_file("bee/ccc/dee/eee.so") + directory.add_file("eff/jee/aych.pyd") + + expected = { + directory.root() + "/ayy", + directory.root() + "/bee", + directory.root() + "/bee/ccc", + directory.root() + "/bee/ccc/dee", + directory.root() + "/eff", + directory.root() + "/eff/jee", + } + actual = namespace_pkgs.implicit_namespace_packages(directory.root()) + self.assertPathsEqual(actual, expected) + + def test_skips_ignored_directories(self): + directory = TempDir() + directory.add_file("foo/boo/my_module.py") + directory.add_file("foo/bar/another_module.py") + + expected = { + directory.root() + "/foo", + directory.root() + "/foo/bar", + } + actual = namespace_pkgs.implicit_namespace_packages( + directory.root(), + ignored_dirnames=[directory.root() + "/foo/boo"], + ) + self.assertPathsEqual(actual, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py new file mode 100644 index 0000000000..6d3673669b --- /dev/null +++ b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py @@ -0,0 +1,122 @@ +""" +A tool that invokes pypa/build to build the given sdist tarball. +""" + +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any + +from absl import app +from absl.flags import argparse_flags +from installer import install +from installer.destinations import SchemeDictionaryDestination +from installer.sources import WheelFile +from pycross.private.tools import namespace_pkgs + + +def setup_namespace_pkg_compatibility(wheel_dir: Path) -> None: + """Converts native namespace packages to pkgutil-style packages + + Namespace packages can be created in one of three ways. They are detailed here: + https://packaging.python.org/guides/packaging-namespace-packages/#creating-a-namespace-package + + 'pkgutil-style namespace packages' (2) and 'pkg_resources-style namespace packages' (3) works in Bazel, but + 'native namespace packages' (1) do not. + + We ensure compatibility with Bazel of method 1 by converting them into method 2. + + Args: + wheel_dir: the directory of the wheel to convert + """ + + namespace_pkg_dirs = namespace_pkgs.implicit_namespace_packages( + str(wheel_dir), + ignored_dirnames=["%s/bin" % wheel_dir], + ) + + for ns_pkg_dir in namespace_pkg_dirs: + namespace_pkgs.add_pkgutil_style_namespace_pkg_init(ns_pkg_dir) + + +def main(args: Any) -> None: + dest_dir = args.directory + lib_dir = dest_dir / "site-packages" + destination = SchemeDictionaryDestination( + scheme_dict={ + "platlib": str(lib_dir), + "purelib": str(lib_dir), + "headers": str(dest_dir / "include"), + "scripts": str(dest_dir / "bin"), + "data": str(dest_dir / "data"), + }, + interpreter="/usr/bin/env python3", # Generic; it's not feasible to run these scripts directly. + script_kind="posix", + bytecode_optimization_levels=[0, 1], + ) + + link_dir = Path(tempfile.mkdtemp()) + if args.wheel_name_file: + with open(args.wheel_name_file, "r") as f: + wheel_name = f.read().strip() + else: + wheel_name = os.path.basename(args.wheel) + + link_path = link_dir / wheel_name + os.symlink(os.path.join(os.getcwd(), args.wheel), link_path) + + try: + with WheelFile.open(link_path) as source: + install( + source=source, + destination=destination, + # Additional metadata that is generated by the installation tool. + additional_metadata={ + "INSTALLER": b"https://github.com/jvolkman/rules_pycross", + }, + ) + finally: + shutil.rmtree(link_dir, ignore_errors=True) + + setup_namespace_pkg_compatibility(lib_dir) + + +def parse_flags(argv) -> Any: + parser = argparse_flags.ArgumentParser(description="Extract a Python wheel.") + + parser.add_argument( + "--wheel", + type=Path, + required=True, + help="The wheel file path.", + ) + + parser.add_argument( + "--wheel-name-file", + type=Path, + required=False, + help="A file containing the canonical name of the wheel.", + ) + + parser.add_argument( + "--enable-implicit-namespace-pkgs", + action="store_true", + help="If true, disables conversion of implicit namespace packages and will unzip as-is.", + ) + + parser.add_argument( + "--directory", + type=Path, + help="The output path.", + ) + + return parser.parse_args(argv[1:]) + + +if __name__ == "__main__": + # When under `bazel run`, change to the actual working dir. + if "BUILD_WORKING_DIRECTORY" in os.environ: + os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) + + app.run(main, flags_parser=parse_flags) diff --git a/third_party/rules_pycross/pycross/private/wheel_library.bzl b/third_party/rules_pycross/pycross/private/wheel_library.bzl new file mode 100644 index 0000000000..25a2497abe --- /dev/null +++ b/third_party/rules_pycross/pycross/private/wheel_library.bzl @@ -0,0 +1,137 @@ +# Copyright 2023 Jeremy Volkman. All rights reserved. +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of the pycross_wheel_library rule.""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@rules_python//python:defs.bzl", "PyInfo") +load(":providers.bzl", "PycrossWheelInfo") + +def _pycross_wheel_library_impl(ctx): + out = ctx.actions.declare_directory(ctx.attr.name) + + wheel_target = ctx.attr.wheel + if PycrossWheelInfo in wheel_target: + wheel_file = wheel_target[PycrossWheelInfo].wheel_file + name_file = wheel_target[PycrossWheelInfo].name_file + else: + wheel_file = ctx.file.wheel + name_file = None + + args = ctx.actions.args().use_param_file("--flagfile=%s") + args.add("--wheel", wheel_file) + args.add("--directory", out.path) + + inputs = [wheel_file] + if name_file: + inputs.append(name_file) + args.add("--wheel-name-file", name_file) + + if ctx.attr.enable_implicit_namespace_pkgs: + args.add("--enable-implicit-namespace-pkgs") + + ctx.actions.run( + inputs = inputs, + outputs = [out], + executable = ctx.executable._tool, + arguments = [args], + # Set environment variables to make generated .pyc files reproducible. + env = { + "PYTHONHASHSEED": "0", + "SOURCE_DATE_EPOCH": "315532800", + }, + mnemonic = "WheelInstall", + progress_message = "Installing %s" % ctx.file.wheel.basename, + ) + + has_py2_only_sources = ctx.attr.python_version == "PY2" + has_py3_only_sources = ctx.attr.python_version == "PY3" + if not has_py2_only_sources: + for d in ctx.attr.deps: + if d[PyInfo].has_py2_only_sources: + has_py2_only_sources = True + break + if not has_py3_only_sources: + for d in ctx.attr.deps: + if d[PyInfo].has_py3_only_sources: + has_py3_only_sources = True + break + + # TODO: Is there a more correct way to get this runfiles-relative import path? + imp = paths.join( + ctx.label.workspace_name or ctx.workspace_name, # Default to the local workspace. + ctx.label.package, + ctx.label.name, + "site-packages", # we put lib files in this subdirectory. + ) + + imports = depset( + direct = [imp], + transitive = [d[PyInfo].imports for d in ctx.attr.deps], + ) + transitive_sources = depset( + direct = [out], + transitive = [dep[PyInfo].transitive_sources for dep in ctx.attr.deps if PyInfo in dep], + ) + runfiles = ctx.runfiles(files = [out]) + for d in ctx.attr.deps: + runfiles = runfiles.merge(d[DefaultInfo].default_runfiles) + + return [ + DefaultInfo( + files = depset(direct = [out]), + runfiles = runfiles, + ), + PyInfo( + has_py2_only_sources = has_py2_only_sources, + has_py3_only_sources = has_py3_only_sources, + imports = imports, + transitive_sources = transitive_sources, + uses_shared_libraries = True, # Docs say this is unused + ), + ] + +pycross_wheel_library = rule( + implementation = _pycross_wheel_library_impl, + attrs = { + "deps": attr.label_list( + doc = "A list of this wheel's Python library dependencies.", + providers = [DefaultInfo, PyInfo], + ), + "enable_implicit_namespace_pkgs": attr.bool( + default = True, + doc = """ +If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary +and py_test targets must specify either `legacy_create_init=False` or the global Bazel option +`--incompatible_default_to_explicit_init_py` to prevent `__init__.py` being automatically generated in every directory. +This option is required to support some packages which cannot handle the conversion to pkg-util style. + """, + ), + "python_version": attr.string( + doc = "The python version required for this wheel ('PY2' or 'PY3')", + values = ["PY2", "PY3", ""], + ), + "wheel": attr.label( + doc = "The wheel file.", + allow_single_file = [".whl"], + mandatory = True, + ), + "_tool": attr.label( + default = Label("//pycross/private/tools:wheel_installer"), + cfg = "exec", + executable = True, + ), + }, +) From 5ea804f7f32fd79a36d866faa34050dab8a3da03 Mon Sep 17 00:00:00 2001 From: Chris Lewis Date: Mon, 11 Sep 2023 04:17:55 -0700 Subject: [PATCH 09/41] refactor: upgrade certifi (#1397) Older versions of certifi allow for revoked HTTPS certificates. This change updates usages of certifi to the first known-good version. See https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 and https://nvd.nist.gov/vuln/detail/CVE-2023-37920 --------- Co-authored-by: Greg --- examples/pip_parse_vendored/requirements.bzl | 2 +- examples/pip_parse_vendored/requirements.in | 1 + examples/pip_parse_vendored/requirements.txt | 10 ++++++---- examples/pip_repository_annotations/requirements.in | 1 + examples/pip_repository_annotations/requirements.txt | 10 ++++++---- tests/pip_repository_entry_points/requirements.in | 5 ++++- tests/pip_repository_entry_points/requirements.txt | 10 ++++++---- .../requirements_windows.txt | 10 ++++++---- 8 files changed, 31 insertions(+), 18 deletions(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 7bf5170120..4e83555d6c 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -13,7 +13,7 @@ all_whl_requirements = ["@pip_certifi//:whl", "@pip_charset_normalizer//:whl", " all_data_requirements = ["@pip_certifi//:data", "@pip_charset_normalizer//:data", "@pip_idna//:data", "@pip_requests//:data", "@pip_urllib3//:data"] -_packages = [("pip_certifi", "certifi==2022.12.7 --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] +_packages = [("pip_certifi", "certifi==2023.7.22 --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"), ("pip_charset_normalizer", "charset-normalizer==2.1.1 --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"), ("pip_idna", "idna==3.4 --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"), ("pip_requests", "requests==2.28.1 --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"), ("pip_urllib3", "urllib3==1.26.13 --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8")] _config = {"download_only": False, "enable_implicit_namespace_pkgs": False, "environment": {}, "extra_pip_args": [], "isolated": True, "pip_data_exclude": [], "python_interpreter": "python3", "python_interpreter_target": interpreter, "quiet": True, "repo": "pip", "repo_prefix": "pip_", "timeout": 600} _annotations = {} diff --git a/examples/pip_parse_vendored/requirements.in b/examples/pip_parse_vendored/requirements.in index f2293605cf..7ec4233fa4 100644 --- a/examples/pip_parse_vendored/requirements.in +++ b/examples/pip_parse_vendored/requirements.in @@ -1 +1,2 @@ requests +certifi>=2023.7.22 # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 diff --git a/examples/pip_parse_vendored/requirements.txt b/examples/pip_parse_vendored/requirements.txt index ff1a3633a2..75b45a1ce3 100644 --- a/examples/pip_parse_vendored/requirements.txt +++ b/examples/pip_parse_vendored/requirements.txt @@ -4,10 +4,12 @@ # # bazel run //:requirements.update # -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 - # via requests +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via + # -r requirements.in + # requests charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f diff --git a/examples/pip_repository_annotations/requirements.in b/examples/pip_repository_annotations/requirements.in index fd3f75c888..29419a216c 100644 --- a/examples/pip_repository_annotations/requirements.in +++ b/examples/pip_repository_annotations/requirements.in @@ -2,5 +2,6 @@ # `pip_repository` rules. --extra-index-url https://pypi.python.org/simple/ +certifi>=2023.7.22 # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 wheel requests[security]>=2.8.1 diff --git a/examples/pip_repository_annotations/requirements.txt b/examples/pip_repository_annotations/requirements.txt index 9fde0a922f..04379ebe24 100644 --- a/examples/pip_repository_annotations/requirements.txt +++ b/examples/pip_repository_annotations/requirements.txt @@ -6,10 +6,12 @@ # --extra-index-url https://pypi.python.org/simple/ -certifi==2022.12.7 \ - --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \ - --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18 - # via requests +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via + # -r requirements.in + # requests charset-normalizer==2.1.1 \ --hash=sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845 \ --hash=sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f diff --git a/tests/pip_repository_entry_points/requirements.in b/tests/pip_repository_entry_points/requirements.in index 2cc4625577..7f999c8837 100644 --- a/tests/pip_repository_entry_points/requirements.in +++ b/tests/pip_repository_entry_points/requirements.in @@ -1,5 +1,8 @@ sphinx==4.3.2 yamllint>=1.28.0 -# Last avialable for ubuntu python3.6 +# Last available for Ubuntu python3.6 setuptools==59.6.0 + +certifi>=2023.7.22 # https://security.snyk.io/vuln/SNYK-PYTHON-CERTIFI-5805047 + diff --git a/tests/pip_repository_entry_points/requirements.txt b/tests/pip_repository_entry_points/requirements.txt index a93facc03b..20114b2838 100644 --- a/tests/pip_repository_entry_points/requirements.txt +++ b/tests/pip_repository_entry_points/requirements.txt @@ -12,10 +12,12 @@ babel==2.9.1 \ --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 # via sphinx -certifi==2021.10.8 \ - --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ - --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 - # via requests +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via + # -r requirements.in + # requests charset-normalizer==2.0.10 \ --hash=sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd \ --hash=sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455 diff --git a/tests/pip_repository_entry_points/requirements_windows.txt b/tests/pip_repository_entry_points/requirements_windows.txt index 651e2b5e56..075c960d60 100644 --- a/tests/pip_repository_entry_points/requirements_windows.txt +++ b/tests/pip_repository_entry_points/requirements_windows.txt @@ -12,10 +12,12 @@ babel==2.9.1 \ --hash=sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9 \ --hash=sha256:bc0c176f9f6a994582230df350aa6e05ba2ebe4b3ac317eab29d9be5d2768da0 # via sphinx -certifi==2021.10.8 \ - --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \ - --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 - # via requests +certifi==2023.7.22 \ + --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \ + --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9 + # via + # -r requirements.in + # requests charset-normalizer==2.0.10 \ --hash=sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd \ --hash=sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455 From 425082473e33d1c01c1581681ae14553790d9012 Mon Sep 17 00:00:00 2001 From: Ivo List Date: Tue, 12 Sep 2023 19:59:39 +0200 Subject: [PATCH 10/41] fix: don't set distribs in version transitioning rule (#1412) This makes the version-aware transition rule compatible with an upcoming Bazel change that disallows setting unknown attributes to None (the `distribs` attribute, in this case). The `distribs` attribute was common to all rules, but it has been long deprecated and it won't be part of every rule in upcoming Bazel versions. The previous implementation resulted in setting `distribs = None` on the target. Bazel won't support setting undefined attributes to None. Addresses: https://github.com/bazelbuild/bazel/issues/19403 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 4 ++++ python/config_settings/transition.bzl | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def9aa0e5f..72e2171fce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ A brief description of the categories of changes: ## Unreleased +### Changed +* (multi-version) The `distribs` attribute is no longer propagated. This + attribute has been long deprecated by Bazel and shouldn't be used. + ### Added * (bzlmod, entry_point) Added diff --git a/python/config_settings/transition.bzl b/python/config_settings/transition.bzl index f9f19f2940..cb25965f76 100644 --- a/python/config_settings/transition.bzl +++ b/python/config_settings/transition.bzl @@ -152,7 +152,6 @@ def _py_rule(rule_impl, transition_rule, name, python_version, **kwargs): # https://bazel.build/reference/be/common-definitions#common-attributes compatible_with = kwargs.pop("compatible_with", None) deprecation = kwargs.pop("deprecation", None) - distribs = kwargs.pop("distribs", None) exec_compatible_with = kwargs.pop("exec_compatible_with", None) exec_properties = kwargs.pop("exec_properties", None) features = kwargs.pop("features", None) @@ -166,7 +165,6 @@ def _py_rule(rule_impl, transition_rule, name, python_version, **kwargs): common_attrs = { "compatible_with": compatible_with, "deprecation": deprecation, - "distribs": distribs, "exec_compatible_with": exec_compatible_with, "exec_properties": exec_properties, "features": features, From 4769fea786d0ceefab384fa9febf789be685ad23 Mon Sep 17 00:00:00 2001 From: Gowroji Sunil Date: Wed, 13 Sep 2023 20:35:40 +0530 Subject: [PATCH 11/41] fix(gazelle): upgrade rules_go: 0.39.1 -> 0.41.0 to work with upcoming Bazel versions (#1410) This makes rules_python's usage of Go work with upcoming Bazel versions. The `_whitelist_function_transition` attribute in the Go rules is being removed, per https://github.com/bazelbuild/bazel/issues/19493. Newer Go rule releases are compatible with the upcoming Bazel versions. Fixes : [1409](https://github.com/bazelbuild/rules_python/issues/1409) --- CHANGELOG.md | 4 ++++ examples/build_file_generation/WORKSPACE | 6 +++--- gazelle/MODULE.bazel | 2 +- gazelle/WORKSPACE | 6 +++--- internal_deps.bzl | 6 +++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72e2171fce..491a0804c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,13 @@ A brief description of the categories of changes: ## Unreleased ### Changed + +* (deps) Upgrade rules_go 0.39.1 -> 0.41.0; this is so gazelle integration works with upcoming Bazel versions + * (multi-version) The `distribs` attribute is no longer propagated. This attribute has been long deprecated by Bazel and shouldn't be used. + ### Added * (bzlmod, entry_point) Added diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 7c74835870..fa11380dde 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -20,10 +20,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", + sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", ], ) diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index ae94a5f863..c70955dd16 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -5,7 +5,7 @@ module( ) bazel_dep(name = "rules_python", version = "0.18.0") -bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go") +bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go") bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index eef16e924d..b9ba91d9f8 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa", + sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", ], ) diff --git a/internal_deps.bzl b/internal_deps.bzl index f50d2bfae1..fd2d91edc1 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -74,10 +74,10 @@ def rules_python_internal_deps(): maybe( http_archive, name = "io_bazel_rules_go", - sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", + sha256 = "278b7ff5a826f3dc10f04feaf0b70d48b68748ccd512d7f98bf442077f043fe3", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.41.0/rules_go-v0.41.0.zip", ], ) From 67da3e5537bc1c63bd51ac46eab66d0273eae63e Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 15 Sep 2023 11:06:51 +0200 Subject: [PATCH 12/41] fix: gazelle: Fix non-hermetic runfiles lookup (#1415) `bazel.Runfiles` is a deprecated way to look up runfiles that can result in non-hermetic lookups. `github.com/bazelbuild/rules_go/go/runfiles` is the recommended package for this. --- gazelle/MODULE.bazel | 2 +- gazelle/go.mod | 2 +- gazelle/go.sum | 4 ++-- gazelle/python/BUILD.bazel | 2 +- gazelle/python/parser.go | 4 ++-- gazelle/python/std_modules.go | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index c70955dd16..8c6ad19c7b 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -6,7 +6,7 @@ module( bazel_dep(name = "rules_python", version = "0.18.0") bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go") -bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") +bazel_dep(name = "gazelle", version = "0.33.0", repo_name = "bazel_gazelle") go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") go_deps.from_file(go_mod = "//:go.mod") diff --git a/gazelle/go.mod b/gazelle/go.mod index 1d1cee75f5..6789aa152b 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( github.com/bazelbuild/bazel-gazelle v0.31.1 github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d - github.com/bazelbuild/rules_go v0.39.1 + github.com/bazelbuild/rules_go v0.41.0 github.com/bmatcuk/doublestar v1.3.4 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 diff --git a/gazelle/go.sum b/gazelle/go.sum index ba2c8bf688..5617f9b822 100644 --- a/gazelle/go.sum +++ b/gazelle/go.sum @@ -4,8 +4,8 @@ github.com/bazelbuild/bazel-gazelle v0.31.1 h1:ROyUyUHzoEdvoOs1e0haxJx1l5EjZX6AO github.com/bazelbuild/bazel-gazelle v0.31.1/go.mod h1:Ul0pqz50f5wxz0QNzsZ+mrEu4AVAVJZEB5xLnHgIG9c= github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d h1:Fl1FfItZp34QIQmmDTbZXHB5XA6JfbNNfH7tRRGWvQo= github.com/bazelbuild/buildtools v0.0.0-20230510134650-37bd1811516d/go.mod h1:689QdV3hBP7Vo9dJMmzhoYIyo/9iMhEmHkJcnaPRCbo= -github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8= -github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= +github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0vK4I+k= +github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index fcfe81bd61..4cb755de25 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -36,7 +36,7 @@ go_library( "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_emirpasic_gods//sets/treeset", "@com_github_emirpasic_gods//utils", - "@io_bazel_rules_go//go/tools/bazel:go_default_library", + "@io_bazel_rules_go//go/runfiles", ], ) diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 7f10a754bf..c45aef139a 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -26,7 +26,7 @@ import ( "strings" "sync" - "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/bazelbuild/rules_go/go/runfiles" "github.com/emirpasic/gods/sets/treeset" godsutils "github.com/emirpasic/gods/utils" ) @@ -38,7 +38,7 @@ var ( ) func startParserProcess(ctx context.Context) { - parseScriptRunfile, err := bazel.Runfile("python/parse") + parseScriptRunfile, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/parse") if err != nil { log.Printf("failed to initialize parser: %v\n", err) os.Exit(1) diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index c537184c74..15ef766ff2 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go @@ -26,7 +26,7 @@ import ( "strings" "sync" - "github.com/bazelbuild/rules_go/go/tools/bazel" + "github.com/bazelbuild/rules_go/go/runfiles" ) var ( @@ -39,7 +39,7 @@ var ( func startStdModuleProcess(ctx context.Context) { stdModulesSeen = make(map[string]struct{}) - stdModulesScriptRunfile, err := bazel.Runfile("python/std_modules") + stdModulesScriptRunfile, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/std_modules") if err != nil { log.Printf("failed to initialize std_modules: %v\n", err) os.Exit(1) From abc3c9f1bd2edb025d5748bae1460e9b9526df4c Mon Sep 17 00:00:00 2001 From: Ivo List Date: Fri, 15 Sep 2023 23:40:29 +0200 Subject: [PATCH 13/41] feat: create toolchain type for py_proto_library (#1416) This is to eventually allow py_proto_library to use toolchain resolution for getting the Python-specific protobuf information necessary to create protos. Design doc: https://docs.google.com/document/d/1CE6wJHNfKbUPBr7-mmk_0Yo3a4TaqcTPE0OWNuQkhPs/edit#heading=h.5mcn15i0e1ch --- python/proto/BUILD.bazel | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 python/proto/BUILD.bazel diff --git a/python/proto/BUILD.bazel b/python/proto/BUILD.bazel new file mode 100644 index 0000000000..9f60574f26 --- /dev/null +++ b/python/proto/BUILD.bazel @@ -0,0 +1,18 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//visibility:public"]) + +# Toolchain type provided by proto_lang_toolchain rule and used by py_proto_library +toolchain_type(name = "toolchain_type") From 3a57e4aa165c617a1d9e1c8874047eb3256d3dbe Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 18 Sep 2023 16:41:06 -0700 Subject: [PATCH 14/41] internal: copy Starlark rule implementation from Bazel (#1418) This is a copy of the Starlark implementation of the Python rules from Bazel. This code isn't loaded and won't work as-is. Modifications to make it work will be made in subsequent changes. It's almost pristine; changes are made to satisfy the buildifier check. Work towards #1069 --- python/private/common/attributes.bzl | 227 +++++ python/private/common/attributes_bazel.bzl | 30 + python/private/common/common.bzl | 528 +++++++++++ python/private/common/common_bazel.bzl | 104 +++ python/private/common/providers.bzl | 212 +++++ python/private/common/py_binary_bazel.bzl | 48 + python/private/common/py_binary_macro.bzl | 21 + python/private/common/py_executable.bzl | 845 ++++++++++++++++++ python/private/common/py_executable_bazel.bzl | 480 ++++++++++ python/private/common/py_library.bzl | 99 ++ python/private/common/py_library_bazel.bzl | 59 ++ python/private/common/py_library_macro.bzl | 19 + python/private/common/py_runtime_macro.bzl | 22 + python/private/common/py_runtime_rule.bzl | 214 +++++ python/private/common/py_test_bazel.bzl | 55 ++ python/private/common/py_test_macro.bzl | 21 + python/private/common/semantics.bzl | 34 + 17 files changed, 3018 insertions(+) create mode 100644 python/private/common/attributes.bzl create mode 100644 python/private/common/attributes_bazel.bzl create mode 100644 python/private/common/common.bzl create mode 100644 python/private/common/common_bazel.bzl create mode 100644 python/private/common/providers.bzl create mode 100644 python/private/common/py_binary_bazel.bzl create mode 100644 python/private/common/py_binary_macro.bzl create mode 100644 python/private/common/py_executable.bzl create mode 100644 python/private/common/py_executable_bazel.bzl create mode 100644 python/private/common/py_library.bzl create mode 100644 python/private/common/py_library_bazel.bzl create mode 100644 python/private/common/py_library_macro.bzl create mode 100644 python/private/common/py_runtime_macro.bzl create mode 100644 python/private/common/py_runtime_rule.bzl create mode 100644 python/private/common/py_test_bazel.bzl create mode 100644 python/private/common/py_test_macro.bzl create mode 100644 python/private/common/semantics.bzl diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl new file mode 100644 index 0000000000..7e28ed9d69 --- /dev/null +++ b/python/private/common/attributes.bzl @@ -0,0 +1,227 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Attributes for Python rules.""" + +load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") +load(":common/python/common.bzl", "union_attrs") +load(":common/python/providers.bzl", "PyInfo") +load( + ":common/python/semantics.bzl", + "DEPS_ATTR_ALLOW_RULES", + "PLATFORMS_LOCATION", + "SRCS_ATTR_ALLOW_FILES", + "TOOLS_REPO", +) + +PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo + +_STAMP_VALUES = [-1, 0, 1] + +def create_stamp_attr(**kwargs): + return {"stamp": attr.int(values = _STAMP_VALUES, **kwargs)} + +def create_srcs_attr(*, mandatory): + return { + "srcs": attr.label_list( + # Google builds change the set of allowed files. + allow_files = SRCS_ATTR_ALLOW_FILES, + mandatory = mandatory, + # Necessary for --compile_one_dependency to work. + flags = ["DIRECT_COMPILE_TIME_INPUT"], + ), + } + +SRCS_VERSION_ALL_VALUES = ["PY2", "PY2ONLY", "PY2AND3", "PY3", "PY3ONLY"] +SRCS_VERSION_NON_CONVERSION_VALUES = ["PY2AND3", "PY2ONLY", "PY3ONLY"] + +def create_srcs_version_attr(values): + return { + "srcs_version": attr.string( + default = "PY2AND3", + values = values, + ), + } + +def copy_common_binary_kwargs(kwargs): + return { + key: kwargs[key] + for key in BINARY_ATTR_NAMES + if key in kwargs + } + +def copy_common_test_kwargs(kwargs): + return { + key: kwargs[key] + for key in TEST_ATTR_NAMES + if key in kwargs + } + +CC_TOOLCHAIN = { + # NOTE: The `cc_helper.find_cpp_toolchain()` function expects the attribute + # name to be this name. + "_cc_toolchain": attr.label(default = "@" + TOOLS_REPO + "//tools/cpp:current_cc_toolchain"), +} + +# The common "data" attribute definition. +DATA_ATTRS = { + # NOTE: The "flags" attribute is deprecated, but there isn't an alternative + # way to specify that constraints should be ignored. + "data": attr.label_list( + allow_files = True, + flags = ["SKIP_CONSTRAINTS_OVERRIDE"], + ), +} + +NATIVE_RULES_ALLOWLIST_ATTRS = { + "_native_rules_allowlist": attr.label( + default = configuration_field( + fragment = "py", + name = "native_rules_allowlist", + ), + providers = [PackageSpecificationInfo], + ), +} + +# Attributes common to all rules. +COMMON_ATTRS = union_attrs( + DATA_ATTRS, + NATIVE_RULES_ALLOWLIST_ATTRS, + { + # NOTE: This attribute is deprecated and slated for removal. + "distribs": attr.string_list(), + # TODO(b/148103851): This attribute is deprecated and slated for + # removal. + # NOTE: The license attribute is missing in some Java integration tests, + # so fallback to a regular string_list for that case. + # buildifier: disable=attr-license + "licenses": attr.license() if hasattr(attr, "license") else attr.string_list(), + }, + allow_none = True, +) + +# Attributes common to rules accepting Python sources and deps. +PY_SRCS_ATTRS = union_attrs( + { + "deps": attr.label_list( + providers = [[PyInfo], [_CcInfo]], + # TODO(b/228692666): Google-specific; remove these allowances once + # the depot is cleaned up. + allow_rules = DEPS_ATTR_ALLOW_RULES, + ), + # Required attribute, but details vary by rule. + # Use create_srcs_attr to create one. + "srcs": None, + # NOTE: In Google, this attribute is deprecated, and can only + # effectively be PY3 or PY3ONLY. Externally, with Bazel, this attribute + # has a separate story. + # Required attribute, but the details vary by rule. + # Use create_srcs_version_attr to create one. + "srcs_version": None, + }, + allow_none = True, +) + +# Attributes specific to Python executable-equivalent rules. Such rules may not +# accept Python sources (e.g. some packaged-version of a py_test/py_binary), but +# still accept Python source-agnostic settings. +AGNOSTIC_EXECUTABLE_ATTRS = union_attrs( + DATA_ATTRS, + { + "env": attr.string_dict( + doc = """\ +Dictionary of strings; optional; values are subject to `$(location)` and "Make +variable" substitution. + +Specifies additional environment variables to set when the target is executed by +`test` or `run`. +""", + ), + # The value is required, but varies by rule and/or rule type. Use + # create_stamp_attr to create one. + "stamp": None, + }, + allow_none = True, +) + +# Attributes specific to Python test-equivalent executable rules. Such rules may +# not accept Python sources (e.g. some packaged-version of a py_test/py_binary), +# but still accept Python source-agnostic settings. +AGNOSTIC_TEST_ATTRS = union_attrs( + AGNOSTIC_EXECUTABLE_ATTRS, + # Tests have stamping disabled by default. + create_stamp_attr(default = 0), + { + "env_inherit": attr.string_list( + doc = """\ +List of strings; optional + +Specifies additional environment variables to inherit from the external +environment when the test is executed by bazel test. +""", + ), + # TODO(b/176993122): Remove when Bazel automatically knows to run on darwin. + "_apple_constraints": attr.label_list( + default = [ + PLATFORMS_LOCATION + "/os:ios", + PLATFORMS_LOCATION + "/os:macos", + PLATFORMS_LOCATION + "/os:tvos", + PLATFORMS_LOCATION + "/os:visionos", + PLATFORMS_LOCATION + "/os:watchos", + ], + ), + }, +) + +# Attributes specific to Python binary-equivalent executable rules. Such rules may +# not accept Python sources (e.g. some packaged-version of a py_test/py_binary), +# but still accept Python source-agnostic settings. +AGNOSTIC_BINARY_ATTRS = union_attrs( + AGNOSTIC_EXECUTABLE_ATTRS, + create_stamp_attr(default = -1), +) + +# Attribute names common to all Python rules +COMMON_ATTR_NAMES = [ + "compatible_with", + "deprecation", + "distribs", # NOTE: Currently common to all rules, but slated for removal + "exec_compatible_with", + "exec_properties", + "features", + "restricted_to", + "tags", + "target_compatible_with", + # NOTE: The testonly attribute requires careful handling: None/unset means + # to use the `package(default_testonly`) value, which isn't observable + # during the loading phase. + "testonly", + "toolchains", + "visibility", +] + COMMON_ATTRS.keys() + +# Attribute names common to all test=True rules +TEST_ATTR_NAMES = COMMON_ATTR_NAMES + [ + "args", + "size", + "timeout", + "flaky", + "shard_count", + "local", +] + AGNOSTIC_TEST_ATTRS.keys() + +# Attribute names common to all executable=True rules +BINARY_ATTR_NAMES = COMMON_ATTR_NAMES + [ + "args", + "output_licenses", # NOTE: Common to all rules, but slated for removal +] + AGNOSTIC_BINARY_ATTRS.keys() diff --git a/python/private/common/attributes_bazel.bzl b/python/private/common/attributes_bazel.bzl new file mode 100644 index 0000000000..f87245d6ff --- /dev/null +++ b/python/private/common/attributes_bazel.bzl @@ -0,0 +1,30 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Attributes specific to the Bazel implementation of the Python rules.""" + +IMPORTS_ATTRS = { + "imports": attr.string_list( + doc = """ +List of import directories to be added to the PYTHONPATH. + +Subject to "Make variable" substitution. These import directories will be added +for this rule and all rules that depend on it (note: not the rules this rule +depends on. Each directory will be added to `PYTHONPATH` by `py_binary` rules +that depend on this rule. The strings are repo-runfiles-root relative, + +Absolute paths (paths that start with `/`) and paths that references a path +above the execution root are not allowed and will result in an error. +""", + ), +} diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl new file mode 100644 index 0000000000..97ed3e3ee6 --- /dev/null +++ b/python/private/common/common.bzl @@ -0,0 +1,528 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Various things common to Bazel and Google rule implementations.""" + +load(":common/cc/cc_helper.bzl", "cc_helper") +load( + ":common/python/providers.bzl", + "PyInfo", +) +load( + ":common/python/semantics.bzl", + "NATIVE_RULES_MIGRATION_FIX_CMD", + "NATIVE_RULES_MIGRATION_HELP_URL", + "TOOLS_REPO", +) + +_testing = _builtins.toplevel.testing +_platform_common = _builtins.toplevel.platform_common +_coverage_common = _builtins.toplevel.coverage_common +_py_builtins = _builtins.internal.py_builtins +PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo + +TOOLCHAIN_TYPE = "@" + TOOLS_REPO + "//tools/python:toolchain_type" + +# Extensions without the dot +_PYTHON_SOURCE_EXTENSIONS = ["py"] + +# NOTE: Must stay in sync with the value used in rules_python +_MIGRATION_TAG = "__PYTHON_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__" + +def create_binary_semantics_struct( + *, + create_executable, + get_cc_details_for_binary, + get_central_uncachable_version_file, + get_coverage_deps, + get_debugger_deps, + get_extra_common_runfiles_for_binary, + get_extra_providers, + get_extra_write_build_data_env, + get_interpreter_path, + get_imports, + get_native_deps_dso_name, + get_native_deps_user_link_flags, + get_stamp_flag, + maybe_precompile, + should_build_native_deps_dso, + should_create_init_files, + should_include_build_data): + """Helper to ensure a semantics struct has all necessary fields. + + Call this instead of a raw call to `struct(...)`; it'll help ensure all + the necessary functions are being correctly provided. + + Args: + create_executable: Callable; creates a binary's executable output. See + py_executable.bzl#py_executable_base_impl for details. + get_cc_details_for_binary: Callable that returns a `CcDetails` struct; see + `create_cc_detail_struct`. + get_central_uncachable_version_file: Callable that returns an optional + Artifact; this artifact is special: it is never cached and is a copy + of `ctx.version_file`; see py_builtins.copy_without_caching + get_coverage_deps: Callable that returns a list of Targets for making + coverage work; only called if coverage is enabled. + get_debugger_deps: Callable that returns a list of Targets that provide + custom debugger support; only called for target-configuration. + get_extra_common_runfiles_for_binary: Callable that returns a runfiles + object of extra runfiles a binary should include. + get_extra_providers: Callable that returns extra providers; see + py_executable.bzl#_create_providers for details. + get_extra_write_build_data_env: Callable that returns a dict[str, str] + of additional environment variable to pass to build data generation. + get_interpreter_path: Callable that returns an optional string, which is + the path to the Python interpreter to use for running the binary. + get_imports: Callable that returns a list of the target's import + paths (from the `imports` attribute, so just the target's own import + path strings, not from dependencies). + get_native_deps_dso_name: Callable that returns a string, which is the + basename (with extension) of the native deps DSO library. + get_native_deps_user_link_flags: Callable that returns a list of strings, + which are any extra linker flags to pass onto the native deps DSO + linking action. + get_stamp_flag: Callable that returns bool of if the --stamp flag was + enabled or not. + maybe_precompile: Callable that may optional precompile the input `.py` + sources and returns the full set of desired outputs derived from + the source files (e.g., both py and pyc, only one of them, etc). + should_build_native_deps_dso: Callable that returns bool; True if + building a native deps DSO is supported, False if not. + should_create_init_files: Callable that returns bool; True if + `__init__.py` files should be generated, False if not. + should_include_build_data: Callable that returns bool; True if + build data should be generated, False if not. + Returns: + A "BinarySemantics" struct. + """ + return struct( + # keep-sorted + create_executable = create_executable, + get_cc_details_for_binary = get_cc_details_for_binary, + get_central_uncachable_version_file = get_central_uncachable_version_file, + get_coverage_deps = get_coverage_deps, + get_debugger_deps = get_debugger_deps, + get_extra_common_runfiles_for_binary = get_extra_common_runfiles_for_binary, + get_extra_providers = get_extra_providers, + get_extra_write_build_data_env = get_extra_write_build_data_env, + get_imports = get_imports, + get_interpreter_path = get_interpreter_path, + get_native_deps_dso_name = get_native_deps_dso_name, + get_native_deps_user_link_flags = get_native_deps_user_link_flags, + get_stamp_flag = get_stamp_flag, + maybe_precompile = maybe_precompile, + should_build_native_deps_dso = should_build_native_deps_dso, + should_create_init_files = should_create_init_files, + should_include_build_data = should_include_build_data, + ) + +def create_library_semantics_struct( + *, + get_cc_info_for_library, + get_imports, + maybe_precompile): + """Create a `LibrarySemantics` struct. + + Call this instead of a raw call to `struct(...)`; it'll help ensure all + the necessary functions are being correctly provided. + + Args: + get_cc_info_for_library: Callable that returns a CcInfo for the library; + see py_library_impl for arg details. + get_imports: Callable; see create_binary_semantics_struct. + maybe_precompile: Callable; see create_binary_semantics_struct. + Returns: + a `LibrarySemantics` struct. + """ + return struct( + # keep sorted + get_cc_info_for_library = get_cc_info_for_library, + get_imports = get_imports, + maybe_precompile = maybe_precompile, + ) + +def create_cc_details_struct( + *, + cc_info_for_propagating, + cc_info_for_self_link, + cc_info_with_extra_link_time_libraries, + extra_runfiles, + cc_toolchain): + """Creates a CcDetails struct. + + Args: + cc_info_for_propagating: CcInfo that is propagated out of the target + by returning it within a PyCcLinkParamsProvider object. + cc_info_for_self_link: CcInfo that is used when linking for the + binary (or its native deps DSO) itself. This may include extra + information that isn't propagating (e.g. a custom malloc) + cc_info_with_extra_link_time_libraries: CcInfo of extra link time + libraries that MUST come after `cc_info_for_self_link` (or possibly + always last; not entirely clear) when passed to + `link.linking_contexts`. + extra_runfiles: runfiles of extra files needed at runtime, usually as + part of `cc_info_with_extra_link_time_libraries`; should be added to + runfiles. + cc_toolchain: CcToolchain that should be used when building. + + Returns: + A `CcDetails` struct. + """ + return struct( + cc_info_for_propagating = cc_info_for_propagating, + cc_info_for_self_link = cc_info_for_self_link, + cc_info_with_extra_link_time_libraries = cc_info_with_extra_link_time_libraries, + extra_runfiles = extra_runfiles, + cc_toolchain = cc_toolchain, + ) + +def create_executable_result_struct(*, extra_files_to_build, output_groups): + """Creates a `CreateExecutableResult` struct. + + This is the return value type of the semantics create_executable function. + + Args: + extra_files_to_build: depset of File; additional files that should be + included as default outputs. + output_groups: dict[str, depset[File]]; additional output groups that + should be returned. + + Returns: + A `CreateExecutableResult` struct. + """ + return struct( + extra_files_to_build = extra_files_to_build, + output_groups = output_groups, + ) + +def union_attrs(*attr_dicts, allow_none = False): + """Helper for combining and building attriute dicts for rules. + + Similar to dict.update, except: + * Duplicate keys raise an error if they aren't equal. This is to prevent + unintentionally replacing an attribute with a potentially incompatible + definition. + * None values are special: They mean the attribute is required, but the + value should be provided by another attribute dict (depending on the + `allow_none` arg). + Args: + *attr_dicts: The dicts to combine. + allow_none: bool, if True, then None values are allowed. If False, + then one of `attrs_dicts` must set a non-None value for keys + with a None value. + + Returns: + dict of attributes. + """ + result = {} + missing = {} + for attr_dict in attr_dicts: + for attr_name, value in attr_dict.items(): + if value == None and not allow_none: + if attr_name not in result: + missing[attr_name] = None + else: + if attr_name in missing: + missing.pop(attr_name) + + if attr_name not in result or result[attr_name] == None: + result[attr_name] = value + elif value != None and result[attr_name] != value: + fail("Duplicate attribute name: '{}': existing={}, new={}".format( + attr_name, + result[attr_name], + value, + )) + + # Else, they're equal, so do nothing. This allows merging dicts + # that both define the same key from a common place. + + if missing and not allow_none: + fail("Required attributes missing: " + csv(missing.keys())) + return result + +def csv(values): + """Convert a list of strings to comma separated value string.""" + return ", ".join(sorted(values)) + +def filter_to_py_srcs(srcs): + """Filters .py files from the given list of files""" + + # TODO(b/203567235): Get the set of recognized extensions from + # elsewhere, as there may be others. e.g. Bazel recognizes .py3 + # as a valid extension. + return [f for f in srcs if f.extension == "py"] + +def collect_imports(ctx, semantics): + return depset(direct = semantics.get_imports(ctx), transitive = [ + dep[PyInfo].imports + for dep in ctx.attr.deps + if PyInfo in dep + ]) + +def collect_runfiles(ctx, files): + """Collects the necessary files from the rule's context. + + This presumes the ctx is for a py_binary, py_test, or py_library rule. + + Args: + ctx: rule ctx + files: depset of extra files to include in the runfiles. + Returns: + runfiles necessary for the ctx's target. + """ + return ctx.runfiles( + transitive_files = files, + # This little arg carries a lot of weight, but because Starlark doesn't + # have a way to identify if a target is just a File, the equivalent + # logic can't be re-implemented in pure-Starlark. + # + # Under the hood, it calls the Java `Runfiles#addRunfiles(ctx, + # DEFAULT_RUNFILES)` method, which is the what the Java implementation + # of the Python rules originally did, and the details of how that method + # works have become relied on in various ways. Specifically, what it + # does is visit the srcs, deps, and data attributes in the following + # ways: + # + # For each target in the "data" attribute... + # If the target is a File, then add that file to the runfiles. + # Otherwise, add the target's **data runfiles** to the runfiles. + # + # Note that, contray to best practice, the default outputs of the + # targets in `data` are *not* added, nor are the default runfiles. + # + # This ends up being important for several reasons, some of which are + # specific to Google-internal features of the rules. + # * For Python executables, we have to use `data_runfiles` to avoid + # conflicts for the build data files. Such files have + # target-specific content, but uses a fixed location, so if a + # binary has another binary in `data`, and both try to specify a + # file for that file path, then a warning is printed and an + # arbitrary one will be used. + # * For rules with _entirely_ different sets of files in data runfiles + # vs default runfiles vs default outputs. For example, + # proto_library: documented behavior of this rule is that putting it + # in the `data` attribute will cause the transitive closure of + # `.proto` source files to be included. This set of sources is only + # in the `data_runfiles` (`default_runfiles` is empty). + # * For rules with a _subset_ of files in data runfiles. For example, + # a certain Google rule used for packaging arbitrary binaries will + # generate multiple versions of a binary (e.g. different archs, + # stripped vs un-stripped, etc) in its default outputs, but only + # one of them in the runfiles; this helps avoid large, unused + # binaries contributing to remote executor input limits. + # + # Unfortunately, the above behavior also results in surprising behavior + # in some cases. For example, simple custom rules that only return their + # files in their default outputs won't have their files included. Such + # cases must either return their files in runfiles, or use `filegroup()` + # which will do so for them. + # + # For each target in "srcs" and "deps"... + # Add the default runfiles of the target to the runfiles. While this + # is desirable behavior, it also ends up letting a `py_library` + # be put in `srcs` and still mostly work. + # TODO(b/224640180): Reject py_library et al rules in srcs. + collect_default = True, + ) + +def create_py_info(ctx, *, direct_sources, imports): + """Create PyInfo provider. + + Args: + ctx: rule ctx. + direct_sources: depset of Files; the direct, raw `.py` sources for the + target. This should only be Python source files. It should not + include pyc files. + imports: depset of strings; the import path values to propagate. + + Returns: + A tuple of the PyInfo instance and a depset of the + transitive sources collected from dependencies (the latter is only + necessary for deprecated extra actions support). + """ + uses_shared_libraries = False + has_py2_only_sources = ctx.attr.srcs_version in ("PY2", "PY2ONLY") + has_py3_only_sources = ctx.attr.srcs_version in ("PY3", "PY3ONLY") + transitive_sources_depsets = [] # list of depsets + transitive_sources_files = [] # list of Files + for target in ctx.attr.deps: + # PyInfo may not be present for e.g. cc_library rules. + if PyInfo in target: + info = target[PyInfo] + transitive_sources_depsets.append(info.transitive_sources) + uses_shared_libraries = uses_shared_libraries or info.uses_shared_libraries + has_py2_only_sources = has_py2_only_sources or info.has_py2_only_sources + has_py3_only_sources = has_py3_only_sources or info.has_py3_only_sources + else: + # TODO(b/228692666): Remove this once non-PyInfo targets are no + # longer supported in `deps`. + files = target.files.to_list() + for f in files: + if f.extension == "py": + transitive_sources_files.append(f) + uses_shared_libraries = ( + uses_shared_libraries or + cc_helper.is_valid_shared_library_artifact(f) + ) + deps_transitive_sources = depset( + direct = transitive_sources_files, + transitive = transitive_sources_depsets, + ) + + # We only look at data to calculate uses_shared_libraries, if it's already + # true, then we don't need to waste time looping over it. + if not uses_shared_libraries: + # Similar to the above, except we only calculate uses_shared_libraries + for target in ctx.attr.data: + # TODO(b/234730058): Remove checking for PyInfo in data once depot + # cleaned up. + if PyInfo in target: + info = target[PyInfo] + uses_shared_libraries = info.uses_shared_libraries + else: + files = target.files.to_list() + for f in files: + uses_shared_libraries = cc_helper.is_valid_shared_library_artifact(f) + if uses_shared_libraries: + break + if uses_shared_libraries: + break + + # TODO(b/203567235): Set `uses_shared_libraries` field, though the Bazel + # docs indicate it's unused in Bazel and may be removed. + py_info = PyInfo( + transitive_sources = depset( + transitive = [deps_transitive_sources, direct_sources], + ), + imports = imports, + # NOTE: This isn't strictly correct, but with Python 2 gone, + # the srcs_version logic is largely defunct, so shouldn't matter in + # practice. + has_py2_only_sources = has_py2_only_sources, + has_py3_only_sources = has_py3_only_sources, + uses_shared_libraries = uses_shared_libraries, + ) + return py_info, deps_transitive_sources + +def create_instrumented_files_info(ctx): + return _coverage_common.instrumented_files_info( + ctx, + source_attributes = ["srcs"], + dependency_attributes = ["deps", "data"], + extensions = _PYTHON_SOURCE_EXTENSIONS, + ) + +def create_output_group_info(transitive_sources, extra_groups): + return OutputGroupInfo( + compilation_prerequisites_INTERNAL_ = transitive_sources, + compilation_outputs = transitive_sources, + **extra_groups + ) + +def maybe_add_test_execution_info(providers, ctx): + """Adds ExecutionInfo, if necessary for proper test execution. + + Args: + providers: Mutable list of providers; may have ExecutionInfo + provider appended. + ctx: Rule ctx. + """ + + # When built for Apple platforms, require the execution to be on a Mac. + # TODO(b/176993122): Remove when bazel automatically knows to run on darwin. + if target_platform_has_any_constraint(ctx, ctx.attr._apple_constraints): + providers.append(_testing.ExecutionInfo({"requires-darwin": ""})) + +_BOOL_TYPE = type(True) + +def is_bool(v): + return type(v) == _BOOL_TYPE + +def target_platform_has_any_constraint(ctx, constraints): + """Check if target platform has any of a list of constraints. + + Args: + ctx: rule context. + constraints: label_list of constraints. + + Returns: + True if target platform has at least one of the constraints. + """ + for constraint in constraints: + constraint_value = constraint[_platform_common.ConstraintValueInfo] + if ctx.target_platform_has_constraint(constraint_value): + return True + return False + +def check_native_allowed(ctx): + """Check if the usage of the native rule is allowed. + + Args: + ctx: rule context to check + """ + if not ctx.fragments.py.disallow_native_rules: + return + + if _MIGRATION_TAG in ctx.attr.tags: + return + + # NOTE: The main repo name is empty in *labels*, but not in + # ctx.workspace_name + is_main_repo = not bool(ctx.label.workspace_name) + if is_main_repo: + check_label = ctx.label + else: + # package_group doesn't allow @repo syntax, so we work around that + # by prefixing external repos with a fake package path. This also + # makes it easy to enable or disable all external repos. + check_label = Label("@//__EXTERNAL_REPOS__/{workspace}/{package}".format( + workspace = ctx.label.workspace_name, + package = ctx.label.package, + )) + allowlist = ctx.attr._native_rules_allowlist + if allowlist: + allowed = ctx.attr._native_rules_allowlist[PackageSpecificationInfo].contains(check_label) + allowlist_help = str(allowlist.label).replace("@//", "//") + else: + allowed = False + allowlist_help = ("no allowlist specified; all disallowed; specify one " + + "with --python_native_rules_allowlist") + if not allowed: + if ctx.attr.generator_function: + generator = "{generator_function}(name={generator_name}) in {generator_location}".format( + generator_function = ctx.attr.generator_function, + generator_name = ctx.attr.generator_name, + generator_location = ctx.attr.generator_location, + ) + else: + generator = "No generator (called directly in BUILD file)" + + msg = ( + "{target} not allowed to use native.{rule}\n" + + "Generated by: {generator}\n" + + "Allowlist: {allowlist}\n" + + "Migrate to using @rules_python, see {help_url}\n" + + "FIXCMD: {fix_cmd} --target={target} --rule={rule} " + + "--generator_name={generator_name} --location={generator_location}" + ) + fail(msg.format( + target = str(ctx.label).replace("@//", "//"), + rule = _py_builtins.get_rule_name(ctx), + generator = generator, + allowlist = allowlist_help, + generator_name = ctx.attr.generator_name, + generator_location = ctx.attr.generator_location, + help_url = NATIVE_RULES_MIGRATION_HELP_URL, + fix_cmd = NATIVE_RULES_MIGRATION_FIX_CMD, + )) diff --git a/python/private/common/common_bazel.bzl b/python/private/common/common_bazel.bzl new file mode 100644 index 0000000000..51b06fb832 --- /dev/null +++ b/python/private/common/common_bazel.bzl @@ -0,0 +1,104 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common functions that are specific to Bazel rule implementation""" + +load(":common/cc/cc_common.bzl", _cc_common = "cc_common") +load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") +load(":common/paths.bzl", "paths") +load(":common/python/common.bzl", "is_bool") +load(":common/python/providers.bzl", "PyCcLinkParamsProvider") + +_py_builtins = _builtins.internal.py_builtins + +def collect_cc_info(ctx, extra_deps = []): + """Collect C++ information from dependencies for Bazel. + + Args: + ctx: Rule ctx; must have `deps` attribute. + extra_deps: list of Target to also collect C+ information from. + + Returns: + CcInfo provider of merged information. + """ + deps = ctx.attr.deps + if extra_deps: + deps = list(deps) + deps.extend(extra_deps) + cc_infos = [] + for dep in deps: + if _CcInfo in dep: + cc_infos.append(dep[_CcInfo]) + + if PyCcLinkParamsProvider in dep: + cc_infos.append(dep[PyCcLinkParamsProvider].cc_info) + + return _cc_common.merge_cc_infos(cc_infos = cc_infos) + +def maybe_precompile(ctx, srcs): + """Computes all the outputs (maybe precompiled) from the input srcs. + + See create_binary_semantics_struct for details about this function. + + Args: + ctx: Rule ctx. + srcs: List of Files; the inputs to maybe precompile. + + Returns: + List of Files; the desired output files derived from the input sources. + """ + _ = ctx # @unused + + # Precompilation isn't implemented yet, so just return srcs as-is + return srcs + +def get_imports(ctx): + """Gets the imports from a rule's `imports` attribute. + + See create_binary_semantics_struct for details about this function. + + Args: + ctx: Rule ctx. + + Returns: + List of strings. + """ + prefix = "{}/{}".format( + ctx.workspace_name, + _py_builtins.get_label_repo_runfiles_path(ctx.label), + ) + result = [] + for import_str in ctx.attr.imports: + import_str = ctx.expand_make_variables("imports", import_str, {}) + if import_str.startswith("/"): + continue + + # To prevent "escaping" out of the runfiles tree, we normalize + # the path and ensure it doesn't have up-level references. + import_path = paths.normalize("{}/{}".format(prefix, import_str)) + if import_path.startswith("../") or import_path == "..": + fail("Path '{}' references a path above the execution root".format( + import_str, + )) + result.append(import_path) + return result + +def convert_legacy_create_init_to_int(kwargs): + """Convert "legacy_create_init" key to int, in-place. + + Args: + kwargs: The kwargs to modify. The key "legacy_create_init", if present + and bool, will be converted to its integer value, in place. + """ + if is_bool(kwargs.get("legacy_create_init")): + kwargs["legacy_create_init"] = 1 if kwargs["legacy_create_init"] else 0 diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl new file mode 100644 index 0000000000..a9df61bda4 --- /dev/null +++ b/python/private/common/providers.bzl @@ -0,0 +1,212 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Providers for Python rules.""" + +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_CcInfo = _builtins.toplevel.CcInfo + +# NOTE: This is copied to PyRuntimeInfo.java +DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" + +# NOTE: This is copied to PyRuntimeInfo.java +DEFAULT_BOOTSTRAP_TEMPLATE = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt" +_PYTHON_VERSION_VALUES = ["PY2", "PY3"] + +def _PyRuntimeInfo_init( + *, + interpreter_path = None, + interpreter = None, + files = None, + coverage_tool = None, + coverage_files = None, + python_version, + stub_shebang = None, + bootstrap_template = None): + if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): + fail("exactly one of interpreter or interpreter_path must be specified") + + if interpreter_path and files != None: + fail("cannot specify 'files' if 'interpreter_path' is given") + + if (coverage_tool and not coverage_files) or (not coverage_tool and coverage_files): + fail( + "coverage_tool and coverage_files must both be set or neither must be set, " + + "got coverage_tool={}, coverage_files={}".format( + coverage_tool, + coverage_files, + ), + ) + + if python_version not in _PYTHON_VERSION_VALUES: + fail("invalid python_version: '{}'; must be one of {}".format( + python_version, + _PYTHON_VERSION_VALUES, + )) + + if files != None and type(files) != type(depset()): + fail("invalid files: got value of type {}, want depset".format(type(files))) + + if interpreter: + if files == None: + files = depset() + else: + files = None + + if coverage_files == None: + coverage_files = depset() + + if not stub_shebang: + stub_shebang = DEFAULT_STUB_SHEBANG + + return { + "bootstrap_template": bootstrap_template, + "coverage_files": coverage_files, + "coverage_tool": coverage_tool, + "files": files, + "interpreter": interpreter, + "interpreter_path": interpreter_path, + "python_version": python_version, + "stub_shebang": stub_shebang, + } + +# TODO(#15897): Rename this to PyRuntimeInfo when we're ready to replace the Java +# implemented provider with the Starlark one. +PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( + doc = """Contains information about a Python runtime, as returned by the `py_runtime` +rule. + +A Python runtime describes either a *platform runtime* or an *in-build runtime*. +A platform runtime accesses a system-installed interpreter at a known path, +whereas an in-build runtime points to a `File` that acts as the interpreter. In +both cases, an "interpreter" is really any executable binary or wrapper script +that is capable of running a Python script passed on the command line, following +the same conventions as the standard CPython interpreter. +""", + init = _PyRuntimeInfo_init, + fields = { + "bootstrap_template": ( + "See py_runtime_rule.bzl%py_runtime.bootstrap_template for docs." + ), + "coverage_files": ( + "The files required at runtime for using `coverage_tool`. " + + "Will be `None` if no `coverage_tool` was provided." + ), + "coverage_tool": ( + "If set, this field is a `File` representing tool used for collecting code coverage information from python tests. Otherwise, this is `None`." + ), + "files": ( + "If this is an in-build runtime, this field is a `depset` of `File`s" + + "that need to be added to the runfiles of an executable target that " + + "uses this runtime (in particular, files needed by `interpreter`). " + + "The value of `interpreter` need not be included in this field. If " + + "this is a platform runtime then this field is `None`." + ), + "interpreter": ( + "If this is an in-build runtime, this field is a `File` representing " + + "the interpreter. Otherwise, this is `None`. Note that an in-build " + + "runtime can use either a prebuilt, checked-in interpreter or an " + + "interpreter built from source." + ), + "interpreter_path": ( + "If this is a platform runtime, this field is the absolute " + + "filesystem path to the interpreter on the target platform. " + + "Otherwise, this is `None`." + ), + "python_version": ( + "Indicates whether this runtime uses Python major version 2 or 3. " + + "Valid values are (only) `\"PY2\"` and " + + "`\"PY3\"`." + ), + "stub_shebang": ( + "\"Shebang\" expression prepended to the bootstrapping Python stub " + + "script used when executing `py_binary` targets. Does not " + + "apply to Windows." + ), + }, +) + +def _check_arg_type(name, required_type, value): + value_type = type(value) + if value_type != required_type: + fail("parameter '{}' got value of type '{}', want '{}'".format( + name, + value_type, + required_type, + )) + +def _PyInfo_init( + *, + transitive_sources, + uses_shared_libraries = False, + imports = depset(), + has_py2_only_sources = False, + has_py3_only_sources = False): + _check_arg_type("transitive_sources", "depset", transitive_sources) + + # Verify it's postorder compatible, but retain is original ordering. + depset(transitive = [transitive_sources], order = "postorder") + + _check_arg_type("uses_shared_libraries", "bool", uses_shared_libraries) + _check_arg_type("imports", "depset", imports) + _check_arg_type("has_py2_only_sources", "bool", has_py2_only_sources) + _check_arg_type("has_py3_only_sources", "bool", has_py3_only_sources) + return { + "has_py2_only_sources": has_py2_only_sources, + "has_py3_only_sources": has_py2_only_sources, + "imports": imports, + "transitive_sources": transitive_sources, + "uses_shared_libraries": uses_shared_libraries, + } + +PyInfo, _unused_raw_py_info_ctor = provider( + "Encapsulates information provided by the Python rules.", + init = _PyInfo_init, + fields = { + "has_py2_only_sources": "Whether any of this target's transitive sources requires a Python 2 runtime.", + "has_py3_only_sources": "Whether any of this target's transitive sources requires a Python 3 runtime.", + "imports": """\ +A depset of import path strings to be added to the `PYTHONPATH` of executable +Python targets. These are accumulated from the transitive `deps`. +The order of the depset is not guaranteed and may be changed in the future. It +is recommended to use `default` order (the default). +""", + "transitive_sources": """\ +A (`postorder`-compatible) depset of `.py` files appearing in the target's +`srcs` and the `srcs` of the target's transitive `deps`. +""", + "uses_shared_libraries": """ +Whether any of this target's transitive `deps` has a shared library file (such +as a `.so` file). + +This field is currently unused in Bazel and may go away in the future. +""", + }, +) + +def _PyCcLinkParamsProvider_init(cc_info): + return { + "cc_info": _CcInfo(linking_context = cc_info.linking_context), + } + +# buildifier: disable=name-conventions +PyCcLinkParamsProvider, _unused_raw_py_cc_link_params_provider_ctor = provider( + doc = ("Python-wrapper to forward CcInfo.linking_context. This is to " + + "allow Python targets to propagate C++ linking information, but " + + "without the Python target appearing to be a valid C++ rule dependency"), + init = _PyCcLinkParamsProvider_init, + fields = { + "cc_info": "A CcInfo instance; it has only linking_context set", + }, +) diff --git a/python/private/common/py_binary_bazel.bzl b/python/private/common/py_binary_bazel.bzl new file mode 100644 index 0000000000..3a5df737b9 --- /dev/null +++ b/python/private/common/py_binary_bazel.bzl @@ -0,0 +1,48 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule implementation of py_binary for Bazel.""" + +load(":common/python/attributes.bzl", "AGNOSTIC_BINARY_ATTRS") +load( + ":common/python/py_executable_bazel.bzl", + "create_executable_rule", + "py_executable_bazel_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_PY_TEST_ATTRS = { + "_collect_cc_coverage": attr.label( + default = "@" + TOOLS_REPO + "//tools/test:collect_cc_coverage", + executable = True, + cfg = "exec", + ), + "_lcov_merger": attr.label( + default = configuration_field(fragment = "coverage", name = "output_generator"), + executable = True, + cfg = "exec", + ), +} + +def _py_binary_impl(ctx): + return py_executable_bazel_impl( + ctx = ctx, + is_test = False, + inherited_environment = [], + ) + +py_binary = create_executable_rule( + implementation = _py_binary_impl, + attrs = AGNOSTIC_BINARY_ATTRS | _PY_TEST_ATTRS, + executable = True, +) diff --git a/python/private/common/py_binary_macro.bzl b/python/private/common/py_binary_macro.bzl new file mode 100644 index 0000000000..24e5c6dbe3 --- /dev/null +++ b/python/private/common/py_binary_macro.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_binary rule.""" + +load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":common/python/py_binary_bazel.bzl", py_binary_rule = "py_binary") + +def py_binary(**kwargs): + convert_legacy_create_init_to_int(kwargs) + py_binary_rule(**kwargs) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl new file mode 100644 index 0000000000..9db92b18e5 --- /dev/null +++ b/python/private/common/py_executable.bzl @@ -0,0 +1,845 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common functionality between test/binary executables.""" + +load(":common/cc/cc_common.bzl", _cc_common = "cc_common") +load(":common/cc/cc_helper.bzl", "cc_helper") +load( + ":common/python/attributes.bzl", + "AGNOSTIC_EXECUTABLE_ATTRS", + "COMMON_ATTRS", + "PY_SRCS_ATTRS", + "SRCS_VERSION_ALL_VALUES", + "create_srcs_attr", + "create_srcs_version_attr", +) +load( + ":common/python/common.bzl", + "TOOLCHAIN_TYPE", + "check_native_allowed", + "collect_imports", + "collect_runfiles", + "create_instrumented_files_info", + "create_output_group_info", + "create_py_info", + "csv", + "filter_to_py_srcs", + "union_attrs", +) +load( + ":common/python/providers.bzl", + "PyCcLinkParamsProvider", + "PyRuntimeInfo", +) +load( + ":common/python/semantics.bzl", + "ALLOWED_MAIN_EXTENSIONS", + "BUILD_DATA_SYMLINK_PATH", + "IS_BAZEL", + "PY_RUNTIME_ATTR_NAME", +) + +_py_builtins = _builtins.internal.py_builtins + +# Non-Google-specific attributes for executables +# These attributes are for rules that accept Python sources. +EXECUTABLE_ATTRS = union_attrs( + COMMON_ATTRS, + AGNOSTIC_EXECUTABLE_ATTRS, + PY_SRCS_ATTRS, + { + # TODO(b/203567235): In the Java impl, any file is allowed. While marked + # label, it is more treated as a string, and doesn't have to refer to + # anything that exists because it gets treated as suffix-search string + # over `srcs`. + "main": attr.label( + allow_single_file = True, + doc = """\ +Optional; the name of the source file that is the main entry point of the +application. This file must also be listed in `srcs`. If left unspecified, +`name`, with `.py` appended, is used instead. If `name` does not match any +filename in `srcs`, `main` must be specified. +""", + ), + # TODO(b/203567235): In Google, this attribute is deprecated, and can + # only effectively be PY3. Externally, with Bazel, this attribute has + # a separate story. + "python_version": attr.string( + # TODO(b/203567235): In the Java impl, the default comes from + # --python_version. Not clear what the Starlark equivalent is. + default = "PY3", + # NOTE: Some tests care about the order of these values. + values = ["PY2", "PY3"], + ), + }, + create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), + create_srcs_attr(mandatory = True), + allow_none = True, +) + +def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = []): + """Base rule implementation for a Python executable. + + Google and Bazel call this common base and apply customizations using the + semantics object. + + Args: + ctx: The rule ctx + semantics: BinarySemantics struct; see create_binary_semantics_struct() + is_test: bool, True if the rule is a test rule (has `test=True`), + False if not (has `executable=True`) + inherited_environment: List of str; additional environment variable + names that should be inherited from the runtime environment when the + executable is run. + Returns: + DefaultInfo provider for the executable + """ + _validate_executable(ctx) + + main_py = determine_main(ctx) + direct_sources = filter_to_py_srcs(ctx.files.srcs) + output_sources = semantics.maybe_precompile(ctx, direct_sources) + imports = collect_imports(ctx, semantics) + executable, files_to_build = _compute_outputs(ctx, output_sources) + + runtime_details = _get_runtime_details(ctx, semantics) + if ctx.configuration.coverage_enabled: + extra_deps = semantics.get_coverage_deps(ctx, runtime_details) + else: + extra_deps = [] + + # The debugger dependency should be prevented by select() config elsewhere, + # but just to be safe, also guard against adding it to the output here. + if not _is_tool_config(ctx): + extra_deps.extend(semantics.get_debugger_deps(ctx, runtime_details)) + + cc_details = semantics.get_cc_details_for_binary(ctx, extra_deps = extra_deps) + native_deps_details = _get_native_deps_details( + ctx, + semantics = semantics, + cc_details = cc_details, + is_test = is_test, + ) + runfiles_details = _get_base_runfiles_for_binary( + ctx, + executable = executable, + extra_deps = extra_deps, + files_to_build = files_to_build, + extra_common_runfiles = [ + runtime_details.runfiles, + cc_details.extra_runfiles, + native_deps_details.runfiles, + semantics.get_extra_common_runfiles_for_binary(ctx), + ], + semantics = semantics, + ) + exec_result = semantics.create_executable( + ctx, + executable = executable, + main_py = main_py, + imports = imports, + is_test = is_test, + runtime_details = runtime_details, + cc_details = cc_details, + native_deps_details = native_deps_details, + runfiles_details = runfiles_details, + ) + files_to_build = depset(transitive = [ + exec_result.extra_files_to_build, + files_to_build, + ]) + extra_exec_runfiles = ctx.runfiles(transitive_files = files_to_build) + runfiles_details = struct( + default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles), + data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles), + ) + + legacy_providers, modern_providers = _create_providers( + ctx = ctx, + executable = executable, + runfiles_details = runfiles_details, + main_py = main_py, + imports = imports, + direct_sources = direct_sources, + files_to_build = files_to_build, + runtime_details = runtime_details, + cc_info = cc_details.cc_info_for_propagating, + inherited_environment = inherited_environment, + semantics = semantics, + output_groups = exec_result.output_groups, + ) + return struct( + legacy_providers = legacy_providers, + providers = modern_providers, + ) + +def _validate_executable(ctx): + if ctx.attr.python_version != "PY3": + fail("It is not allowed to use Python 2") + check_native_allowed(ctx) + +def _compute_outputs(ctx, output_sources): + # TODO: This should use the configuration instead of the Bazel OS. + if _py_builtins.get_current_os_name() == "windows": + executable = ctx.actions.declare_file(ctx.label.name + ".exe") + else: + executable = ctx.actions.declare_file(ctx.label.name) + + # TODO(b/208657718): Remove output_sources from the default outputs + # once the depot is cleaned up. + return executable, depset([executable] + output_sources) + +def _get_runtime_details(ctx, semantics): + """Gets various information about the Python runtime to use. + + While most information comes from the toolchain, various legacy and + compatibility behaviors require computing some other information. + + Args: + ctx: Rule ctx + semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct` + + Returns: + A struct; see inline-field comments of the return value for details. + """ + + # Bazel has --python_path. This flag has a computed default of "python" when + # its actual default is null (see + # BazelPythonConfiguration.java#getPythonPath). This flag is only used if + # toolchains are not enabled and `--python_top` isn't set. Note that Google + # used to have a variant of this named --python_binary, but it has since + # been removed. + # + # TOOD(bazelbuild/bazel#7901): Remove this once --python_path flag is removed. + + if IS_BAZEL: + flag_interpreter_path = ctx.fragments.bazel_py.python_path + toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) + if not effective_runtime: + # Clear these just in case + toolchain_runtime = None + effective_runtime = None + + else: # Google code path + flag_interpreter_path = None + toolchain_runtime, effective_runtime = _maybe_get_runtime_from_ctx(ctx) + if not effective_runtime: + fail("Unable to find Python runtime") + + if effective_runtime: + direct = [] # List of files + transitive = [] # List of depsets + if effective_runtime.interpreter: + direct.append(effective_runtime.interpreter) + transitive.append(effective_runtime.files) + + if ctx.configuration.coverage_enabled: + if effective_runtime.coverage_tool: + direct.append(effective_runtime.coverage_tool) + if effective_runtime.coverage_files: + transitive.append(effective_runtime.coverage_files) + runtime_files = depset(direct = direct, transitive = transitive) + else: + runtime_files = depset() + + executable_interpreter_path = semantics.get_interpreter_path( + ctx, + runtime = effective_runtime, + flag_interpreter_path = flag_interpreter_path, + ) + + return struct( + # Optional PyRuntimeInfo: The runtime found from toolchain resolution. + # This may be None because, within Google, toolchain resolution isn't + # yet enabled. + toolchain_runtime = toolchain_runtime, + # Optional PyRuntimeInfo: The runtime that should be used. When + # toolchain resolution is enabled, this is the same as + # `toolchain_resolution`. Otherwise, this probably came from the + # `_python_top` attribute that the Google implementation still uses. + # This is separate from `toolchain_runtime` because toolchain_runtime + # is propagated as a provider, while non-toolchain runtimes are not. + effective_runtime = effective_runtime, + # str; Path to the Python interpreter to use for running the executable + # itself (not the bootstrap script). Either an absolute path (which + # means it is platform-specific), or a runfiles-relative path (which + # means the interpreter should be within `runtime_files`) + executable_interpreter_path = executable_interpreter_path, + # runfiles: Additional runfiles specific to the runtime that should + # be included. For in-build runtimes, this shold include the interpreter + # and any supporting files. + runfiles = ctx.runfiles(transitive_files = runtime_files), + ) + +def _maybe_get_runtime_from_ctx(ctx): + """Finds the PyRuntimeInfo from the toolchain or attribute, if available. + + Returns: + 2-tuple of toolchain_runtime, effective_runtime + """ + if ctx.fragments.py.use_toolchains: + toolchain = ctx.toolchains[TOOLCHAIN_TYPE] + + if not hasattr(toolchain, "py3_runtime"): + fail("Python toolchain field 'py3_runtime' is missing") + if not toolchain.py3_runtime: + fail("Python toolchain missing py3_runtime") + py3_runtime = toolchain.py3_runtime + + # Hack around the fact that the autodetecting Python toolchain, which is + # automatically registered, does not yet support Windows. In this case, + # we want to return null so that _get_interpreter_path falls back on + # --python_path. See tools/python/toolchain.bzl. + # TODO(#7844): Remove this hack when the autodetecting toolchain has a + # Windows implementation. + if py3_runtime.interpreter_path == "/_magic_pyruntime_sentinel_do_not_use": + return None, None + + if py3_runtime.python_version != "PY3": + fail("Python toolchain py3_runtime must be python_version=PY3, got {}".format( + py3_runtime.python_version, + )) + toolchain_runtime = toolchain.py3_runtime + effective_runtime = toolchain_runtime + else: + toolchain_runtime = None + attr_target = getattr(ctx.attr, PY_RUNTIME_ATTR_NAME) + + # In Bazel, --python_top is null by default. + if attr_target and PyRuntimeInfo in attr_target: + effective_runtime = attr_target[PyRuntimeInfo] + else: + return None, None + + return toolchain_runtime, effective_runtime + +def _get_base_runfiles_for_binary( + ctx, + *, + executable, + extra_deps, + files_to_build, + extra_common_runfiles, + semantics): + """Returns the set of runfiles necessary prior to executable creation. + + NOTE: The term "common runfiles" refers to the runfiles that both the + default and data runfiles have in common. + + Args: + ctx: The rule ctx. + executable: The main executable output. + extra_deps: List of Targets; additional targets whose runfiles + will be added to the common runfiles. + files_to_build: depset of File of the default outputs to add into runfiles. + extra_common_runfiles: List of runfiles; additional runfiles that + will be added to the common runfiles. + semantics: A `BinarySemantics` struct; see `create_binary_semantics_struct`. + + Returns: + struct with attributes: + * default_runfiles: The default runfiles + * data_runfiles: The data runfiles + """ + common_runfiles = collect_runfiles(ctx, depset( + direct = [executable], + transitive = [files_to_build], + )) + if extra_deps: + common_runfiles = common_runfiles.merge_all([ + t[DefaultInfo].default_runfiles + for t in extra_deps + ]) + common_runfiles = common_runfiles.merge_all(extra_common_runfiles) + + if semantics.should_create_init_files(ctx): + common_runfiles = _py_builtins.merge_runfiles_with_generated_inits_empty_files_supplier( + ctx = ctx, + runfiles = common_runfiles, + ) + + # Don't include build_data.txt in data runfiles. This allows binaries to + # contain other binaries while still using the same fixed location symlink + # for the build_data.txt file. Really, the fixed location symlink should be + # removed and another way found to locate the underlying build data file. + data_runfiles = common_runfiles + + if is_stamping_enabled(ctx, semantics) and semantics.should_include_build_data(ctx): + default_runfiles = common_runfiles.merge(_create_runfiles_with_build_data( + ctx, + semantics.get_central_uncachable_version_file(ctx), + semantics.get_extra_write_build_data_env(ctx), + )) + else: + default_runfiles = common_runfiles + + return struct( + default_runfiles = default_runfiles, + data_runfiles = data_runfiles, + ) + +def _create_runfiles_with_build_data( + ctx, + central_uncachable_version_file, + extra_write_build_data_env): + return ctx.runfiles( + symlinks = { + BUILD_DATA_SYMLINK_PATH: _write_build_data( + ctx, + central_uncachable_version_file, + extra_write_build_data_env, + ), + }, + ) + +def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_data_env): + # TODO: Remove this logic when a central file is always available + if not central_uncachable_version_file: + version_file = ctx.actions.declare_file(ctx.label.name + "-uncachable_version_file.txt") + _py_builtins.copy_without_caching( + ctx = ctx, + read_from = ctx.version_file, + write_to = version_file, + ) + else: + version_file = central_uncachable_version_file + + direct_inputs = [ctx.info_file, version_file] + + # A "constant metadata" file is basically a special file that doesn't + # support change detection logic and reports that it is unchanged. i.e., it + # behaves like ctx.version_file and is ignored when computing "what inputs + # changed" (see https://bazel.build/docs/user-manual#workspace-status). + # + # We do this so that consumers of the final build data file don't have + # to transitively rebuild everything -- the `uncachable_version_file` file + # isn't cachable, which causes the build data action to always re-run. + # + # While this technically means a binary could have stale build info, + # it ends up not mattering in practice because the volatile information + # doesn't meaningfully effect other outputs. + # + # This is also done for performance and Make It work reasons: + # * Passing the transitive dependencies into the action requires passing + # the runfiles, but actions don't directly accept runfiles. While + # flattening the depsets can be deferred, accessing the + # `runfiles.empty_filenames` attribute will will invoke the empty + # file supplier a second time, which is too much of a memory and CPU + # performance hit. + # * Some targets specify a directory in `data`, which is unsound, but + # mostly works. Google's RBE, unfortunately, rejects it. + # * A binary's transitive closure may be so large that it exceeds + # Google RBE limits for action inputs. + build_data = _py_builtins.declare_constant_metadata_file( + ctx = ctx, + name = ctx.label.name + ".build_data.txt", + root = ctx.bin_dir, + ) + + ctx.actions.run( + executable = ctx.executable._build_data_gen, + env = { + # NOTE: ctx.info_file is undocumented; see + # https://github.com/bazelbuild/bazel/issues/9363 + "INFO_FILE": ctx.info_file.path, + "OUTPUT": build_data.path, + "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, + "TARGET": str(ctx.label), + "VERSION_FILE": version_file.path, + } | extra_write_build_data_env, + inputs = depset( + direct = direct_inputs, + ), + outputs = [build_data], + mnemonic = "PyWriteBuildData", + progress_message = "Generating %{label} build_data.txt", + ) + return build_data + +def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): + if not semantics.should_build_native_deps_dso(ctx): + return struct(dso = None, runfiles = ctx.runfiles()) + + cc_info = cc_details.cc_info_for_self_link + + if not cc_info.linking_context.linker_inputs: + return struct(dso = None, runfiles = ctx.runfiles()) + + dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) + share_native_deps = ctx.fragments.cpp.share_native_deps() + cc_feature_config = cc_configure_features( + ctx, + cc_toolchain = cc_details.cc_toolchain, + # See b/171276569#comment18: this feature string is just to allow + # Google's RBE to know the link action is for the Python case so it can + # take special actions (though as of Jun 2022, no special action is + # taken). + extra_features = ["native_deps_link"], + ) + if share_native_deps: + linked_lib = _create_shared_native_deps_dso( + ctx, + cc_info = cc_info, + is_test = is_test, + requested_features = cc_feature_config.requested_features, + feature_configuration = cc_feature_config.feature_configuration, + ) + ctx.actions.symlink( + output = dso, + target_file = linked_lib, + progress_message = "Symlinking shared native deps for %{label}", + ) + else: + linked_lib = dso + _cc_common.link( + name = ctx.label.name, + actions = ctx.actions, + linking_contexts = [cc_info.linking_context], + output_type = "dynamic_library", + never_link = True, + native_deps = True, + feature_configuration = cc_feature_config.feature_configuration, + cc_toolchain = cc_details.cc_toolchain, + test_only_target = is_test, + stamp = 1 if is_stamping_enabled(ctx, semantics) else 0, + main_output = linked_lib, + use_shareable_artifact_factory = True, + # NOTE: Only flags not captured by cc_info.linking_context need to + # be manually passed + user_link_flags = semantics.get_native_deps_user_link_flags(ctx), + ) + return struct( + dso = dso, + runfiles = ctx.runfiles(files = [dso]), + ) + +def _create_shared_native_deps_dso( + ctx, + *, + cc_info, + is_test, + feature_configuration, + requested_features): + linkstamps = cc_info.linking_context.linkstamps() + + partially_disabled_thin_lto = ( + _cc_common.is_enabled( + feature_name = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", + feature_configuration = feature_configuration, + ) and not _cc_common.is_enabled( + feature_name = "thin_lto_all_linkstatic_use_shared_nonlto_backends", + feature_configuration = feature_configuration, + ) + ) + dso_hash = _get_shared_native_deps_hash( + linker_inputs = cc_helper.get_static_mode_params_for_dynamic_library_libraries( + depset([ + lib + for linker_input in cc_info.linking_context.linker_inputs.to_list() + for lib in linker_input.libraries + ]), + ), + link_opts = [ + flag + for input in cc_info.linking_context.linker_inputs.to_list() + for flag in input.user_link_flags + ], + linkstamps = [linkstamp.file() for linkstamp in linkstamps.to_list()], + build_info_artifacts = _cc_common.get_build_info(ctx) if linkstamps else [], + features = requested_features, + is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, + ) + return ctx.actions.declare_shareable_artifact("_nativedeps/%x.so" % dso_hash) + +# This is a minimal version of NativeDepsHelper.getSharedNativeDepsPath, see +# com.google.devtools.build.lib.rules.nativedeps.NativeDepsHelper#getSharedNativeDepsPath +# The basic idea is to take all the inputs that affect linking and encode (via +# hashing) them into the filename. +# TODO(b/234232820): The settings that affect linking must be kept in sync with the actual +# C++ link action. For more information, see the large descriptive comment on +# NativeDepsHelper#getSharedNativeDepsPath. +def _get_shared_native_deps_hash( + *, + linker_inputs, + link_opts, + linkstamps, + build_info_artifacts, + features, + is_test_target_partially_disabled_thin_lto): + # NOTE: We use short_path because the build configuration root in which + # files are always created already captures the configuration-specific + # parts, so no need to include them manually. + parts = [] + for artifact in linker_inputs: + parts.append(artifact.short_path) + parts.append(str(len(link_opts))) + parts.extend(link_opts) + for artifact in linkstamps: + parts.append(artifact.short_path) + for artifact in build_info_artifacts: + parts.append(artifact.short_path) + parts.extend(sorted(features)) + + # Sharing of native dependencies may cause an {@link + # ActionConflictException} when ThinLTO is disabled for test and test-only + # targets that are statically linked, but enabled for other statically + # linked targets. This happens in case the artifacts for the shared native + # dependency are output by {@link Action}s owned by the non-test and test + # targets both. To fix this, we allow creation of multiple artifacts for the + # shared native library - one shared among the test and test-only targets + # where ThinLTO is disabled, and the other shared among other targets where + # ThinLTO is enabled. See b/138118275 + parts.append("1" if is_test_target_partially_disabled_thin_lto else "0") + + return hash("".join(parts)) + +def determine_main(ctx): + """Determine the main entry point .py source file. + + Args: + ctx: The rule ctx. + + Returns: + Artifact; the main file. If one can't be found, an error is raised. + """ + if ctx.attr.main: + proposed_main = ctx.attr.main.label.name + if not proposed_main.endswith(tuple(ALLOWED_MAIN_EXTENSIONS)): + fail("main must end in '.py'") + else: + if ctx.label.name.endswith(".py"): + fail("name must not end in '.py'") + proposed_main = ctx.label.name + ".py" + + main_files = [src for src in ctx.files.srcs if _path_endswith(src.short_path, proposed_main)] + if not main_files: + if ctx.attr.main: + fail("could not find '{}' as specified by 'main' attribute".format(proposed_main)) + else: + fail(("corresponding default '{}' does not appear in srcs. Add " + + "it or override default file name with a 'main' attribute").format( + proposed_main, + )) + + elif len(main_files) > 1: + if ctx.attr.main: + fail(("file name '{}' specified by 'main' attributes matches multiple files. " + + "Matches: {}").format( + proposed_main, + csv([f.short_path for f in main_files]), + )) + else: + fail(("default main file '{}' matches multiple files in srcs. Perhaps specify " + + "an explicit file with 'main' attribute? Matches were: {}").format( + proposed_main, + csv([f.short_path for f in main_files]), + )) + return main_files[0] + +def _path_endswith(path, endswith): + # Use slash to anchor each path to prevent e.g. + # "ab/c.py".endswith("b/c.py") from incorrectly matching. + return ("/" + path).endswith("/" + endswith) + +def is_stamping_enabled(ctx, semantics): + """Tells if stamping is enabled or not. + + Args: + ctx: The rule ctx + semantics: a semantics struct (see create_semantics_struct). + Returns: + bool; True if stamping is enabled, False if not. + """ + if _is_tool_config(ctx): + return False + + stamp = ctx.attr.stamp + if stamp == 1: + return True + elif stamp == 0: + return False + elif stamp == -1: + return semantics.get_stamp_flag(ctx) + else: + fail("Unsupported `stamp` value: {}".format(stamp)) + +def _is_tool_config(ctx): + # NOTE: The is_tool_configuration() function is only usable by builtins. + # See https://github.com/bazelbuild/bazel/issues/14444 for the FR for + # a more public API. Outside of builtins, ctx.bin_dir.path can be + # checked for `/host/` or `-exec-`. + return ctx.configuration.is_tool_configuration() + +def _create_providers( + *, + ctx, + executable, + main_py, + direct_sources, + files_to_build, + runfiles_details, + imports, + cc_info, + inherited_environment, + runtime_details, + output_groups, + semantics): + """Creates the providers an executable should return. + + Args: + ctx: The rule ctx. + executable: File; the target's executable file. + main_py: File; the main .py entry point. + direct_sources: list of Files; the direct, raw `.py` sources for the target. + This should only be Python source files. It should not include pyc + files. + files_to_build: depset of Files; the files for DefaultInfo.files + runfiles_details: runfiles that will become the default and data runfiles. + imports: depset of strings; the import paths to propagate + cc_info: optional CcInfo; Linking information to propagate as + PyCcLinkParamsProvider. Note that only the linking information + is propagated, not the whole CcInfo. + inherited_environment: list of strings; Environment variable names + that should be inherited from the environment the executuble + is run within. + runtime_details: struct of runtime information; see _get_runtime_details() + output_groups: dict[str, depset[File]]; used to create OutputGroupInfo + semantics: BinarySemantics struct; see create_binary_semantics() + + Returns: + A two-tuple of: + 1. A dict of legacy providers. + 2. A list of modern providers. + """ + providers = [ + DefaultInfo( + executable = executable, + files = files_to_build, + default_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( + ctx, + runfiles_details.default_runfiles, + ), + data_runfiles = _py_builtins.make_runfiles_respect_legacy_external_runfiles( + ctx, + runfiles_details.data_runfiles, + ), + ), + create_instrumented_files_info(ctx), + _create_run_environment_info(ctx, inherited_environment), + ] + + # TODO(b/265840007): Make this non-conditional once Google enables + # --incompatible_use_python_toolchains. + if runtime_details.toolchain_runtime: + providers.append(runtime_details.toolchain_runtime) + + # TODO(b/163083591): Remove the PyCcLinkParamsProvider once binaries-in-deps + # are cleaned up. + if cc_info: + providers.append( + PyCcLinkParamsProvider(cc_info = cc_info), + ) + + py_info, deps_transitive_sources = create_py_info( + ctx, + direct_sources = depset(direct_sources), + imports = imports, + ) + + # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 + listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) + if listeners_enabled: + _py_builtins.add_py_extra_pseudo_action( + ctx = ctx, + dependency_transitive_python_sources = deps_transitive_sources, + ) + + providers.append(py_info) + providers.append(create_output_group_info(py_info.transitive_sources, output_groups)) + + extra_legacy_providers, extra_providers = semantics.get_extra_providers( + ctx, + main_py = main_py, + runtime_details = runtime_details, + ) + providers.extend(extra_providers) + return extra_legacy_providers, providers + +def _create_run_environment_info(ctx, inherited_environment): + expanded_env = {} + for key, value in ctx.attr.env.items(): + expanded_env[key] = _py_builtins.expand_location_and_make_variables( + ctx = ctx, + attribute_name = "env[{}]".format(key), + expression = value, + targets = ctx.attr.data, + ) + return RunEnvironmentInfo( + environment = expanded_env, + inherited_environment = inherited_environment, + ) + +def create_base_executable_rule(*, attrs, fragments = [], **kwargs): + """Create a function for defining for Python binary/test targets. + + Args: + attrs: Rule attributes + fragments: List of str; extra config fragments that are required. + **kwargs: Additional args to pass onto `rule()` + + Returns: + A rule function + """ + if "py" not in fragments: + # The list might be frozen, so use concatentation + fragments = fragments + ["py"] + return rule( + # TODO: add ability to remove attrs, i.e. for imports attr + attrs = EXECUTABLE_ATTRS | attrs, + toolchains = [TOOLCHAIN_TYPE] + cc_helper.use_cpp_toolchain(), + fragments = fragments, + **kwargs + ) + +def cc_configure_features(ctx, *, cc_toolchain, extra_features): + """Configure C++ features for Python purposes. + + Args: + ctx: Rule ctx + cc_toolchain: The CcToolchain the target is using. + extra_features: list of strings; additional features to request be + enabled. + + Returns: + struct of the feature configuration and all requested features. + """ + requested_features = ["static_linking_mode"] + requested_features.extend(extra_features) + requested_features.extend(ctx.features) + if "legacy_whole_archive" not in ctx.disabled_features: + requested_features.append("legacy_whole_archive") + feature_configuration = _cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = requested_features, + unsupported_features = ctx.disabled_features, + ) + return struct( + feature_configuration = feature_configuration, + requested_features = requested_features, + ) + +only_exposed_for_google_internal_reason = struct( + create_runfiles_with_build_data = _create_runfiles_with_build_data, +) diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl new file mode 100644 index 0000000000..7c7ecb01d1 --- /dev/null +++ b/python/private/common/py_executable_bazel.bzl @@ -0,0 +1,480 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation for Bazel Python executable.""" + +load(":common/paths.bzl", "paths") +load(":common/python/attributes_bazel.bzl", "IMPORTS_ATTRS") +load( + ":common/python/common.bzl", + "create_binary_semantics_struct", + "create_cc_details_struct", + "create_executable_result_struct", + "union_attrs", +) +load(":common/python/common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") +load(":common/python/providers.bzl", "DEFAULT_STUB_SHEBANG") +load( + ":common/python/py_executable.bzl", + "create_base_executable_rule", + "py_executable_base_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_py_builtins = _builtins.internal.py_builtins +_EXTERNAL_PATH_PREFIX = "external" +_ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" + +BAZEL_EXECUTABLE_ATTRS = union_attrs( + IMPORTS_ATTRS, + { + "legacy_create_init": attr.int( + default = -1, + values = [-1, 0, 1], + doc = """\ +Whether to implicitly create empty `__init__.py` files in the runfiles tree. +These are created in every directory containing Python source code or shared +libraries, and every parent directory of those directories, excluding the repo +root directory. The default, `-1` (auto), means true unless +`--incompatible_default_to_explicit_init_py` is used. If false, the user is +responsible for creating (possibly empty) `__init__.py` files and adding them to +the `srcs` of Python targets as required. + """, + ), + "_bootstrap_template": attr.label( + allow_single_file = True, + default = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt", + ), + "_launcher": attr.label( + cfg = "target", + default = "@" + TOOLS_REPO + "//tools/launcher:launcher", + executable = True, + ), + "_py_interpreter": attr.label( + default = configuration_field( + fragment = "bazel_py", + name = "python_top", + ), + ), + # TODO: This appears to be vestigial. It's only added because + # GraphlessQueryTest.testLabelsOperator relies on it to test for + # query behavior of implicit dependencies. + "_py_toolchain_type": attr.label( + default = "@" + TOOLS_REPO + "//tools/python:toolchain_type", + ), + "_windows_launcher_maker": attr.label( + default = "@" + TOOLS_REPO + "//tools/launcher:launcher_maker", + cfg = "exec", + executable = True, + ), + "_zipper": attr.label( + cfg = "exec", + executable = True, + default = "@" + TOOLS_REPO + "//tools/zip:zipper", + ), + }, +) + +def create_executable_rule(*, attrs, **kwargs): + return create_base_executable_rule( + attrs = BAZEL_EXECUTABLE_ATTRS | attrs, + fragments = ["py", "bazel_py"], + **kwargs + ) + +def py_executable_bazel_impl(ctx, *, is_test, inherited_environment): + """Common code for executables for Baze.""" + result = py_executable_base_impl( + ctx = ctx, + semantics = create_binary_semantics_bazel(), + is_test = is_test, + inherited_environment = inherited_environment, + ) + return struct( + providers = result.providers, + **result.legacy_providers + ) + +def create_binary_semantics_bazel(): + return create_binary_semantics_struct( + # keep-sorted start + create_executable = _create_executable, + get_cc_details_for_binary = _get_cc_details_for_binary, + get_central_uncachable_version_file = lambda ctx: None, + get_coverage_deps = _get_coverage_deps, + get_debugger_deps = _get_debugger_deps, + get_extra_common_runfiles_for_binary = lambda ctx: ctx.runfiles(), + get_extra_providers = _get_extra_providers, + get_extra_write_build_data_env = lambda ctx: {}, + get_imports = get_imports, + get_interpreter_path = _get_interpreter_path, + get_native_deps_dso_name = _get_native_deps_dso_name, + get_native_deps_user_link_flags = _get_native_deps_user_link_flags, + get_stamp_flag = _get_stamp_flag, + maybe_precompile = maybe_precompile, + should_build_native_deps_dso = lambda ctx: False, + should_create_init_files = _should_create_init_files, + should_include_build_data = lambda ctx: False, + # keep-sorted end + ) + +def _get_coverage_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_debugger_deps(ctx, runtime_details): + _ = ctx, runtime_details # @unused + return [] + +def _get_extra_providers(ctx, main_py, runtime_details): + _ = ctx, main_py, runtime_details # @unused + return {}, [] + +def _get_stamp_flag(ctx): + # NOTE: Undocumented API; private to builtins + return ctx.configuration.stamp_binaries + +def _should_create_init_files(ctx): + if ctx.attr.legacy_create_init == -1: + return not ctx.fragments.py.default_to_explicit_init_py + else: + return bool(ctx.attr.legacy_create_init) + +def _create_executable( + ctx, + *, + executable, + main_py, + imports, + is_test, + runtime_details, + cc_details, + native_deps_details, + runfiles_details): + _ = is_test, cc_details, native_deps_details # @unused + + common_bootstrap_template_kwargs = dict( + main_py = main_py, + imports = imports, + runtime_details = runtime_details, + ) + + # TODO: This should use the configuration instead of the Bazel OS. + # This is just legacy behavior. + is_windows = _py_builtins.get_current_os_name() == "windows" + + if is_windows: + if not executable.extension == "exe": + fail("Should not happen: somehow we are generating a non-.exe file on windows") + base_executable_name = executable.basename[0:-4] + else: + base_executable_name = executable.basename + + zip_bootstrap = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) + zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) + + _expand_bootstrap_template( + ctx, + output = zip_bootstrap, + is_for_zip = True, + **common_bootstrap_template_kwargs + ) + _create_zip_file( + ctx, + output = zip_file, + original_nonzip_executable = executable, + executable_for_zip_file = zip_bootstrap, + runfiles = runfiles_details.default_runfiles, + ) + + extra_files_to_build = [] + + # NOTE: --build_python_zip defauls to true on Windows + build_zip_enabled = ctx.fragments.py.build_python_zip + + # When --build_python_zip is enabled, then the zip file becomes + # one of the default outputs. + if build_zip_enabled: + extra_files_to_build.append(zip_file) + + # The logic here is a bit convoluted. Essentially, there are 3 types of + # executables produced: + # 1. (non-Windows) A bootstrap template based program. + # 2. (non-Windows) A self-executable zip file of a bootstrap template based program. + # 3. (Windows) A native Windows executable that finds and launches + # the actual underlying Bazel program (one of the above). Note that + # it implicitly assumes one of the above is located next to it, and + # that --build_python_zip defaults to true for Windows. + + should_create_executable_zip = False + bootstrap_output = None + if not is_windows: + if build_zip_enabled: + should_create_executable_zip = True + else: + bootstrap_output = executable + else: + _create_windows_exe_launcher( + ctx, + output = executable, + use_zip_file = build_zip_enabled, + python_binary_path = runtime_details.executable_interpreter_path, + ) + if not build_zip_enabled: + # On Windows, the main executable has an "exe" extension, so + # here we re-use the un-extensioned name for the bootstrap output. + bootstrap_output = ctx.actions.declare_file(base_executable_name) + + # The launcher looks for the non-zip executable next to + # itself, so add it to the default outputs. + extra_files_to_build.append(bootstrap_output) + + if should_create_executable_zip: + if bootstrap_output != None: + fail("Should not occur: bootstrap_output should not be used " + + "when creating an executable zip") + _create_executable_zip_file(ctx, output = executable, zip_file = zip_file) + elif bootstrap_output: + _expand_bootstrap_template( + ctx, + output = bootstrap_output, + is_for_zip = build_zip_enabled, + **common_bootstrap_template_kwargs + ) + else: + # Otherwise, this should be the Windows case of launcher + zip. + # Double check this just to make sure. + if not is_windows or not build_zip_enabled: + fail(("Should not occur: The non-executable-zip and " + + "non-boostrap-template case should have windows and zip " + + "both true, but got " + + "is_windows={is_windows} " + + "build_zip_enabled={build_zip_enabled}").format( + is_windows = is_windows, + build_zip_enabled = build_zip_enabled, + )) + + return create_executable_result_struct( + extra_files_to_build = depset(extra_files_to_build), + output_groups = {"python_zip_file": depset([zip_file])}, + ) + +def _expand_bootstrap_template( + ctx, + *, + output, + main_py, + imports, + is_for_zip, + runtime_details): + runtime = runtime_details.effective_runtime + if (ctx.configuration.coverage_enabled and + runtime and + runtime.coverage_tool): + coverage_tool_runfiles_path = "{}/{}".format( + ctx.workspace_name, + runtime.coverage_tool.short_path, + ) + else: + coverage_tool_runfiles_path = "" + + if runtime: + shebang = runtime.stub_shebang + template = runtime.bootstrap_template + else: + shebang = DEFAULT_STUB_SHEBANG + template = ctx.file._bootstrap_template + + ctx.actions.expand_template( + template = template, + output = output, + substitutions = { + "%coverage_tool%": coverage_tool_runfiles_path, + "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", + "%imports%": ":".join(imports.to_list()), + "%is_zipfile%": "True" if is_for_zip else "False", + "%main%": "{}/{}".format( + ctx.workspace_name, + main_py.short_path, + ), + "%python_binary%": runtime_details.executable_interpreter_path, + "%shebang%": shebang, + "%target%": str(ctx.label), + "%workspace_name%": ctx.workspace_name, + }, + is_executable = True, + ) + +def _create_windows_exe_launcher( + ctx, + *, + output, + python_binary_path, + use_zip_file): + launch_info = ctx.actions.args() + launch_info.use_param_file("%s", use_always = True) + launch_info.set_param_file_format("multiline") + launch_info.add("binary_type=Python") + launch_info.add(ctx.workspace_name, format = "workspace_name=%s") + launch_info.add( + "1" if ctx.configuration.runfiles_enabled() else "0", + format = "symlink_runfiles_enabled=%s", + ) + launch_info.add(python_binary_path, format = "python_bin_path=%s") + launch_info.add("1" if use_zip_file else "0", format = "use_zip_file=%s") + + ctx.actions.run( + executable = ctx.executable._windows_launcher_maker, + arguments = [ctx.executable._launcher.path, launch_info, output.path], + inputs = [ctx.executable._launcher], + outputs = [output], + mnemonic = "PyBuildLauncher", + progress_message = "Creating launcher for %{label}", + # Needed to inherit PATH when using non-MSVC compilers like MinGW + use_default_shell_env = True, + ) + +def _create_zip_file(ctx, *, output, original_nonzip_executable, executable_for_zip_file, runfiles): + workspace_name = ctx.workspace_name + legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) + + manifest = ctx.actions.args() + manifest.use_param_file("@%s", use_always = True) + manifest.set_param_file_format("multiline") + + manifest.add("__main__.py={}".format(executable_for_zip_file.path)) + manifest.add("__init__.py=") + manifest.add( + "{}=".format( + _get_zip_runfiles_path("__init__.py", workspace_name, legacy_external_runfiles), + ), + ) + for path in runfiles.empty_filenames.to_list(): + manifest.add("{}=".format(_get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles))) + + def map_zip_runfiles(file): + if file != original_nonzip_executable and file != output: + return "{}={}".format( + _get_zip_runfiles_path(file.short_path, workspace_name, legacy_external_runfiles), + file.path, + ) + else: + return None + + manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) + + inputs = [executable_for_zip_file] + if _py_builtins.is_bzlmod_enabled(ctx): + zip_repo_mapping_manifest = ctx.actions.declare_file( + output.basename + ".repo_mapping", + sibling = output, + ) + _py_builtins.create_repo_mapping_manifest( + ctx = ctx, + runfiles = runfiles, + output = zip_repo_mapping_manifest, + ) + manifest.add("{}/_repo_mapping={}".format( + _ZIP_RUNFILES_DIRECTORY_NAME, + zip_repo_mapping_manifest.path, + )) + inputs.append(zip_repo_mapping_manifest) + + for artifact in runfiles.files.to_list(): + # Don't include the original executable because it isn't used by the + # zip file, so no need to build it for the action. + # Don't include the zipfile itself because it's an output. + if artifact != original_nonzip_executable and artifact != output: + inputs.append(artifact) + + zip_cli_args = ctx.actions.args() + zip_cli_args.add("cC") + zip_cli_args.add(output) + + ctx.actions.run( + executable = ctx.executable._zipper, + arguments = [zip_cli_args, manifest], + inputs = depset(inputs), + outputs = [output], + use_default_shell_env = True, + mnemonic = "PythonZipper", + progress_message = "Building Python zip: %{label}", + ) + +def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): + if legacy_external_runfiles and path.startswith(_EXTERNAL_PATH_PREFIX): + zip_runfiles_path = paths.relativize(path, _EXTERNAL_PATH_PREFIX) + else: + # NOTE: External runfiles (artifacts in other repos) will have a leading + # path component of "../" so that they refer outside the main workspace + # directory and into the runfiles root. By normalizing, we simplify e.g. + # "workspace/../foo/bar" to simply "foo/bar". + zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) + return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) + +def _create_executable_zip_file(ctx, *, output, zip_file): + ctx.actions.run_shell( + command = "echo '{shebang}' | cat - {zip} > {output}".format( + shebang = "#!/usr/bin/env python3", + zip = zip_file.path, + output = output.path, + ), + inputs = [zip_file], + outputs = [output], + use_default_shell_env = True, + mnemonic = "BuildBinary", + progress_message = "Build Python zip executable: %{label}", + ) + +def _get_cc_details_for_binary(ctx, extra_deps): + cc_info = collect_cc_info(ctx, extra_deps = extra_deps) + return create_cc_details_struct( + cc_info_for_propagating = cc_info, + cc_info_for_self_link = cc_info, + cc_info_with_extra_link_time_libraries = None, + extra_runfiles = ctx.runfiles(), + # Though the rules require the CcToolchain, it isn't actually used. + cc_toolchain = None, + ) + +def _get_interpreter_path(ctx, *, runtime, flag_interpreter_path): + if runtime: + if runtime.interpreter_path: + interpreter_path = runtime.interpreter_path + else: + interpreter_path = "{}/{}".format( + ctx.workspace_name, + runtime.interpreter.short_path, + ) + + # NOTE: External runfiles (artifacts in other repos) will have a + # leading path component of "../" so that they refer outside the + # main workspace directory and into the runfiles root. By + # normalizing, we simplify e.g. "workspace/../foo/bar" to simply + # "foo/bar" + interpreter_path = paths.normalize(interpreter_path) + + elif flag_interpreter_path: + interpreter_path = flag_interpreter_path + else: + fail("Unable to determine interpreter path") + + return interpreter_path + +def _get_native_deps_dso_name(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") + +def _get_native_deps_user_link_flags(ctx): + _ = ctx # @unused + fail("Building native deps DSO not supported.") diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl new file mode 100644 index 0000000000..62f974f4b1 --- /dev/null +++ b/python/private/common/py_library.bzl @@ -0,0 +1,99 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_library rule.""" + +load( + ":common/python/attributes.bzl", + "COMMON_ATTRS", + "PY_SRCS_ATTRS", + "SRCS_VERSION_ALL_VALUES", + "create_srcs_attr", + "create_srcs_version_attr", +) +load( + ":common/python/common.bzl", + "check_native_allowed", + "collect_imports", + "collect_runfiles", + "create_instrumented_files_info", + "create_output_group_info", + "create_py_info", + "filter_to_py_srcs", + "union_attrs", +) +load(":common/python/providers.bzl", "PyCcLinkParamsProvider") + +_py_builtins = _builtins.internal.py_builtins + +LIBRARY_ATTRS = union_attrs( + COMMON_ATTRS, + PY_SRCS_ATTRS, + create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), + create_srcs_attr(mandatory = False), +) + +def py_library_impl(ctx, *, semantics): + """Abstract implementation of py_library rule. + + Args: + ctx: The rule ctx + semantics: A `LibrarySemantics` struct; see `create_library_semantics_struct` + + Returns: + A list of modern providers to propagate. + """ + check_native_allowed(ctx) + direct_sources = filter_to_py_srcs(ctx.files.srcs) + output_sources = depset(semantics.maybe_precompile(ctx, direct_sources)) + runfiles = collect_runfiles(ctx = ctx, files = output_sources) + + cc_info = semantics.get_cc_info_for_library(ctx) + py_info, deps_transitive_sources = create_py_info( + ctx, + direct_sources = depset(direct_sources), + imports = collect_imports(ctx, semantics), + ) + + # TODO(b/253059598): Remove support for extra actions; https://github.com/bazelbuild/bazel/issues/16455 + listeners_enabled = _py_builtins.are_action_listeners_enabled(ctx) + if listeners_enabled: + _py_builtins.add_py_extra_pseudo_action( + ctx = ctx, + dependency_transitive_python_sources = deps_transitive_sources, + ) + + return [ + DefaultInfo(files = output_sources, runfiles = runfiles), + py_info, + create_instrumented_files_info(ctx), + PyCcLinkParamsProvider(cc_info = cc_info), + create_output_group_info(py_info.transitive_sources, extra_groups = {}), + ] + +def create_py_library_rule(*, attrs = {}, **kwargs): + """Creates a py_library rule. + + Args: + attrs: dict of rule attributes. + **kwargs: Additional kwargs to pass onto the rule() call. + Returns: + A rule object + """ + return rule( + attrs = LIBRARY_ATTRS | attrs, + # TODO(b/253818097): fragments=py is only necessary so that + # RequiredConfigFragmentsTest passes + fragments = ["py"], + **kwargs + ) diff --git a/python/private/common/py_library_bazel.bzl b/python/private/common/py_library_bazel.bzl new file mode 100644 index 0000000000..b844b97e9f --- /dev/null +++ b/python/private/common/py_library_bazel.bzl @@ -0,0 +1,59 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_library for Bazel.""" + +load( + ":common/python/attributes_bazel.bzl", + "IMPORTS_ATTRS", +) +load( + ":common/python/common.bzl", + "create_library_semantics_struct", + "union_attrs", +) +load( + ":common/python/common_bazel.bzl", + "collect_cc_info", + "get_imports", + "maybe_precompile", +) +load( + ":common/python/py_library.bzl", + "LIBRARY_ATTRS", + "create_py_library_rule", + bazel_py_library_impl = "py_library_impl", +) + +_BAZEL_LIBRARY_ATTRS = union_attrs( + LIBRARY_ATTRS, + IMPORTS_ATTRS, +) + +def create_library_semantics_bazel(): + return create_library_semantics_struct( + get_imports = get_imports, + maybe_precompile = maybe_precompile, + get_cc_info_for_library = collect_cc_info, + ) + +def _py_library_impl(ctx): + return bazel_py_library_impl( + ctx, + semantics = create_library_semantics_bazel(), + ) + +py_library = create_py_library_rule( + implementation = _py_library_impl, + attrs = _BAZEL_LIBRARY_ATTRS, +) diff --git a/python/private/common/py_library_macro.bzl b/python/private/common/py_library_macro.bzl new file mode 100644 index 0000000000..729c426f15 --- /dev/null +++ b/python/private/common/py_library_macro.bzl @@ -0,0 +1,19 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_library rule.""" + +load(":common/python/py_library_bazel.bzl", py_library_rule = "py_library") + +def py_library(**kwargs): + py_library_rule(**kwargs) diff --git a/python/private/common/py_runtime_macro.bzl b/python/private/common/py_runtime_macro.bzl new file mode 100644 index 0000000000..6b27bccfcc --- /dev/null +++ b/python/private/common/py_runtime_macro.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Macro to wrap the py_runtime rule.""" + +load(":common/python/py_runtime_rule.bzl", py_runtime_rule = "py_runtime") + +# NOTE: The function name is purposefully selected to match the underlying +# rule name so that e.g. 'generator_function' shows as the same name so +# that it is less confusing to users. +def py_runtime(**kwargs): + py_runtime_rule(**kwargs) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl new file mode 100644 index 0000000000..22efaa6b77 --- /dev/null +++ b/python/private/common/py_runtime_rule.bzl @@ -0,0 +1,214 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of py_runtime rule.""" + +load(":common/paths.bzl", "paths") +load(":common/python/attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS") +load(":common/python/common.bzl", "check_native_allowed") +load(":common/python/providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", _PyRuntimeInfo = "PyRuntimeInfo") + +_py_builtins = _builtins.internal.py_builtins + +def _py_runtime_impl(ctx): + check_native_allowed(ctx) + interpreter_path = ctx.attr.interpreter_path or None # Convert empty string to None + interpreter = ctx.file.interpreter + if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): + fail("exactly one of the 'interpreter' or 'interpreter_path' attributes must be specified") + + runtime_files = depset(transitive = [ + t[DefaultInfo].files + for t in ctx.attr.files + ]) + + hermetic = bool(interpreter) + if not hermetic: + if runtime_files: + fail("if 'interpreter_path' is given then 'files' must be empty") + if not paths.is_absolute(interpreter_path): + fail("interpreter_path must be an absolute path") + + if ctx.attr.coverage_tool: + coverage_di = ctx.attr.coverage_tool[DefaultInfo] + + if _py_builtins.is_singleton_depset(coverage_di.files): + coverage_tool = coverage_di.files.to_list()[0] + elif coverage_di.files_to_run and coverage_di.files_to_run.executable: + coverage_tool = coverage_di.files_to_run.executable + else: + fail("coverage_tool must be an executable target or must produce exactly one file.") + + coverage_files = depset(transitive = [ + coverage_di.files, + coverage_di.default_runfiles.files, + ]) + else: + coverage_tool = None + coverage_files = None + + python_version = ctx.attr.python_version + if python_version == "_INTERNAL_SENTINEL": + if ctx.fragments.py.use_toolchains: + fail( + "When using Python toolchains, this attribute must be set explicitly to either 'PY2' " + + "or 'PY3'. See https://github.com/bazelbuild/bazel/issues/7899 for more " + + "information. You can temporarily avoid this error by reverting to the legacy " + + "Python runtime mechanism (`--incompatible_use_python_toolchains=false`).", + ) + else: + python_version = ctx.fragments.py.default_python_version + + # TODO: Uncomment this after --incompatible_python_disable_py2 defaults to true + # if ctx.fragments.py.disable_py2 and python_version == "PY2": + # fail("Using Python 2 is not supported and disabled; see " + + # "https://github.com/bazelbuild/bazel/issues/15684") + + return [ + _PyRuntimeInfo( + interpreter_path = interpreter_path or None, + interpreter = interpreter, + files = runtime_files if hermetic else None, + coverage_tool = coverage_tool, + coverage_files = coverage_files, + python_version = python_version, + stub_shebang = ctx.attr.stub_shebang, + bootstrap_template = ctx.file.bootstrap_template, + ), + DefaultInfo( + files = runtime_files, + runfiles = ctx.runfiles(), + ), + ] + +# Bind to the name "py_runtime" to preserve the kind/rule_class it shows up +# as elsewhere. +py_runtime = rule( + implementation = _py_runtime_impl, + doc = """ +Represents a Python runtime used to execute Python code. + +A `py_runtime` target can represent either a *platform runtime* or an *in-build +runtime*. A platform runtime accesses a system-installed interpreter at a known +path, whereas an in-build runtime points to an executable target that acts as +the interpreter. In both cases, an "interpreter" means any executable binary or +wrapper script that is capable of running a Python script passed on the command +line, following the same conventions as the standard CPython interpreter. + +A platform runtime is by its nature non-hermetic. It imposes a requirement on +the target platform to have an interpreter located at a specific path. An +in-build runtime may or may not be hermetic, depending on whether it points to +a checked-in interpreter or a wrapper script that accesses the system +interpreter. + +# Example + +``` +py_runtime( + name = "python-2.7.12", + files = glob(["python-2.7.12/**"]), + interpreter = "python-2.7.12/bin/python", +) + +py_runtime( + name = "python-3.6.0", + interpreter_path = "/opt/pyenv/versions/3.6.0/bin/python", +) +``` +""", + fragments = ["py"], + attrs = NATIVE_RULES_ALLOWLIST_ATTRS | { + "bootstrap_template": attr.label( + allow_single_file = True, + default = DEFAULT_BOOTSTRAP_TEMPLATE, + doc = """ +The bootstrap script template file to use. Should have %python_binary%, +%workspace_name%, %main%, and %imports%. + +This template, after expansion, becomes the executable file used to start the +process, so it is responsible for initial bootstrapping actions such as finding +the Python interpreter, runfiles, and constructing an environment to run the +intended Python application. + +While this attribute is currently optional, it will become required when the +Python rules are moved out of Bazel itself. + +The exact variable names expanded is an unstable API and is subject to change. +The API will become more stable when the Python rules are moved out of Bazel +itself. + +See @bazel_tools//tools/python:python_bootstrap_template.txt for more variables. +""", + ), + "coverage_tool": attr.label( + allow_files = False, + doc = """ +This is a target to use for collecting code coverage information from `py_binary` +and `py_test` targets. + +If set, the target must either produce a single file or be an executable target. +The path to the single file, or the executable if the target is executable, +determines the entry point for the python coverage tool. The target and its +runfiles will be added to the runfiles when coverage is enabled. + +The entry point for the tool must be loadable by a Python interpreter (e.g. a +`.py` or `.pyc` file). It must accept the command line arguments +of coverage.py (https://coverage.readthedocs.io), at least including +the `run` and `lcov` subcommands. +""", + ), + "files": attr.label_list( + allow_files = True, + doc = """ +For an in-build runtime, this is the set of files comprising this runtime. +These files will be added to the runfiles of Python binaries that use this +runtime. For a platform runtime this attribute must not be set. +""", + ), + "interpreter": attr.label( + allow_single_file = True, + doc = """ +For an in-build runtime, this is the target to invoke as the interpreter. For a +platform runtime this attribute must not be set. +""", + ), + "interpreter_path": attr.string(doc = """ +For a platform runtime, this is the absolute path of a Python interpreter on +the target platform. For an in-build runtime this attribute must not be set. +"""), + "python_version": attr.string( + default = "_INTERNAL_SENTINEL", + values = ["PY2", "PY3", "_INTERNAL_SENTINEL"], + doc = """ +Whether this runtime is for Python major version 2 or 3. Valid values are `"PY2"` +and `"PY3"`. + +The default value is controlled by the `--incompatible_py3_is_default` flag. +However, in the future this attribute will be mandatory and have no default +value. + """, + ), + "stub_shebang": attr.string( + default = DEFAULT_STUB_SHEBANG, + doc = """ +"Shebang" expression prepended to the bootstrapping Python stub script +used when executing `py_binary` targets. + +See https://github.com/bazelbuild/bazel/issues/8685 for +motivation. + +Does not apply to Windows. +""", + ), + }, +) diff --git a/python/private/common/py_test_bazel.bzl b/python/private/common/py_test_bazel.bzl new file mode 100644 index 0000000000..fde3a5a47d --- /dev/null +++ b/python/private/common/py_test_bazel.bzl @@ -0,0 +1,55 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule implementation of py_test for Bazel.""" + +load(":common/python/attributes.bzl", "AGNOSTIC_TEST_ATTRS") +load(":common/python/common.bzl", "maybe_add_test_execution_info") +load( + ":common/python/py_executable_bazel.bzl", + "create_executable_rule", + "py_executable_bazel_impl", +) +load(":common/python/semantics.bzl", "TOOLS_REPO") + +_BAZEL_PY_TEST_ATTRS = { + # This *might* be a magic attribute to help C++ coverage work. There's no + # docs about this; see TestActionBuilder.java + "_collect_cc_coverage": attr.label( + default = "@" + TOOLS_REPO + "//tools/test:collect_cc_coverage", + executable = True, + cfg = "exec", + ), + # This *might* be a magic attribute to help C++ coverage work. There's no + # docs about this; see TestActionBuilder.java + "_lcov_merger": attr.label( + default = configuration_field(fragment = "coverage", name = "output_generator"), + cfg = "exec", + executable = True, + ), +} + +def _py_test_impl(ctx): + providers = py_executable_bazel_impl( + ctx = ctx, + is_test = True, + inherited_environment = ctx.attr.env_inherit, + ) + maybe_add_test_execution_info(providers.providers, ctx) + return providers + +py_test = create_executable_rule( + implementation = _py_test_impl, + attrs = AGNOSTIC_TEST_ATTRS | _BAZEL_PY_TEST_ATTRS, + test = True, +) diff --git a/python/private/common/py_test_macro.bzl b/python/private/common/py_test_macro.bzl new file mode 100644 index 0000000000..4faede68ad --- /dev/null +++ b/python/private/common/py_test_macro.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Implementation of macro-half of py_test rule.""" + +load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":common/python/py_test_bazel.bzl", py_test_rule = "py_test") + +def py_test(**kwargs): + convert_legacy_create_init_to_int(kwargs) + py_test_rule(**kwargs) diff --git a/python/private/common/semantics.bzl b/python/private/common/semantics.bzl new file mode 100644 index 0000000000..487ff303ef --- /dev/null +++ b/python/private/common/semantics.bzl @@ -0,0 +1,34 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains constants that vary between Bazel and Google-internal""" + +IMPORTS_ATTR_SUPPORTED = True + +TOOLS_REPO = "bazel_tools" +PLATFORMS_LOCATION = "@platforms/" + +SRCS_ATTR_ALLOW_FILES = [".py", ".py3"] + +DEPS_ATTR_ALLOW_RULES = None + +PY_RUNTIME_ATTR_NAME = "_py_interpreter" + +BUILD_DATA_SYMLINK_PATH = None + +IS_BAZEL = True + +NATIVE_RULES_MIGRATION_HELP_URL = "https://github.com/bazelbuild/bazel/issues/17773" +NATIVE_RULES_MIGRATION_FIX_CMD = "add_python_loads" + +ALLOWED_MAIN_EXTENSIONS = [".py"] From 6cb77cdd953a6c90a63881bb023b4209557f9d59 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 19 Sep 2023 10:49:22 +0900 Subject: [PATCH 15/41] feat: add new Python toolchain versions (#1414) Towards #1396, defaults will be bumped in a separate PR. --- CHANGELOG.md | 2 ++ python/versions.bzl | 50 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 491a0804c4..ed78e625c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ A brief description of the categories of changes: [`py_console_script_binary`](./docs/py_console_script_binary.md), which allows adding custom dependencies to a package's entry points and customizing the `py_binary` rule used to build it. +* New Python versions available: `3.8.17`, `3.9.18`, `3.10.13`, `3.11.5` using + https://github.com/indygreg/python-build-standalone/releases/tag/20230826. ### Removed diff --git a/python/versions.bzl b/python/versions.bzl index 1ef3172588..20dcbea1fa 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -97,6 +97,17 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.8.17": { + "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "c6f7a130d0044a78e39648f4dae56dcff5a41eba91888a99f6e560507162e6a1", + "aarch64-unknown-linux-gnu": "9f6d585091fe26906ff1dbb80437a3fe37a1e3db34d6ecc0098f3d6a78356682", + "x86_64-apple-darwin": "155b06821607bae1a58ecc60a7d036b358c766f19e493b8876190765c883a5c2", + "x86_64-pc-windows-msvc": "6428e1b4e0b4482d390828de7d4c82815257443416cb786abe10cb2466ca68cd", + "x86_64-unknown-linux-gnu": "8d3e1826c0bb7821ec63288038644808a2d45553245af106c685ef5892fabcd8", + }, + "strip_prefix": "python", + }, "3.9.10": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -166,6 +177,19 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.9.18": { + "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "44000d3bd79a6c689f3b6cae846d302d9a4e974c46d078b1bc79cc0c706a0718", + "aarch64-unknown-linux-gnu": "2161e834aa4334cc8bb55335767a073aafff3338cf37392d2a9123b4972276f9", + "ppc64le-unknown-linux-gnu": "1e95c15627cea707156b41d653af994283876162f14ac9280cc1fb8023cf56b3", + "s390x-unknown-linux-gnu": "476d1ba8f85ae8a0e0b5ae7f0e204dd9376fe55afd9c6a7ae7b18bd84a223bf6", + "x86_64-apple-darwin": "ce03b97a41be6d548698baaf5804fff2ce96bf49237fb73f8692aca3f5798454", + "x86_64-pc-windows-msvc": "709c1aabf712aa4553c53c4879a459ebe8575a996d68ccbce492af03db8a6ee0", + "x86_64-unknown-linux-gnu": "377da2aebc3b58c5af901899e8efeb2c91b35b0ea92c8b447036767e529fc5b2", + }, + "strip_prefix": "python", + }, "3.10.2": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -246,6 +270,19 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.10.13": { + "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "142332021441ee1ab04eb126baa6c6690dc41699d4af608b72b399a786f6ee71", + "aarch64-unknown-linux-gnu": "0479cf10254adbf7a554453874e91bb526ba62cbac8a758f6865cdcdbef20f2d", + "ppc64le-unknown-linux-gnu": "355ec3d0983e1e454d7175c9c8581221472d4597f6a93d676b60ed4e1655c299", + "s390x-unknown-linux-gnu": "a61ff760d39e2b06794cdcf8b2f62c39d58b97f5a1ddd0e112741f60d6fe712f", + "x86_64-apple-darwin": "3a5d50b98e4981af4fc23cf3fc53a38ef3f9a8f32453849e295e747aa9936b2b", + "x86_64-pc-windows-msvc": "2ae0ee39450d428ce2aa4bea9ad41c96916d4f92fe641a3bf6d6f80d360677c3", + "x86_64-unknown-linux-gnu": "ba512bcca3ac6cb6d834f496cd0a66416f0a53ff20b05c4794fa82ece185b85a", + }, + "strip_prefix": "python", + }, "3.11.1": { "url": "20230116/cpython-{python_version}+20230116-{platform}-{build}.tar.gz", "sha256": { @@ -282,6 +319,19 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.5": { + "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "dab64b3580118ad2073babd7c29fd2053b616479df5c107d31fe2af1f45e948b", + "aarch64-unknown-linux-gnu": "bb5c5d1ea0f199fe2d3f0996fff4b48ca6ddc415a3dbd98f50bff7fce48aac80", + "ppc64le-unknown-linux-gnu": "14121b53e9c8c6d0741f911ae00102a35adbcf5c3cdf732687ef7617b7d7304d", + "s390x-unknown-linux-gnu": "fe459da39874443579d6fe88c68777c6d3e331038e1fb92a0451879fb6beb16d", + "x86_64-apple-darwin": "4a4efa7378c72f1dd8ebcce1afb99b24c01b07023aa6b8fea50eaedb50bf2bfc", + "x86_64-pc-windows-msvc": "00f002263efc8aea896bcfaaf906b1f4dab3e5cd3db53e2b69ab9a10ba220b97", + "x86_64-unknown-linux-gnu": "fbed6f7694b2faae5d7c401a856219c945397f772eea5ca50c6eb825cbc9d1e1", + }, + "strip_prefix": "python", + }, } # buildifier: disable=unsorted-dict-items From 09109e345700c123ad1aa1eff76cccebf28558a7 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 20 Sep 2023 15:40:51 -0700 Subject: [PATCH 16/41] internal(pystar): make starlark impl (mostly) loadable (#1422) This just makes the files able to get passed the loading stage under Bazel 7+. This mostly involves fixing load statements, but also exposed a couple places where py_internal needs some small changes. * Also renames files to better distinguish rule vs macro vs Bazel-specific. This makes it easier to patch them within Google and more clear about which file is doing what. Work towards #1069 --- python/private/common/BUILD.bazel | 13 ++++++++++ python/private/common/attributes.bzl | 14 ++++++----- python/private/common/cc_helper.bzl | 23 +++++++++++++++++ python/private/common/common.bzl | 20 +++++++-------- python/private/common/common_bazel.bzl | 17 ++++++++----- python/private/common/providers.bzl | 7 +++--- ...ry_macro.bzl => py_binary_macro_bazel.bzl} | 4 +-- ...ary_bazel.bzl => py_binary_rule_bazel.bzl} | 6 ++--- python/private/common/py_executable.bzl | 22 ++++++++-------- python/private/common/py_executable_bazel.bzl | 17 +++++++------ python/private/common/py_internal.bzl | 24 ++++++++++++++++++ python/private/common/py_library.bzl | 9 ++++--- ...y_macro.bzl => py_library_macro_bazel.bzl} | 2 +- ...ry_bazel.bzl => py_library_rule_bazel.bzl} | 20 +++------------ python/private/common/py_runtime_macro.bzl | 2 +- python/private/common/py_runtime_rule.bzl | 11 ++++---- ...test_macro.bzl => py_test_macro_bazel.bzl} | 4 +-- ..._test_bazel.bzl => py_test_rule_bazel.bzl} | 8 +++--- tools/build_defs/python/private/BUILD.bazel | 13 ++++++++++ .../python/private/py_internal_renamed.bzl | 25 +++++++++++++++++++ 20 files changed, 178 insertions(+), 83 deletions(-) create mode 100644 python/private/common/BUILD.bazel create mode 100644 python/private/common/cc_helper.bzl rename python/private/common/{py_binary_macro.bzl => py_binary_macro_bazel.bzl} (83%) rename python/private/common/{py_binary_bazel.bzl => py_binary_rule_bazel.bzl} (89%) create mode 100644 python/private/common/py_internal.bzl rename python/private/common/{py_library_macro.bzl => py_library_macro_bazel.bzl} (90%) rename python/private/common/{py_library_bazel.bzl => py_library_rule_bazel.bzl} (80%) rename python/private/common/{py_test_macro.bzl => py_test_macro_bazel.bzl} (83%) rename python/private/common/{py_test_bazel.bzl => py_test_rule_bazel.bzl} (88%) create mode 100644 tools/build_defs/python/private/BUILD.bazel create mode 100644 tools/build_defs/python/private/py_internal_renamed.bzl diff --git a/python/private/common/BUILD.bazel b/python/private/common/BUILD.bazel new file mode 100644 index 0000000000..aa21042e25 --- /dev/null +++ b/python/private/common/BUILD.bazel @@ -0,0 +1,13 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl index 7e28ed9d69..ea43ceafb1 100644 --- a/python/private/common/attributes.bzl +++ b/python/private/common/attributes.bzl @@ -13,18 +13,20 @@ # limitations under the License. """Attributes for Python rules.""" -load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") -load(":common/python/common.bzl", "union_attrs") -load(":common/python/providers.bzl", "PyInfo") +load(":common.bzl", "union_attrs") +load(":providers.bzl", "PyInfo") +load(":py_internal.bzl", "py_internal") load( - ":common/python/semantics.bzl", + ":semantics.bzl", "DEPS_ATTR_ALLOW_RULES", "PLATFORMS_LOCATION", "SRCS_ATTR_ALLOW_FILES", "TOOLS_REPO", ) -PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo +# TODO: Load CcInfo from rules_cc +_CcInfo = CcInfo +_PackageSpecificationInfo = py_internal.PackageSpecificationInfo _STAMP_VALUES = [-1, 0, 1] @@ -89,7 +91,7 @@ NATIVE_RULES_ALLOWLIST_ATTRS = { fragment = "py", name = "native_rules_allowlist", ), - providers = [PackageSpecificationInfo], + providers = [_PackageSpecificationInfo], ), } diff --git a/python/private/common/cc_helper.bzl b/python/private/common/cc_helper.bzl new file mode 100644 index 0000000000..cef1ab169d --- /dev/null +++ b/python/private/common/cc_helper.bzl @@ -0,0 +1,23 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PYTHON RULE IMPLEMENTATION ONLY: Do not use outside of the rule implementations and their tests. + +Adapter for accessing Bazel's internal cc_helper. + +These may change at any time and are closely coupled to the rule implementation. +""" + +load(":py_internal.bzl", "py_internal") + +cc_helper = py_internal.cc_helper diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index 97ed3e3ee6..8522d80606 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -13,23 +13,21 @@ # limitations under the License. """Various things common to Bazel and Google rule implementations.""" -load(":common/cc/cc_helper.bzl", "cc_helper") +load(":cc_helper.bzl", "cc_helper") +load(":providers.bzl", "PyInfo") +load(":py_internal.bzl", "py_internal") load( - ":common/python/providers.bzl", - "PyInfo", -) -load( - ":common/python/semantics.bzl", + ":semantics.bzl", "NATIVE_RULES_MIGRATION_FIX_CMD", "NATIVE_RULES_MIGRATION_HELP_URL", "TOOLS_REPO", ) -_testing = _builtins.toplevel.testing -_platform_common = _builtins.toplevel.platform_common -_coverage_common = _builtins.toplevel.coverage_common -_py_builtins = _builtins.internal.py_builtins -PackageSpecificationInfo = _builtins.toplevel.PackageSpecificationInfo +_testing = testing +_platform_common = platform_common +_coverage_common = coverage_common +_py_builtins = py_internal +PackageSpecificationInfo = py_internal.PackageSpecificationInfo TOOLCHAIN_TYPE = "@" + TOOLS_REPO + "//tools/python:toolchain_type" diff --git a/python/private/common/common_bazel.bzl b/python/private/common/common_bazel.bzl index 51b06fb832..7277337849 100644 --- a/python/private/common/common_bazel.bzl +++ b/python/private/common/common_bazel.bzl @@ -13,13 +13,18 @@ # limitations under the License. """Common functions that are specific to Bazel rule implementation""" -load(":common/cc/cc_common.bzl", _cc_common = "cc_common") -load(":common/cc/cc_info.bzl", _CcInfo = "CcInfo") -load(":common/paths.bzl", "paths") -load(":common/python/common.bzl", "is_bool") -load(":common/python/providers.bzl", "PyCcLinkParamsProvider") +load("@bazel_skylib//lib:paths.bzl", "paths") +load(":common.bzl", "is_bool") +load(":providers.bzl", "PyCcLinkParamsProvider") +load(":py_internal.bzl", "py_internal") -_py_builtins = _builtins.internal.py_builtins +# TODO: Load cc_common from rules_cc +_cc_common = cc_common + +# TODO: Load CcInfo from rules_cc +_CcInfo = CcInfo + +_py_builtins = py_internal def collect_cc_info(ctx, extra_deps = []): """Collect C++ information from dependencies for Bazel. diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl index a9df61bda4..237a3e4d20 100644 --- a/python/private/common/providers.bzl +++ b/python/private/common/providers.bzl @@ -13,14 +13,13 @@ # limitations under the License. """Providers for Python rules.""" -load(":common/python/semantics.bzl", "TOOLS_REPO") +load(":semantics.bzl", "TOOLS_REPO") -_CcInfo = _builtins.toplevel.CcInfo +# TODO: load CcInfo from rules_cc +_CcInfo = CcInfo -# NOTE: This is copied to PyRuntimeInfo.java DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" -# NOTE: This is copied to PyRuntimeInfo.java DEFAULT_BOOTSTRAP_TEMPLATE = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt" _PYTHON_VERSION_VALUES = ["PY2", "PY3"] diff --git a/python/private/common/py_binary_macro.bzl b/python/private/common/py_binary_macro_bazel.bzl similarity index 83% rename from python/private/common/py_binary_macro.bzl rename to python/private/common/py_binary_macro_bazel.bzl index 24e5c6dbe3..a6c4e97dac 100644 --- a/python/private/common/py_binary_macro.bzl +++ b/python/private/common/py_binary_macro_bazel.bzl @@ -13,8 +13,8 @@ # limitations under the License. """Implementation of macro-half of py_binary rule.""" -load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") -load(":common/python/py_binary_bazel.bzl", py_binary_rule = "py_binary") +load(":common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":py_binary_rule_bazel.bzl", py_binary_rule = "py_binary") def py_binary(**kwargs): convert_legacy_create_init_to_int(kwargs) diff --git a/python/private/common/py_binary_bazel.bzl b/python/private/common/py_binary_rule_bazel.bzl similarity index 89% rename from python/private/common/py_binary_bazel.bzl rename to python/private/common/py_binary_rule_bazel.bzl index 3a5df737b9..6c324d8bc5 100644 --- a/python/private/common/py_binary_bazel.bzl +++ b/python/private/common/py_binary_rule_bazel.bzl @@ -13,13 +13,13 @@ # limitations under the License. """Rule implementation of py_binary for Bazel.""" -load(":common/python/attributes.bzl", "AGNOSTIC_BINARY_ATTRS") +load(":attributes.bzl", "AGNOSTIC_BINARY_ATTRS") load( - ":common/python/py_executable_bazel.bzl", + ":py_executable_bazel.bzl", "create_executable_rule", "py_executable_bazel_impl", ) -load(":common/python/semantics.bzl", "TOOLS_REPO") +load(":semantics.bzl", "TOOLS_REPO") _PY_TEST_ATTRS = { "_collect_cc_coverage": attr.label( diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 9db92b18e5..7a50a75c11 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -13,10 +13,8 @@ # limitations under the License. """Common functionality between test/binary executables.""" -load(":common/cc/cc_common.bzl", _cc_common = "cc_common") -load(":common/cc/cc_helper.bzl", "cc_helper") load( - ":common/python/attributes.bzl", + ":attributes.bzl", "AGNOSTIC_EXECUTABLE_ATTRS", "COMMON_ATTRS", "PY_SRCS_ATTRS", @@ -24,8 +22,9 @@ load( "create_srcs_attr", "create_srcs_version_attr", ) +load(":cc_helper.bzl", "cc_helper") load( - ":common/python/common.bzl", + ":common.bzl", "TOOLCHAIN_TYPE", "check_native_allowed", "collect_imports", @@ -38,19 +37,23 @@ load( "union_attrs", ) load( - ":common/python/providers.bzl", + ":providers.bzl", "PyCcLinkParamsProvider", "PyRuntimeInfo", ) +load(":py_internal.bzl", "py_internal") load( - ":common/python/semantics.bzl", + ":semantics.bzl", "ALLOWED_MAIN_EXTENSIONS", "BUILD_DATA_SYMLINK_PATH", "IS_BAZEL", "PY_RUNTIME_ATTR_NAME", ) -_py_builtins = _builtins.internal.py_builtins +# TODO: Load cc_common from rules_cc +_cc_common = cc_common + +_py_builtins = py_internal # Non-Google-specific attributes for executables # These attributes are for rules that accept Python sources. @@ -677,9 +680,8 @@ def is_stamping_enabled(ctx, semantics): def _is_tool_config(ctx): # NOTE: The is_tool_configuration() function is only usable by builtins. # See https://github.com/bazelbuild/bazel/issues/14444 for the FR for - # a more public API. Outside of builtins, ctx.bin_dir.path can be - # checked for `/host/` or `-exec-`. - return ctx.configuration.is_tool_configuration() + # a more public API. Until that's available, py_internal to the rescue. + return py_internal.is_tool_configuration(ctx) def _create_providers( *, diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index 7c7ecb01d1..a145d421a6 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -13,25 +13,26 @@ # limitations under the License. """Implementation for Bazel Python executable.""" -load(":common/paths.bzl", "paths") -load(":common/python/attributes_bazel.bzl", "IMPORTS_ATTRS") +load("@bazel_skylib//lib:paths.bzl", "paths") +load(":attributes_bazel.bzl", "IMPORTS_ATTRS") load( - ":common/python/common.bzl", + ":common.bzl", "create_binary_semantics_struct", "create_cc_details_struct", "create_executable_result_struct", "union_attrs", ) -load(":common/python/common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") -load(":common/python/providers.bzl", "DEFAULT_STUB_SHEBANG") +load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") +load(":providers.bzl", "DEFAULT_STUB_SHEBANG") load( - ":common/python/py_executable.bzl", + ":py_executable.bzl", "create_base_executable_rule", "py_executable_base_impl", ) -load(":common/python/semantics.bzl", "TOOLS_REPO") +load(":py_internal.bzl", "py_internal") +load(":semantics.bzl", "TOOLS_REPO") -_py_builtins = _builtins.internal.py_builtins +_py_builtins = py_internal _EXTERNAL_PATH_PREFIX = "external" _ZIP_RUNFILES_DIRECTORY_NAME = "runfiles" diff --git a/python/private/common/py_internal.bzl b/python/private/common/py_internal.bzl new file mode 100644 index 0000000000..c17bbf0522 --- /dev/null +++ b/python/private/common/py_internal.bzl @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PYTHON RULE IMPLEMENTATION ONLY: Do not use outside of the rule implementations and their tests. + +Re-exports the restricted-use py_internal helper under its original name. + +These may change at any time and are closely coupled to the rule implementation. +""" + +# buildifier: disable=bzl-visibility +load("//tools/build_defs/python/private:py_internal_renamed.bzl", "py_internal_renamed") + +py_internal = py_internal_renamed diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl index 62f974f4b1..ca71e72443 100644 --- a/python/private/common/py_library.bzl +++ b/python/private/common/py_library.bzl @@ -14,7 +14,7 @@ """Implementation of py_library rule.""" load( - ":common/python/attributes.bzl", + ":attributes.bzl", "COMMON_ATTRS", "PY_SRCS_ATTRS", "SRCS_VERSION_ALL_VALUES", @@ -22,7 +22,7 @@ load( "create_srcs_version_attr", ) load( - ":common/python/common.bzl", + ":common.bzl", "check_native_allowed", "collect_imports", "collect_runfiles", @@ -32,9 +32,10 @@ load( "filter_to_py_srcs", "union_attrs", ) -load(":common/python/providers.bzl", "PyCcLinkParamsProvider") +load(":providers.bzl", "PyCcLinkParamsProvider") +load(":py_internal.bzl", "py_internal") -_py_builtins = _builtins.internal.py_builtins +_py_builtins = py_internal LIBRARY_ATTRS = union_attrs( COMMON_ATTRS, diff --git a/python/private/common/py_library_macro.bzl b/python/private/common/py_library_macro_bazel.bzl similarity index 90% rename from python/private/common/py_library_macro.bzl rename to python/private/common/py_library_macro_bazel.bzl index 729c426f15..b4f51eff1d 100644 --- a/python/private/common/py_library_macro.bzl +++ b/python/private/common/py_library_macro_bazel.bzl @@ -13,7 +13,7 @@ # limitations under the License. """Implementation of macro-half of py_library rule.""" -load(":common/python/py_library_bazel.bzl", py_library_rule = "py_library") +load(":py_library_rule_bazel.bzl", py_library_rule = "py_library") def py_library(**kwargs): py_library_rule(**kwargs) diff --git a/python/private/common/py_library_bazel.bzl b/python/private/common/py_library_rule_bazel.bzl similarity index 80% rename from python/private/common/py_library_bazel.bzl rename to python/private/common/py_library_rule_bazel.bzl index b844b97e9f..453abcb816 100644 --- a/python/private/common/py_library_bazel.bzl +++ b/python/private/common/py_library_rule_bazel.bzl @@ -13,23 +13,11 @@ # limitations under the License. """Implementation of py_library for Bazel.""" +load(":attributes_bazel.bzl", "IMPORTS_ATTRS") +load(":common.bzl", "create_library_semantics_struct", "union_attrs") +load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") load( - ":common/python/attributes_bazel.bzl", - "IMPORTS_ATTRS", -) -load( - ":common/python/common.bzl", - "create_library_semantics_struct", - "union_attrs", -) -load( - ":common/python/common_bazel.bzl", - "collect_cc_info", - "get_imports", - "maybe_precompile", -) -load( - ":common/python/py_library.bzl", + ":py_library.bzl", "LIBRARY_ATTRS", "create_py_library_rule", bazel_py_library_impl = "py_library_impl", diff --git a/python/private/common/py_runtime_macro.bzl b/python/private/common/py_runtime_macro.bzl index 6b27bccfcc..7d04388fd6 100644 --- a/python/private/common/py_runtime_macro.bzl +++ b/python/private/common/py_runtime_macro.bzl @@ -13,7 +13,7 @@ # limitations under the License. """Macro to wrap the py_runtime rule.""" -load(":common/python/py_runtime_rule.bzl", py_runtime_rule = "py_runtime") +load(":py_runtime_rule.bzl", py_runtime_rule = "py_runtime") # NOTE: The function name is purposefully selected to match the underlying # rule name so that e.g. 'generator_function' shows as the same name so diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index 22efaa6b77..4bffb876c9 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -13,12 +13,13 @@ # limitations under the License. """Implementation of py_runtime rule.""" -load(":common/paths.bzl", "paths") -load(":common/python/attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS") -load(":common/python/common.bzl", "check_native_allowed") -load(":common/python/providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", _PyRuntimeInfo = "PyRuntimeInfo") +load("@bazel_skylib//lib:paths.bzl", "paths") +load(":attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS") +load(":common.bzl", "check_native_allowed") +load(":providers.bzl", "DEFAULT_BOOTSTRAP_TEMPLATE", "DEFAULT_STUB_SHEBANG", _PyRuntimeInfo = "PyRuntimeInfo") +load(":py_internal.bzl", "py_internal") -_py_builtins = _builtins.internal.py_builtins +_py_builtins = py_internal def _py_runtime_impl(ctx): check_native_allowed(ctx) diff --git a/python/private/common/py_test_macro.bzl b/python/private/common/py_test_macro_bazel.bzl similarity index 83% rename from python/private/common/py_test_macro.bzl rename to python/private/common/py_test_macro_bazel.bzl index 4faede68ad..24b78fef96 100644 --- a/python/private/common/py_test_macro.bzl +++ b/python/private/common/py_test_macro_bazel.bzl @@ -13,8 +13,8 @@ # limitations under the License. """Implementation of macro-half of py_test rule.""" -load(":common/python/common_bazel.bzl", "convert_legacy_create_init_to_int") -load(":common/python/py_test_bazel.bzl", py_test_rule = "py_test") +load(":common_bazel.bzl", "convert_legacy_create_init_to_int") +load(":py_test_rule_bazel.bzl", py_test_rule = "py_test") def py_test(**kwargs): convert_legacy_create_init_to_int(kwargs) diff --git a/python/private/common/py_test_bazel.bzl b/python/private/common/py_test_rule_bazel.bzl similarity index 88% rename from python/private/common/py_test_bazel.bzl rename to python/private/common/py_test_rule_bazel.bzl index fde3a5a47d..de1aa4581c 100644 --- a/python/private/common/py_test_bazel.bzl +++ b/python/private/common/py_test_rule_bazel.bzl @@ -13,14 +13,14 @@ # limitations under the License. """Rule implementation of py_test for Bazel.""" -load(":common/python/attributes.bzl", "AGNOSTIC_TEST_ATTRS") -load(":common/python/common.bzl", "maybe_add_test_execution_info") +load(":attributes.bzl", "AGNOSTIC_TEST_ATTRS") +load(":common.bzl", "maybe_add_test_execution_info") load( - ":common/python/py_executable_bazel.bzl", + ":py_executable_bazel.bzl", "create_executable_rule", "py_executable_bazel_impl", ) -load(":common/python/semantics.bzl", "TOOLS_REPO") +load(":semantics.bzl", "TOOLS_REPO") _BAZEL_PY_TEST_ATTRS = { # This *might* be a magic attribute to help C++ coverage work. There's no diff --git a/tools/build_defs/python/private/BUILD.bazel b/tools/build_defs/python/private/BUILD.bazel new file mode 100644 index 0000000000..aa21042e25 --- /dev/null +++ b/tools/build_defs/python/private/BUILD.bazel @@ -0,0 +1,13 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/build_defs/python/private/py_internal_renamed.bzl b/tools/build_defs/python/private/py_internal_renamed.bzl new file mode 100644 index 0000000000..abab31c45e --- /dev/null +++ b/tools/build_defs/python/private/py_internal_renamed.bzl @@ -0,0 +1,25 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PYTHON RULE IMPLEMENTATION ONLY: Do not use outside of the rule implementations and their tests. + +Re-exports the restricted-use py_internal helper under another name. This is +necessary because `py_internal = py_internal` results in an error (trying +to bind a local symbol to itself before its defined). + +This is to allow the rule implementation in the //python directory to access +the internal helpers only rules_python is allowed to use. + +These may change at any time and are closely coupled to the rule implementation. +""" +py_internal_renamed = py_internal From daca84397aa87f605105010ef9bc0e86c96a6b04 Mon Sep 17 00:00:00 2001 From: raylu <90059+raylu@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:30:21 -0700 Subject: [PATCH 17/41] feat: generate py_library per file (#1398) fixes #1150 fixes #1323 you can no longer pre-define the name of the target by creating an empty `py_library` (see 3c84655). I don't think this was being used and it's straightforward to rename the generated per-project or per-package target if you want --- CHANGELOG.md | 2 + gazelle/README.md | 16 ++++--- gazelle/python/configure.go | 5 +++ gazelle/python/generate.go | 42 ++++++++++++++----- gazelle/python/kinds.go | 3 +- gazelle/python/resolve.go | 8 ++-- .../testdata/dont_rename_target/BUILD.in | 1 + gazelle/python/testdata/per_file/BUILD.in | 11 +++++ gazelle/python/testdata/per_file/BUILD.out | 24 +++++++++++ gazelle/python/testdata/per_file/README.md | 5 +++ gazelle/python/testdata/per_file/WORKSPACE | 1 + gazelle/python/testdata/per_file/__init__.py | 0 gazelle/python/testdata/per_file/bar.py | 15 +++++++ gazelle/python/testdata/per_file/baz.py | 15 +++++++ gazelle/python/testdata/per_file/foo.py | 15 +++++++ gazelle/python/testdata/per_file/test.yaml | 15 +++++++ .../testdata/per_file_non_empty_init/BUILD.in | 3 ++ .../per_file_non_empty_init/BUILD.out | 16 +++++++ .../per_file_non_empty_init/README.md | 3 ++ .../per_file_non_empty_init/WORKSPACE | 1 + .../per_file_non_empty_init/__init__.py | 15 +++++++ .../testdata/per_file_non_empty_init/foo.py | 15 +++++++ .../per_file_non_empty_init/test.yaml | 15 +++++++ .../python/testdata/per_file_subdirs/BUILD.in | 3 ++ .../testdata/per_file_subdirs/BUILD.out | 10 +++++ .../testdata/per_file_subdirs/README.md | 3 ++ .../testdata/per_file_subdirs/WORKSPACE | 1 + .../testdata/per_file_subdirs/bar/BUILD.in | 0 .../testdata/per_file_subdirs/bar/BUILD.out | 13 ++++++ .../testdata/per_file_subdirs/bar/__init__.py | 15 +++++++ .../testdata/per_file_subdirs/bar/foo.py | 16 +++++++ .../testdata/per_file_subdirs/baz/baz.py | 15 +++++++ .../python/testdata/per_file_subdirs/foo.py | 15 +++++++ .../testdata/per_file_subdirs/test.yaml | 15 +++++++ gazelle/pythonconfig/pythonconfig.go | 16 +++++++ 35 files changed, 346 insertions(+), 22 deletions(-) create mode 100644 gazelle/python/testdata/per_file/BUILD.in create mode 100644 gazelle/python/testdata/per_file/BUILD.out create mode 100644 gazelle/python/testdata/per_file/README.md create mode 100644 gazelle/python/testdata/per_file/WORKSPACE create mode 100644 gazelle/python/testdata/per_file/__init__.py create mode 100644 gazelle/python/testdata/per_file/bar.py create mode 100644 gazelle/python/testdata/per_file/baz.py create mode 100644 gazelle/python/testdata/per_file/foo.py create mode 100644 gazelle/python/testdata/per_file/test.yaml create mode 100644 gazelle/python/testdata/per_file_non_empty_init/BUILD.in create mode 100644 gazelle/python/testdata/per_file_non_empty_init/BUILD.out create mode 100644 gazelle/python/testdata/per_file_non_empty_init/README.md create mode 100644 gazelle/python/testdata/per_file_non_empty_init/WORKSPACE create mode 100644 gazelle/python/testdata/per_file_non_empty_init/__init__.py create mode 100644 gazelle/python/testdata/per_file_non_empty_init/foo.py create mode 100644 gazelle/python/testdata/per_file_non_empty_init/test.yaml create mode 100644 gazelle/python/testdata/per_file_subdirs/BUILD.in create mode 100644 gazelle/python/testdata/per_file_subdirs/BUILD.out create mode 100644 gazelle/python/testdata/per_file_subdirs/README.md create mode 100644 gazelle/python/testdata/per_file_subdirs/WORKSPACE create mode 100644 gazelle/python/testdata/per_file_subdirs/bar/BUILD.in create mode 100644 gazelle/python/testdata/per_file_subdirs/bar/BUILD.out create mode 100644 gazelle/python/testdata/per_file_subdirs/bar/__init__.py create mode 100644 gazelle/python/testdata/per_file_subdirs/bar/foo.py create mode 100644 gazelle/python/testdata/per_file_subdirs/baz/baz.py create mode 100644 gazelle/python/testdata/per_file_subdirs/foo.py create mode 100644 gazelle/python/testdata/per_file_subdirs/test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index ed78e625c3..c380066651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,8 @@ A brief description of the categories of changes: the `py_binary` rule used to build it. * New Python versions available: `3.8.17`, `3.9.18`, `3.10.13`, `3.11.5` using https://github.com/indygreg/python-build-standalone/releases/tag/20230826. +* (gazelle) New `# gazelle:python_generation_mode file` directive to support + generating one `py_library` per file. ### Removed diff --git a/gazelle/README.md b/gazelle/README.md index ba8520d36b..4728e4c429 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -189,9 +189,9 @@ Python-specific directives are as follows: | `# gazelle:python_validate_import_statements`| `true` | | Controls whether the Python import statements should be validated. Can be "true" or "false" | | | `# gazelle:python_generation_mode`| `package` | -| Controls the target generation mode. Can be "package" or "project" | | +| Controls the target generation mode. Can be "file", "package", or "project" | | | `# gazelle:python_library_naming_convention`| `$package_name$` | -| Controls the `py_library` naming convention. It interpolates $package_name$ with the Bazel package name. E.g. if the Bazel package name is `foo`, setting this to `$package_name$_my_lib` would result in a generated target named `foo_my_lib`. | | +| Controls the `py_library` naming convention. It interpolates \$package_name\$ with the Bazel package name. E.g. if the Bazel package name is `foo`, setting this to `$package_name$_my_lib` would result in a generated target named `foo_my_lib`. | | | `# gazelle:python_binary_naming_convention` | `$package_name$_bin` | | Controls the `py_binary` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | | | `# gazelle:python_test_naming_convention` | `$package_name$_test` | @@ -206,11 +206,15 @@ Python source files are those ending in `.py` but not ending in `_test.py`. First, we look for the nearest ancestor BUILD file starting from the folder containing the Python source file. -If there is no `py_library` in this BUILD file, one is created, using the -package name as the target's name. This makes it the default target in the -package. +In package generation mode, if there is no `py_library` in this BUILD file, one +is created using the package name as the target's name. This makes it the +default target in the package. Next, all source files are collected into the +`srcs` of the `py_library`. -Next, all source files are collected into the `srcs` of the `py_library`. +In project generation mode, all source files in subdirectories (that don't have +BUILD files) are also collected. + +In file generation mode, each file is given its own target. Finally, the `import` statements in the source files are parsed, and dependencies are added to the `deps` attribute. diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index 32f9ab0a11..2d3880571c 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -137,8 +137,13 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { switch pythonconfig.GenerationModeType(strings.TrimSpace(d.Value)) { case pythonconfig.GenerationModePackage: config.SetCoarseGrainedGeneration(false) + config.SetPerFileGeneration(false) + case pythonconfig.GenerationModeFile: + config.SetCoarseGrainedGeneration(false) + config.SetPerFileGeneration(true) case pythonconfig.GenerationModeProject: config.SetCoarseGrainedGeneration(true) + config.SetPerFileGeneration(false) default: err := fmt.Errorf("invalid value for directive %q: %s", pythonconfig.GenerationMode, d.Value) diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index fb41324fd6..ede4d2a222 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -153,12 +153,17 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes if entry.IsDir() { // If we are visiting a directory, we determine if we should // halt digging the tree based on a few criterias: - // 1. The directory has a BUILD or BUILD.bazel files. Then + // 1. We are using per-file generation. + // 2. The directory has a BUILD or BUILD.bazel files. Then // it doesn't matter at all what it has since it's a // separate Bazel package. - // 2. (only for fine-grained generation) The directory has - // an __init__.py, __main__.py or __test__.py, meaning - // a BUILD file will be generated. + // 3. (only for package generation) The directory has an + // __init__.py, __main__.py or __test__.py, meaning a + // BUILD file will be generated. + if cfg.PerFileGeneration() { + return fs.SkipDir + } + if isBazelPackage(path) { boundaryPackages[path] = struct{}{} return nil @@ -213,15 +218,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors := singlylinkedlist.New() - var pyLibrary *rule.Rule - if !pyLibraryFilenames.Empty() { - deps, err := parser.parse(pyLibraryFilenames) + appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) { + deps, err := parser.parse(srcs) if err != nil { log.Fatalf("ERROR: %v\n", err) } - pyLibraryTargetName := cfg.RenderLibraryName(packageName) - // Check if a target with the same name we are generating already // exists, and if it is of a different kind from the one we are // generating. If so, we have to throw an error since Gazelle won't @@ -239,9 +241,9 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } } - pyLibrary = newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames). + pyLibrary := newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames). addVisibility(visibility). - addSrcs(pyLibraryFilenames). + addSrcs(srcs). addModuleDependencies(deps). generateImportsAttribute(). build() @@ -249,6 +251,24 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes result.Gen = append(result.Gen, pyLibrary) result.Imports = append(result.Imports, pyLibrary.PrivateAttr(config.GazelleImportsKey)) } + if cfg.PerFileGeneration() { + pyLibraryFilenames.Each(func(index int, filename interface{}) { + if filename == pyLibraryEntrypointFilename { + stat, err := os.Stat(filepath.Join(args.Dir, filename.(string))) + if err != nil { + log.Fatalf("ERROR: %v\n", err) + } + if stat.Size() == 0 { + return // ignore empty __init__.py + } + } + srcs := treeset.NewWith(godsutils.StringComparator, filename) + pyLibraryTargetName := strings.TrimSuffix(filepath.Base(filename.(string)), ".py") + appendPyLibrary(srcs, pyLibraryTargetName) + }) + } else if !pyLibraryFilenames.Empty() { + appendPyLibrary(pyLibraryFilenames, cfg.RenderLibraryName(packageName)) + } if hasPyBinary { deps, err := parser.parseSingle(pyBinaryEntrypointFilename) diff --git a/gazelle/python/kinds.go b/gazelle/python/kinds.go index ab1afb7d55..941b45b5c6 100644 --- a/gazelle/python/kinds.go +++ b/gazelle/python/kinds.go @@ -49,7 +49,8 @@ var pyKinds = map[string]rule.KindInfo{ }, }, pyLibraryKind: { - MatchAny: true, + MatchAny: false, + MatchAttrs: []string{"srcs"}, NonEmptyAttrs: map[string]bool{ "deps": true, "srcs": true, diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 46014e50ec..87eed76ec3 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -151,10 +151,10 @@ func (py *Resolver) Resolve( for len(moduleParts) > 1 { // Iterate back through the possible imports until // a match is found. - // For example, "from foo.bar import baz" where bar is a variable, we should try - // `foo.bar.baz` first, then `foo.bar`, then `foo`. In the first case, the import could be file `baz.py` - // in the directory `foo/bar`. - // Or, the import could be variable `bar` in file `foo/bar.py`. + // For example, "from foo.bar import baz" where baz is a module, we should try `foo.bar.baz` first, then + // `foo.bar`, then `foo`. + // In the first case, the import could be file `baz.py` in the directory `foo/bar`. + // Or, the import could be variable `baz` in file `foo/bar.py`. // The import could also be from a standard module, e.g. `six.moves`, where // the dependency is actually `six`. moduleParts = moduleParts[:len(moduleParts)-1] diff --git a/gazelle/python/testdata/dont_rename_target/BUILD.in b/gazelle/python/testdata/dont_rename_target/BUILD.in index 33e8ec25cb..e9bc0e6e29 100644 --- a/gazelle/python/testdata/dont_rename_target/BUILD.in +++ b/gazelle/python/testdata/dont_rename_target/BUILD.in @@ -2,4 +2,5 @@ load("@rules_python//python:defs.bzl", "py_library") py_library( name = "my_custom_target", + srcs = ["__init__.py"], ) diff --git a/gazelle/python/testdata/per_file/BUILD.in b/gazelle/python/testdata/per_file/BUILD.in new file mode 100644 index 0000000000..01b0904d50 --- /dev/null +++ b/gazelle/python/testdata/per_file/BUILD.in @@ -0,0 +1,11 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +# This target should be kept unmodified by Gazelle. +py_library( + name = "custom", + srcs = ["bar.py"], + visibility = ["//visibility:private"], + tags = ["cant_touch_this"], +) diff --git a/gazelle/python/testdata/per_file/BUILD.out b/gazelle/python/testdata/per_file/BUILD.out new file mode 100644 index 0000000000..2ec825b207 --- /dev/null +++ b/gazelle/python/testdata/per_file/BUILD.out @@ -0,0 +1,24 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +# This target should be kept unmodified by Gazelle. +py_library( + name = "custom", + srcs = ["bar.py"], + tags = ["cant_touch_this"], + visibility = ["//visibility:private"], +) + +py_library( + name = "baz", + srcs = ["baz.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], + deps = [":custom"], +) diff --git a/gazelle/python/testdata/per_file/README.md b/gazelle/python/testdata/per_file/README.md new file mode 100644 index 0000000000..3ddeb213fc --- /dev/null +++ b/gazelle/python/testdata/per_file/README.md @@ -0,0 +1,5 @@ +# Per-file generation + +This test case generates one `py_library` per file. + +`__init__.py` is left empty so no target is generated for it. diff --git a/gazelle/python/testdata/per_file/WORKSPACE b/gazelle/python/testdata/per_file/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/per_file/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/per_file/__init__.py b/gazelle/python/testdata/per_file/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/per_file/bar.py b/gazelle/python/testdata/per_file/bar.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/per_file/bar.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/per_file/baz.py b/gazelle/python/testdata/per_file/baz.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/per_file/baz.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/per_file/foo.py b/gazelle/python/testdata/per_file/foo.py new file mode 100644 index 0000000000..c000990002 --- /dev/null +++ b/gazelle/python/testdata/per_file/foo.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import bar diff --git a/gazelle/python/testdata/per_file/test.yaml b/gazelle/python/testdata/per_file/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/per_file/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/python/testdata/per_file_non_empty_init/BUILD.in b/gazelle/python/testdata/per_file_non_empty_init/BUILD.in new file mode 100644 index 0000000000..a5853f6c5c --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/BUILD.in @@ -0,0 +1,3 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/per_file_non_empty_init/BUILD.out b/gazelle/python/testdata/per_file_non_empty_init/BUILD.out new file mode 100644 index 0000000000..8733dbd971 --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/BUILD.out @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], + deps = [":foo"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/per_file_non_empty_init/README.md b/gazelle/python/testdata/per_file_non_empty_init/README.md new file mode 100644 index 0000000000..6e6e9e245d --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/README.md @@ -0,0 +1,3 @@ +# Per-file generation + +This test case generates one `py_library` per file, including `__init__.py`. diff --git a/gazelle/python/testdata/per_file_non_empty_init/WORKSPACE b/gazelle/python/testdata/per_file_non_empty_init/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/per_file_non_empty_init/__init__.py b/gazelle/python/testdata/per_file_non_empty_init/__init__.py new file mode 100644 index 0000000000..492cbc0260 --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import foo diff --git a/gazelle/python/testdata/per_file_non_empty_init/foo.py b/gazelle/python/testdata/per_file_non_empty_init/foo.py new file mode 100644 index 0000000000..730755995d --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/foo.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For test purposes only. diff --git a/gazelle/python/testdata/per_file_non_empty_init/test.yaml b/gazelle/python/testdata/per_file_non_empty_init/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/per_file_non_empty_init/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/python/testdata/per_file_subdirs/BUILD.in b/gazelle/python/testdata/per_file_subdirs/BUILD.in new file mode 100644 index 0000000000..a5853f6c5c --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/BUILD.in @@ -0,0 +1,3 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file diff --git a/gazelle/python/testdata/per_file_subdirs/BUILD.out b/gazelle/python/testdata/per_file_subdirs/BUILD.out new file mode 100644 index 0000000000..69c42e01a9 --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/BUILD.out @@ -0,0 +1,10 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_generation_mode file + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], + deps = ["//bar:__init__"], +) diff --git a/gazelle/python/testdata/per_file_subdirs/README.md b/gazelle/python/testdata/per_file_subdirs/README.md new file mode 100644 index 0000000000..9eda2fac28 --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/README.md @@ -0,0 +1,3 @@ +# Per-file generation + +This test case generates one `py_library` per file in subdirectories. diff --git a/gazelle/python/testdata/per_file_subdirs/WORKSPACE b/gazelle/python/testdata/per_file_subdirs/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/per_file_subdirs/bar/BUILD.in b/gazelle/python/testdata/per_file_subdirs/bar/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/per_file_subdirs/bar/BUILD.out b/gazelle/python/testdata/per_file_subdirs/bar/BUILD.out new file mode 100644 index 0000000000..7258d27524 --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/bar/BUILD.out @@ -0,0 +1,13 @@ +load("@rules_python//python:defs.bzl", "py_library") + +py_library( + name = "__init__", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "foo", + srcs = ["foo.py"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/per_file_subdirs/bar/__init__.py b/gazelle/python/testdata/per_file_subdirs/bar/__init__.py new file mode 100644 index 0000000000..579915261d --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/bar/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .foo import func diff --git a/gazelle/python/testdata/per_file_subdirs/bar/foo.py b/gazelle/python/testdata/per_file_subdirs/bar/foo.py new file mode 100644 index 0000000000..59eb08c42f --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/bar/foo.py @@ -0,0 +1,16 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def func(): + pass diff --git a/gazelle/python/testdata/per_file_subdirs/baz/baz.py b/gazelle/python/testdata/per_file_subdirs/baz/baz.py new file mode 100644 index 0000000000..5256394021 --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/baz/baz.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from bar.foo import func diff --git a/gazelle/python/testdata/per_file_subdirs/foo.py b/gazelle/python/testdata/per_file_subdirs/foo.py new file mode 100644 index 0000000000..b5e6cff5c6 --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/foo.py @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from bar import func diff --git a/gazelle/python/testdata/per_file_subdirs/test.yaml b/gazelle/python/testdata/per_file_subdirs/test.yaml new file mode 100644 index 0000000000..fcea77710f --- /dev/null +++ b/gazelle/python/testdata/per_file_subdirs/test.yaml @@ -0,0 +1,15 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index c7cd7c1a28..a266804fab 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -78,6 +78,7 @@ const ( // GenerationModeProject defines the mode in which a coarse-grained target will // be generated englobing sub-directories containing Python files. GenerationModeProject GenerationModeType = "project" + GenerationModeFile GenerationModeType = "file" ) const ( @@ -126,6 +127,7 @@ type Config struct { ignoreDependencies map[string]struct{} validateImportStatements bool coarseGrainedGeneration bool + perFileGeneration bool libraryNamingConvention string binaryNamingConvention string testNamingConvention string @@ -145,6 +147,7 @@ func New( ignoreDependencies: make(map[string]struct{}), validateImportStatements: true, coarseGrainedGeneration: false, + perFileGeneration: false, libraryNamingConvention: packageNameNamingConventionSubstitution, binaryNamingConvention: fmt.Sprintf("%s_bin", packageNameNamingConventionSubstitution), testNamingConvention: fmt.Sprintf("%s_test", packageNameNamingConventionSubstitution), @@ -169,6 +172,7 @@ func (c *Config) NewChild() *Config { ignoreDependencies: make(map[string]struct{}), validateImportStatements: c.validateImportStatements, coarseGrainedGeneration: c.coarseGrainedGeneration, + perFileGeneration: c.perFileGeneration, libraryNamingConvention: c.libraryNamingConvention, binaryNamingConvention: c.binaryNamingConvention, testNamingConvention: c.testNamingConvention, @@ -327,6 +331,18 @@ func (c *Config) CoarseGrainedGeneration() bool { return c.coarseGrainedGeneration } +// SetPerFileGneration sets whether a separate py_library target should be +// generated for each file. +func (c *Config) SetPerFileGeneration(perFile bool) { + c.perFileGeneration = perFile +} + +// PerFileGeneration returns whether a separate py_library target should be +// generated for each file. +func (c *Config) PerFileGeneration() bool { + return c.perFileGeneration +} + // SetLibraryNamingConvention sets the py_library target naming convention. func (c *Config) SetLibraryNamingConvention(libraryNamingConvention string) { c.libraryNamingConvention = libraryNamingConvention From a07f3006c30f131389084dab7d17fae5631c598d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:17:59 +0900 Subject: [PATCH 18/41] chore: bump default python versions (#1425) Work towards #1396 --- CHANGELOG.md | 6 ++++++ WORKSPACE | 2 +- python/versions.bzl | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c380066651..1385adeee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,12 @@ A brief description of the categories of changes: ### Changed +* Python version patch level bumps: + * 3.8.15 -> 3.8.17 + * 3.9.17 -> 3.9.18 + * 3.10.12 -> 3.10.13 + * 3.11.4 -> 3.11.5 + * (deps) Upgrade rules_go 0.39.1 -> 0.41.0; this is so gazelle integration works with upcoming Bazel versions * (multi-version) The `distribs` attribute is no longer propagated. This diff --git a/WORKSPACE b/WORKSPACE index 7438bb8257..94123677ff 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -72,7 +72,7 @@ _py_gazelle_deps() # Install twine for our own runfiles wheel publishing. # Eventually we might want to install twine automatically for users too, see: # https://github.com/bazelbuild/rules_python/issues/1016. -load("@python//3.11.4:defs.bzl", "interpreter") +load("@python//3.11.5:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( diff --git a/python/versions.bzl b/python/versions.bzl index 20dcbea1fa..a79ba91293 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -336,10 +336,10 @@ TOOL_VERSIONS = { # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { - "3.8": "3.8.15", - "3.9": "3.9.17", - "3.10": "3.10.12", - "3.11": "3.11.4", + "3.8": "3.8.17", + "3.9": "3.9.18", + "3.10": "3.10.13", + "3.11": "3.11.5", } PLATFORMS = { From 0d0e1838e5a297a416a875f11cb11049c33257e4 Mon Sep 17 00:00:00 2001 From: LINKIWI Date: Thu, 21 Sep 2023 15:58:24 -0700 Subject: [PATCH 19/41] feat: Support netrc-based authentication for python_repository rule (#1417) This change introduces support for `netrc` and `auth_patterns` attributes in `python_repository` (and by extension, `python_register_toolchains`). This allows consuming projects to fetch custom Python toolchain binaries from a private/authenticated HTTP host when specified directly by URL in `python_register_toolchains`. The implementation proposed here mirrors that of `http_archive`: https://github.com/bazelbuild/bazel/blob/1cf392ff3918386858b8c038f82c013b1e04be98/tools/build_defs/repo/http.bzl#L116 Fixes #1215. --- CHANGELOG.md | 5 +++-- python/repositories.bzl | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1385adeee0..e319d28fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,9 @@ A brief description of the categories of changes: * (gazelle) New `# gazelle:python_generation_mode file` directive to support generating one `py_library` per file. +* (python_repository) Support `netrc` and `auth_patterns` attributes to enable + authentication against private HTTP hosts serving Python toolchain binaries. + ### Removed * (bzlmod) The `entry_point` macro is no longer supported and has been removed @@ -118,5 +121,3 @@ A brief description of the categories of changes: * Expose Python C headers through the toolchain. [0.24.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.24.0 - - diff --git a/python/repositories.bzl b/python/repositories.bzl index fbe23bc2e3..ea4a9275db 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -18,7 +18,7 @@ For historic reasons, pip_repositories() is defined in //python:pip.bzl. """ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe", "read_netrc", "read_user_netrc", "use_netrc") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") load( @@ -85,6 +85,28 @@ def is_standalone_interpreter(rctx, python_interpreter_path): ), ]).return_code == 0 +def _get_auth(rctx, urls): + """Utility for retrieving netrc-based authentication parameters for repository download rules used in python_repository. + + The implementation below is copied directly from Bazel's implementation of `http_archive`. + Accordingly, the return value of this function should be used identically as the `auth` parameter of `http_archive`. + Reference: https://github.com/bazelbuild/bazel/blob/6.3.2/tools/build_defs/repo/http.bzl#L109 + + Args: + rctx (repository_ctx): The repository rule's context object. + urls: A list of URLs from which assets will be downloaded. + + Returns: + dict: A map of authentication parameters by URL. + """ + if rctx.attr.netrc: + netrc = read_netrc(rctx, rctx.attr.netrc) + elif "NETRC" in rctx.os.environ: + netrc = read_netrc(rctx, rctx.os.environ["NETRC"]) + else: + netrc = read_user_netrc(rctx) + return use_netrc(netrc, urls, rctx.attr.auth_patterns) + def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") @@ -96,12 +118,14 @@ def _python_repository_impl(rctx): python_short_version = python_version.rpartition(".")[0] release_filename = rctx.attr.release_filename urls = rctx.attr.urls or [rctx.attr.url] + auth = _get_auth(rctx, urls) if release_filename.endswith(".zst"): rctx.download( url = urls, sha256 = rctx.attr.sha256, output = release_filename, + auth = auth, ) unzstd = rctx.which("unzstd") if not unzstd: @@ -109,6 +133,7 @@ def _python_repository_impl(rctx): rctx.download_and_extract( url = url, sha256 = rctx.attr.zstd_sha256, + auth = auth, ) working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) @@ -146,6 +171,7 @@ def _python_repository_impl(rctx): url = urls, sha256 = rctx.attr.sha256, stripPrefix = rctx.attr.strip_prefix, + auth = auth, ) patches = rctx.attr.patches @@ -348,11 +374,13 @@ py_cc_toolchain( rctx.file("BUILD.bazel", build_content) attrs = { + "auth_patterns": rctx.attr.auth_patterns, "coverage_tool": rctx.attr.coverage_tool, "distutils": rctx.attr.distutils, "distutils_content": rctx.attr.distutils_content, "ignore_root_user_error": rctx.attr.ignore_root_user_error, "name": rctx.attr.name, + "netrc": rctx.attr.netrc, "patches": rctx.attr.patches, "platform": platform, "python_version": python_version, @@ -372,6 +400,9 @@ python_repository = repository_rule( _python_repository_impl, doc = "Fetches the external tools needed for the Python toolchain.", attrs = { + "auth_patterns": attr.string_dict( + doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", + ), "coverage_tool": attr.string( # Mirrors the definition at # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl @@ -412,6 +443,9 @@ For more information see the official bazel docs doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", mandatory = False, ), + "netrc": attr.string( + doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", + ), "patches": attr.label_list( doc = "A list of patch files to apply to the unpacked interpreter", mandatory = False, From d8966b81fc996ee34dbc7af040346a0f80cc7645 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 25 Sep 2023 10:39:38 -0700 Subject: [PATCH 20/41] refactor(pystar): load (but don't use) Starlark implementation. (#1428) Always loading the code provides several benefits: * It's easier to reason about what code paths are taken. * Iteratively working on them is simply changing an environment variable instead of editing several files. * Ensures the files are loadable on older versions of Bazel. Usage of the Starlark implemenation is controlled by an environment variable, `RULES_PYTHON_ENABLE_PYSTAR=1`. An environment variable must be used because the decision about which implementation to use must be made before regular build flags are able to run (loading phase logic is affected). The Starlark implementation is almost entirely compatible with pre-Bazel 7, except for the `py_internal` symbol. This symbol is special in a couple ways: * It only exists within the `@rules_python` repo * It does not exist prior to Bazel 7. This requires using a repo rule, `@rules_python_internal`, to do some feature/version detection to generate a shim bzl file so that the `py_internal` symbol is always loadable. Regular rules_python code then loads the shim and can act accordingly. Also fixes some other loading-time issues (beyond simply py_internal being None): * `configuration_field()` args are validated at time of call, so those must be guarded so Bazel 5.4 doesn't fail on them. * The `init` arg of `provider()` isn't supported under Bazel 5.4; change them to no-op stubs behind a guard. * The `|` operator for dicts isn't supported under Bazel 5.4; change to use skylib's `dicts.add` Work towards #1069 --- .bazelignore | 4 + CHANGELOG.md | 5 + MODULE.bazel | 1 + examples/build_file_generation/WORKSPACE | 7 +- gazelle/WORKSPACE | 4 +- internal_setup.bzl | 3 + python/BUILD.bazel | 36 +++- python/extensions/private/internal_deps.bzl | 2 + python/private/BUILD.bazel | 6 +- python/private/common/BUILD.bazel | 191 ++++++++++++++++++ python/private/common/attributes.bzl | 25 ++- python/private/common/cc_helper.bzl | 2 +- python/private/common/common.bzl | 2 +- python/private/common/providers.bzl | 21 +- .../private/common/py_binary_rule_bazel.bzl | 3 +- python/private/common/py_executable.bzl | 9 +- python/private/common/py_executable_bazel.bzl | 8 +- python/private/common/py_internal.bzl | 8 +- python/private/common/py_library.bzl | 3 +- python/private/common/py_runtime_rule.bzl | 5 +- python/private/common/py_test_rule_bazel.bzl | 3 +- python/private/internal_config_repo.bzl | 99 +++++++++ python/py_binary.bzl | 8 +- python/py_info.bzl | 4 +- python/py_library.bzl | 8 +- python/py_runtime.bzl | 8 +- python/py_runtime_info.bzl | 4 +- python/py_test.bzl | 7 +- python/repositories.bzl | 5 + .../workspace_template/WORKSPACE.tmpl | 4 +- tests/ignore_root_user_error/WORKSPACE | 4 +- tools/build_defs/python/private/BUILD.bazel | 14 ++ .../python/private/py_internal_renamed.bzl | 5 + 33 files changed, 471 insertions(+), 47 deletions(-) create mode 100644 python/private/internal_config_repo.bzl diff --git a/.bazelignore b/.bazelignore index 135f709824..025277a60e 100644 --- a/.bazelignore +++ b/.bazelignore @@ -6,6 +6,10 @@ bazel-rules_python bazel-bin bazel-out bazel-testlogs +# Prevent the convenience symlinks within the examples from being +# treated as directories with valid BUILD files for the main repo. examples/bzlmod/bazel-bzlmod +examples/bzlmod/other_module/bazel-other_module examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation +examples/pip_parse/bazel-pip_parse examples/py_proto_library/bazel-py_proto_library diff --git a/CHANGELOG.md b/CHANGELOG.md index e319d28fb5..2609bb2596 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,11 @@ A brief description of the categories of changes: * (multi-version) The `distribs` attribute is no longer propagated. This attribute has been long deprecated by Bazel and shouldn't be used. +* Calling `//python:repositories.bzl#py_repositories()` is required. It has + always been documented as necessary, but it was possible to omit it in certain + cases. An error about `@rules_python_internal` means the `py_repositories()` + call is missing in `WORKSPACE`. + ### Added diff --git a/MODULE.bazel b/MODULE.bazel index aaa5c86912..ab7b597518 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -15,6 +15,7 @@ internal_deps = use_extension("@rules_python//python/extensions/private:internal internal_deps.install() use_repo( internal_deps, + "rules_python_internal", # START: maintained by 'bazel run //tools/private:update_pip_deps' "pypi__build", "pypi__click", diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index fa11380dde..03085d86b5 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -71,8 +71,11 @@ local_repository( path = "../../gazelle", ) -# Next we load the toolchain from rules_python. -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +# Next we load the setup and toolchain from rules_python. +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +# Perform general setup +py_repositories() # We now register a hermetic Python interpreter rather than relying on a system-installed interpreter. # This toolchain will allow bazel to download a specific python version, and use that version diff --git a/gazelle/WORKSPACE b/gazelle/WORKSPACE index b9ba91d9f8..fe7ac3ec53 100644 --- a/gazelle/WORKSPACE +++ b/gazelle/WORKSPACE @@ -34,7 +34,9 @@ local_repository( path = "..", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() python_register_toolchains( name = "python39", diff --git a/internal_setup.bzl b/internal_setup.bzl index c3a7ad452d..0c9d6c48a6 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -20,10 +20,13 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") load("//python/pip_install:repositories.bzl", "pip_install_dependencies") +load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility def rules_python_internal_setup(): """Setup for rules_python tests and tools.""" + internal_config_repo(name = "rules_python_internal") + # Because we don't use the pip_install rule, we have to call this to fetch its deps pip_install_dependencies() diff --git a/python/BUILD.bazel b/python/BUILD.bazel index aa8c8bf3e4..3e0919c4df 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -84,7 +84,11 @@ bzl_library( bzl_library( name = "py_binary_bzl", srcs = ["py_binary.bzl"], - deps = ["//python/private:util_bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/common:py_binary_macro_bazel_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( @@ -101,19 +105,31 @@ bzl_library( bzl_library( name = "py_info_bzl", srcs = ["py_info.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = [ + "//python/private:reexports_bzl", + "//python/private/common:providers_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( name = "py_library_bzl", srcs = ["py_library.bzl"], - deps = ["//python/private:util_bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/common:py_library_macro_bazel_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( name = "py_runtime_bzl", srcs = ["py_runtime.bzl"], - deps = ["//python/private:util_bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/common:py_runtime_macro_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( @@ -125,13 +141,21 @@ bzl_library( bzl_library( name = "py_runtime_info_bzl", srcs = ["py_runtime_info.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = [ + "//python/private:reexports_bzl", + "//python/private/common:providers_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( name = "py_test_bzl", srcs = ["py_test.bzl"], - deps = ["//python/private:util_bzl"], + deps = [ + "//python/private:util_bzl", + "//python/private/common:py_test_macro_bazel_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) # NOTE: Remember to add bzl_library targets to //tests:bzl_libraries diff --git a/python/extensions/private/internal_deps.bzl b/python/extensions/private/internal_deps.bzl index 8a98b82827..aadf2cc997 100644 --- a/python/extensions/private/internal_deps.bzl +++ b/python/extensions/private/internal_deps.bzl @@ -9,9 +9,11 @@ "Python toolchain module extension for internal rule use" load("//python/pip_install:repositories.bzl", "pip_install_dependencies") +load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=unused-variable def _internal_deps_impl(module_ctx): + internal_config_repo(name = "rules_python_internal") pip_install_dependencies() internal_deps = module_extension( diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 48c3f8c73b..5dd7b35f1a 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -22,7 +22,11 @@ licenses(["notice"]) filegroup( name = "distribution", - srcs = glob(["**"]) + ["//python/private/proto:distribution"], + srcs = glob(["**"]) + [ + "//python/private/common:distribution", + "//python/private/proto:distribution", + "//tools/build_defs/python/private:distribution", + ], visibility = ["//python:__pkg__"], ) diff --git a/python/private/common/BUILD.bazel b/python/private/common/BUILD.bazel index aa21042e25..f20e682e26 100644 --- a/python/private/common/BUILD.bazel +++ b/python/private/common/BUILD.bazel @@ -11,3 +11,194 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = ["//python:__subpackages__"], +) + +bzl_library( + name = "attributes_bazel_bzl", + srcs = ["attributes_bazel.bzl"], +) + +bzl_library( + name = "attributes_bzl", + srcs = ["attributes.bzl"], + deps = [ + ":common_bzl", + ":providers_bzl", + ":py_internal_bzl", + ":semantics_bzl", + ], +) + +bzl_library( + name = "cc_helper_bzl", + srcs = ["cc_helper.bzl"], + deps = [":py_internal_bzl"], +) + +bzl_library( + name = "common_bazel_bzl", + srcs = ["common_bazel.bzl"], + deps = [ + ":common_bzl", + ":providers_bzl", + ":py_internal_bzl", + "@bazel_skylib//lib:paths", + ], +) + +bzl_library( + name = "common_bzl", + srcs = ["common.bzl"], + deps = [ + ":cc_helper_bzl", + ":providers_bzl", + ":py_internal_bzl", + ":semantics_bzl", + ], +) + +filegroup( + name = "distribution", + srcs = glob(["**"]), +) + +bzl_library( + name = "providers_bzl", + srcs = ["providers.bzl"], + deps = [ + ":semantics_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], +) + +bzl_library( + name = "py_binary_macro_bazel_bzl", + srcs = ["py_binary_macro_bazel.bzl"], + deps = [ + ":common_bzl", + ":py_binary_rule_bazel_bzl", + ], +) + +bzl_library( + name = "py_binary_rule_bazel_bzl", + srcs = ["py_binary_rule_bazel.bzl"], + deps = [ + ":attributes_bzl", + ":py_executable_bazel_bzl", + ":semantics_bzl", + "@bazel_skylib//lib:dicts", + ], +) + +bzl_library( + name = "py_executable_bazel_bzl", + srcs = ["py_executable_bazel.bzl"], + deps = [ + ":attributes_bazel_bzl", + ":common_bazel_bzl", + ":common_bzl", + ":providers_bzl", + ":py_executable_bzl", + ":py_internal_bzl", + ":semantics_bzl", + ], +) + +bzl_library( + name = "py_executable_bzl", + srcs = ["py_executable.bzl"], + deps = [ + ":attributes_bzl", + ":cc_helper_bzl", + ":common_bzl", + ":providers_bzl", + ":py_internal_bzl", + "@bazel_skylib//lib:dicts", + ], +) + +bzl_library( + name = "py_internal_bzl", + srcs = ["py_internal.bzl"], + deps = ["@rules_python_internal//:py_internal_bzl"], +) + +bzl_library( + name = "py_library_bzl", + srcs = ["py_library.bzl"], + deps = [ + ":attributes_bzl", + ":common_bzl", + ":providers_bzl", + ":py_internal_bzl", + "@bazel_skylib//lib:dicts", + ], +) + +bzl_library( + name = "py_library_macro_bazel_bzl", + srcs = ["py_library_macro_bazel.bzl"], + deps = [":py_library_rule_bazel_bzl"], +) + +bzl_library( + name = "py_library_rule_bazel_bzl", + srcs = ["py_library_rule_bazel.bzl"], + deps = [ + ":attributes_bazel_bzl", + ":common_bazel_bzl", + ":common_bzl", + ":py_library_bzl", + ], +) + +bzl_library( + name = "py_runtime_macro_bzl", + srcs = ["py_runtime_macro.bzl"], + deps = [":py_runtime_rule_bzl"], +) + +bzl_library( + name = "py_runtime_rule_bzl", + srcs = ["py_runtime_rule.bzl"], + deps = [ + ":attributes_bzl", + ":common_bzl", + ":providers_bzl", + ":py_internal_bzl", + "@bazel_skylib//lib:dicts", + "@bazel_skylib//lib:paths", + ], +) + +bzl_library( + name = "py_test_macro_bazel_bzl", + srcs = ["py_test_macro_bazel.bzl"], + deps = [ + ":common_bazel_bzl", + ":py_test_rule_bazel_bzl", + ], +) + +bzl_library( + name = "py_test_rule_bazel_bzl", + srcs = ["py_test_rule_bazel.bzl"], + deps = [ + ":attributes_bzl", + ":common_bzl", + ":py_executable_bazel_bzl", + ":semantics_bzl", + "@bazel_skylib//lib:dicts", + ], +) + +bzl_library( + name = "semantics_bzl", + srcs = ["semantics.bzl"], +) diff --git a/python/private/common/attributes.bzl b/python/private/common/attributes.bzl index ea43ceafb1..6e184c0c8f 100644 --- a/python/private/common/attributes.bzl +++ b/python/private/common/attributes.bzl @@ -26,7 +26,7 @@ load( # TODO: Load CcInfo from rules_cc _CcInfo = CcInfo -_PackageSpecificationInfo = py_internal.PackageSpecificationInfo +_PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None) _STAMP_VALUES = [-1, 0, 1] @@ -85,15 +85,28 @@ DATA_ATTRS = { ), } -NATIVE_RULES_ALLOWLIST_ATTRS = { - "_native_rules_allowlist": attr.label( +def _create_native_rules_allowlist_attrs(): + if py_internal: + # The fragment and name are validated when configuration_field is called default = configuration_field( fragment = "py", name = "native_rules_allowlist", + ) + + # A None provider isn't allowed + providers = [_PackageSpecificationInfo] + else: + default = None + providers = [] + + return { + "_native_rules_allowlist": attr.label( + default = default, + providers = providers, ), - providers = [_PackageSpecificationInfo], - ), -} + } + +NATIVE_RULES_ALLOWLIST_ATTRS = _create_native_rules_allowlist_attrs() # Attributes common to all rules. COMMON_ATTRS = union_attrs( diff --git a/python/private/common/cc_helper.bzl b/python/private/common/cc_helper.bzl index cef1ab169d..552b42eae8 100644 --- a/python/private/common/cc_helper.bzl +++ b/python/private/common/cc_helper.bzl @@ -20,4 +20,4 @@ These may change at any time and are closely coupled to the rule implementation. load(":py_internal.bzl", "py_internal") -cc_helper = py_internal.cc_helper +cc_helper = getattr(py_internal, "cc_helper", None) diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index 8522d80606..bffbf6f0cf 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -27,7 +27,7 @@ _testing = testing _platform_common = platform_common _coverage_common = coverage_common _py_builtins = py_internal -PackageSpecificationInfo = py_internal.PackageSpecificationInfo +PackageSpecificationInfo = getattr(py_internal, "PackageSpecificationInfo", None) TOOLCHAIN_TYPE = "@" + TOOLS_REPO + "//tools/python:toolchain_type" diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl index 237a3e4d20..8a5089d976 100644 --- a/python/private/common/providers.bzl +++ b/python/private/common/providers.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Providers for Python rules.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load(":semantics.bzl", "TOOLS_REPO") # TODO: load CcInfo from rules_cc @@ -23,6 +24,18 @@ DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" DEFAULT_BOOTSTRAP_TEMPLATE = "@" + TOOLS_REPO + "//tools/python:python_bootstrap_template.txt" _PYTHON_VERSION_VALUES = ["PY2", "PY3"] +# Helper to make the provider definitions not crash under Bazel 5.4: +# Bazel 5.4 doesn't support the `init` arg of `provider()`, so we have to +# not pass that when using Bazel 5.4. But, not passing the `init` arg +# changes the return value from a two-tuple to a single value, which then +# breaks Bazel 6+ code. +# This isn't actually used under Bazel 5.4, so just stub out the values +# to get past the loading phase. +def _define_provider(doc, fields, **kwargs): + if not config.enable_pystar: + return provider("Stub, not used", fields = []), None + return provider(doc = doc, fields = fields, **kwargs) + def _PyRuntimeInfo_init( *, interpreter_path = None, @@ -82,7 +95,7 @@ def _PyRuntimeInfo_init( # TODO(#15897): Rename this to PyRuntimeInfo when we're ready to replace the Java # implemented provider with the Starlark one. -PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = provider( +PyRuntimeInfo, _unused_raw_py_runtime_info_ctor = _define_provider( doc = """Contains information about a Python runtime, as returned by the `py_runtime` rule. @@ -169,8 +182,8 @@ def _PyInfo_init( "uses_shared_libraries": uses_shared_libraries, } -PyInfo, _unused_raw_py_info_ctor = provider( - "Encapsulates information provided by the Python rules.", +PyInfo, _unused_raw_py_info_ctor = _define_provider( + doc = "Encapsulates information provided by the Python rules.", init = _PyInfo_init, fields = { "has_py2_only_sources": "Whether any of this target's transitive sources requires a Python 2 runtime.", @@ -200,7 +213,7 @@ def _PyCcLinkParamsProvider_init(cc_info): } # buildifier: disable=name-conventions -PyCcLinkParamsProvider, _unused_raw_py_cc_link_params_provider_ctor = provider( +PyCcLinkParamsProvider, _unused_raw_py_cc_link_params_provider_ctor = _define_provider( doc = ("Python-wrapper to forward CcInfo.linking_context. This is to " + "allow Python targets to propagate C++ linking information, but " + "without the Python target appearing to be a valid C++ rule dependency"), diff --git a/python/private/common/py_binary_rule_bazel.bzl b/python/private/common/py_binary_rule_bazel.bzl index 6c324d8bc5..491d9050da 100644 --- a/python/private/common/py_binary_rule_bazel.bzl +++ b/python/private/common/py_binary_rule_bazel.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Rule implementation of py_binary for Bazel.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_BINARY_ATTRS") load( ":py_executable_bazel.bzl", @@ -43,6 +44,6 @@ def _py_binary_impl(ctx): py_binary = create_executable_rule( implementation = _py_binary_impl, - attrs = AGNOSTIC_BINARY_ATTRS | _PY_TEST_ATTRS, + attrs = dicts.add(AGNOSTIC_BINARY_ATTRS, _PY_TEST_ATTRS), executable = True, ) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 7a50a75c11..98a29bedde 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Common functionality between test/binary executables.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load( ":attributes.bzl", "AGNOSTIC_EXECUTABLE_ATTRS", @@ -452,7 +453,7 @@ def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_da ctx.actions.run( executable = ctx.executable._build_data_gen, - env = { + env = dicts.add({ # NOTE: ctx.info_file is undocumented; see # https://github.com/bazelbuild/bazel/issues/9363 "INFO_FILE": ctx.info_file.path, @@ -460,7 +461,7 @@ def _write_build_data(ctx, central_uncachable_version_file, extra_write_build_da "PLATFORM": cc_helper.find_cpp_toolchain(ctx).toolchain_id, "TARGET": str(ctx.label), "VERSION_FILE": version_file.path, - } | extra_write_build_data_env, + }, extra_write_build_data_env), inputs = depset( direct = direct_inputs, ), @@ -808,8 +809,8 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs): fragments = fragments + ["py"] return rule( # TODO: add ability to remove attrs, i.e. for imports attr - attrs = EXECUTABLE_ATTRS | attrs, - toolchains = [TOOLCHAIN_TYPE] + cc_helper.use_cpp_toolchain(), + attrs = dicts.add(EXECUTABLE_ATTRS, attrs), + toolchains = [TOOLCHAIN_TYPE] + (cc_helper.use_cpp_toolchain() if cc_helper else []), fragments = fragments, **kwargs ) diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index a145d421a6..6c50b75b71 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Implementation for Bazel Python executable.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_skylib//lib:paths.bzl", "paths") load(":attributes_bazel.bzl", "IMPORTS_ATTRS") load( @@ -62,10 +63,13 @@ the `srcs` of Python targets as required. executable = True, ), "_py_interpreter": attr.label( + # The configuration_field args are validated when called; + # we use the precense of py_internal to indicate this Bazel + # build has that fragment and name. default = configuration_field( fragment = "bazel_py", name = "python_top", - ), + ) if py_internal else None, ), # TODO: This appears to be vestigial. It's only added because # GraphlessQueryTest.testLabelsOperator relies on it to test for @@ -88,7 +92,7 @@ the `srcs` of Python targets as required. def create_executable_rule(*, attrs, **kwargs): return create_base_executable_rule( - attrs = BAZEL_EXECUTABLE_ATTRS | attrs, + attrs = dicts.add(BAZEL_EXECUTABLE_ATTRS, attrs), fragments = ["py", "bazel_py"], **kwargs ) diff --git a/python/private/common/py_internal.bzl b/python/private/common/py_internal.bzl index c17bbf0522..429637253f 100644 --- a/python/private/common/py_internal.bzl +++ b/python/private/common/py_internal.bzl @@ -18,7 +18,9 @@ Re-exports the restricted-use py_internal helper under its original name. These may change at any time and are closely coupled to the rule implementation. """ -# buildifier: disable=bzl-visibility -load("//tools/build_defs/python/private:py_internal_renamed.bzl", "py_internal_renamed") +# The py_internal global is only available in Bazel 7+, so loading of it +# must go through a repo rule with Bazel version detection logic. +load("@rules_python_internal//:py_internal.bzl", "py_internal_impl") -py_internal = py_internal_renamed +# NOTE: This is None prior to Bazel 7, as set by @rules_python_internal +py_internal = py_internal_impl diff --git a/python/private/common/py_library.bzl b/python/private/common/py_library.bzl index ca71e72443..8d09c51092 100644 --- a/python/private/common/py_library.bzl +++ b/python/private/common/py_library.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Implementation of py_library rule.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load( ":attributes.bzl", "COMMON_ATTRS", @@ -92,7 +93,7 @@ def create_py_library_rule(*, attrs = {}, **kwargs): A rule object """ return rule( - attrs = LIBRARY_ATTRS | attrs, + attrs = dicts.add(LIBRARY_ATTRS, attrs), # TODO(b/253818097): fragments=py is only necessary so that # RequiredConfigFragmentsTest passes fragments = ["py"], diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index 4bffb876c9..39434042ea 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Implementation of py_runtime rule.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_skylib//lib:paths.bzl", "paths") load(":attributes.bzl", "NATIVE_RULES_ALLOWLIST_ATTRS") load(":common.bzl", "check_native_allowed") @@ -128,7 +129,7 @@ py_runtime( ``` """, fragments = ["py"], - attrs = NATIVE_RULES_ALLOWLIST_ATTRS | { + attrs = dicts.add(NATIVE_RULES_ALLOWLIST_ATTRS, { "bootstrap_template": attr.label( allow_single_file = True, default = DEFAULT_BOOTSTRAP_TEMPLATE, @@ -211,5 +212,5 @@ motivation. Does not apply to Windows. """, ), - }, + }), ) diff --git a/python/private/common/py_test_rule_bazel.bzl b/python/private/common/py_test_rule_bazel.bzl index de1aa4581c..348935edee 100644 --- a/python/private/common/py_test_rule_bazel.bzl +++ b/python/private/common/py_test_rule_bazel.bzl @@ -13,6 +13,7 @@ # limitations under the License. """Rule implementation of py_test for Bazel.""" +load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_TEST_ATTRS") load(":common.bzl", "maybe_add_test_execution_info") load( @@ -50,6 +51,6 @@ def _py_test_impl(ctx): py_test = create_executable_rule( implementation = _py_test_impl, - attrs = AGNOSTIC_TEST_ATTRS | _BAZEL_PY_TEST_ATTRS, + attrs = dicts.add(AGNOSTIC_TEST_ATTRS, _BAZEL_PY_TEST_ATTRS), test = True, ) diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl new file mode 100644 index 0000000000..cfc7616de9 --- /dev/null +++ b/python/private/internal_config_repo.bzl @@ -0,0 +1,99 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Repository to generate configuration settings info from the environment. + +This handles settings that can't be encoded as regular build configuration flags, +such as globals available to Bazel versions, or propagating user environment +settings for rules to later use. +""" + +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") + +_ENABLE_PYSTAR_ENVVAR_NAME = "RULES_PYTHON_ENABLE_PYSTAR" +_ENABLE_PYSTAR_DEFAULT = "0" + +_CONFIG_TEMPLATE = """\ +config = struct( + enable_pystar = {enable_pystar}, +) +""" + +# The py_internal symbol is only accessible from within @rules_python, so we have to +# load it from there and re-export it so that rules_python can later load it. +_PY_INTERNAL_SHIM = """\ +load("@rules_python//tools/build_defs/python/private:py_internal_renamed.bzl", "py_internal_renamed") +py_internal_impl = py_internal_renamed +""" + +ROOT_BUILD_TEMPLATE = """\ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = [ + "{visibility}", + ] +) + +bzl_library( + name = "rules_python_config_bzl", + srcs = ["rules_python_config.bzl"] +) + +bzl_library( + name = "py_internal_bzl", + srcs = ["py_internal.bzl"], + deps = [{py_internal_dep}], +) +""" + +def _internal_config_repo_impl(rctx): + enable_pystar = _bool_from_environ(rctx, _ENABLE_PYSTAR_ENVVAR_NAME, _ENABLE_PYSTAR_DEFAULT) + rctx.file("rules_python_config.bzl", _CONFIG_TEMPLATE.format( + enable_pystar = enable_pystar, + )) + + if enable_pystar or ( + # Bazel 7+ (dev and later) has native.starlark_doc_extract, and thus the py_internal global + hasattr(native, "starlark_doc_extract") and + # The logic to allow the symbol doesn't work properly under bzlmod, + # even if the symbol is otherwise functional. + not BZLMOD_ENABLED + ): + shim_content = _PY_INTERNAL_SHIM + py_internal_dep = '"@rules_python//tools/build_defs/python/private:py_internal_renamed_bzl"' + else: + shim_content = "py_internal_impl = None\n" + py_internal_dep = "" + + # Bazel 5 doesn't support repository visibility, so just use public + # as a stand-in + if native.bazel_version.startswith("5."): + visibility = "//visibility:public" + else: + visibility = "@rules_python//:__subpackages__" + + rctx.file("BUILD", ROOT_BUILD_TEMPLATE.format( + py_internal_dep = py_internal_dep, + visibility = visibility, + )) + rctx.file("py_internal.bzl", shim_content) + return None + +internal_config_repo = repository_rule( + implementation = _internal_config_repo_impl, + environ = [_ENABLE_PYSTAR_ENVVAR_NAME], +) + +def _bool_from_environ(rctx, key, default): + return bool(int(rctx.os.environ.get(key, default))) diff --git a/python/py_binary.bzl b/python/py_binary.bzl index 6b6f7e0f8a..6dcb4ad40c 100644 --- a/python/py_binary.bzl +++ b/python/py_binary.bzl @@ -14,7 +14,12 @@ """Public entry point for py_binary.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:util.bzl", "add_migration_tag") +load("//python/private/common:py_binary_macro_bazel.bzl", _starlark_py_binary = "py_binary") + +# buildifier: disable=native-python +_py_binary_impl = _starlark_py_binary if config.enable_pystar else native.py_binary def py_binary(**attrs): """See the Bazel core [py_binary](https://docs.bazel.build/versions/master/be/python.html#py_binary) documentation. @@ -27,5 +32,4 @@ def py_binary(**attrs): if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - # buildifier: disable=native-python - native.py_binary(**add_migration_tag(attrs)) + _py_binary_impl(**add_migration_tag(attrs)) diff --git a/python/py_info.bzl b/python/py_info.bzl index 2c3997dee2..cbf145d07d 100644 --- a/python/py_info.bzl +++ b/python/py_info.bzl @@ -14,6 +14,8 @@ """Public entry point for PyInfo.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:reexports.bzl", "internal_PyInfo") +load("//python/private/common:providers.bzl", _starlark_PyInfo = "PyInfo") -PyInfo = internal_PyInfo +PyInfo = _starlark_PyInfo if config.enable_pystar else internal_PyInfo diff --git a/python/py_library.bzl b/python/py_library.bzl index d54cbb2958..ef4c3c3969 100644 --- a/python/py_library.bzl +++ b/python/py_library.bzl @@ -14,7 +14,12 @@ """Public entry point for py_library.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:util.bzl", "add_migration_tag") +load("//python/private/common:py_library_macro_bazel.bzl", _starlark_py_library = "py_library") + +# buildifier: disable=native-python +_py_library_impl = _starlark_py_library if config.enable_pystar else native.py_library def py_library(**attrs): """See the Bazel core [py_library](https://docs.bazel.build/versions/master/be/python.html#py_library) documentation. @@ -25,5 +30,4 @@ def py_library(**attrs): if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - # buildifier: disable=native-python - native.py_library(**add_migration_tag(attrs)) + _py_library_impl(**add_migration_tag(attrs)) diff --git a/python/py_runtime.bzl b/python/py_runtime.bzl index b70f9d4ec4..ac8b090c94 100644 --- a/python/py_runtime.bzl +++ b/python/py_runtime.bzl @@ -14,7 +14,12 @@ """Public entry point for py_runtime.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:util.bzl", "add_migration_tag") +load("//python/private/common:py_runtime_macro.bzl", _starlark_py_runtime = "py_runtime") + +# buildifier: disable=native-python +_py_runtime_impl = _starlark_py_runtime if config.enable_pystar else native.py_runtime def py_runtime(**attrs): """See the Bazel core [py_runtime](https://docs.bazel.build/versions/master/be/python.html#py_runtime) documentation. @@ -25,5 +30,4 @@ def py_runtime(**attrs): if attrs.get("python_version") == "PY2": fail("Python 2 is no longer supported: see https://github.com/bazelbuild/rules_python/issues/886") - # buildifier: disable=native-python - native.py_runtime(**add_migration_tag(attrs)) + _py_runtime_impl(**add_migration_tag(attrs)) diff --git a/python/py_runtime_info.bzl b/python/py_runtime_info.bzl index 15598ee903..699b31d6df 100644 --- a/python/py_runtime_info.bzl +++ b/python/py_runtime_info.bzl @@ -14,6 +14,8 @@ """Public entry point for PyRuntimeInfo.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:reexports.bzl", "internal_PyRuntimeInfo") +load("//python/private/common:providers.bzl", _starlark_PyRuntimeInfo = "PyRuntimeInfo") -PyRuntimeInfo = internal_PyRuntimeInfo +PyRuntimeInfo = _starlark_PyRuntimeInfo if config.enable_pystar else internal_PyRuntimeInfo diff --git a/python/py_test.bzl b/python/py_test.bzl index 09580c01c4..ad9bdc06ad 100644 --- a/python/py_test.bzl +++ b/python/py_test.bzl @@ -14,7 +14,12 @@ """Public entry point for py_test.""" +load("@rules_python_internal//:rules_python_config.bzl", "config") load("//python/private:util.bzl", "add_migration_tag") +load("//python/private/common:py_test_macro_bazel.bzl", _starlark_py_test = "py_test") + +# buildifier: disable=native-python +_py_test_impl = _starlark_py_test if config.enable_pystar else native.py_test def py_test(**attrs): """See the Bazel core [py_test](https://docs.bazel.build/versions/master/be/python.html#py_test) documentation. @@ -28,4 +33,4 @@ def py_test(**attrs): fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") # buildifier: disable=native-python - native.py_test(**add_migration_tag(attrs)) + _py_test_impl(**add_migration_tag(attrs)) diff --git a/python/repositories.bzl b/python/repositories.bzl index ea4a9275db..9b3926ac15 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -21,6 +21,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archi load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe", "read_netrc", "read_user_netrc", "use_netrc") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") +load("//python/private:internal_config_repo.bzl", "internal_config_repo") load( "//python/private:toolchains_repo.bzl", "multi_toolchain_aliases", @@ -46,6 +47,10 @@ def py_repositories(): This function should be loaded and called in the user's WORKSPACE. With bzlmod enabled, this function is not needed since MODULE.bazel handles transitive deps. """ + maybe( + internal_config_repo, + name = "rules_python_internal", + ) http_archive( name = "bazel_skylib", sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", diff --git a/python/tests/toolchains/workspace_template/WORKSPACE.tmpl b/python/tests/toolchains/workspace_template/WORKSPACE.tmpl index 973e020c1e..3335f4b063 100644 --- a/python/tests/toolchains/workspace_template/WORKSPACE.tmpl +++ b/python/tests/toolchains/workspace_template/WORKSPACE.tmpl @@ -19,7 +19,9 @@ local_repository( path = "", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load("@rules_python//python:repositories.bzl", "python_register_toolchains", "py_repositories") + +py_repositories() python_register_toolchains( name = "python", diff --git a/tests/ignore_root_user_error/WORKSPACE b/tests/ignore_root_user_error/WORKSPACE index e0528e4047..d1249feab2 100644 --- a/tests/ignore_root_user_error/WORKSPACE +++ b/tests/ignore_root_user_error/WORKSPACE @@ -3,7 +3,9 @@ local_repository( path = "../..", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() python_register_toolchains( name = "python39", diff --git a/tools/build_defs/python/private/BUILD.bazel b/tools/build_defs/python/private/BUILD.bazel index aa21042e25..0a7f308f02 100644 --- a/tools/build_defs/python/private/BUILD.bazel +++ b/tools/build_defs/python/private/BUILD.bazel @@ -11,3 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python:__subpackages__"], +) + +bzl_library( + name = "py_internal_renamed_bzl", + srcs = ["py_internal_renamed.bzl"], + visibility = ["@rules_python_internal//:__subpackages__"], +) diff --git a/tools/build_defs/python/private/py_internal_renamed.bzl b/tools/build_defs/python/private/py_internal_renamed.bzl index abab31c45e..a12fc2d14e 100644 --- a/tools/build_defs/python/private/py_internal_renamed.bzl +++ b/tools/build_defs/python/private/py_internal_renamed.bzl @@ -13,6 +13,10 @@ # limitations under the License. """PYTHON RULE IMPLEMENTATION ONLY: Do not use outside of the rule implementations and their tests. +NOTE: This file is only loaded by @rules_python_internal//:py_internal.bzl. This +is because the `py_internal` global symbol is only present in Bazel 7+, so +a repo rule has to conditionally load this depending on the Bazel version. + Re-exports the restricted-use py_internal helper under another name. This is necessary because `py_internal = py_internal` results in an error (trying to bind a local symbol to itself before its defined). @@ -22,4 +26,5 @@ the internal helpers only rules_python is allowed to use. These may change at any time and are closely coupled to the rule implementation. """ + py_internal_renamed = py_internal From 48daf52bdf85d46c529a873cc1f51e08672fd6d7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:21:15 +0900 Subject: [PATCH 21/41] fix(gazelle): runfiles discovery (#1429) Pass the environment generated by the `runfiles` helper so that the Python launcher can find the runfiles correctly as implemented in bazelbuild/bazel#14740. Fixes #1426 --- CHANGELOG.md | 2 ++ gazelle/python/parser.go | 9 ++++++++- gazelle/python/std_modules.go | 11 +++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2609bb2596..82717d394d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,8 @@ A brief description of the categories of changes: dependencies improving initial build times involving external dependency fetching. +* (gazelle) Improve runfiles lookup hermeticity. + ## [0.25.0] - 2023-08-22 ### Changed diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index c45aef139a..60a3c24269 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -38,13 +38,20 @@ var ( ) func startParserProcess(ctx context.Context) { - parseScriptRunfile, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/parse") + rfiles, err := runfiles.New() + if err != nil { + log.Printf("failed to create a runfiles object: %v\n", err) + os.Exit(1) + } + + parseScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/parse") if err != nil { log.Printf("failed to initialize parser: %v\n", err) os.Exit(1) } cmd := exec.CommandContext(ctx, parseScriptRunfile) + cmd.Env = append(os.Environ(), rfiles.Env()...) cmd.Stderr = os.Stderr diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index 15ef766ff2..a87deec366 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go @@ -39,7 +39,13 @@ var ( func startStdModuleProcess(ctx context.Context) { stdModulesSeen = make(map[string]struct{}) - stdModulesScriptRunfile, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/std_modules") + rfiles, err := runfiles.New() + if err != nil { + log.Printf("failed to create a runfiles object: %v\n", err) + os.Exit(1) + } + + stdModulesScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/std_modules") if err != nil { log.Printf("failed to initialize std_modules: %v\n", err) os.Exit(1) @@ -49,7 +55,8 @@ func startStdModuleProcess(ctx context.Context) { cmd.Stderr = os.Stderr // All userland site-packages should be ignored. - cmd.Env = []string{"PYTHONNOUSERSITE=1"} + cmd.Env = append([]string{"PYTHONNOUSERSITE=1"}, rfiles.Env()...) + stdin, err := cmd.StdinPipe() if err != nil { log.Printf("failed to initialize std_modules: %v\n", err) From 49b21ced3f1cb97199d8ba269583a19f85c5acc0 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 26 Sep 2023 15:29:59 -0700 Subject: [PATCH 22/41] feat, refactor(pystar): bzl_library for packaging.bzl; fix pystar doc building and py_wheel (#1432) Changed `py_wheel` to load `py_binary` instead of using `native.py_binary`. This caused the doc generation to fail because of the additional loads(), so the doc libraries were refactored to represent the additional loads. This adds `//python:packaging_bzl` as a public target. Docs were failing to build because Stardoc wasn't able to process the value `cc_helper.use_cpp_toolchains()` returned. For some reason, under Stardoc, the value is some mocked out value that it can't handle. This is easily fixed by just using the regular way for referencing an optional toolchain; the labels the two use are the same. Work towards #1069 --- .bazelignore | 3 +++ CHANGELOG.md | 3 ++- docs/BUILD.bazel | 16 +--------------- python/BUILD.bazel | 12 ++++++++++++ python/packaging.bzl | 3 ++- python/private/BUILD.bazel | 24 +++++++++++++++++++++++- python/private/common/py_executable.bzl | 9 ++++++++- 7 files changed, 51 insertions(+), 19 deletions(-) diff --git a/.bazelignore b/.bazelignore index 025277a60e..564eb06195 100644 --- a/.bazelignore +++ b/.bazelignore @@ -8,8 +8,11 @@ bazel-out bazel-testlogs # Prevent the convenience symlinks within the examples from being # treated as directories with valid BUILD files for the main repo. +# Any directory with a WORKSPACE in it should be added here, with +# an entry like `bazel-{workspacename}` examples/bzlmod/bazel-bzlmod examples/bzlmod/other_module/bazel-other_module examples/bzlmod_build_file_generation/bazel-bzlmod_build_file_generation examples/pip_parse/bazel-pip_parse examples/py_proto_library/bazel-py_proto_library +tests/ignore_root_user_error/bazel-ignore_root_user_error diff --git a/CHANGELOG.md b/CHANGELOG.md index 82717d394d..0e1bf1faa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,9 +48,10 @@ A brief description of the categories of changes: https://github.com/indygreg/python-build-standalone/releases/tag/20230826. * (gazelle) New `# gazelle:python_generation_mode file` directive to support generating one `py_library` per file. - * (python_repository) Support `netrc` and `auth_patterns` attributes to enable authentication against private HTTP hosts serving Python toolchain binaries. +* `//python:packaging_bzl` added, a `bzl_library` for the Starlark + files `//python:packaging.bzl` requires. ### Removed diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 3a222ab8d2..6ddf54aeba 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -74,20 +74,6 @@ bzl_library( ], ) -bzl_library( - name = "packaging_bzl", - srcs = [ - "//python:packaging.bzl", - "//python/private:py_package.bzl", - "//python/private:py_wheel.bzl", - "//python/private:stamp.bzl", - "//python/private:util.bzl", - ], - deps = [ - "//python/private:util_bzl", - ], -) - # TODO: Stardoc does not guarantee consistent outputs accross platforms (Unix/Windows). # As a result we do not build or test docs on Windows. _NOT_WINDOWS = select({ @@ -144,7 +130,7 @@ stardoc( out = "packaging.md_", input = "//python:packaging.bzl", target_compatible_with = _NOT_WINDOWS, - deps = [":packaging_bzl"], + deps = ["//python:packaging_bzl"], ) stardoc( diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 3e0919c4df..f9c93e5539 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -70,6 +70,18 @@ bzl_library( ], ) +bzl_library( + name = "packaging_bzl", + srcs = ["packaging.bzl"], + deps = [ + ":py_binary_bzl", + "//python/private:py_package.bzl", + "//python/private:py_wheel_bzl", + "//python/private:stamp_bzl", + "//python/private:util_bzl", + ], +) + bzl_library( name = "proto_bzl", srcs = [ diff --git a/python/packaging.bzl b/python/packaging.bzl index d9b9d02711..48423e307f 100644 --- a/python/packaging.bzl +++ b/python/packaging.bzl @@ -14,6 +14,7 @@ """Public API for for building wheels.""" +load("//python:py_binary.bzl", "py_binary") load("//python/private:py_package.bzl", "py_package_lib") load("//python/private:py_wheel.bzl", _PyWheelInfo = "PyWheelInfo", _py_wheel = "py_wheel") load("//python/private:util.bzl", "copy_propagating_kwargs") @@ -167,7 +168,7 @@ def py_wheel(name, twine = None, publish_args = [], **kwargs): # TODO: use py_binary from //python:defs.bzl after our stardoc setup is less brittle # buildifier: disable=native-py - native.py_binary( + py_binary( name = "{}.publish".format(name), srcs = [twine_main], args = twine_args, diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 5dd7b35f1a..af121cf66d 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -106,6 +106,28 @@ bzl_library( ], ) +bzl_library( + name = "py_package_bzl", + srcs = ["py_package.bzl"], + visibility = ["//:__subpackages__"], +) + +bzl_library( + name = "py_wheel_bzl", + srcs = ["py_wheel.bzl"], + visibility = ["//:__subpackages__"], + deps = [ + ":py_package_bzl", + ":stamp_bzl", + ], +) + +bzl_library( + name = "stamp_bzl", + srcs = ["stamp.bzl"], + visibility = ["//:__subpackages__"], +) + # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", @@ -130,7 +152,7 @@ exports_files( "util.bzl", "py_cc_toolchain_rule.bzl", ], - visibility = ["//docs:__pkg__"], + visibility = ["//:__subpackages__"], ) # Used to determine the use of `--stamp` in Starlark rules diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 98a29bedde..1782f8db7f 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -49,6 +49,7 @@ load( "BUILD_DATA_SYMLINK_PATH", "IS_BAZEL", "PY_RUNTIME_ATTR_NAME", + "TOOLS_REPO", ) # TODO: Load cc_common from rules_cc @@ -56,6 +57,12 @@ _cc_common = cc_common _py_builtins = py_internal +# Bazel 5.4 doesn't have config_common.toolchain_type +_CC_TOOLCHAINS = [config_common.toolchain_type( + "@" + TOOLS_REPO + "//tools/cpp:toolchain_type", + mandatory = False, +)] if hasattr(config_common, "toolchain_type") else [] + # Non-Google-specific attributes for executables # These attributes are for rules that accept Python sources. EXECUTABLE_ATTRS = union_attrs( @@ -810,7 +817,7 @@ def create_base_executable_rule(*, attrs, fragments = [], **kwargs): return rule( # TODO: add ability to remove attrs, i.e. for imports attr attrs = dicts.add(EXECUTABLE_ATTRS, attrs), - toolchains = [TOOLCHAIN_TYPE] + (cc_helper.use_cpp_toolchain() if cc_helper else []), + toolchains = [TOOLCHAIN_TYPE] + _CC_TOOLCHAINS, fragments = fragments, **kwargs ) From 6726875822ab883fa128818a053a5a41ded4d577 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:50:35 +0900 Subject: [PATCH 23/41] refactor(toolchain): use a helper method to convert an X.Y version to X.Y.Z (#1423) Before this PR in numerous places we would check the MINOR_MAPPING dict. This PR adds a function that also fails if the X.Y format is not in the MINOR_MAPPING dict making the code more robust. Split from #1340 to unblock #1364. --------- Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/extensions/private/pythons_hub.bzl | 11 ++---- python/pip.bzl | 4 +-- python/private/full_version.bzl | 43 +++++++++++++++++++++++ python/repositories.bzl | 5 ++- 4 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 python/private/full_version.bzl diff --git a/python/extensions/private/pythons_hub.bzl b/python/extensions/private/pythons_hub.bzl index a64f203bd6..f36ce45521 100644 --- a/python/extensions/private/pythons_hub.bzl +++ b/python/extensions/private/pythons_hub.bzl @@ -14,7 +14,8 @@ "Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels" -load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME") +load("//python:versions.bzl", "WINDOWS_NAME") +load("//python/private:full_version.bzl", "full_version") load( "//python/private:toolchains_repo.bzl", "get_host_os_arch", @@ -28,12 +29,6 @@ def _have_same_length(*lists): fail("expected at least one list") return len({len(length): None for length in lists}) == 1 -def _get_version(python_version): - # we need to get the MINOR_MAPPING or use the full version - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] - return python_version - def _python_toolchain_build_file_content( prefixes, python_versions, @@ -55,7 +50,7 @@ def _python_toolchain_build_file_content( # build the toolchain content by calling python_toolchain_build_file_content return "\n".join([python_toolchain_build_file_content( prefix = prefixes[i], - python_version = _get_version(python_versions[i]), + python_version = full_version(python_versions[i]), set_python_version_constraint = set_python_version_constraints[i], user_repository_name = user_repository_names[i], rules_python = rules_python, diff --git a/python/pip.bzl b/python/pip.bzl index 0c6e90f577..fb842cc4ce 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -17,8 +17,8 @@ load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annot load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") +load("//python/private:full_version.bzl", "full_version") load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE") -load(":versions.bzl", "MINOR_MAPPING") compile_pip_requirements = _compile_pip_requirements package_annotation = _package_annotation @@ -295,7 +295,7 @@ alias( for [python_version, repo_prefix] in version_map: alias.append("""\ "@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format( - full_python_version = MINOR_MAPPING[python_version] if python_version in MINOR_MAPPING else python_version, + full_python_version = full_version(python_version), actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format( repo_prefix = repo_prefix, wheel_name = wheel_name, diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl new file mode 100644 index 0000000000..68c969416e --- /dev/null +++ b/python/private/full_version.bzl @@ -0,0 +1,43 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A small helper to ensure that we are working with full versions.""" + +load("//python:versions.bzl", "MINOR_MAPPING") + +def full_version(version): + """Return a full version. + + Args: + version: the version in `X.Y` or `X.Y.Z` format. + + Returns: + a full version given the version string. If the string is already a + major version then we return it as is. + """ + if version in MINOR_MAPPING: + return MINOR_MAPPING[version] + + parts = version.split(".") + if len(parts) == 3: + return version + elif len(parts) == 2: + fail( + "Unknown Python version '{}', available values are: {}".format( + version, + ",".join(MINOR_MAPPING.keys()), + ), + ) + else: + fail("Unknown version format: {}".format(version)) diff --git a/python/repositories.bzl b/python/repositories.bzl index 9b3926ac15..050ba14a76 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -21,6 +21,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archi load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe", "read_netrc", "read_user_netrc", "use_netrc") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") load("//python/private:coverage_deps.bzl", "coverage_dep") +load("//python/private:full_version.bzl", "full_version") load("//python/private:internal_config_repo.bzl", "internal_config_repo") load( "//python/private:toolchains_repo.bzl", @@ -32,7 +33,6 @@ load("//python/private:which.bzl", "which_with_fail") load( ":versions.bzl", "DEFAULT_RELEASE_BASE_URL", - "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS", "get_release_info", @@ -534,8 +534,7 @@ def python_register_toolchains( base_url = kwargs.pop("base_url", DEFAULT_RELEASE_BASE_URL) - if python_version in MINOR_MAPPING: - python_version = MINOR_MAPPING[python_version] + python_version = full_version(python_version) toolchain_repo_name = "{name}_toolchains".format(name = name) From f4b62fc61126b827896d041f5617d942c4e81152 Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Tue, 26 Sep 2023 21:53:36 -0700 Subject: [PATCH 24/41] pycross: Rename `pycross_wheel_library` and make it work (#1413) This patch changes the name of the rule to reflect the fact that it's not exactly the same as the one in rules_pycross. I also took this opportunity to delete the superfluous `namespace_pkgs.py` library (plus test) since we have a nearly identical version already in the main repo. I added a test to validate that the rule functions at a basic level. References: #1360 --- WORKSPACE | 13 +- .../tools/wheel_installer/BUILD.bazel | 1 + tests/pycross/BUILD.bazel | 34 ++++ tests/pycross/py_wheel_library_test.py | 48 +++++ .../rules_pycross/pycross/private/BUILD.bazel | 14 ++ .../pycross/private/providers.bzl | 6 +- .../pycross/private/tools/BUILD.bazel | 33 +--- .../pycross/private/tools/namespace_pkgs.py | 109 ----------- .../private/tools/namespace_pkgs_test.py | 179 ------------------ .../pycross/private/tools/wheel_installer.py | 28 ++- .../pycross/private/wheel_library.bzl | 20 +- 11 files changed, 147 insertions(+), 338 deletions(-) create mode 100644 tests/pycross/BUILD.bazel create mode 100644 tests/pycross/py_wheel_library_test.py create mode 100644 third_party/rules_pycross/pycross/private/BUILD.bazel delete mode 100644 third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py delete mode 100644 third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py diff --git a/WORKSPACE b/WORKSPACE index 94123677ff..6e1e5fc620 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -34,7 +34,7 @@ python_register_multi_toolchains( python_versions = MINOR_MAPPING.values(), ) -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") # Used for Bazel CI http_archive( @@ -86,3 +86,14 @@ pip_parse( load("@publish_deps//:requirements.bzl", "install_deps") install_deps() + +# This wheel is purely here to validate the wheel extraction code. It's not +# intended for anything else. +http_file( + name = "wheel_for_testing", + downloaded_file_path = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + sha256 = "0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2", + urls = [ + "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + ], +) diff --git a/python/pip_install/tools/wheel_installer/BUILD.bazel b/python/pip_install/tools/wheel_installer/BUILD.bazel index 6360ca5c70..0eadcc25f6 100644 --- a/python/pip_install/tools/wheel_installer/BUILD.bazel +++ b/python/pip_install/tools/wheel_installer/BUILD.bazel @@ -9,6 +9,7 @@ py_library( "wheel.py", "wheel_installer.py", ], + visibility = ["//third_party/rules_pycross/pycross/private:__subpackages__"], deps = [ requirement("installer"), requirement("pip"), diff --git a/tests/pycross/BUILD.bazel b/tests/pycross/BUILD.bazel new file mode 100644 index 0000000000..4f01272b7c --- /dev/null +++ b/tests/pycross/BUILD.bazel @@ -0,0 +1,34 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//python:defs.bzl", "py_test") +load("//third_party/rules_pycross/pycross/private:wheel_library.bzl", "py_wheel_library") # buildifier: disable=bzl-visibility + +py_wheel_library( + name = "extracted_wheel_for_testing", + wheel = "@wheel_for_testing//file", +) + +py_test( + name = "py_wheel_library_test", + srcs = [ + "py_wheel_library_test.py", + ], + data = [ + ":extracted_wheel_for_testing", + ], + deps = [ + "//python/runfiles", + ], +) diff --git a/tests/pycross/py_wheel_library_test.py b/tests/pycross/py_wheel_library_test.py new file mode 100644 index 0000000000..fa8e20e563 --- /dev/null +++ b/tests/pycross/py_wheel_library_test.py @@ -0,0 +1,48 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from pathlib import Path + +from python.runfiles import runfiles + +RUNFILES = runfiles.Create() + + +class TestPyWheelLibrary(unittest.TestCase): + def setUp(self): + self.extraction_dir = Path( + RUNFILES.Rlocation( + "rules_python/tests/pycross/extracted_wheel_for_testing" + ) + ) + self.assertTrue(self.extraction_dir.exists(), self.extraction_dir) + self.assertTrue(self.extraction_dir.is_dir(), self.extraction_dir) + + def test_file_presence(self): + """Validate that the basic file layout looks good.""" + for path in ( + "bin/f2py", + "site-packages/numpy.libs/libgfortran-daac5196.so.5.0.0", + "site-packages/numpy/dtypes.py", + "site-packages/numpy/core/_umath_tests.cpython-311-aarch64-linux-gnu.so", + ): + print(self.extraction_dir / path) + self.assertTrue( + (self.extraction_dir / path).exists(), f"{path} does not exist" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/third_party/rules_pycross/pycross/private/BUILD.bazel b/third_party/rules_pycross/pycross/private/BUILD.bazel new file mode 100644 index 0000000000..f59b087027 --- /dev/null +++ b/third_party/rules_pycross/pycross/private/BUILD.bazel @@ -0,0 +1,14 @@ +# Copyright 2023 Jeremy Volkman. All rights reserved. +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/third_party/rules_pycross/pycross/private/providers.bzl b/third_party/rules_pycross/pycross/private/providers.bzl index f55e98693a..47fc9f7271 100644 --- a/third_party/rules_pycross/pycross/private/providers.bzl +++ b/third_party/rules_pycross/pycross/private/providers.bzl @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Pycross providers.""" +"""Python providers.""" -PycrossWheelInfo = provider( +PyWheelInfo = provider( doc = "Information about a Python wheel.", fields = { "name_file": "File: A file containing the canonical name of the wheel.", @@ -23,7 +23,7 @@ PycrossWheelInfo = provider( }, ) -PycrossTargetEnvironmentInfo = provider( +PyTargetEnvironmentInfo = provider( doc = "A target environment description.", fields = { "file": "The JSON file containing target environment information.", diff --git a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel index 867b771aae..a87e6aa67e 100644 --- a/third_party/rules_pycross/pycross/private/tools/BUILD.bazel +++ b/third_party/rules_pycross/pycross/private/tools/BUILD.bazel @@ -13,41 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") - -package(default_visibility = ["//visibility:private"]) - -py_library( - name = "namespace_pkgs", - srcs = [ - "namespace_pkgs.py", - ], -) - -py_test( - name = "namespace_pkgs_test", - size = "small", - srcs = [ - "namespace_pkgs_test.py", - ], - tags = [ - "unit", - # TODO(philsc): Make this work. - "manual", - ], - deps = [ - ":namespace_pkgs", - ], -) +load("//python:defs.bzl", "py_binary") py_binary( name = "wheel_installer", srcs = ["wheel_installer.py"], visibility = ["//visibility:public"], deps = [ - ":namespace_pkgs", - # TODO(philsc): Make this work with what's available in rules_python. - #"@rules_pycross_pypi_deps_absl_py//:pkg", - #"@rules_pycross_pypi_deps_installer//:pkg", + "//python/pip_install/tools/wheel_installer:lib", + "@pypi__installer//:lib", ], ) diff --git a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py deleted file mode 100644 index 59300ffcd1..0000000000 --- a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Utility functions to discover python package types""" -import os -import textwrap -from pathlib import Path # supported in >= 3.4 -from typing import List -from typing import Optional -from typing import Set - - -def implicit_namespace_packages( - directory: str, ignored_dirnames: Optional[List[str]] = None -) -> Set[Path]: - """Discovers namespace packages implemented using the 'native namespace packages' method. - - AKA 'implicit namespace packages', which has been supported since Python 3.3. - See: https://packaging.python.org/guides/packaging-namespace-packages/#native-namespace-packages - - Args: - directory: The root directory to recursively find packages in. - ignored_dirnames: A list of directories to exclude from the search - - Returns: - The set of directories found under root to be packages using the native namespace method. - """ - namespace_pkg_dirs: Set[Path] = set() - standard_pkg_dirs: Set[Path] = set() - directory_path = Path(directory) - ignored_dirname_paths: List[Path] = [Path(p) for p in ignored_dirnames or ()] - # Traverse bottom-up because a directory can be a namespace pkg because its child contains module files. - for dirpath, dirnames, filenames in map( - lambda t: (Path(t[0]), *t[1:]), os.walk(directory_path, topdown=False) - ): - if "__init__.py" in filenames: - standard_pkg_dirs.add(dirpath) - continue - elif ignored_dirname_paths: - is_ignored_dir = dirpath in ignored_dirname_paths - child_of_ignored_dir = any( - d in dirpath.parents for d in ignored_dirname_paths - ) - if is_ignored_dir or child_of_ignored_dir: - continue - - dir_includes_py_modules = _includes_python_modules(filenames) - parent_of_namespace_pkg = any( - Path(dirpath, d) in namespace_pkg_dirs for d in dirnames - ) - parent_of_standard_pkg = any( - Path(dirpath, d) in standard_pkg_dirs for d in dirnames - ) - parent_of_pkg = parent_of_namespace_pkg or parent_of_standard_pkg - if ( - (dir_includes_py_modules or parent_of_pkg) - and - # The root of the directory should never be an implicit namespace - dirpath != directory_path - ): - namespace_pkg_dirs.add(dirpath) - return namespace_pkg_dirs - - -def add_pkgutil_style_namespace_pkg_init(dir_path: Path) -> None: - """Adds 'pkgutil-style namespace packages' init file to the given directory - - See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages - - Args: - dir_path: The directory to create an __init__.py for. - - Raises: - ValueError: If the directory already contains an __init__.py file - """ - ns_pkg_init_filepath = os.path.join(dir_path, "__init__.py") - - if os.path.isfile(ns_pkg_init_filepath): - raise ValueError("%s already contains an __init__.py file." % dir_path) - - with open(ns_pkg_init_filepath, "w") as ns_pkg_init_f: - # See https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages - ns_pkg_init_f.write( - textwrap.dedent( - """\ - # __path__ manipulation added by bazelbuild/rules_python to support namespace pkgs. - __path__ = __import__('pkgutil').extend_path(__path__, __name__) - """ - ) - ) - - -def _includes_python_modules(files: List[str]) -> bool: - """ - In order to only transform directories that Python actually considers namespace pkgs - we need to detect if a directory includes Python modules. - - Which files are loadable as modules is extension based, and the particular set of extensions - varies by platform. - - See: - 1. https://github.com/python/cpython/blob/7d9d25dbedfffce61fc76bc7ccbfa9ae901bf56f/Lib/importlib/machinery.py#L19 - 2. PEP 420 -- Implicit Namespace Packages, Specification - https://www.python.org/dev/peps/pep-0420/#specification - 3. dynload_shlib.c and dynload_win.c in python/cpython. - """ - module_suffixes = { - ".py", # Source modules - ".pyc", # Compiled bytecode modules - ".so", # Unix extension modules - ".pyd", # https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll - } - return any(Path(f).suffix in module_suffixes for f in files) diff --git a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py b/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py deleted file mode 100644 index 49945f9b8c..0000000000 --- a/third_party/rules_pycross/pycross/private/tools/namespace_pkgs_test.py +++ /dev/null @@ -1,179 +0,0 @@ -import os -import pathlib -import shutil -import tempfile -import unittest -from typing import Optional -from typing import Set - -from pycross.private.tools import namespace_pkgs - - -class TempDir: - def __init__(self) -> None: - self.dir = tempfile.mkdtemp() - - def root(self) -> str: - return self.dir - - def add_dir(self, rel_path: str) -> None: - d = pathlib.Path(self.dir, rel_path) - d.mkdir(parents=True) - - def add_file(self, rel_path: str, contents: Optional[str] = None) -> None: - f = pathlib.Path(self.dir, rel_path) - f.parent.mkdir(parents=True, exist_ok=True) - if contents: - with open(str(f), "w") as writeable_f: - writeable_f.write(contents) - else: - f.touch() - - def remove(self) -> None: - shutil.rmtree(self.dir) - - -class TestImplicitNamespacePackages(unittest.TestCase): - def assertPathsEqual(self, actual: Set[pathlib.Path], expected: Set[str]) -> None: - self.assertEqual(actual, {pathlib.Path(p) for p in expected}) - - def test_in_current_directory(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - cwd = os.getcwd() - os.chdir(directory.root()) - expected = { - "foo", - "foo/bar", - "foo/bee", - } - try: - actual = namespace_pkgs.implicit_namespace_packages(".") - self.assertPathsEqual(actual, expected) - finally: - os.chdir(cwd) - directory.remove() - - def test_finds_correct_namespace_packages(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_file("foo/bee/boo.py") - directory.add_file("foo/buu/__init__.py") - directory.add_file("foo/buu/bii.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - directory.root() + "/foo/bee", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_ignores_empty_directories(self) -> None: - directory = TempDir() - directory.add_file("foo/bar/biz.py") - directory.add_dir("foo/cat") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_empty_case(self) -> None: - directory = TempDir() - directory.add_file("foo/__init__.py") - directory.add_file("foo/bar/__init__.py") - directory.add_file("foo/bar/biz.py") - - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - def test_ignores_non_module_files_in_directories(self) -> None: - directory = TempDir() - directory.add_file("foo/__init__.pyi") - directory.add_file("foo/py.typed") - - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertEqual(actual, set()) - - def test_parent_child_relationship_of_namespace_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/biff/my_module.py") - directory.add_file("foo/bar/biff/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - directory.root() + "/foo/bar/biff", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_parent_child_relationship_of_namespace_and_standard_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/biff/__init__.py") - directory.add_file("foo/bar/biff/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_parent_child_relationship_of_namespace_and_nested_standard_pkgs(self): - directory = TempDir() - directory.add_file("foo/bar/__init__.py") - directory.add_file("foo/bar/biff/another_module.py") - directory.add_file("foo/bar/biff/__init__.py") - directory.add_file("foo/bar/boof/big_module.py") - directory.add_file("foo/bar/boof/__init__.py") - directory.add_file("fim/in_a_ns_pkg.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/fim", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_recognized_all_nonstandard_module_types(self): - directory = TempDir() - directory.add_file("ayy/my_module.pyc") - directory.add_file("bee/ccc/dee/eee.so") - directory.add_file("eff/jee/aych.pyd") - - expected = { - directory.root() + "/ayy", - directory.root() + "/bee", - directory.root() + "/bee/ccc", - directory.root() + "/bee/ccc/dee", - directory.root() + "/eff", - directory.root() + "/eff/jee", - } - actual = namespace_pkgs.implicit_namespace_packages(directory.root()) - self.assertPathsEqual(actual, expected) - - def test_skips_ignored_directories(self): - directory = TempDir() - directory.add_file("foo/boo/my_module.py") - directory.add_file("foo/bar/another_module.py") - - expected = { - directory.root() + "/foo", - directory.root() + "/foo/bar", - } - actual = namespace_pkgs.implicit_namespace_packages( - directory.root(), - ignored_dirnames=[directory.root() + "/foo/boo"], - ) - self.assertPathsEqual(actual, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py index 6d3673669b..8367f08d41 100644 --- a/third_party/rules_pycross/pycross/private/tools/wheel_installer.py +++ b/third_party/rules_pycross/pycross/private/tools/wheel_installer.py @@ -1,19 +1,35 @@ +# Copyright 2023 Jeremy Volkman. All rights reserved. +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ A tool that invokes pypa/build to build the given sdist tarball. """ +import argparse import os import shutil +import sys import tempfile from pathlib import Path from typing import Any -from absl import app -from absl.flags import argparse_flags from installer import install from installer.destinations import SchemeDictionaryDestination from installer.sources import WheelFile -from pycross.private.tools import namespace_pkgs + +from python.pip_install.tools.wheel_installer import namespace_pkgs def setup_namespace_pkg_compatibility(wheel_dir: Path) -> None: @@ -73,7 +89,7 @@ def main(args: Any) -> None: destination=destination, # Additional metadata that is generated by the installation tool. additional_metadata={ - "INSTALLER": b"https://github.com/jvolkman/rules_pycross", + "INSTALLER": b"https://github.com/bazelbuild/rules_python/tree/main/third_party/rules_pycross", }, ) finally: @@ -83,7 +99,7 @@ def main(args: Any) -> None: def parse_flags(argv) -> Any: - parser = argparse_flags.ArgumentParser(description="Extract a Python wheel.") + parser = argparse.ArgumentParser(description="Extract a Python wheel.") parser.add_argument( "--wheel", @@ -119,4 +135,4 @@ def parse_flags(argv) -> Any: if "BUILD_WORKING_DIRECTORY" in os.environ: os.chdir(os.environ["BUILD_WORKING_DIRECTORY"]) - app.run(main, flags_parser=parse_flags) + main(parse_flags(sys.argv)) diff --git a/third_party/rules_pycross/pycross/private/wheel_library.bzl b/third_party/rules_pycross/pycross/private/wheel_library.bzl index 25a2497abe..381511a2f1 100644 --- a/third_party/rules_pycross/pycross/private/wheel_library.bzl +++ b/third_party/rules_pycross/pycross/private/wheel_library.bzl @@ -13,19 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Implementation of the pycross_wheel_library rule.""" +"""Implementation of the py_wheel_library rule.""" load("@bazel_skylib//lib:paths.bzl", "paths") -load("@rules_python//python:defs.bzl", "PyInfo") -load(":providers.bzl", "PycrossWheelInfo") +load("//python:defs.bzl", "PyInfo") +load(":providers.bzl", "PyWheelInfo") -def _pycross_wheel_library_impl(ctx): +def _py_wheel_library_impl(ctx): out = ctx.actions.declare_directory(ctx.attr.name) wheel_target = ctx.attr.wheel - if PycrossWheelInfo in wheel_target: - wheel_file = wheel_target[PycrossWheelInfo].wheel_file - name_file = wheel_target[PycrossWheelInfo].name_file + if PyWheelInfo in wheel_target: + wheel_file = wheel_target[PyWheelInfo].wheel_file + name_file = wheel_target[PyWheelInfo].name_file else: wheel_file = ctx.file.wheel name_file = None @@ -103,8 +103,8 @@ def _pycross_wheel_library_impl(ctx): ), ] -pycross_wheel_library = rule( - implementation = _pycross_wheel_library_impl, +py_wheel_library = rule( + implementation = _py_wheel_library_impl, attrs = { "deps": attr.label_list( doc = "A list of this wheel's Python library dependencies.", @@ -129,7 +129,7 @@ This option is required to support some packages which cannot handle the convers mandatory = True, ), "_tool": attr.label( - default = Label("//pycross/private/tools:wheel_installer"), + default = Label("//third_party/rules_pycross/pycross/private/tools:wheel_installer"), cfg = "exec", executable = True, ), From 7bb1f4a931884fb4cc1d1eba71436dc31b6d31a7 Mon Sep 17 00:00:00 2001 From: Matt Date: Thu, 28 Sep 2023 04:36:15 +1000 Subject: [PATCH 25/41] fix: Skip printing unneccesary warning. (#1407) If the python version is explicitly provided by the root module, they should not be warned for choosing the same version that rules_python provides as default. --- python/extensions/python.bzl | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 2d007267b1..c7c2c82c05 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -86,16 +86,22 @@ def _python_impl(module_ctx): # much else that can be done. Modules don't know and can't control # what other modules do, so the first in the dependency graph wins. if toolchain_version in global_toolchain_versions: - _warn_duplicate_global_toolchain_version( - toolchain_version, - first = global_toolchain_versions[toolchain_version], - second_toolchain_name = toolchain_name, - second_module_name = mod.name, - ) + # If the python version is explicitly provided by the root + # module, they should not be warned for choosing the same + # version that rules_python provides as default. + first = global_toolchain_versions[toolchain_version] + if mod.name != "rules_python" or not first.is_root: + _warn_duplicate_global_toolchain_version( + toolchain_version, + first = first, + second_toolchain_name = toolchain_name, + second_module_name = mod.name, + ) continue global_toolchain_versions[toolchain_version] = struct( toolchain_name = toolchain_name, module_name = mod.name, + is_root = mod.is_root, ) # Only the root module and rules_python are allowed to specify the default From f2a4dd5e70b7e31d06599bf8b1237bb8e45318f1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 28 Sep 2023 05:29:59 +0900 Subject: [PATCH 26/41] refactor(bzlmod)!: simplify pip.parse repository layout (#1395) Before this PR we would generate extra `alias` repos and the extra `hub` repo for the `entry_point` macro usage. This PR removes the extras and delegates the creation of version-aware aliases to the `render_pkg_aliases` internal function. This reduces the number of repositories created by the `pip.parse` extension. Fixes #1255. BREAKING CHANGE: Note that this only affects bzlmod support, which is still beta. * Bzlmod `pip.parse` no longer generates `{hub_name}_{py_version}` hub repos. * Bzlmod `pip.parse` no longer generates `{hub_name}_{distribution}` hub repos. These repos aren't part of a public API, but were typically used for the `entry_point` macros. Instead, use `py_console_script_binary`, which is the supported replacement for entry points under bzlmod. Directly referencing the underlying distribution repos remains unsupported. --- CHANGELOG.md | 5 + docs/pip_repository.md | 30 +--- python/extensions/pip.bzl | 136 ++++++------------ ...ub_repository_requirements_bzlmod.bzl.tmpl | 29 ---- python/pip_install/pip_repository.bzl | 78 +++------- ...ip_repository_requirements_bzlmod.bzl.tmpl | 3 +- 6 files changed, 72 insertions(+), 209 deletions(-) delete mode 100644 python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1bf1faa2..ed3a60d889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,11 @@ A brief description of the categories of changes: * (bzlmod) The `entry_point` macro is no longer supported and has been removed in favour of the `py_console_script_binary` macro for `bzlmod` users. +* (bzlmod) The `pip.parse` no longer generates `{hub_name}_{py_version}` hub repos + as the `entry_point` macro has been superseded by `py_console_script_binary`. + +* (bzlmod) The `pip.parse` no longer generates `{hub_name}_{distribution}` hub repos. + ### Fixed * (whl_library) No longer restarts repository rule when fetching external diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 853605276f..453ca29713 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -7,7 +7,7 @@ ## pip_hub_repository_bzlmod 
-pip_hub_repository_bzlmod(name, repo_mapping, repo_name, whl_library_alias_names)
+pip_hub_repository_bzlmod(name, default_version, repo_mapping, repo_name, whl_map)
A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY. @@ -18,9 +18,10 @@ A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this repository. | Name | required | | +| default_version | This is the default python version in the format of X.Y.Z. This should match what is setup by the 'python' extension using the 'is_default = True' setting. | String | required | | | repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | | repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name. | String | required | | -| whl_library_alias_names | The list of whl alias that we use to build aliases and the whl names | List of strings | required | | +| whl_map | The wheel map where values are python versions | Dictionary: String -> List of strings | required | | @@ -101,31 +102,6 @@ py_binary( | timeout | Timeout (in seconds) on the rule's execution duration. | Integer | optional | 600 | - - -## pip_repository_bzlmod - -
-pip_repository_bzlmod(name, repo_mapping, repo_name, requirements_darwin, requirements_linux,
-                      requirements_lock, requirements_windows)
-
- -A rule for bzlmod pip_repository creation. Intended for private use only. - -**ATTRIBUTES** - - -| Name | Description | Type | Mandatory | Default | -| :------------- | :------------- | :------------- | :------------- | :------------- | -| name | A unique name for this repository. | Name | required | | -| repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | -| repo_name | The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name | String | required | | -| requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | -| requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | -| requirements_lock | A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. | Label | optional | None | -| requirements_windows | Override the requirements_lock attribute when the host platform is Windows | Label | optional | None | - - ## whl_library diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index 3ba0d3eb58..f94f18c619 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -15,17 +15,16 @@ "pip module extension for use with bzlmod" load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") -load("//python:pip.bzl", "whl_library_alias") load( "//python/pip_install:pip_repository.bzl", "locked_requirements_label", "pip_hub_repository_bzlmod", "pip_repository_attrs", - "pip_repository_bzlmod", "use_isolated", "whl_library", ) load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") +load("//python/private:full_version.bzl", "full_version") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version_label.bzl", "version_label") @@ -78,11 +77,11 @@ You cannot use both the additive_build_content and additive_build_content_file a whl_mods = whl_mods, ) -def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): +def _create_whl_repos(module_ctx, pip_attr, whl_map): python_interpreter_target = pip_attr.python_interpreter_target # if we do not have the python_interpreter set in the attributes - # we programtically find it. + # we programmatically find it. hub_name = pip_attr.hub_name if python_interpreter_target == None: python_name = "python_" + version_label(pip_attr.python_version, sep = "_") @@ -104,23 +103,12 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): requrements_lock = locked_requirements_label(module_ctx, pip_attr) # Parse the requirements file directly in starlark to get the information - # needed for the whl_libary declarations below. This is needed to contain - # the pip_repository logic to a single module extension. + # needed for the whl_libary declarations below. requirements_lock_content = module_ctx.read(requrements_lock) parse_result = parse_requirements(requirements_lock_content) requirements = parse_result.requirements extra_pip_args = pip_attr.extra_pip_args + parse_result.options - # Create the repository where users load the `requirement` macro. Under bzlmod - # this does not create the install_deps() macro. - # TODO: we may not need this repository once we have entry points - # supported. For now a user can access this repository and use - # the entrypoint functionality. - pip_repository_bzlmod( - name = pip_name, - repo_name = pip_name, - requirements_lock = pip_attr.requirements_lock, - ) if hub_name not in whl_map: whl_map[hub_name] = {} @@ -157,12 +145,12 @@ def _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, whl_map): if whl_name not in whl_map[hub_name]: whl_map[hub_name][whl_name] = {} - whl_map[hub_name][whl_name][pip_attr.python_version] = pip_name + "_" + whl_map[hub_name][whl_name][full_version(pip_attr.python_version)] = pip_name + "_" def _pip_impl(module_ctx): - """Implementation of a class tag that creates the pip hub(s) and corresponding pip spoke, alias and whl repositories. + """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. - This implmentation iterates through all of the `pip.parse` calls and creates + This implementation iterates through all of the `pip.parse` calls and creates different pip hub repositories based on the "hub_name". Each of the pip calls create spoke repos that uses a specific Python interpreter. @@ -196,52 +184,33 @@ def _pip_impl(module_ctx): Both of these pip spokes contain requirements files that includes websocket and its dependencies. - Two different repositories are created for the two spokes: - - - @@rules_python~override~pip~pip_39 - - @@rules_python~override~pip~pip_310 - - The different spoke names are a combination of the hub_name and the Python version. - In the future we may remove this repository, but we do not support entry points. - yet, and that functionality exists in these repos. - We also need repositories for the wheels that the different pip spokes contain. For each Python version a different wheel repository is created. In our example - each pip spoke had a requirments file that contained websockets. We + each pip spoke had a requirements file that contained websockets. We then create two different wheel repositories that are named the following. - @@rules_python~override~pip~pip_39_websockets - @@rules_python~override~pip~pip_310_websockets - And if the wheel has any other dependies subsequest wheels are created in the same fashion. - - We also create a repository for the wheel alias. We want to just use the syntax - 'requirement("websockets")' we need to have an alias repository that is named: + And if the wheel has any other dependencies subsequent wheels are created in the same fashion. - - @@rules_python~override~pip~pip_websockets - - This repository contains alias statements for the different wheel components (pkg, data, etc). - Each of those aliases has a select that resolves to a spoke repository depending on - the Python version. + The hub repository has aliases for `pkg`, `data`, etc, which have a select that resolves to + a spoke repository depending on the Python version. Also we may have more than one hub as defined in a MODULES.bazel file. So we could have multiple hubs pointing to various different pip spokes. - Some other business rules notes. A hub can only have one spoke per Python version. We cannot + Some other business rules notes. A hub can only have one spoke per Python version. We cannot have a hub named "pip" that has two spokes that use the Python 3.9 interpreter. Second - we cannot have the same hub name used in submodules. The hub name has to be globally + we cannot have the same hub name used in sub-modules. The hub name has to be globally unique. - This implementation reuses elements of non-bzlmod code and also reuses the first implementation - of pip bzlmod, but adds the capability to have multiple pip.parse calls. - This implementation also handles the creation of whl_modification JSON files that are used - during the creation of wheel libraries. These JSON files used via the annotations argument + during the creation of wheel libraries. These JSON files used via the annotations argument when calling wheel_installer.py. Args: module_ctx: module contents - """ # Build all of the wheel modifications if the tag class is called. @@ -259,63 +228,46 @@ def _pip_impl(module_ctx): for mod in module_ctx.modules: for pip_attr in mod.tags.parse: hub_name = pip_attr.hub_name - if hub_name in pip_hub_map: - # We cannot have two hubs with the same name in different - # modules. - if pip_hub_map[hub_name].module_name != mod.name: - fail(( - "Duplicate cross-module pip hub named '{hub}': pip hub " + - "names must be unique across modules. First defined " + - "by module '{first_module}', second attempted by " + - "module '{second_module}'" - ).format( - hub = hub_name, - first_module = pip_hub_map[hub_name].module_name, - second_module = mod.name, - )) - - if pip_attr.python_version in pip_hub_map[hub_name].python_versions: - fail(( - "Duplicate pip python version '{version}' for hub " + - "'{hub}' in module '{module}': the Python versions " + - "used for a hub must be unique" - ).format( - hub = hub_name, - module = mod.name, - version = pip_attr.python_version, - )) - else: - pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) - else: + if hub_name not in pip_hub_map: pip_hub_map[pip_attr.hub_name] = struct( module_name = mod.name, python_versions = [pip_attr.python_version], ) + elif pip_hub_map[hub_name].module_name != mod.name: + # We cannot have two hubs with the same name in different + # modules. + fail(( + "Duplicate cross-module pip hub named '{hub}': pip hub " + + "names must be unique across modules. First defined " + + "by module '{first_module}', second attempted by " + + "module '{second_module}'" + ).format( + hub = hub_name, + first_module = pip_hub_map[hub_name].module_name, + second_module = mod.name, + )) - _create_versioned_pip_and_whl_repos(module_ctx, pip_attr, hub_whl_map) + elif pip_attr.python_version in pip_hub_map[hub_name].python_versions: + fail(( + "Duplicate pip python version '{version}' for hub " + + "'{hub}' in module '{module}': the Python versions " + + "used for a hub must be unique" + ).format( + hub = hub_name, + module = mod.name, + version = pip_attr.python_version, + )) + else: + pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version) + + _create_whl_repos(module_ctx, pip_attr, hub_whl_map) for hub_name, whl_map in hub_whl_map.items(): - for whl_name, version_map in whl_map.items(): - if DEFAULT_PYTHON_VERSION in version_map: - whl_default_version = DEFAULT_PYTHON_VERSION - else: - whl_default_version = None - - # Create the alias repositories which contains different select - # statements These select statements point to the different pip - # whls that are based on a specific version of Python. - whl_library_alias( - name = hub_name + "_" + whl_name, - wheel_name = whl_name, - default_version = whl_default_version, - version_map = version_map, - ) - - # Create the hub repository for pip. pip_hub_repository_bzlmod( name = hub_name, repo_name = hub_name, - whl_library_alias_names = whl_map.keys(), + whl_map = whl_map, + default_version = full_version(DEFAULT_PYTHON_VERSION), ) def _pip_parse_ext_attrs(): diff --git a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl deleted file mode 100644 index 53d4ee99c4..0000000000 --- a/python/pip_install/pip_hub_repository_requirements_bzlmod.bzl.tmpl +++ /dev/null @@ -1,29 +0,0 @@ -"""Starlark representation of locked requirements. - -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. - -This file is different from the other bzlmod template -because we do not support entry_point yet. -""" - -all_requirements = %%ALL_REQUIREMENTS%% - -all_whl_requirements = %%ALL_WHL_REQUIREMENTS%% - -all_data_requirements = %%ALL_DATA_REQUIREMENTS%% - -def _clean_name(name): - return name.replace("-", "_").replace(".", "_").lower() - -def requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "pkg") - -def whl_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "whl") - -def data_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "data") - -def dist_info_requirement(name): - return "%%MACRO_TMPL%%".format(_clean_name(name), "dist_info") diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index abe3ca787c..ea8b9eb5ac 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -267,10 +267,14 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile """) return requirements_txt -def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): - repo_name = rctx.attr.repo_name - build_contents = _BUILD_FILE_CONTENTS - aliases = render_pkg_aliases(repo_name = repo_name, bzl_packages = bzl_packages) +def _pip_hub_repository_bzlmod_impl(rctx): + bzl_packages = rctx.attr.whl_map.keys() + aliases = render_pkg_aliases( + repo_name = rctx.attr.repo_name, + rules_python = rctx.attr._template.workspace_name, + default_version = rctx.attr.default_version, + whl_map = rctx.attr.whl_map, + ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -280,7 +284,7 @@ def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): # `requirement`, et al. macros. macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) - rctx.file("BUILD.bazel", build_contents) + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": _format_repr_list([ macro_tmpl.format(p, "data") @@ -296,24 +300,26 @@ def _create_pip_repository_bzlmod(rctx, bzl_packages, requirements): ]), "%%MACRO_TMPL%%": macro_tmpl, "%%NAME%%": rctx.attr.name, - "%%REQUIREMENTS_LOCK%%": requirements, }) -def _pip_hub_repository_bzlmod_impl(rctx): - bzl_packages = rctx.attr.whl_library_alias_names - _create_pip_repository_bzlmod(rctx, bzl_packages, "") - pip_hub_repository_bzlmod_attrs = { + "default_version": attr.string( + mandatory = True, + doc = """\ +This is the default python version in the format of X.Y.Z. This should match +what is setup by the 'python' extension using the 'is_default = True' +setting.""", + ), "repo_name": attr.string( mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), - "whl_library_alias_names": attr.string_list( + "whl_map": attr.string_list_dict( mandatory = True, - doc = "The list of whl alias that we use to build aliases and the whl names", + doc = "The wheel map where values are python versions", ), "_template": attr.label( - default = ":pip_hub_repository_requirements_bzlmod.bzl.tmpl", + default = ":pip_repository_requirements_bzlmod.bzl.tmpl", ), } @@ -323,52 +329,6 @@ pip_hub_repository_bzlmod = repository_rule( implementation = _pip_hub_repository_bzlmod_impl, ) -def _pip_repository_bzlmod_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] - - bzl_packages = sorted([name for name, _ in packages]) - _create_pip_repository_bzlmod(rctx, bzl_packages, str(requirements_txt)) - -pip_repository_bzlmod_attrs = { - "repo_name": attr.string( - mandatory = True, - doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name", - ), - "requirements_darwin": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Mac OS", - ), - "requirements_linux": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Linux", - ), - "requirements_lock": attr.label( - allow_single_file = True, - doc = """ -A fully resolved 'requirements.txt' pip requirement file containing the transitive set of your dependencies. If this file is passed instead -of 'requirements' no resolve will take place and pip_repository will create individual repositories for each of your dependencies so that -wheels are fetched/built only for the targets specified by 'build/run/test'. -""", - ), - "requirements_windows": attr.label( - allow_single_file = True, - doc = "Override the requirements_lock attribute when the host platform is Windows", - ), - "_template": attr.label( - default = ":pip_repository_requirements_bzlmod.bzl.tmpl", - ), -} - -pip_repository_bzlmod = repository_rule( - attrs = pip_repository_bzlmod_attrs, - doc = """A rule for bzlmod pip_repository creation. Intended for private use only.""", - implementation = _pip_repository_bzlmod_impl, -) - def _pip_repository_impl(rctx): requirements_txt = locked_requirements_label(rctx, rctx.attr) content = rctx.read(requirements_txt) diff --git a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl index 00580f5593..c72187c7ee 100644 --- a/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl +++ b/python/pip_install/pip_repository_requirements_bzlmod.bzl.tmpl @@ -1,7 +1,6 @@ """Starlark representation of locked requirements. -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%%. +@generated by rules_python pip.parse bzlmod extension. """ all_requirements = %%ALL_REQUIREMENTS%% From cdb5902c3e5b67f2b65316d3efdb7597b2e52f56 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 28 Sep 2023 14:01:18 +0900 Subject: [PATCH 27/41] feat(bzlmod): mark pip extension as os/arch dependent (#1433) This ensures that under bzlmod with `--lockfile-mode=update` we would generate an entry per os/arch, which is needed because the hermetic toolchain interpreter path is os/arch dependent. Summary: - add bazel_features dep - mark the pip extension as arch/os dependent Related: bazelbuild/bazel#19154 --------- Co-authored-by: Richard Levasseur --- CHANGELOG.md | 3 +++ MODULE.bazel | 3 ++- python/extensions/pip.bzl | 13 +++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3a60d889..59bdac1b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ A brief description of the categories of changes: cases. An error about `@rules_python_internal` means the `py_repositories()` call is missing in `WORKSPACE`. +* (bzlmod) The `pip.parse` extension will generate os/arch specific lock + file entries on `bazel>=6.4`. + ### Added diff --git a/MODULE.bazel b/MODULE.bazel index ab7b597518..23e78c025e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,8 +4,9 @@ module( compatibility_level = 1, ) -bazel_dep(name = "platforms", version = "0.0.4") +bazel_dep(name = "bazel_features", version = "1.1.0") bazel_dep(name = "bazel_skylib", version = "1.3.0") +bazel_dep(name = "platforms", version = "0.0.4") # Those are loaded only when using py_proto_library bazel_dep(name = "rules_proto", version = "5.3.0-21.7") diff --git a/python/extensions/pip.bzl b/python/extensions/pip.bzl index f94f18c619..a0559ffe97 100644 --- a/python/extensions/pip.bzl +++ b/python/extensions/pip.bzl @@ -14,6 +14,7 @@ "pip module extension for use with bzlmod" +load("@bazel_features//:features.bzl", "bazel_features") load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") load( "//python/pip_install:pip_repository.bzl", @@ -380,6 +381,17 @@ cannot have a child module that uses the same `hub_name`. } return attrs +def _extension_extra_args(): + args = {} + + if bazel_features.external_deps.module_extension_has_os_arch_dependent: + args = args | { + "arch_dependent": True, + "os_dependent": True, + } + + return args + pip = module_extension( doc = """\ This extension is used to make dependencies from pip available. @@ -422,6 +434,7 @@ extension. """, ), }, + **_extension_extra_args() ) def _whl_mods_repo_impl(rctx): From 9a8c4479aab3e9f481aec9c5f88b1c92c2838d58 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 28 Sep 2023 17:55:26 +0900 Subject: [PATCH 28/41] chore: bump internal_deps (#1322) Before this PR the dependencies were out of date and the `pyproject_hooks` was missing allowing `pip-compile` to use `pyproject.toml` instead of `requirements.in`. With the update we are more likely to support the new features but they are not tested in this PR as the main goal is to update all of the files that need regenerating due to upgraded `pip-tools`. Here we: * Upgrade all of the deps to the latest versions. * Update all of the requirements and gazelle manifest files. * Add a quirk to ensure fix `pip_compile` on Windows. Summary of version changes: * build: 0.9.0 -> 0.10.0 * click: 8.0.1 -> 8.1.6 * importlib_metadata: 1.4.0 -> 6.8.0 * more_itertools: 8.13.0 -> 10.1.0 * packaging: 22.0 -> 23.1 * pip: 22.3.1 -> 23.2.1 * pip_tools: 6.12.1 -> 7.2.0 * pyproject_hooks: 1.0.0 * setuptools: 60.10.0 -> 68.0.0 * wheel: 0.38.4 -> 0.41.0 * zipp: 1.0.0 -> 3.16.2 Fixes #1351 --------- Co-authored-by: Greg --- MODULE.bazel | 1 + examples/build_file_generation/BUILD.bazel | 1 - examples/bzlmod/BUILD.bazel | 2 - examples/bzlmod/requirements_lock_3_9.txt | 10 +- examples/bzlmod/requirements_windows_3_9.txt | 10 +- .../bzlmod_build_file_generation/BUILD.bazel | 1 - .../gazelle_python.yaml | 3 +- .../requirements_lock.txt | 10 +- .../requirements_windows.txt | 10 +- .../requirements/BUILD.bazel | 4 - .../requirements/requirements_lock_3_10.txt | 120 +++++++++++------- .../requirements/requirements_lock_3_11.txt | 120 +++++++++++------- .../requirements/requirements_lock_3_8.txt | 120 +++++++++++------- .../requirements/requirements_lock_3_9.txt | 120 +++++++++++------- examples/pip_install/BUILD.bazel | 3 +- examples/pip_install/pip_install_test.py | 2 +- examples/pip_install/requirements.in | 2 +- examples/pip_install/requirements.txt | 15 ++- examples/pip_install/requirements_windows.txt | 15 ++- examples/pip_parse/BUILD.bazel | 1 - examples/pip_parse/pip_parse_test.py | 2 +- examples/pip_parse/requirements.in | 2 +- examples/pip_parse/requirements_lock.txt | 15 ++- .../pip_repository_annotations/BUILD.bazel | 1 - python/pip_install/repositories.bzl | 45 ++++--- python/pip_install/requirements.bzl | 7 +- python/pip_install/tools/requirements.txt | 20 +-- tests/compile_pip_requirements/BUILD.bazel | 12 -- .../requirements_lock.txt | 2 + .../requirements_lock_darwin.txt | 2 + .../requirements_lock_linux.txt | 2 + .../requirements_lock_windows.txt | 2 + .../requirements_nohashes_lock.txt | 2 + tests/pip_repository_entry_points/BUILD.bazel | 1 - .../requirements.txt | 16 ++- .../requirements_windows.txt | 16 ++- 36 files changed, 410 insertions(+), 307 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 23e78c025e..e9b06d66ef 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -28,6 +28,7 @@ use_repo( "pypi__pep517", "pypi__pip", "pypi__pip_tools", + "pypi__pyproject_hooks", "pypi__setuptools", "pypi__tomli", "pypi__wheel", diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 928fb128e2..79f62519df 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -12,7 +12,6 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 3db7751f72..ff14016b85 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -16,7 +16,6 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") # with pip-compile. compile_pip_requirements_3_9( name = "requirements_3_9", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_9.txt", requirements_windows = "requirements_windows_3_9.txt", @@ -26,7 +25,6 @@ compile_pip_requirements_3_9( # with pip-compile. compile_pip_requirements_3_10( name = "requirements_3_10", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_10.txt", requirements_windows = "requirements_windows_3_10.txt", diff --git a/examples/bzlmod/requirements_lock_3_9.txt b/examples/bzlmod/requirements_lock_3_9.txt index 79c18127ec..a3bfba22ad 100644 --- a/examples/bzlmod/requirements_lock_3_9.txt +++ b/examples/bzlmod/requirements_lock_3_9.txt @@ -133,10 +133,6 @@ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 # via -r requirements.in -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -309,3 +305,9 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/bzlmod/requirements_windows_3_9.txt b/examples/bzlmod/requirements_windows_3_9.txt index 790e3d5d11..2681ff2a00 100644 --- a/examples/bzlmod/requirements_windows_3_9.txt +++ b/examples/bzlmod/requirements_windows_3_9.txt @@ -137,10 +137,6 @@ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 # via -r requirements.in -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -313,3 +309,9 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index bd2fc80933..9b2e5bdce4 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -17,7 +17,6 @@ load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") # with pip-compile. compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock.txt", requirements_windows = "requirements_windows.txt", diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml index e33021b9c8..0c7f14876e 100644 --- a/examples/bzlmod_build_file_generation/gazelle_python.yaml +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -232,7 +232,6 @@ manifest: isort.wrap: isort isort.wrap_modes: isort lazy_object_proxy: lazy_object_proxy - lazy_object_proxy.cext: lazy_object_proxy lazy_object_proxy.compat: lazy_object_proxy lazy_object_proxy.simple: lazy_object_proxy lazy_object_proxy.slots: lazy_object_proxy @@ -588,4 +587,4 @@ manifest: pip_repository: name: pip use_pip_repository_aliases: true -integrity: cee7684391c4a8a1ff219cd354deae61cdcdee70f2076789aabd5249f3c4eca9 +integrity: 369584d55f2168d92c415f4c4ab4bc9d2d21a7fb0b0a6749437fcc771fd2f254 diff --git a/examples/bzlmod_build_file_generation/requirements_lock.txt b/examples/bzlmod_build_file_generation/requirements_lock.txt index 2160fe1163..3fd053f777 100644 --- a/examples/bzlmod_build_file_generation/requirements_lock.txt +++ b/examples/bzlmod_build_file_generation/requirements_lock.txt @@ -125,10 +125,6 @@ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 # via -r requirements.in -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -225,3 +221,9 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/bzlmod_build_file_generation/requirements_windows.txt b/examples/bzlmod_build_file_generation/requirements_windows.txt index 06cfdc332c..15e92288dc 100644 --- a/examples/bzlmod_build_file_generation/requirements_windows.txt +++ b/examples/bzlmod_build_file_generation/requirements_windows.txt @@ -129,10 +129,6 @@ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 # via -r requirements.in -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -229,3 +225,9 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/multi_python_versions/requirements/BUILD.bazel b/examples/multi_python_versions/requirements/BUILD.bazel index e3184c8ac5..e3e821a68d 100644 --- a/examples/multi_python_versions/requirements/BUILD.bazel +++ b/examples/multi_python_versions/requirements/BUILD.bazel @@ -5,28 +5,24 @@ load("@python//3.9:defs.bzl", compile_pip_requirements_3_9 = "compile_pip_requir compile_pip_requirements_3_8( name = "requirements_3_8", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_8.txt", ) compile_pip_requirements_3_9( name = "requirements_3_9", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_9.txt", ) compile_pip_requirements_3_10( name = "requirements_3_10", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_10.txt", ) compile_pip_requirements_3_11( name = "requirements_3_11", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock_3_11.txt", ) diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt index 6bee4e0030..4910d13844 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_10.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_10.txt @@ -4,53 +4,75 @@ # # bazel run //requirements:requirements_3_10.update # -websockets==10.3 \ - --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \ - --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \ - --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \ - --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \ - --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \ - --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \ - --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \ - --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \ - --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \ - --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \ - --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \ - --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \ - --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \ - --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \ - --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \ - --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \ - --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \ - --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \ - --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \ - --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \ - --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \ - --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \ - --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \ - --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \ - --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \ - --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \ - --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \ - --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \ - --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \ - --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \ - --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \ - --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \ - --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \ - --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \ - --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \ - --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \ - --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \ - --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \ - --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \ - --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \ - --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \ - --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \ - --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \ - --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \ - --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \ - --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \ - --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \ - --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4 +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 # via -r requirements/requirements.in diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt index a437a397d0..35666b54b1 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_11.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_11.txt @@ -4,53 +4,75 @@ # # bazel run //requirements:requirements_3_11.update # -websockets==10.3 \ - --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \ - --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \ - --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \ - --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \ - --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \ - --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \ - --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \ - --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \ - --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \ - --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \ - --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \ - --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \ - --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \ - --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \ - --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \ - --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \ - --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \ - --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \ - --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \ - --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \ - --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \ - --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \ - --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \ - --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \ - --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \ - --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \ - --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \ - --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \ - --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \ - --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \ - --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \ - --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \ - --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \ - --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \ - --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \ - --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \ - --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \ - --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \ - --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \ - --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \ - --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \ - --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \ - --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \ - --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \ - --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \ - --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \ - --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \ - --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4 +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 # via -r requirements/requirements.in diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_8.txt b/examples/multi_python_versions/requirements/requirements_lock_3_8.txt index 19303f8eff..10b5df4830 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_8.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_8.txt @@ -4,53 +4,75 @@ # # bazel run //requirements:requirements_3_8.update # -websockets==10.3 \ - --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \ - --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \ - --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \ - --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \ - --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \ - --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \ - --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \ - --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \ - --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \ - --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \ - --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \ - --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \ - --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \ - --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \ - --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \ - --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \ - --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \ - --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \ - --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \ - --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \ - --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \ - --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \ - --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \ - --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \ - --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \ - --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \ - --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \ - --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \ - --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \ - --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \ - --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \ - --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \ - --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \ - --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \ - --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \ - --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \ - --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \ - --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \ - --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \ - --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \ - --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \ - --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \ - --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \ - --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \ - --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \ - --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \ - --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \ - --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4 +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 # via -r requirements/requirements.in diff --git a/examples/multi_python_versions/requirements/requirements_lock_3_9.txt b/examples/multi_python_versions/requirements/requirements_lock_3_9.txt index 4af42ca277..0001f88d48 100644 --- a/examples/multi_python_versions/requirements/requirements_lock_3_9.txt +++ b/examples/multi_python_versions/requirements/requirements_lock_3_9.txt @@ -4,53 +4,75 @@ # # bazel run //requirements:requirements_3_9.update # -websockets==10.3 \ - --hash=sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af \ - --hash=sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c \ - --hash=sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76 \ - --hash=sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47 \ - --hash=sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69 \ - --hash=sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079 \ - --hash=sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c \ - --hash=sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55 \ - --hash=sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02 \ - --hash=sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559 \ - --hash=sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3 \ - --hash=sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e \ - --hash=sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978 \ - --hash=sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98 \ - --hash=sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae \ - --hash=sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755 \ - --hash=sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d \ - --hash=sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991 \ - --hash=sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1 \ - --hash=sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680 \ - --hash=sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247 \ - --hash=sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f \ - --hash=sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2 \ - --hash=sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7 \ - --hash=sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4 \ - --hash=sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667 \ - --hash=sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb \ - --hash=sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094 \ - --hash=sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36 \ - --hash=sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79 \ - --hash=sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500 \ - --hash=sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e \ - --hash=sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582 \ - --hash=sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442 \ - --hash=sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd \ - --hash=sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6 \ - --hash=sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731 \ - --hash=sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4 \ - --hash=sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d \ - --hash=sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8 \ - --hash=sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f \ - --hash=sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677 \ - --hash=sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8 \ - --hash=sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9 \ - --hash=sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e \ - --hash=sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b \ - --hash=sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916 \ - --hash=sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4 +websockets==11.0.3 \ + --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \ + --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \ + --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \ + --hash=sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82 \ + --hash=sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788 \ + --hash=sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa \ + --hash=sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f \ + --hash=sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4 \ + --hash=sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7 \ + --hash=sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f \ + --hash=sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd \ + --hash=sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69 \ + --hash=sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb \ + --hash=sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b \ + --hash=sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016 \ + --hash=sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac \ + --hash=sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4 \ + --hash=sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb \ + --hash=sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99 \ + --hash=sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e \ + --hash=sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54 \ + --hash=sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf \ + --hash=sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007 \ + --hash=sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3 \ + --hash=sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6 \ + --hash=sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86 \ + --hash=sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1 \ + --hash=sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61 \ + --hash=sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11 \ + --hash=sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8 \ + --hash=sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f \ + --hash=sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931 \ + --hash=sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526 \ + --hash=sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016 \ + --hash=sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae \ + --hash=sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd \ + --hash=sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b \ + --hash=sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311 \ + --hash=sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af \ + --hash=sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152 \ + --hash=sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288 \ + --hash=sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de \ + --hash=sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97 \ + --hash=sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d \ + --hash=sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d \ + --hash=sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca \ + --hash=sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0 \ + --hash=sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9 \ + --hash=sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b \ + --hash=sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e \ + --hash=sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128 \ + --hash=sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d \ + --hash=sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c \ + --hash=sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5 \ + --hash=sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6 \ + --hash=sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b \ + --hash=sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b \ + --hash=sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280 \ + --hash=sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c \ + --hash=sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c \ + --hash=sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f \ + --hash=sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20 \ + --hash=sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8 \ + --hash=sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb \ + --hash=sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602 \ + --hash=sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf \ + --hash=sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0 \ + --hash=sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74 \ + --hash=sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0 \ + --hash=sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564 # via -r requirements/requirements.in diff --git a/examples/pip_install/BUILD.bazel b/examples/pip_install/BUILD.bazel index 35f5a9338a..87c5aa7f8c 100644 --- a/examples/pip_install/BUILD.bazel +++ b/examples/pip_install/BUILD.bazel @@ -61,7 +61,6 @@ alias( # Check that our compiled requirements are up-to-date compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], requirements_windows = ":requirements_windows.txt", ) @@ -88,7 +87,7 @@ py_test( genquery( name = "yamllint_lib_by_version", expression = """ - attr("tags", "\\bpypi_version=1.26.3\\b", "@pip_yamllint//:pkg") + attr("tags", "\\bpypi_version=1.28.0\\b", "@pip_yamllint//:pkg") intersect attr("tags", "\\bpypi_name=yamllint\\b", "@pip_yamllint//:pkg") """, diff --git a/examples/pip_install/pip_install_test.py b/examples/pip_install/pip_install_test.py index 3e1b085ed4..f49422bb83 100644 --- a/examples/pip_install/pip_install_test.py +++ b/examples/pip_install/pip_install_test.py @@ -43,7 +43,7 @@ def test_entry_point(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.28.0") def test_data(self): env = os.environ.get("WHEEL_DATA_CONTENTS") diff --git a/examples/pip_install/requirements.in b/examples/pip_install/requirements.in index 11ede3c44a..3480175020 100644 --- a/examples/pip_install/requirements.in +++ b/examples/pip_install/requirements.in @@ -1,4 +1,4 @@ boto3~=1.14.51 s3cmd~=2.1.0 -yamllint~=1.26.3 +yamllint~=1.28.0 tree-sitter==0.20.0 ; sys_platform != "win32" diff --git a/examples/pip_install/requirements.txt b/examples/pip_install/requirements.txt index 495a32a637..00fe860169 100644 --- a/examples/pip_install/requirements.txt +++ b/examples/pip_install/requirements.txt @@ -89,10 +89,6 @@ s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 # via boto3 -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -105,6 +101,13 @@ urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e # via botocore -yamllint==1.26.3 \ - --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e +yamllint==1.28.0 \ + --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ + --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/pip_install/requirements_windows.txt b/examples/pip_install/requirements_windows.txt index b87192f9d0..298f31f996 100644 --- a/examples/pip_install/requirements_windows.txt +++ b/examples/pip_install/requirements_windows.txt @@ -89,10 +89,6 @@ s3transfer==0.3.7 \ --hash=sha256:35627b86af8ff97e7ac27975fe0a98a312814b46c6333d8a6b889627bcd80994 \ --hash=sha256:efa5bd92a897b6a8d5c1383828dca3d52d0790e0756d49740563a3fb6ed03246 # via boto3 -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -101,6 +97,13 @@ urllib3==1.25.11 \ --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e # via botocore -yamllint==1.26.3 \ - --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e +yamllint==1.28.0 \ + --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ + --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/pip_parse/BUILD.bazel b/examples/pip_parse/BUILD.bazel index 653f75ce2b..b7aa5b172b 100644 --- a/examples/pip_parse/BUILD.bazel +++ b/examples/pip_parse/BUILD.bazel @@ -58,7 +58,6 @@ alias( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], requirements_in = "requirements.in", requirements_txt = "requirements_lock.txt", ) diff --git a/examples/pip_parse/pip_parse_test.py b/examples/pip_parse/pip_parse_test.py index f319cb898f..199879065c 100644 --- a/examples/pip_parse/pip_parse_test.py +++ b/examples/pip_parse/pip_parse_test.py @@ -41,7 +41,7 @@ def test_entry_point(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.26.3") + self.assertEqual(proc.stdout.decode("utf-8").strip(), "yamllint 1.28.0") def test_data(self): env = os.environ.get("WHEEL_DATA_CONTENTS") diff --git a/examples/pip_parse/requirements.in b/examples/pip_parse/requirements.in index ec2102fdda..279dd6068e 100644 --- a/examples/pip_parse/requirements.in +++ b/examples/pip_parse/requirements.in @@ -1,3 +1,3 @@ requests~=2.25.1 s3cmd~=2.1.0 -yamllint~=1.26.3 +yamllint~=1.28.0 diff --git a/examples/pip_parse/requirements_lock.txt b/examples/pip_parse/requirements_lock.txt index 3cbe57f28c..8b68356b29 100644 --- a/examples/pip_parse/requirements_lock.txt +++ b/examples/pip_parse/requirements_lock.txt @@ -78,10 +78,6 @@ s3cmd==2.1.0 \ --hash=sha256:49cd23d516b17974b22b611a95ce4d93fe326feaa07320bd1d234fed68cbccfa \ --hash=sha256:966b0a494a916fc3b4324de38f089c86c70ee90e8e1cae6d59102103a4c0cc03 # via -r requirements.in -setuptools==65.6.3 \ - --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ - --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 - # via yamllint six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 @@ -90,6 +86,13 @@ urllib3==1.26.13 \ --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 # via requests -yamllint==1.26.3 \ - --hash=sha256:3934dcde484374596d6b52d8db412929a169f6d9e52e20f9ade5bf3523d9b96e +yamllint==1.28.0 \ + --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ + --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==65.6.3 \ + --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \ + --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75 + # via yamllint diff --git a/examples/pip_repository_annotations/BUILD.bazel b/examples/pip_repository_annotations/BUILD.bazel index 84089f77d0..77b5ab0698 100644 --- a/examples/pip_repository_annotations/BUILD.bazel +++ b/examples/pip_repository_annotations/BUILD.bazel @@ -10,7 +10,6 @@ exports_files( # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], ) py_test( diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 4b209b304c..b322a7007e 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -23,13 +23,13 @@ _RULE_DEPS = [ # START: maintained by 'bazel run //tools/private:update_pip_deps' ( "pypi__build", - "https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl", - "38a7a2b7a0bdc61a42a0a67509d88c71ecfc37b393baba770fae34e20929ff69", + "https://files.pythonhosted.org/packages/58/91/17b00d5fac63d3dca605f1b8269ba3c65e98059e1fd99d00283e42a454f0/build-0.10.0-py3-none-any.whl", + "af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171", ), ( "pypi__click", - "https://files.pythonhosted.org/packages/76/0a/b6c5f311e32aeb3b406e03c079ade51e905ea630fc19d1262a46249c1c86/click-8.0.1-py3-none-any.whl", - "fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6", + "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", ), ( "pypi__colorama", @@ -38,8 +38,8 @@ _RULE_DEPS = [ ), ( "pypi__importlib_metadata", - "https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl", - "bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "https://files.pythonhosted.org/packages/cc/37/db7ba97e676af155f5fcb1a35466f446eadc9104e25b83366e8088c9c926/importlib_metadata-6.8.0-py3-none-any.whl", + "3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb", ), ( "pypi__installer", @@ -48,13 +48,13 @@ _RULE_DEPS = [ ), ( "pypi__more_itertools", - "https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl", - "c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb", + "https://files.pythonhosted.org/packages/5a/cb/6dce742ea14e47d6f565589e859ad225f2a5de576d7696e0623b784e226b/more_itertools-10.1.0-py3-none-any.whl", + "64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6", ), ( "pypi__packaging", - "https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl", - "957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3", + "https://files.pythonhosted.org/packages/ab/c3/57f0601a2d4fe15de7a553c00adbc901425661bf048f2a22dfc500caf121/packaging-23.1-py3-none-any.whl", + "994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", ), ( "pypi__pep517", @@ -63,18 +63,23 @@ _RULE_DEPS = [ ), ( "pypi__pip", - "https://files.pythonhosted.org/packages/09/bd/2410905c76ee14c62baf69e3f4aa780226c1bbfc9485731ad018e35b0cb5/pip-22.3.1-py3-none-any.whl", - "908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077", + "https://files.pythonhosted.org/packages/50/c2/e06851e8cc28dcad7c155f4753da8833ac06a5c704c109313b8d5a62968a/pip-23.2.1-py3-none-any.whl", + "7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", ), ( "pypi__pip_tools", - "https://files.pythonhosted.org/packages/5e/e8/f6d7d1847c7351048da870417724ace5c4506e816b38db02f4d7c675c189/pip_tools-6.12.1-py3-none-any.whl", - "f0c0c0ec57b58250afce458e2e6058b1f30a4263db895b7d72fd6311bf1dc6f7", + "https://files.pythonhosted.org/packages/e8/df/47e6267c6b5cdae867adbdd84b437393e6202ce4322de0a5e0b92960e1d6/pip_tools-7.3.0-py3-none-any.whl", + "8717693288720a8c6ebd07149c93ab0be1fced0b5191df9e9decd3263e20d85e", + ), + ( + "pypi__pyproject_hooks", + "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl", + "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8", ), ( "pypi__setuptools", - "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", - "782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96", + "https://files.pythonhosted.org/packages/4f/ab/0bcfebdfc3bfa8554b2b2c97a555569c4c1ebc74ea288741ea8326c51906/setuptools-68.1.2-py3-none-any.whl", + "3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b", ), ( "pypi__tomli", @@ -83,13 +88,13 @@ _RULE_DEPS = [ ), ( "pypi__wheel", - "https://files.pythonhosted.org/packages/bd/7c/d38a0b30ce22fc26ed7dbc087c6d00851fb3395e9d0dac40bec1f905030c/wheel-0.38.4-py3-none-any.whl", - "b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8", + "https://files.pythonhosted.org/packages/b8/8b/31273bf66016be6ad22bb7345c37ff350276cfd46e389a0c2ac5da9d9073/wheel-0.41.2-py3-none-any.whl", + "75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8", ), ( "pypi__zipp", - "https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl", - "8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", + "https://files.pythonhosted.org/packages/8c/08/d3006317aefe25ea79d3b76c9650afabaf6d63d1c8443b236e7405447503/zipp-3.16.2-py3-none-any.whl", + "679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0", ), # END: maintained by 'bazel run //tools/private:update_pip_deps' ] diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index 84ee203ffd..a48718151f 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -90,20 +90,23 @@ def compile_pip_requirements( loc.format(requirements_darwin) if requirements_darwin else "None", loc.format(requirements_windows) if requirements_windows else "None", "//%s:%s.update" % (native.package_name(), name), + "--resolver=backtracking", + "--allow-unsafe", ] + (["--generate-hashes"] if generate_hashes else []) + extra_args deps = [ requirement("build"), requirement("click"), requirement("colorama"), + requirement("importlib_metadata"), + requirement("more_itertools"), requirement("pep517"), requirement("pip"), requirement("pip_tools"), + requirement("pyproject_hooks"), requirement("setuptools"), requirement("tomli"), - requirement("importlib_metadata"), requirement("zipp"), - requirement("more_itertools"), Label("//python/runfiles:runfiles"), ] + extra_deps diff --git a/python/pip_install/tools/requirements.txt b/python/pip_install/tools/requirements.txt index e8de11216e..bf9fe46afd 100755 --- a/python/pip_install/tools/requirements.txt +++ b/python/pip_install/tools/requirements.txt @@ -1,14 +1,14 @@ -build==0.9 -click==8.0.1 +build +click colorama -importlib_metadata==1.4.0 +importlib_metadata installer -more_itertools==8.13.0 -packaging==22.0 +more_itertools +packaging pep517 -pip==22.3.1 -pip_tools==6.12.1 -setuptools==60.10 +pip +pip_tools +setuptools tomli -wheel==0.38.4 -zipp==1.0.0 +wheel +zipp diff --git a/tests/compile_pip_requirements/BUILD.bazel b/tests/compile_pip_requirements/BUILD.bazel index ad5ee1a9d7..cadb59a3e8 100644 --- a/tests/compile_pip_requirements/BUILD.bazel +++ b/tests/compile_pip_requirements/BUILD.bazel @@ -25,10 +25,6 @@ compile_pip_requirements( "requirements.in", "requirements_extra.in", ], - extra_args = [ - "--allow-unsafe", - "--resolver=backtracking", - ], requirements_in = "requirements.txt", requirements_txt = "requirements_lock.txt", ) @@ -39,10 +35,6 @@ compile_pip_requirements( "requirements.in", "requirements_extra.in", ], - extra_args = [ - "--allow-unsafe", - "--resolver=backtracking", - ], generate_hashes = False, requirements_in = "requirements.txt", requirements_txt = "requirements_nohashes_lock.txt", @@ -67,10 +59,6 @@ compile_pip_requirements( "requirements_extra.in", "requirements_os_specific.in", ], - extra_args = [ - "--allow-unsafe", - "--resolver=backtracking", - ], requirements_darwin = "requirements_lock_darwin.txt", requirements_in = "requirements_os_specific.in", requirements_linux = "requirements_lock_linux.txt", diff --git a/tests/compile_pip_requirements/requirements_lock.txt b/tests/compile_pip_requirements/requirements_lock.txt index 4ca4a11f3e..5ce7d3bf71 100644 --- a/tests/compile_pip_requirements/requirements_lock.txt +++ b/tests/compile_pip_requirements/requirements_lock.txt @@ -4,6 +4,8 @@ # # bazel run //:requirements.update # + +# The following packages are considered to be unsafe in a requirements file: pip==22.3.1 \ --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \ --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077 diff --git a/tests/compile_pip_requirements/requirements_lock_darwin.txt b/tests/compile_pip_requirements/requirements_lock_darwin.txt index 7b580a2a03..0428dc05b2 100644 --- a/tests/compile_pip_requirements/requirements_lock_darwin.txt +++ b/tests/compile_pip_requirements/requirements_lock_darwin.txt @@ -4,6 +4,8 @@ # # bazel run //:os_specific_requirements.update # + +# The following packages are considered to be unsafe in a requirements file: pip==22.2.2 ; sys_platform == "darwin" \ --hash=sha256:3fd1929db052f056d7a998439176d3333fa1b3f6c1ad881de1885c0717608a4b \ --hash=sha256:b61a374b5bc40a6e982426aede40c9b5a08ff20e640f5b56977f4f91fed1e39a diff --git a/tests/compile_pip_requirements/requirements_lock_linux.txt b/tests/compile_pip_requirements/requirements_lock_linux.txt index 54eca176c5..37c4d49839 100644 --- a/tests/compile_pip_requirements/requirements_lock_linux.txt +++ b/tests/compile_pip_requirements/requirements_lock_linux.txt @@ -4,6 +4,8 @@ # # bazel run //:os_specific_requirements.update # + +# The following packages are considered to be unsafe in a requirements file: pip==22.3 ; sys_platform == "linux" \ --hash=sha256:1daab4b8d3b97d1d763caeb01a4640a2250a0ea899e257b1e44b9eded91e15ab \ --hash=sha256:8182aec21dad6c0a49a2a3d121a87cd524b950e0b6092b181625f07ebdde7530 diff --git a/tests/compile_pip_requirements/requirements_lock_windows.txt b/tests/compile_pip_requirements/requirements_lock_windows.txt index 5803d8e620..5f8e0faa6c 100644 --- a/tests/compile_pip_requirements/requirements_lock_windows.txt +++ b/tests/compile_pip_requirements/requirements_lock_windows.txt @@ -4,6 +4,8 @@ # # bazel run //:os_specific_requirements.update # + +# The following packages are considered to be unsafe in a requirements file: pip==22.2.1 ; sys_platform == "win32" \ --hash=sha256:0bbbc87dfbe6eed217beff0021f8b7dea04c8f4a0baa9d31dc4cff281ffc5b2b \ --hash=sha256:50516e47a2b79e77446f0d05649f0d53772c192571486236b1905492bfc24bac diff --git a/tests/compile_pip_requirements/requirements_nohashes_lock.txt b/tests/compile_pip_requirements/requirements_nohashes_lock.txt index 2b08a8eb6c..f6f0d86ceb 100644 --- a/tests/compile_pip_requirements/requirements_nohashes_lock.txt +++ b/tests/compile_pip_requirements/requirements_nohashes_lock.txt @@ -4,6 +4,8 @@ # # bazel run //:requirements_nohashes.update # + +# The following packages are considered to be unsafe in a requirements file: pip==22.3.1 # via -r requirements.in setuptools==65.6.3 diff --git a/tests/pip_repository_entry_points/BUILD.bazel b/tests/pip_repository_entry_points/BUILD.bazel index 81c01c316c..2e2e2dcf99 100644 --- a/tests/pip_repository_entry_points/BUILD.bazel +++ b/tests/pip_repository_entry_points/BUILD.bazel @@ -6,7 +6,6 @@ load("@rules_python//python:pip.bzl", "compile_pip_requirements") # This rule adds a convenient way to update the requirements file. compile_pip_requirements( name = "requirements", - extra_args = ["--allow-unsafe"], requirements_windows = ":requirements_windows.txt", ) diff --git a/tests/pip_repository_entry_points/requirements.txt b/tests/pip_repository_entry_points/requirements.txt index 20114b2838..d663c358f3 100644 --- a/tests/pip_repository_entry_points/requirements.txt +++ b/tests/pip_repository_entry_points/requirements.txt @@ -168,13 +168,6 @@ requests==2.27.1 \ --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d # via sphinx -setuptools==59.6.0 \ - --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ - --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e - # via - # -r requirements.in - # sphinx - # yamllint snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a @@ -215,3 +208,12 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.6.0 \ + --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ + --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e + # via + # -r requirements.in + # sphinx + # yamllint diff --git a/tests/pip_repository_entry_points/requirements_windows.txt b/tests/pip_repository_entry_points/requirements_windows.txt index 075c960d60..fc5779bebd 100644 --- a/tests/pip_repository_entry_points/requirements_windows.txt +++ b/tests/pip_repository_entry_points/requirements_windows.txt @@ -172,13 +172,6 @@ requests==2.27.1 \ --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \ --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d # via sphinx -setuptools==59.6.0 \ - --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ - --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e - # via - # -r requirements.in - # sphinx - # yamllint snowballstemmer==2.2.0 \ --hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \ --hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a @@ -219,3 +212,12 @@ yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b # via -r requirements.in + +# The following packages are considered to be unsafe in a requirements file: +setuptools==59.6.0 \ + --hash=sha256:22c7348c6d2976a52632c67f7ab0cdf40147db7789f9aed18734643fe9cf3373 \ + --hash=sha256:4ce92f1e1f8f01233ee9952c04f6b81d1e02939d6e1b488428154974a4d0783e + # via + # -r requirements.in + # sphinx + # yamllint From 4a0ba3b136825ad0fa7bb6bd2fba09211a0851bc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 28 Sep 2023 11:17:26 -0700 Subject: [PATCH 29/41] tests(pystar): CI configs that uses Starlark implementation of rules (#1435) Since Bazel 7 isn't yet available, and we need a couple recent commits in Bazel, the last_green release is used. This will eventually be changed to "rolling" once the necessary commits are available in the next rolling release. Because we're at the limit of 80 test jobs, some existing jobs have to be removed and only a limited number of cases can be currently covered. I opted to remove the pip_install tests, since they're redundant with the pip_parse tests, which frees up 4 slots. Two test configs are added: one using workspace, the other using bzlmod; both run the default tests on Ubuntu. This will provide some basic coverage; more slots will free up when Bazel 5.4 support is dropped and Bazel 7 is released. Work toward #1069 --- .bazelci/presubmit.yml | 59 +++++++++++++++++++++--------------------- docs/BUILD.bazel | 31 +++++++++++++--------- 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 1da4d9fb6b..c9b8bc286d 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -93,6 +93,36 @@ tasks: <<: *reusable_config name: Default test on Ubuntu platform: ubuntu2004 + ubuntu_bazel_rolling: + <<: *reusable_config + name: "Default test: Ubuntu, Pystar, workspace" + platform: ubuntu2004 + # TODO: Change to "rolling" once + # https://github.com/bazelbuild/bazel/commit/f3aafea59ae021c6a12086cb2cd34c5fa782faf1 + # is available in rolling. + bazel: "last_green" + environment: + RULES_PYTHON_ENABLE_PYSTAR: "1" + test_flags: + # The doc check tests fail because the Starlark implementation makes the + # PyInfo and PyRuntimeInfo symbols become documented. + - "--test_tag_filters=-integration-test,-doc_check_test" + ubuntu_bazel_rolling_bzlmod: + <<: *reusable_config + <<: *common_bzlmod_flags + name: "Default test: Ubuntu, Pystar, bzlmod" + platform: ubuntu2004 + # TODO: Change to "rolling" once + # https://github.com/bazelbuild/bazel/commit/f3aafea59ae021c6a12086cb2cd34c5fa782faf1 + # is available in rolling. + bazel: "last_green" + environment: + RULES_PYTHON_ENABLE_PYSTAR: "1" + test_flags: + # The doc check tests fail because the Starlark implementation makes the + # PyInfo and PyRuntimeInfo symbols become documented. + - "--test_tag_filters=-integration-test,-doc_check_test" + debian: <<: *reusable_config name: Default test on Debian @@ -107,7 +137,6 @@ tasks: platform: windows test_flags: - "--test_tag_filters=-integration-test,-fix-windows" - rbe_min: <<: *minimum_supported_version <<: *reusable_config @@ -262,34 +291,6 @@ tasks: name: multi_python_versions integration tests on Windows working_directory: examples/multi_python_versions platform: windows - - integration_test_pip_install_ubuntu_min: - <<: *minimum_supported_version - <<: *reusable_build_test_all - name: pip_install integration tests on Ubuntu using minimum supported Bazel version - working_directory: examples/pip_install - platform: ubuntu2004 - integration_test_pip_install_ubuntu: - <<: *reusable_build_test_all - name: pip_install integration tests on Ubuntu - working_directory: examples/pip_install - platform: ubuntu2004 - integration_test_pip_install_debian: - <<: *reusable_build_test_all - name: pip_install integration tests on Debian - working_directory: examples/pip_install - platform: debian11 - integration_test_pip_install_macos: - <<: *reusable_build_test_all - name: pip_install integration tests on macOS - working_directory: examples/pip_install - platform: macos - integration_test_pip_install_windows: - <<: *reusable_build_test_all - name: pip_install integration tests on Windows - working_directory: examples/pip_install - platform: windows - integration_test_pip_parse_ubuntu_min: <<: *minimum_supported_version <<: *reusable_build_test_all diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 6ddf54aeba..5e0357f2ee 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -16,6 +16,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:diff_test.bzl", "diff_test") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@io_bazel_stardoc//stardoc:stardoc.bzl", "stardoc") +load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility package(default_visibility = ["//visibility:public"]) @@ -74,11 +75,16 @@ bzl_library( ], ) +# Empty list means "compatible with all". +# Stardoc+bzlmod doesn't currently work with our docs, so disable trying to +# build it for now. +_COMPATIBLE_PLATFORM = [] if not BZLMOD_ENABLED else ["@platforms//:incompatible"] + # TODO: Stardoc does not guarantee consistent outputs accross platforms (Unix/Windows). # As a result we do not build or test docs on Windows. -_NOT_WINDOWS = select({ - "@platforms//os:linux": [], - "@platforms//os:macos": [], +_TARGET_COMPATIBLE_WITH = select({ + "@platforms//os:linux": _COMPATIBLE_PLATFORM, + "@platforms//os:macos": _COMPATIBLE_PLATFORM, "//conditions:default": ["@platforms//:incompatible"], }) @@ -86,7 +92,7 @@ stardoc( name = "core-docs", out = "python.md_", input = "//python:defs.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [":defs"], ) @@ -94,7 +100,7 @@ stardoc( name = "pip-docs", out = "pip.md_", input = "//python:pip.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ ":bazel_repo_tools", ":pip_install_bzl", @@ -106,7 +112,7 @@ stardoc( name = "pip-repository", out = "pip_repository.md_", input = "//python/pip_install:pip_repository.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ ":bazel_repo_tools", ":pip_install_bzl", @@ -119,7 +125,7 @@ stardoc( name = "py-console-script-binary", out = "py_console_script_binary.md_", input = "//python/entry_points:py_console_script_binary.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ "//python/entry_points:py_console_script_binary_bzl", ], @@ -129,7 +135,7 @@ stardoc( name = "packaging-docs", out = "packaging.md_", input = "//python:packaging.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = ["//python:packaging_bzl"], ) @@ -141,7 +147,7 @@ stardoc( # doesn't do anything interesting to users, so bypass it to avoid having to # copy/paste all the rule's doc in the macro. input = "//python/private:py_cc_toolchain_rule.bzl", - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = ["//python/private:py_cc_toolchain_bzl"], ) @@ -158,7 +164,8 @@ stardoc( failure_message = "Please run: bazel run //docs:update", file1 = k + ".md", file2 = k + ".md_", - target_compatible_with = _NOT_WINDOWS, + tags = ["doc_check_test"], + target_compatible_with = _TARGET_COMPATIBLE_WITH, ) for k in _DOCS.keys() ] @@ -173,12 +180,12 @@ write_file( "cp -fv bazel-bin/docs/{0}.md_ docs/{0}.md".format(k) for k in _DOCS.keys() ], - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, ) sh_binary( name = "update", srcs = ["update.sh"], data = _DOCS.values(), - target_compatible_with = _NOT_WINDOWS, + target_compatible_with = _TARGET_COMPATIBLE_WITH, ) From 0c1ce6fc98dba9278df03925056582a74499097e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 29 Sep 2023 16:10:44 -0700 Subject: [PATCH 30/41] internal(pystar): Copy @bazel_tools//tools/python files to rules_python (#1437) This copies the useful pieces from @bazel_tools//tools/python into rules_python. They're copied in relatively as-is, and not yet used. Subsequent commits will make them usable. These pieces are: * Bootstrap template (python_bootstrap_template.txt) * The py_runtime_pair rule (split from toolchain.bzl) * Autodetecting toolchain setup (split from toolchain.bzl) Work towards #1069 --- python/private/autodetecting_toolchain.bzl | 127 +++++ python/private/py_runtime_pair.bzl | 140 +++++ python/private/python_bootstrap_template.txt | 559 +++++++++++++++++++ 3 files changed, 826 insertions(+) create mode 100644 python/private/autodetecting_toolchain.bzl create mode 100644 python/private/py_runtime_pair.bzl create mode 100644 python/private/python_bootstrap_template.txt diff --git a/python/private/autodetecting_toolchain.bzl b/python/private/autodetecting_toolchain.bzl new file mode 100644 index 0000000000..3f31f1fba2 --- /dev/null +++ b/python/private/autodetecting_toolchain.bzl @@ -0,0 +1,127 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Definitions related to the Python toolchain.""" + +load(":utils.bzl", "expand_pyversion_template") + +def define_autodetecting_toolchain( + name, + pywrapper_template, + windows_config_setting): + """Defines the autodetecting Python toolchain. + + This includes both strict and non-strict variants. + + For use only by @bazel_tools//tools/python:BUILD; see the documentation + comment there. + + Args: + name: The name of the toolchain to introduce. Must have value + "autodetecting_toolchain". This param is present only to make the + BUILD file more readable. + pywrapper_template: The label of the pywrapper_template.txt file. + windows_config_setting: The label of a config_setting that matches when + the platform is windows, in which case the toolchain is configured + in a way that triggers a workaround for #7844. + """ + if native.package_name() != "tools/python": + fail("define_autodetecting_toolchain() is private to " + + "@bazel_tools//tools/python") + if name != "autodetecting_toolchain": + fail("Python autodetecting toolchain must be named " + + "'autodetecting_toolchain'") + + expand_pyversion_template( + name = "_generate_wrappers", + template = pywrapper_template, + out2 = ":py2wrapper.sh", + out3 = ":py3wrapper.sh", + out2_nonstrict = ":py2wrapper_nonstrict.sh", + out3_nonstrict = ":py3wrapper_nonstrict.sh", + visibility = ["//visibility:private"], + ) + + # Note that the pywrapper script is a .sh file, not a sh_binary target. If + # we needed to make it a proper shell target, e.g. because it needed to + # access runfiles and needed to depend on the runfiles library, then we'd + # have to use a workaround to allow it to be depended on by py_runtime. See + # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317. + + # buildifier: disable=native-py + py_runtime( + name = "_autodetecting_py3_runtime", + interpreter = ":py3wrapper.sh", + python_version = "PY3", + stub_shebang = "#!/usr/bin/env python3", + visibility = ["//visibility:private"], + ) + + # buildifier: disable=native-py + py_runtime( + name = "_autodetecting_py3_runtime_nonstrict", + interpreter = ":py3wrapper_nonstrict.sh", + python_version = "PY3", + stub_shebang = "#!/usr/bin/env python3", + visibility = ["//visibility:private"], + ) + + # This is a dummy runtime whose interpreter_path triggers the native rule + # logic to use the legacy behavior on Windows. + # TODO(#7844): Remove this target. + # buildifier: disable=native-py + py_runtime( + name = "_magic_sentinel_runtime", + interpreter_path = "/_magic_pyruntime_sentinel_do_not_use", + python_version = "PY3", + visibility = ["//visibility:private"], + ) + + py_runtime_pair( + name = "_autodetecting_py_runtime_pair", + py3_runtime = select({ + # If we're on windows, inject the sentinel to tell native rule logic + # that we attempted to use the autodetecting toolchain and need to + # switch back to legacy behavior. + # TODO(#7844): Remove this hack. + windows_config_setting: ":_magic_sentinel_runtime", + "//conditions:default": ":_autodetecting_py3_runtime", + }), + visibility = ["//visibility:public"], + ) + + py_runtime_pair( + name = "_autodetecting_py_runtime_pair_nonstrict", + py3_runtime = select({ + # Same hack as above. + # TODO(#7844): Remove this hack. + windows_config_setting: ":_magic_sentinel_runtime", + "//conditions:default": ":_autodetecting_py3_runtime_nonstrict", + }), + visibility = ["//visibility:public"], + ) + + native.toolchain( + name = name, + toolchain = ":_autodetecting_py_runtime_pair", + toolchain_type = ":toolchain_type", + visibility = ["//visibility:public"], + ) + + native.toolchain( + name = name + "_nonstrict", + toolchain = ":_autodetecting_py_runtime_pair_nonstrict", + toolchain_type = ":toolchain_type", + visibility = ["//visibility:public"], + ) diff --git a/python/private/py_runtime_pair.bzl b/python/private/py_runtime_pair.bzl new file mode 100644 index 0000000000..58b55193ed --- /dev/null +++ b/python/private/py_runtime_pair.bzl @@ -0,0 +1,140 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of py_runtime_pair.""" + +# TODO: move py_runtime_pair into rules_python (and the rest of @bazel_tools//python) +# py_runtime should be loaded from rules_python, but this creates a circular dep, because py_runtime_pair is imported there. +py_runtime = native.py_runtime + +def _py_runtime_pair_impl(ctx): + if ctx.attr.py2_runtime != None: + py2_runtime = ctx.attr.py2_runtime[PyRuntimeInfo] + if py2_runtime.python_version != "PY2": + fail("The Python runtime in the 'py2_runtime' attribute did not have " + + "version 'PY2'") + else: + py2_runtime = None + + if ctx.attr.py3_runtime != None: + py3_runtime = ctx.attr.py3_runtime[PyRuntimeInfo] + if py3_runtime.python_version != "PY3": + fail("The Python runtime in the 'py3_runtime' attribute did not have " + + "version 'PY3'") + else: + py3_runtime = None + + # TODO: Uncomment this after --incompatible_python_disable_py2 defaults to true + # if _is_py2_disabled(ctx) and py2_runtime != None: + # fail("Using Python 2 is not supported and disabled; see " + + # "https://github.com/bazelbuild/bazel/issues/15684") + + return [platform_common.ToolchainInfo( + py2_runtime = py2_runtime, + py3_runtime = py3_runtime, + )] + +# buildifier: disable=unused-variable +def _is_py2_disabled(ctx): + # In Google, this file isn't bundled with Bazel, so we have to conditionally + # check for this flag. + # TODO: Remove this once a build with the flag is released in Google + if not hasattr(ctx.fragments.py, "disable_py"): + return False + return ctx.fragments.py.disable_py2 + +py_runtime_pair = rule( + implementation = _py_runtime_pair_impl, + attrs = { + # The two runtimes are used by the py_binary at runtime, and so need to + # be built for the target platform. + "py2_runtime": attr.label( + providers = [PyRuntimeInfo], + cfg = "target", + doc = """\ +The runtime to use for Python 2 targets. Must have `python_version` set to +`PY2`. +""", + ), + "py3_runtime": attr.label( + providers = [PyRuntimeInfo], + cfg = "target", + doc = """\ +The runtime to use for Python 3 targets. Must have `python_version` set to +`PY3`. +""", + ), + }, + fragments = ["py"], + doc = """\ +A toolchain rule for Python. + +This wraps up to two Python runtimes, one for Python 2 and one for Python 3. +The rule consuming this toolchain will choose which runtime is appropriate. +Either runtime may be omitted, in which case the resulting toolchain will be +unusable for building Python code using that version. + +Usually the wrapped runtimes are declared using the `py_runtime` rule, but any +rule returning a `PyRuntimeInfo` provider may be used. + +This rule returns a `platform_common.ToolchainInfo` provider with the following +schema: + +```python +platform_common.ToolchainInfo( + py2_runtime = , + py3_runtime = , +) +``` + +Example usage: + +```python +# In your BUILD file... + +load("@rules_python//python:defs.bzl", "py_runtime_pair") + +py_runtime( + name = "my_py2_runtime", + interpreter_path = "/system/python2", + python_version = "PY2", +) + +py_runtime( + name = "my_py3_runtime", + interpreter_path = "/system/python3", + python_version = "PY3", +) + +py_runtime_pair( + name = "my_py_runtime_pair", + py2_runtime = ":my_py2_runtime", + py3_runtime = ":my_py3_runtime", +) + +toolchain( + name = "my_toolchain", + target_compatible_with = <...>, + toolchain = ":my_py_runtime_pair", + toolchain_type = "@rules_python//python:toolchain_type", +) +``` + +```python +# In your WORKSPACE... + +register_toolchains("//my_pkg:my_toolchain") +``` +""", +) diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt new file mode 100644 index 0000000000..92dd6b82fa --- /dev/null +++ b/python/private/python_bootstrap_template.txt @@ -0,0 +1,559 @@ +%shebang% + +# This script must retain compatibility with a wide variety of Python versions +# since it is run for every py_binary target. Currently we guarantee support +# going back to Python 2.7, and try to support even Python 2.6 on a best-effort +# basis. We might abandon 2.6 support once users have the ability to control the +# above shebang string via the Python toolchain (#8685). + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys + +# The Python interpreter unconditionally prepends the directory containing this +# script (following symlinks) to the import path. This is the cause of #9239, +# and is a special case of #7091. We therefore explicitly delete that entry. +# TODO(#7091): Remove this hack when no longer necessary. +del sys.path[0] + +import os +import subprocess + +def IsRunningFromZip(): + return %is_zipfile% + +if IsRunningFromZip(): + import shutil + import tempfile + import zipfile +else: + import re + +# Return True if running on Windows +def IsWindows(): + return os.name == 'nt' + +def GetWindowsPathWithUNCPrefix(path): + """Adds UNC prefix after getting a normalized absolute Windows path. + + No-op for non-Windows platforms or if running under python2. + """ + path = path.strip() + + # No need to add prefix for non-Windows platforms. + # And \\?\ doesn't work in python 2 or on mingw + if not IsWindows() or sys.version_info[0] < 3: + return path + + # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been + # removed from common Win32 file and directory functions. + # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later + import platform + if platform.win32_ver()[1] >= '10.0.14393': + return path + + # import sysconfig only now to maintain python 2.6 compatibility + import sysconfig + if sysconfig.get_platform() == 'mingw': + return path + + # Lets start the unicode fun + unicode_prefix = '\\\\?\\' + if path.startswith(unicode_prefix): + return path + + # os.path.abspath returns a normalized absolute path + return unicode_prefix + os.path.abspath(path) + +def HasWindowsExecutableExtension(path): + return path.endswith('.exe') or path.endswith('.com') or path.endswith('.bat') + +PYTHON_BINARY = '%python_binary%' +if IsWindows() and not HasWindowsExecutableExtension(PYTHON_BINARY): + PYTHON_BINARY = PYTHON_BINARY + '.exe' + +def SearchPath(name): + """Finds a file in a given search path.""" + search_path = os.getenv('PATH', os.defpath).split(os.pathsep) + for directory in search_path: + if directory: + path = os.path.join(directory, name) + if os.path.isfile(path) and os.access(path, os.X_OK): + return path + return None + +def FindPythonBinary(module_space): + """Finds the real Python binary if it's not a normal absolute path.""" + return FindBinary(module_space, PYTHON_BINARY) + +def PrintVerboseCoverage(*args): + """Print output if VERBOSE_COVERAGE is non-empty in the environment.""" + if os.environ.get("VERBOSE_COVERAGE"): + print(*args, file=sys.stderr) + +def FindCoverageEntryPoint(module_space): + cov_tool = '%coverage_tool%' + if cov_tool: + PrintVerboseCoverage('Using toolchain coverage_tool %r' % cov_tool) + else: + cov_tool = os.environ.get('PYTHON_COVERAGE') + if cov_tool: + PrintVerboseCoverage('PYTHON_COVERAGE: %r' % cov_tool) + if cov_tool: + return FindBinary(module_space, cov_tool) + return None + +def FindBinary(module_space, bin_name): + """Finds the real binary if it's not a normal absolute path.""" + if not bin_name: + return None + if bin_name.startswith("//"): + # Case 1: Path is a label. Not supported yet. + raise AssertionError( + "Bazel does not support execution of Python interpreters via labels yet" + ) + elif os.path.isabs(bin_name): + # Case 2: Absolute path. + return bin_name + # Use normpath() to convert slashes to os.sep on Windows. + elif os.sep in os.path.normpath(bin_name): + # Case 3: Path is relative to the repo root. + return os.path.join(module_space, bin_name) + else: + # Case 4: Path has to be looked up in the search path. + return SearchPath(bin_name) + +def CreatePythonPathEntries(python_imports, module_space): + parts = python_imports.split(':') + return [module_space] + ['%s/%s' % (module_space, path) for path in parts] + +def FindModuleSpace(main_rel_path): + """Finds the runfiles tree.""" + # When the calling process used the runfiles manifest to resolve the + # location of this stub script, the path may be expanded. This means + # argv[0] may no longer point to a location inside the runfiles + # directory. We should therefore respect RUNFILES_DIR and + # RUNFILES_MANIFEST_FILE set by the caller. + runfiles_dir = os.environ.get('RUNFILES_DIR', None) + if not runfiles_dir: + runfiles_manifest_file = os.environ.get('RUNFILES_MANIFEST_FILE', '') + if (runfiles_manifest_file.endswith('.runfiles_manifest') or + runfiles_manifest_file.endswith('.runfiles/MANIFEST')): + runfiles_dir = runfiles_manifest_file[:-9] + # Be defensive: the runfiles dir should contain our main entry point. If + # it doesn't, then it must not be our runfiles directory. + if runfiles_dir and os.path.exists(os.path.join(runfiles_dir, main_rel_path)): + return runfiles_dir + + stub_filename = sys.argv[0] + if not os.path.isabs(stub_filename): + stub_filename = os.path.join(os.getcwd(), stub_filename) + + while True: + module_space = stub_filename + ('.exe' if IsWindows() else '') + '.runfiles' + if os.path.isdir(module_space): + return module_space + + runfiles_pattern = r'(.*\.runfiles)' + (r'\\' if IsWindows() else '/') + '.*' + matchobj = re.match(runfiles_pattern, stub_filename) + if matchobj: + return matchobj.group(1) + + if not os.path.islink(stub_filename): + break + target = os.readlink(stub_filename) + if os.path.isabs(target): + stub_filename = target + else: + stub_filename = os.path.join(os.path.dirname(stub_filename), target) + + raise AssertionError('Cannot find .runfiles directory for %s' % sys.argv[0]) + +def ExtractZip(zip_path, dest_dir): + """Extracts the contents of a zip file, preserving the unix file mode bits. + + These include the permission bits, and in particular, the executable bit. + + Ideally the zipfile module should set these bits, but it doesn't. See: + https://bugs.python.org/issue15795. + + Args: + zip_path: The path to the zip file to extract + dest_dir: The path to the destination directory + """ + zip_path = GetWindowsPathWithUNCPrefix(zip_path) + dest_dir = GetWindowsPathWithUNCPrefix(dest_dir) + with zipfile.ZipFile(zip_path) as zf: + for info in zf.infolist(): + zf.extract(info, dest_dir) + # UNC-prefixed paths must be absolute/normalized. See + # https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file#maximum-path-length-limitation + file_path = os.path.abspath(os.path.join(dest_dir, info.filename)) + # The Unix st_mode bits (see "man 7 inode") are stored in the upper 16 + # bits of external_attr. Of those, we set the lower 12 bits, which are the + # file mode bits (since the file type bits can't be set by chmod anyway). + attrs = info.external_attr >> 16 + if attrs != 0: # Rumor has it these can be 0 for zips created on Windows. + os.chmod(file_path, attrs & 0o7777) + +# Create the runfiles tree by extracting the zip file +def CreateModuleSpace(): + temp_dir = tempfile.mkdtemp('', 'Bazel.runfiles_') + ExtractZip(os.path.dirname(__file__), temp_dir) + # IMPORTANT: Later code does `rm -fr` on dirname(module_space) -- it's + # important that deletion code be in sync with this directory structure + return os.path.join(temp_dir, 'runfiles') + +# Returns repository roots to add to the import path. +def GetRepositoriesImports(module_space, import_all): + if import_all: + repo_dirs = [os.path.join(module_space, d) for d in os.listdir(module_space)] + repo_dirs.sort() + return [d for d in repo_dirs if os.path.isdir(d)] + return [os.path.join(module_space, '%workspace_name%')] + +def RunfilesEnvvar(module_space): + """Finds the runfiles manifest or the runfiles directory. + + Returns: + A tuple of (var_name, var_value) where var_name is either 'RUNFILES_DIR' or + 'RUNFILES_MANIFEST_FILE' and var_value is the path to that directory or + file, or (None, None) if runfiles couldn't be found. + """ + # If this binary is the data-dependency of another one, the other sets + # RUNFILES_MANIFEST_FILE or RUNFILES_DIR for our sake. + runfiles = os.environ.get('RUNFILES_MANIFEST_FILE', None) + if runfiles: + return ('RUNFILES_MANIFEST_FILE', runfiles) + + runfiles = os.environ.get('RUNFILES_DIR', None) + if runfiles: + return ('RUNFILES_DIR', runfiles) + + # If running from a zip, there's no manifest file. + if IsRunningFromZip(): + return ('RUNFILES_DIR', module_space) + + # Look for the runfiles "output" manifest, argv[0] + ".runfiles_manifest" + runfiles = module_space + '_manifest' + if os.path.exists(runfiles): + return ('RUNFILES_MANIFEST_FILE', runfiles) + + # Look for the runfiles "input" manifest, argv[0] + ".runfiles/MANIFEST" + # Normally .runfiles_manifest and MANIFEST are both present, but the + # former will be missing for zip-based builds or if someone copies the + # runfiles tree elsewhere. + runfiles = os.path.join(module_space, 'MANIFEST') + if os.path.exists(runfiles): + return ('RUNFILES_MANIFEST_FILE', runfiles) + + # If running in a sandbox and no environment variables are set, then + # Look for the runfiles next to the binary. + if module_space.endswith('.runfiles') and os.path.isdir(module_space): + return ('RUNFILES_DIR', module_space) + + return (None, None) + +def Deduplicate(items): + """Efficiently filter out duplicates, keeping the first element only.""" + seen = set() + for it in items: + if it not in seen: + seen.add(it) + yield it + +def InstrumentedFilePaths(): + """Yields tuples of realpath of each instrumented file with the relative path.""" + manifest_filename = os.environ.get('COVERAGE_MANIFEST') + if not manifest_filename: + return + with open(manifest_filename, "r") as manifest: + for line in manifest: + filename = line.strip() + if not filename: + continue + try: + realpath = os.path.realpath(filename) + except OSError: + print( + "Could not find instrumented file {}".format(filename), + file=sys.stderr) + continue + if realpath != filename: + PrintVerboseCoverage("Fixing up {} -> {}".format(realpath, filename)) + yield (realpath, filename) + +def UnresolveSymlinks(output_filename): + # type: (str) -> None + """Replace realpath of instrumented files with the relative path in the lcov output. + + Though we are asking coveragepy to use relative file names, currently + ignore that for purposes of generating the lcov report (and other reports + which are not the XML report), so we need to go and fix up the report. + + This function is a workaround for that issue. Once that issue is fixed + upstream and the updated version is widely in use, this should be removed. + + See https://github.com/nedbat/coveragepy/issues/963. + """ + substitutions = list(InstrumentedFilePaths()) + if substitutions: + unfixed_file = output_filename + '.tmp' + os.rename(output_filename, unfixed_file) + with open(unfixed_file, "r") as unfixed: + with open(output_filename, "w") as output_file: + for line in unfixed: + if line.startswith('SF:'): + for (realpath, filename) in substitutions: + line = line.replace(realpath, filename) + output_file.write(line) + os.unlink(unfixed_file) + +def ExecuteFile(python_program, main_filename, args, env, module_space, + coverage_entrypoint, workspace, delete_module_space): + # type: (str, str, list[str], dict[str, str], str, str|None, str|None) -> ... + """Executes the given Python file using the various environment settings. + + This will not return, and acts much like os.execv, except is much + more restricted, and handles Bazel-related edge cases. + + Args: + python_program: (str) Path to the Python binary to use for execution + main_filename: (str) The Python file to execute + args: (list[str]) Additional args to pass to the Python file + env: (dict[str, str]) A dict of environment variables to set for the execution + module_space: (str) Path to the module space/runfiles tree directory + coverage_entrypoint: (str|None) Path to the coverage tool entry point file. + workspace: (str|None) Name of the workspace to execute in. This is expected to be a + directory under the runfiles tree. + delete_module_space: (bool), True if the module space should be deleted + after a successful (exit code zero) program run, False if not. + """ + # We want to use os.execv instead of subprocess.call, which causes + # problems with signal passing (making it difficult to kill + # Bazel). However, these conditions force us to run via + # subprocess.call instead: + # + # - On Windows, os.execv doesn't handle arguments with spaces + # correctly, and it actually starts a subprocess just like + # subprocess.call. + # - When running in a workspace or zip file, we need to clean up the + # workspace after the process finishes so control must return here. + # - If we may need to emit a host config warning after execution, we + # can't execv because we need control to return here. This only + # happens for targets built in the host config. + # - For coverage targets, at least coveragepy requires running in + # two invocations, which also requires control to return here. + # + if not (IsWindows() or workspace or coverage_entrypoint or delete_module_space): + _RunExecv(python_program, main_filename, args, env) + + if coverage_entrypoint is not None: + ret_code = _RunForCoverage(python_program, main_filename, args, env, + coverage_entrypoint, workspace) + else: + ret_code = subprocess.call( + [python_program, main_filename] + args, + env=env, + cwd=workspace + ) + + if delete_module_space: + # NOTE: dirname() is called because CreateModuleSpace() creates a + # sub-directory within a temporary directory, and we want to remove the + # whole temporary directory. + shutil.rmtree(os.path.dirname(module_space), True) + sys.exit(ret_code) + +def _RunExecv(python_program, main_filename, args, env): + # type: (str, str, list[str], dict[str, str]) -> ... + """Executes the given Python file using the various environment settings.""" + os.environ.update(env) + os.execv(python_program, [python_program, main_filename] + args) + +def _RunForCoverage(python_program, main_filename, args, env, + coverage_entrypoint, workspace): + # type: (str, str, list[str], dict[str, str], str, str|None) -> int + """Collects coverage infomration for the given Python file. + + Args: + python_program: (str) Path to the Python binary to use for execution + main_filename: (str) The Python file to execute + args: (list[str]) Additional args to pass to the Python file + env: (dict[str, str]) A dict of environment variables to set for the execution + coverage_entrypoint: (str|None) Path to the coverage entry point to execute with. + workspace: (str|None) Name of the workspace to execute in. This is expected to be a + directory under the runfiles tree, and will recursively delete the + runfiles directory if set. + """ + # We need for coveragepy to use relative paths. This can only be configured + # via an rc file, so we need to make one. + rcfile_name = os.path.join(os.environ['COVERAGE_DIR'], '.coveragerc') + with open(rcfile_name, "w") as rcfile: + rcfile.write('''[run] +relative_files = True +''') + PrintVerboseCoverage('Coverage entrypoint:', coverage_entrypoint) + # First run the target Python file via coveragepy to create a .coverage + # database file, from which we can later export lcov. + ret_code = subprocess.call( + [ + python_program, + coverage_entrypoint, + "run", + "--rcfile=" + rcfile_name, + "--append", + "--branch", + main_filename + ] + args, + env=env, + cwd=workspace + ) + output_filename = os.path.join(os.environ['COVERAGE_DIR'], 'pylcov.dat') + + PrintVerboseCoverage('Converting coveragepy database to lcov:', output_filename) + # Run coveragepy again to convert its .coverage database file into lcov. + ret_code = subprocess.call( + [ + python_program, + coverage_entrypoint, + "lcov", + "--rcfile=" + rcfile_name, + "-o", + output_filename + ], + env=env, + cwd=workspace + ) or ret_code + try: + os.unlink(rcfile_name) + except OSError as err: + # It's possible that the profiled program might execute another Python + # binary through a wrapper that would then delete the rcfile. Not much + # we can do about that, besides ignore the failure here. + PrintVerboseCoverage('Error removing temporary coverage rc file:', err) + if os.path.isfile(output_filename): + UnresolveSymlinks(output_filename) + return ret_code + +def Main(): + args = sys.argv[1:] + + new_env = {} + + # The main Python source file. + # The magic string percent-main-percent is replaced with the runfiles-relative + # filename of the main file of the Python binary in BazelPythonSemantics.java. + main_rel_path = '%main%' + if IsWindows(): + main_rel_path = main_rel_path.replace('/', os.sep) + + if IsRunningFromZip(): + module_space = CreateModuleSpace() + delete_module_space = True + else: + module_space = FindModuleSpace(main_rel_path) + delete_module_space = False + + python_imports = '%imports%' + python_path_entries = CreatePythonPathEntries(python_imports, module_space) + python_path_entries += GetRepositoriesImports(module_space, %import_all%) + # Remove duplicates to avoid overly long PYTHONPATH (#10977). Preserve order, + # keep first occurrence only. + python_path_entries = [ + GetWindowsPathWithUNCPrefix(d) + for d in python_path_entries + ] + + old_python_path = os.environ.get('PYTHONPATH') + if old_python_path: + python_path_entries += old_python_path.split(os.pathsep) + + python_path = os.pathsep.join(Deduplicate(python_path_entries)) + + if IsWindows(): + python_path = python_path.replace('/', os.sep) + + new_env['PYTHONPATH'] = python_path + runfiles_envkey, runfiles_envvalue = RunfilesEnvvar(module_space) + if runfiles_envkey: + new_env[runfiles_envkey] = runfiles_envvalue + + # Don't prepend a potentially unsafe path to sys.path + # See: https://docs.python.org/3.11/using/cmdline.html#envvar-PYTHONSAFEPATH + new_env['PYTHONSAFEPATH'] = '1' + + main_filename = os.path.join(module_space, main_rel_path) + main_filename = GetWindowsPathWithUNCPrefix(main_filename) + assert os.path.exists(main_filename), \ + 'Cannot exec() %r: file not found.' % main_filename + assert os.access(main_filename, os.R_OK), \ + 'Cannot exec() %r: file not readable.' % main_filename + + program = python_program = FindPythonBinary(module_space) + if python_program is None: + raise AssertionError('Could not find python binary: ' + PYTHON_BINARY) + + # COVERAGE_DIR is set if coverage is enabled and instrumentation is configured + # for something, though it could be another program executing this one or + # one executed by this one (e.g. an extension module). + if os.environ.get('COVERAGE_DIR'): + cov_tool = FindCoverageEntryPoint(module_space) + if cov_tool is None: + PrintVerboseCoverage('Coverage was enabled, but python coverage tool was not configured.') + else: + # Inhibit infinite recursion: + if 'PYTHON_COVERAGE' in os.environ: + del os.environ['PYTHON_COVERAGE'] + + if not os.path.exists(cov_tool): + raise EnvironmentError( + 'Python coverage tool %r not found. ' + 'Try running with VERBOSE_COVERAGE=1 to collect more information.' + % cov_tool + ) + + # coverage library expects sys.path[0] to contain the library, and replaces + # it with the directory of the program it starts. Our actual sys.path[0] is + # the runfiles directory, which must not be replaced. + # CoverageScript.do_execute() undoes this sys.path[0] setting. + # + # Update sys.path such that python finds the coverage package. The coverage + # entry point is coverage.coverage_main, so we need to do twice the dirname. + python_path_entries = new_env['PYTHONPATH'].split(os.pathsep) + python_path_entries.append(os.path.dirname(os.path.dirname(cov_tool))) + new_env['PYTHONPATH'] = os.pathsep.join(Deduplicate(python_path_entries)) + else: + cov_tool = None + + new_env.update((key, val) for key, val in os.environ.items() if key not in new_env) + + workspace = None + if IsRunningFromZip(): + # If RUN_UNDER_RUNFILES equals 1, it means we need to + # change directory to the right runfiles directory. + # (So that the data files are accessible) + if os.environ.get('RUN_UNDER_RUNFILES') == '1': + workspace = os.path.join(module_space, '%workspace_name%') + + try: + sys.stdout.flush() + # NOTE: ExecuteFile may call execve() and lines after this will never run. + ExecuteFile( + python_program, main_filename, args, new_env, module_space, + cov_tool, workspace, + delete_module_space = delete_module_space, + ) + + except EnvironmentError: + # This works from Python 2.4 all the way to 3.x. + e = sys.exc_info()[1] + # This exception occurs when os.execv() fails for some reason. + if not getattr(e, 'filename', None): + e.filename = program # Add info to error message + raise + +if __name__ == '__main__': + Main() From 4862a8ddd6d3bd9c741723442e0e1254d1cc6cdc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 29 Sep 2023 16:33:42 -0700 Subject: [PATCH 31/41] internal(pystar): Make py_runtime_pair and autodetecting toolchain mostly loadable. (#1439) They aren't quite usable yet. This mostly fixes references and adds surrounding infrastructure to make this mostly loadable and somewhat runnable. Note that the "autodetecting" name is misleading: it doesn't really autodetect anything. But it's the established name and part of the public API. The autodetecting toolchain is trimmed down from what Bazel does. The Bazel version has a Python 2 variant and a "non strict" mode to support that. With Python 2 no longer supported, that code is removed. * Also alphabetically sorts the bzl_libraries in //python/private Work towards #1069 --- python/private/BUILD.bazel | 54 ++++++++++----- python/private/autodetecting_toolchain.bzl | 65 ++----------------- .../autodetecting_toolchain_interpreter.sh | 57 ++++++++++++++++ python/private/py_runtime_pair_macro.bzl | 27 ++++++++ ...time_pair.bzl => py_runtime_pair_rule.bzl} | 8 +-- 5 files changed, 130 insertions(+), 81 deletions(-) create mode 100644 python/private/autodetecting_toolchain_interpreter.sh create mode 100644 python/private/py_runtime_pair_macro.bzl rename python/private/{py_runtime_pair.bzl => py_runtime_pair_rule.bzl} (90%) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index af121cf66d..a67183e5a7 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -44,13 +44,12 @@ filegroup( ) bzl_library( - name = "reexports_bzl", - srcs = ["reexports.bzl"], - visibility = [ - "//docs:__pkg__", - "//python:__pkg__", + name = "autodetecting_toolchain_bzl", + srcs = ["autodetecting_toolchain.bzl"], + deps = [ + "//python:py_runtime_bzl", + "//python:py_runtime_pair_bzl", ], - deps = [":bazel_tools_bzl"], ) bzl_library( @@ -63,15 +62,6 @@ bzl_library( deps = ["@bazel_skylib//lib:types"], ) -bzl_library( - name = "which_bzl", - srcs = ["which.bzl"], - visibility = [ - "//docs:__subpackages__", - "//python:__subpackages__", - ], -) - bzl_library( name = "py_cc_toolchain_bzl", srcs = [ @@ -112,6 +102,21 @@ bzl_library( visibility = ["//:__subpackages__"], ) +bzl_library( + name = "py_runtime_pair_macro_bzl", + srcs = ["py_runtime_pair_rule.bzl"], + deps = [":py_runtime_pair_rule_bzl"], +) + +bzl_library( + name = "py_runtime_pair_rule_bzl", + srcs = ["py_runtime_pair_rule.bzl"], + deps = [ + "//python:py_runtime_bzl", + "//python:py_runtime_info_bzl", + ], +) + bzl_library( name = "py_wheel_bzl", srcs = ["py_wheel.bzl"], @@ -122,12 +127,31 @@ bzl_library( ], ) +bzl_library( + name = "reexports_bzl", + srcs = ["reexports.bzl"], + visibility = [ + "//docs:__pkg__", + "//python:__pkg__", + ], + deps = [":bazel_tools_bzl"], +) + bzl_library( name = "stamp_bzl", srcs = ["stamp.bzl"], visibility = ["//:__subpackages__"], ) +bzl_library( + name = "which_bzl", + srcs = ["which.bzl"], + visibility = [ + "//docs:__subpackages__", + "//python:__subpackages__", + ], +) + # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", diff --git a/python/private/autodetecting_toolchain.bzl b/python/private/autodetecting_toolchain.bzl index 3f31f1fba2..3caa5aa8ca 100644 --- a/python/private/autodetecting_toolchain.bzl +++ b/python/private/autodetecting_toolchain.bzl @@ -14,51 +14,21 @@ """Definitions related to the Python toolchain.""" -load(":utils.bzl", "expand_pyversion_template") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") -def define_autodetecting_toolchain( - name, - pywrapper_template, - windows_config_setting): +def define_autodetecting_toolchain(name): """Defines the autodetecting Python toolchain. - This includes both strict and non-strict variants. - - For use only by @bazel_tools//tools/python:BUILD; see the documentation - comment there. - Args: name: The name of the toolchain to introduce. Must have value "autodetecting_toolchain". This param is present only to make the BUILD file more readable. - pywrapper_template: The label of the pywrapper_template.txt file. - windows_config_setting: The label of a config_setting that matches when - the platform is windows, in which case the toolchain is configured - in a way that triggers a workaround for #7844. """ - if native.package_name() != "tools/python": - fail("define_autodetecting_toolchain() is private to " + - "@bazel_tools//tools/python") if name != "autodetecting_toolchain": fail("Python autodetecting toolchain must be named " + "'autodetecting_toolchain'") - expand_pyversion_template( - name = "_generate_wrappers", - template = pywrapper_template, - out2 = ":py2wrapper.sh", - out3 = ":py3wrapper.sh", - out2_nonstrict = ":py2wrapper_nonstrict.sh", - out3_nonstrict = ":py3wrapper_nonstrict.sh", - visibility = ["//visibility:private"], - ) - - # Note that the pywrapper script is a .sh file, not a sh_binary target. If - # we needed to make it a proper shell target, e.g. because it needed to - # access runfiles and needed to depend on the runfiles library, then we'd - # have to use a workaround to allow it to be depended on by py_runtime. See - # https://github.com/bazelbuild/bazel/issues/4286#issuecomment-475661317. - # buildifier: disable=native-py py_runtime( name = "_autodetecting_py3_runtime", @@ -68,15 +38,6 @@ def define_autodetecting_toolchain( visibility = ["//visibility:private"], ) - # buildifier: disable=native-py - py_runtime( - name = "_autodetecting_py3_runtime_nonstrict", - interpreter = ":py3wrapper_nonstrict.sh", - python_version = "PY3", - stub_shebang = "#!/usr/bin/env python3", - visibility = ["//visibility:private"], - ) - # This is a dummy runtime whose interpreter_path triggers the native rule # logic to use the legacy behavior on Windows. # TODO(#7844): Remove this target. @@ -95,33 +56,15 @@ def define_autodetecting_toolchain( # that we attempted to use the autodetecting toolchain and need to # switch back to legacy behavior. # TODO(#7844): Remove this hack. - windows_config_setting: ":_magic_sentinel_runtime", + "@platforms//os:windows": ":_magic_sentinel_runtime", "//conditions:default": ":_autodetecting_py3_runtime", }), visibility = ["//visibility:public"], ) - py_runtime_pair( - name = "_autodetecting_py_runtime_pair_nonstrict", - py3_runtime = select({ - # Same hack as above. - # TODO(#7844): Remove this hack. - windows_config_setting: ":_magic_sentinel_runtime", - "//conditions:default": ":_autodetecting_py3_runtime_nonstrict", - }), - visibility = ["//visibility:public"], - ) - native.toolchain( name = name, toolchain = ":_autodetecting_py_runtime_pair", toolchain_type = ":toolchain_type", visibility = ["//visibility:public"], ) - - native.toolchain( - name = name + "_nonstrict", - toolchain = ":_autodetecting_py_runtime_pair_nonstrict", - toolchain_type = ":toolchain_type", - visibility = ["//visibility:public"], - ) diff --git a/python/private/autodetecting_toolchain_interpreter.sh b/python/private/autodetecting_toolchain_interpreter.sh new file mode 100644 index 0000000000..5c8a10d601 --- /dev/null +++ b/python/private/autodetecting_toolchain_interpreter.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +# Don't set -e because we don't have robust trapping and printing of errors. +set -u + +# We use /bin/sh rather than /bin/bash for portability. See discussion here: +# https://groups.google.com/forum/?nomobile=true#!topic/bazel-dev/4Ql_7eDcLC0 +# We do lose the ability to set -o pipefail. + +FAILURE_HEADER="\ +Error occurred while attempting to use the default Python toolchain \ +(@rules_python//python:autodetecting_toolchain)." + +die() { + echo "$FAILURE_HEADER" 1>&2 + echo "$1" 1>&2 + exit 1 +} + +# We use `which` to locate the Python interpreter command on PATH. `command -v` +# is another option, but it doesn't check whether the file it finds has the +# executable bit. +# +# A tricky situation happens when this wrapper is invoked as part of running a +# tool, e.g. passing a py_binary target to `ctx.actions.run()`. Bazel will unset +# the PATH variable. Then the shell will see there's no PATH and initialize its +# own, sometimes without exporting it. This causes `which` to fail and this +# script to think there's no Python interpreter installed. To avoid this we +# explicitly pass PATH to each `which` invocation. We can't just export PATH +# because that would modify the environment seen by the final user Python +# program. +# +# See also: +# +# https://github.com/bazelbuild/continuous-integration/issues/578 +# https://github.com/bazelbuild/bazel/issues/8414 +# https://github.com/bazelbuild/bazel/issues/8415 + +# Try the "python3" command name first, then fall back on "python". +PYTHON_BIN="$(PATH="$PATH" which python3 2> /dev/null)" +if [ -z "${PYTHON_BIN:-}" ]; then + PYTHON_BIN="$(PATH="$PATH" which python 2>/dev/null)" +fi +if [ -z "${PYTHON_BIN:-}" ]; then + die "Neither 'python3' nor 'python' were found on the target \ +platform's PATH, which is: + +$PATH + +Please ensure an interpreter is available on this platform (and marked \ +executable), or else register an appropriate Python toolchain as per the \ +documentation for py_runtime_pair \ +(https://github.com/bazelbuild/rules_python/blob/master/docs/python.md#py_runtime_pair)." +fi + +exec "$PYTHON_BIN" "$@" + diff --git a/python/private/py_runtime_pair_macro.bzl b/python/private/py_runtime_pair_macro.bzl new file mode 100644 index 0000000000..3cc359968e --- /dev/null +++ b/python/private/py_runtime_pair_macro.bzl @@ -0,0 +1,27 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of py_runtime_pair macro portion.""" + +load(":py_runtime_pair_rule.bzl", _py_runtime_pair = "py_runtime_pair") + +# A fronting macro is used because macros have user-observable behavior; +# using one from the onset avoids introducing those changes in the future. +def py_runtime_pair(**kwargs): + """Creates a py_runtime_pair target. + + Args: + **kwargs: Keyword args to pass onto underlying rule. + """ + _py_runtime_pair(**kwargs) diff --git a/python/private/py_runtime_pair.bzl b/python/private/py_runtime_pair_rule.bzl similarity index 90% rename from python/private/py_runtime_pair.bzl rename to python/private/py_runtime_pair_rule.bzl index 58b55193ed..d0d8c5b5ee 100644 --- a/python/private/py_runtime_pair.bzl +++ b/python/private/py_runtime_pair_rule.bzl @@ -14,9 +14,7 @@ """Implementation of py_runtime_pair.""" -# TODO: move py_runtime_pair into rules_python (and the rest of @bazel_tools//python) -# py_runtime should be loaded from rules_python, but this creates a circular dep, because py_runtime_pair is imported there. -py_runtime = native.py_runtime +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") def _py_runtime_pair_impl(ctx): if ctx.attr.py2_runtime != None: @@ -47,9 +45,9 @@ def _py_runtime_pair_impl(ctx): # buildifier: disable=unused-variable def _is_py2_disabled(ctx): - # In Google, this file isn't bundled with Bazel, so we have to conditionally + # Because this file isn't bundled with Bazel, so we have to conditionally # check for this flag. - # TODO: Remove this once a build with the flag is released in Google + # TODO: Remove this once all supported Balze versions have this flag. if not hasattr(ctx.fragments.py, "disable_py"): return False return ctx.fragments.py.disable_py2 From 4dfb78dff756e537b5c458575c2cd72c98c36918 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 29 Sep 2023 19:46:58 -0700 Subject: [PATCH 32/41] tests: Move base rule tests under tests instead of //tools/build_defs/python (#1440) The tools/build_defs/python path is an artifact of some Google-internal naming that isn't applicable to the rules_python source tree layout. With the Starlark implementation running in CI and working in some capacity, it's time to move them to a less mysterious location. * Also moves the py_wheel tests out of the base rule tests. Not sure how they ended up there. Work towards #1069 --- .../python/tests => tests/base_rules}/BUILD.bazel | 0 .../python/tests => tests/base_rules}/base_tests.bzl | 4 ++-- .../tests => tests/base_rules}/py_binary/BUILD.bazel | 0 .../base_rules}/py_binary/py_binary_tests.bzl | 2 +- .../base_rules}/py_executable_base_tests.bzl | 4 ++-- .../python/tests => tests/base_rules}/py_info_subject.bzl | 0 .../tests => tests/base_rules}/py_library/BUILD.bazel | 0 .../base_rules}/py_library/py_library_tests.bzl | 4 ++-- .../python/tests => tests/base_rules}/py_test/BUILD.bazel | 0 .../tests => tests/base_rules}/py_test/py_test_tests.bzl | 8 ++++---- .../build_defs/python/tests => tests/base_rules}/util.bzl | 0 .../python/tests => tests/py_wheel}/py_wheel/BUILD.bazel | 0 .../tests => tests/py_wheel}/py_wheel/py_wheel_tests.bzl | 2 +- 13 files changed, 12 insertions(+), 12 deletions(-) rename {tools/build_defs/python/tests => tests/base_rules}/BUILD.bazel (100%) rename {tools/build_defs/python/tests => tests/base_rules}/base_tests.bzl (96%) rename {tools/build_defs/python/tests => tests/base_rules}/py_binary/BUILD.bazel (100%) rename {tools/build_defs/python/tests => tests/base_rules}/py_binary/py_binary_tests.bzl (92%) rename {tools/build_defs/python/tests => tests/base_rules}/py_executable_base_tests.bzl (98%) rename {tools/build_defs/python/tests => tests/base_rules}/py_info_subject.bzl (100%) rename {tools/build_defs/python/tests => tests/base_rules}/py_library/BUILD.bazel (100%) rename {tools/build_defs/python/tests => tests/base_rules}/py_library/py_library_tests.bzl (96%) rename {tools/build_defs/python/tests => tests/base_rules}/py_test/BUILD.bazel (100%) rename {tools/build_defs/python/tests => tests/base_rules}/py_test/py_test_tests.bzl (93%) rename {tools/build_defs/python/tests => tests/base_rules}/util.bzl (100%) rename {tools/build_defs/python/tests => tests/py_wheel}/py_wheel/BUILD.bazel (100%) rename {tools/build_defs/python/tests => tests/py_wheel}/py_wheel/py_wheel_tests.bzl (94%) diff --git a/tools/build_defs/python/tests/BUILD.bazel b/tests/base_rules/BUILD.bazel similarity index 100% rename from tools/build_defs/python/tests/BUILD.bazel rename to tests/base_rules/BUILD.bazel diff --git a/tools/build_defs/python/tests/base_tests.bzl b/tests/base_rules/base_tests.bzl similarity index 96% rename from tools/build_defs/python/tests/base_tests.bzl rename to tests/base_rules/base_tests.bzl index 467611fcd8..53001639f6 100644 --- a/tools/build_defs/python/tests/base_tests.bzl +++ b/tests/base_rules/base_tests.bzl @@ -17,8 +17,8 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", "PREVENT_IMPLICIT_BUILDING_TAGS", rt_util = "util") load("//python:defs.bzl", "PyInfo") -load("//tools/build_defs/python/tests:py_info_subject.bzl", "py_info_subject") -load("//tools/build_defs/python/tests:util.bzl", pt_util = "util") +load("//tests/base_rules:py_info_subject.bzl", "py_info_subject") +load("//tests/base_rules:util.bzl", pt_util = "util") _tests = [] diff --git a/tools/build_defs/python/tests/py_binary/BUILD.bazel b/tests/base_rules/py_binary/BUILD.bazel similarity index 100% rename from tools/build_defs/python/tests/py_binary/BUILD.bazel rename to tests/base_rules/py_binary/BUILD.bazel diff --git a/tools/build_defs/python/tests/py_binary/py_binary_tests.bzl b/tests/base_rules/py_binary/py_binary_tests.bzl similarity index 92% rename from tools/build_defs/python/tests/py_binary/py_binary_tests.bzl rename to tests/base_rules/py_binary/py_binary_tests.bzl index 8d32632610..571955d3c6 100644 --- a/tools/build_defs/python/tests/py_binary/py_binary_tests.bzl +++ b/tests/base_rules/py_binary/py_binary_tests.bzl @@ -15,7 +15,7 @@ load("//python:defs.bzl", "py_binary") load( - "//tools/build_defs/python/tests:py_executable_base_tests.bzl", + "//tests/base_rules:py_executable_base_tests.bzl", "create_executable_tests", ) diff --git a/tools/build_defs/python/tests/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl similarity index 98% rename from tools/build_defs/python/tests/py_executable_base_tests.bzl rename to tests/base_rules/py_executable_base_tests.bzl index c66ea11e00..13ec946be5 100644 --- a/tools/build_defs/python/tests/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -16,8 +16,8 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//tools/build_defs/python/tests:base_tests.bzl", "create_base_tests") -load("//tools/build_defs/python/tests:util.bzl", "WINDOWS_ATTR", pt_util = "util") +load("//tests/base_rules:base_tests.bzl", "create_base_tests") +load("//tests/base_rules:util.bzl", "WINDOWS_ATTR", pt_util = "util") _tests = [] diff --git a/tools/build_defs/python/tests/py_info_subject.bzl b/tests/base_rules/py_info_subject.bzl similarity index 100% rename from tools/build_defs/python/tests/py_info_subject.bzl rename to tests/base_rules/py_info_subject.bzl diff --git a/tools/build_defs/python/tests/py_library/BUILD.bazel b/tests/base_rules/py_library/BUILD.bazel similarity index 100% rename from tools/build_defs/python/tests/py_library/BUILD.bazel rename to tests/base_rules/py_library/BUILD.bazel diff --git a/tools/build_defs/python/tests/py_library/py_library_tests.bzl b/tests/base_rules/py_library/py_library_tests.bzl similarity index 96% rename from tools/build_defs/python/tests/py_library/py_library_tests.bzl rename to tests/base_rules/py_library/py_library_tests.bzl index 1fcb0c19b9..526735af71 100644 --- a/tools/build_defs/python/tests/py_library/py_library_tests.bzl +++ b/tests/base_rules/py_library/py_library_tests.bzl @@ -4,8 +4,8 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:defs.bzl", "PyRuntimeInfo", "py_library") -load("//tools/build_defs/python/tests:base_tests.bzl", "create_base_tests") -load("//tools/build_defs/python/tests:util.bzl", pt_util = "util") +load("//tests/base_rules:base_tests.bzl", "create_base_tests") +load("//tests/base_rules:util.bzl", pt_util = "util") _tests = [] diff --git a/tools/build_defs/python/tests/py_test/BUILD.bazel b/tests/base_rules/py_test/BUILD.bazel similarity index 100% rename from tools/build_defs/python/tests/py_test/BUILD.bazel rename to tests/base_rules/py_test/BUILD.bazel diff --git a/tools/build_defs/python/tests/py_test/py_test_tests.bzl b/tests/base_rules/py_test/py_test_tests.bzl similarity index 93% rename from tools/build_defs/python/tests/py_test/py_test_tests.bzl rename to tests/base_rules/py_test/py_test_tests.bzl index 1ecb2524bf..4d0f7d1c3e 100644 --- a/tools/build_defs/python/tests/py_test/py_test_tests.bzl +++ b/tests/base_rules/py_test/py_test_tests.bzl @@ -17,17 +17,17 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:defs.bzl", "py_test") load( - "//tools/build_defs/python/tests:py_executable_base_tests.bzl", + "//tests/base_rules:py_executable_base_tests.bzl", "create_executable_tests", ) -load("//tools/build_defs/python/tests:util.bzl", pt_util = "util") +load("//tests/base_rules:util.bzl", pt_util = "util") # Explicit Label() calls are required so that it resolves in @rules_python context instead of # @rules_testing context. _FAKE_CC_TOOLCHAIN = Label("//tests/cc:cc_toolchain_suite") _FAKE_CC_TOOLCHAINS = [str(Label("//tests/cc:all"))] -_PLATFORM_MAC = Label("//tools/build_defs/python/tests:mac") -_PLATFORM_LINUX = Label("//tools/build_defs/python/tests:linux") +_PLATFORM_MAC = Label("//tests/base_rules:mac") +_PLATFORM_LINUX = Label("//tests/base_rules:linux") _tests = [] diff --git a/tools/build_defs/python/tests/util.bzl b/tests/base_rules/util.bzl similarity index 100% rename from tools/build_defs/python/tests/util.bzl rename to tests/base_rules/util.bzl diff --git a/tools/build_defs/python/tests/py_wheel/BUILD.bazel b/tests/py_wheel/py_wheel/BUILD.bazel similarity index 100% rename from tools/build_defs/python/tests/py_wheel/BUILD.bazel rename to tests/py_wheel/py_wheel/BUILD.bazel diff --git a/tools/build_defs/python/tests/py_wheel/py_wheel_tests.bzl b/tests/py_wheel/py_wheel/py_wheel_tests.bzl similarity index 94% rename from tools/build_defs/python/tests/py_wheel/py_wheel_tests.bzl rename to tests/py_wheel/py_wheel/py_wheel_tests.bzl index 4408592d32..c70163ef37 100644 --- a/tools/build_defs/python/tests/py_wheel/py_wheel_tests.bzl +++ b/tests/py_wheel/py_wheel/py_wheel_tests.bzl @@ -4,7 +4,7 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:packaging.bzl", "py_wheel") -load("//tools/build_defs/python/tests:util.bzl", pt_util = "util") +load("//tests/base_rules:util.bzl", pt_util = "util") _tests = [] From 21b54b2533571c978a55301903160ef8cd7c2141 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 2 Oct 2023 09:24:43 -0700 Subject: [PATCH 33/41] tests(pystar): py_runtime_pair and py_runtime analysis tests (#1441) These analysis tests verify that `py_runtime` and `py_runtime_pair` are working as intended for both the native Bazel and Starlark implementations. Work towards #1069 --- python/BUILD.bazel | 6 +- python/private/BUILD.bazel | 3 +- python/py_runtime_pair.bzl | 6 +- tests/py_runtime/BUILD.bazel | 17 ++ tests/py_runtime/py_runtime_tests.bzl | 262 ++++++++++++++++++ tests/py_runtime_info_subject.bzl | 101 +++++++ tests/py_runtime_pair/BUILD.bazel | 17 ++ .../py_runtime_pair/py_runtime_pair_tests.bzl | 66 +++++ 8 files changed, 475 insertions(+), 3 deletions(-) create mode 100644 tests/py_runtime/BUILD.bazel create mode 100644 tests/py_runtime/py_runtime_tests.bzl create mode 100644 tests/py_runtime_info_subject.bzl create mode 100644 tests/py_runtime_pair/BUILD.bazel create mode 100644 tests/py_runtime_pair/py_runtime_pair_tests.bzl diff --git a/python/BUILD.bazel b/python/BUILD.bazel index f9c93e5539..3884349f3b 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -147,7 +147,11 @@ bzl_library( bzl_library( name = "py_runtime_pair_bzl", srcs = ["py_runtime_pair.bzl"], - deps = ["//python/private:bazel_tools_bzl"], + deps = [ + "//python/private:bazel_tools_bzl", + "//python/private:py_runtime_pair_macro_bzl", + "@rules_python_internal//:rules_python_config_bzl", + ], ) bzl_library( diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index a67183e5a7..f0eddadc3c 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -104,7 +104,8 @@ bzl_library( bzl_library( name = "py_runtime_pair_macro_bzl", - srcs = ["py_runtime_pair_rule.bzl"], + srcs = ["py_runtime_pair_macro.bzl"], + visibility = ["//:__subpackages__"], deps = [":py_runtime_pair_rule_bzl"], ) diff --git a/python/py_runtime_pair.bzl b/python/py_runtime_pair.bzl index 951c606f4a..c80994c963 100644 --- a/python/py_runtime_pair.bzl +++ b/python/py_runtime_pair.bzl @@ -14,7 +14,11 @@ """Public entry point for py_runtime_pair.""" -load("@bazel_tools//tools/python:toolchain.bzl", _py_runtime_pair = "py_runtime_pair") +load("@bazel_tools//tools/python:toolchain.bzl", _bazel_tools_impl = "py_runtime_pair") +load("@rules_python_internal//:rules_python_config.bzl", "config") +load("//python/private:py_runtime_pair_macro.bzl", _starlark_impl = "py_runtime_pair") + +_py_runtime_pair = _bazel_tools_impl if not config.enable_pystar else _starlark_impl # NOTE: This doc is copy/pasted from the builtin py_runtime_pair rule so our # doc generator gives useful API docs. diff --git a/tests/py_runtime/BUILD.bazel b/tests/py_runtime/BUILD.bazel new file mode 100644 index 0000000000..e097f0df08 --- /dev/null +++ b/tests/py_runtime/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":py_runtime_tests.bzl", "py_runtime_test_suite") + +py_runtime_test_suite(name = "py_runtime_tests") diff --git a/tests/py_runtime/py_runtime_tests.bzl b/tests/py_runtime/py_runtime_tests.bzl new file mode 100644 index 0000000000..662909cca2 --- /dev/null +++ b/tests/py_runtime/py_runtime_tests.bzl @@ -0,0 +1,262 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starlark tests for py_runtime rule.""" + +load("@rules_python_internal//:rules_python_config.bzl", "config") +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_info.bzl", "PyRuntimeInfo") +load("//tests:py_runtime_info_subject.bzl", "py_runtime_info_subject") +load("//tests/base_rules:util.bzl", br_util = "util") + +_tests = [] + +_SKIP_TEST = { + "target_compatible_with": ["@platforms//:incompatible"], +} + +def _test_bootstrap_template(name): + # The bootstrap_template arg isn't present in older Bazel versions, so + # we have to conditionally pass the arg and mark the test incompatible. + if config.enable_pystar: + py_runtime_kwargs = {"bootstrap_template": "bootstrap.txt"} + attr_values = {} + else: + py_runtime_kwargs = {} + attr_values = _SKIP_TEST + + rt_util.helper_target( + py_runtime, + name = name + "_subject", + interpreter_path = "/py", + python_version = "PY3", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_bootstrap_template_impl, + attr_values = attr_values, + ) + +def _test_bootstrap_template_impl(env, target): + env.expect.that_target(target).provider( + PyRuntimeInfo, + factory = py_runtime_info_subject, + ).bootstrap_template().path().contains("bootstrap.txt") + +_tests.append(_test_bootstrap_template) + +def _test_cannot_have_both_inbuild_and_system_interpreter(name): + if br_util.is_bazel_6_or_higher(): + py_runtime_kwargs = { + "interpreter": "fake_interpreter", + "interpreter_path": "/some/path", + } + attr_values = {} + else: + py_runtime_kwargs = { + "interpreter_path": "/some/path", + } + attr_values = _SKIP_TEST + rt_util.helper_target( + py_runtime, + name = name + "_subject", + python_version = "PY3", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_cannot_have_both_inbuild_and_system_interpreter_impl, + expect_failure = True, + attr_values = attr_values, + ) + +def _test_cannot_have_both_inbuild_and_system_interpreter_impl(env, target): + env.expect.that_target(target).failures().contains_predicate( + matching.str_matches("one of*interpreter*interpreter_path"), + ) + +_tests.append(_test_cannot_have_both_inbuild_and_system_interpreter) + +def _test_cannot_specify_files_for_system_interpreter(name): + if br_util.is_bazel_6_or_higher(): + py_runtime_kwargs = {"files": ["foo.txt"]} + attr_values = {} + else: + py_runtime_kwargs = {} + attr_values = _SKIP_TEST + rt_util.helper_target( + py_runtime, + name = name + "_subject", + interpreter_path = "/foo", + python_version = "PY3", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_cannot_specify_files_for_system_interpreter_impl, + expect_failure = True, + attr_values = attr_values, + ) + +def _test_cannot_specify_files_for_system_interpreter_impl(env, target): + env.expect.that_target(target).failures().contains_predicate( + matching.str_matches("files*must be empty"), + ) + +_tests.append(_test_cannot_specify_files_for_system_interpreter) + +def _test_in_build_interpreter(name): + rt_util.helper_target( + py_runtime, + name = name + "_subject", + interpreter = "fake_interpreter", + python_version = "PY3", + files = ["file1.txt"], + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_in_build_interpreter_impl, + ) + +def _test_in_build_interpreter_impl(env, target): + info = env.expect.that_target(target).provider(PyRuntimeInfo, factory = py_runtime_info_subject) + info.python_version().equals("PY3") + info.files().contains_predicate(matching.file_basename_equals("file1.txt")) + info.interpreter().path().contains("fake_interpreter") + +_tests.append(_test_in_build_interpreter) + +def _test_must_have_either_inbuild_or_system_interpreter(name): + if br_util.is_bazel_6_or_higher(): + py_runtime_kwargs = {} + attr_values = {} + else: + py_runtime_kwargs = { + "interpreter_path": "/some/path", + } + attr_values = _SKIP_TEST + rt_util.helper_target( + py_runtime, + name = name + "_subject", + python_version = "PY3", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_must_have_either_inbuild_or_system_interpreter_impl, + expect_failure = True, + attr_values = attr_values, + ) + +def _test_must_have_either_inbuild_or_system_interpreter_impl(env, target): + env.expect.that_target(target).failures().contains_predicate( + matching.str_matches("one of*interpreter*interpreter_path"), + ) + +_tests.append(_test_must_have_either_inbuild_or_system_interpreter) + +def _test_python_version_required(name): + # Bazel 5.4 will entirely crash when python_version is missing. + if br_util.is_bazel_6_or_higher(): + py_runtime_kwargs = {} + attr_values = {} + else: + py_runtime_kwargs = {"python_version": "PY3"} + attr_values = _SKIP_TEST + rt_util.helper_target( + py_runtime, + name = name + "_subject", + interpreter_path = "/math/pi", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_python_version_required_impl, + expect_failure = True, + attr_values = attr_values, + ) + +def _test_python_version_required_impl(env, target): + env.expect.that_target(target).failures().contains_predicate( + matching.str_matches("must be set*PY2*PY3"), + ) + +_tests.append(_test_python_version_required) + +def _test_system_interpreter(name): + rt_util.helper_target( + py_runtime, + name = name + "_subject", + interpreter_path = "/system/python", + python_version = "PY3", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_system_interpreter_impl, + ) + +def _test_system_interpreter_impl(env, target): + env.expect.that_target(target).provider( + PyRuntimeInfo, + factory = py_runtime_info_subject, + ).interpreter_path().equals("/system/python") + +_tests.append(_test_system_interpreter) + +def _test_system_interpreter_must_be_absolute(name): + # Bazel 5.4 will entirely crash when an invalid interpreter_path + # is given. + if br_util.is_bazel_6_or_higher(): + py_runtime_kwargs = {"interpreter_path": "relative/path"} + attr_values = {} + else: + py_runtime_kwargs = {"interpreter_path": "/junk/value/for/bazel5.4"} + attr_values = _SKIP_TEST + rt_util.helper_target( + py_runtime, + name = name + "_subject", + python_version = "PY3", + **py_runtime_kwargs + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_system_interpreter_must_be_absolute_impl, + expect_failure = True, + attr_values = attr_values, + ) + +def _test_system_interpreter_must_be_absolute_impl(env, target): + env.expect.that_target(target).failures().contains_predicate( + matching.str_matches("must be*absolute"), + ) + +_tests.append(_test_system_interpreter_must_be_absolute) + +def py_runtime_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) diff --git a/tests/py_runtime_info_subject.bzl b/tests/py_runtime_info_subject.bzl new file mode 100644 index 0000000000..9f42d3a839 --- /dev/null +++ b/tests/py_runtime_info_subject.bzl @@ -0,0 +1,101 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyRuntimeInfo testing subject.""" + +load("@rules_testing//lib:truth.bzl", "subjects") + +def py_runtime_info_subject(info, *, meta): + """Creates a new `PyRuntimeInfoSubject` for a PyRuntimeInfo provider instance. + + Method: PyRuntimeInfoSubject.new + + Args: + info: The PyRuntimeInfo object + meta: ExpectMeta object. + + Returns: + A `PyRuntimeInfoSubject` struct + """ + + # buildifier: disable=uninitialized + public = struct( + # go/keep-sorted start + bootstrap_template = lambda *a, **k: _py_runtime_info_subject_bootstrap_template(self, *a, **k), + coverage_files = lambda *a, **k: _py_runtime_info_subject_coverage_files(self, *a, **k), + coverage_tool = lambda *a, **k: _py_runtime_info_subject_coverage_tool(self, *a, **k), + files = lambda *a, **k: _py_runtime_info_subject_files(self, *a, **k), + interpreter = lambda *a, **k: _py_runtime_info_subject_interpreter(self, *a, **k), + interpreter_path = lambda *a, **k: _py_runtime_info_subject_interpreter_path(self, *a, **k), + python_version = lambda *a, **k: _py_runtime_info_subject_python_version(self, *a, **k), + stub_shebang = lambda *a, **k: _py_runtime_info_subject_stub_shebang(self, *a, **k), + # go/keep-sorted end + ) + self = struct( + actual = info, + meta = meta, + ) + return public + +def _py_runtime_info_subject_bootstrap_template(self): + return subjects.file( + self.actual.bootstrap_template, + meta = self.meta.derive("bootstrap_template()"), + ) + +def _py_runtime_info_subject_coverage_files(self): + """Returns a `DepsetFileSubject` for the `coverage_files` attribute. + + Args: + self: implicitly added. + """ + return subjects.depset_file( + self.actual.coverage_files, + meta = self.meta.derive("coverage_files()"), + ) + +def _py_runtime_info_subject_coverage_tool(self): + return subjects.file( + self.actual.coverage_tool, + meta = self.meta.derive("coverage_tool()"), + ) + +def _py_runtime_info_subject_files(self): + return subjects.depset_file( + self.actual.files, + meta = self.meta.derive("files()"), + ) + +def _py_runtime_info_subject_interpreter(self): + return subjects.file( + self.actual.interpreter, + meta = self.meta.derive("interpreter()"), + ) + +def _py_runtime_info_subject_interpreter_path(self): + return subjects.str( + self.actual.interpreter_path, + meta = self.meta.derive("interpreter_path()"), + ) + +def _py_runtime_info_subject_python_version(self): + return subjects.str( + self.actual.python_version, + meta = self.meta.derive("python_version()"), + ) + +def _py_runtime_info_subject_stub_shebang(self): + return subjects.str( + self.actual.stub_shebang, + meta = self.meta.derive("stub_shebang()"), + ) diff --git a/tests/py_runtime_pair/BUILD.bazel b/tests/py_runtime_pair/BUILD.bazel new file mode 100644 index 0000000000..6a6a4b91f0 --- /dev/null +++ b/tests/py_runtime_pair/BUILD.bazel @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load(":py_runtime_pair_tests.bzl", "py_runtime_pair_test_suite") + +py_runtime_pair_test_suite(name = "py_runtime_pair_tests") diff --git a/tests/py_runtime_pair/py_runtime_pair_tests.bzl b/tests/py_runtime_pair/py_runtime_pair_tests.bzl new file mode 100644 index 0000000000..e1ff19ee3a --- /dev/null +++ b/tests/py_runtime_pair/py_runtime_pair_tests.bzl @@ -0,0 +1,66 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Starlark tests for py_runtime_pair rule.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:truth.bzl", "matching", "subjects") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_runtime.bzl", "py_runtime") +load("//python:py_runtime_pair.bzl", "py_runtime_pair") +load("//tests:py_runtime_info_subject.bzl", "py_runtime_info_subject") + +_tests = [] + +def _test_basic(name): + rt_util.helper_target( + py_runtime, + name = name + "_runtime", + interpreter = "fake_interpreter", + python_version = "PY3", + files = ["file1.txt"], + ) + rt_util.helper_target( + py_runtime_pair, + name = name + "_subject", + py3_runtime = name + "_runtime", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_basic_impl, + ) + +def _test_basic_impl(env, target): + toolchain = env.expect.that_target(target).provider( + platform_common.ToolchainInfo, + factory = lambda value, meta: subjects.struct( + value, + meta = meta, + attrs = { + "py3_runtime": py_runtime_info_subject, + }, + ), + ) + toolchain.py3_runtime().python_version().equals("PY3") + toolchain.py3_runtime().files().contains_predicate(matching.file_basename_equals("file1.txt")) + toolchain.py3_runtime().interpreter().path().contains("fake_interpreter") + +_tests.append(_test_basic) + +def py_runtime_pair_test_suite(name): + test_suite( + name = name, + tests = _tests, + ) From 4d6ae085df62c6b520b7752aeec67f36d48fe14a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 3 Oct 2023 12:05:23 -0700 Subject: [PATCH 34/41] fix(pystar): Use py_internal for runfiles_enabled, declare_shareable_artifact, share_native_deps (#1443) These are restricted use APIs, so they have to go through py_internal. They aren't caught by CI because tests don't currently cover their code paths; fixing that will be done in a separate change. Work towards #1069 --- python/private/common/py_executable.bzl | 4 ++-- python/private/common/py_executable_bazel.bzl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 1782f8db7f..50be673729 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -488,7 +488,7 @@ def _get_native_deps_details(ctx, *, semantics, cc_details, is_test): return struct(dso = None, runfiles = ctx.runfiles()) dso = ctx.actions.declare_file(semantics.get_native_deps_dso_name(ctx)) - share_native_deps = ctx.fragments.cpp.share_native_deps() + share_native_deps = py_internal.share_native_deps(ctx) cc_feature_config = cc_configure_features( ctx, cc_toolchain = cc_details.cc_toolchain, @@ -571,7 +571,7 @@ def _create_shared_native_deps_dso( features = requested_features, is_test_target_partially_disabled_thin_lto = is_test and partially_disabled_thin_lto, ) - return ctx.actions.declare_shareable_artifact("_nativedeps/%x.so" % dso_hash) + return py_internal.declare_shareable_artifact(ctx, "_nativedeps/%x.so" % dso_hash) # This is a minimal version of NativeDepsHelper.getSharedNativeDepsPath, see # com.google.devtools.build.lib.rules.nativedeps.NativeDepsHelper#getSharedNativeDepsPath diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index 6c50b75b71..3136ab18b7 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -332,7 +332,7 @@ def _create_windows_exe_launcher( launch_info.add("binary_type=Python") launch_info.add(ctx.workspace_name, format = "workspace_name=%s") launch_info.add( - "1" if ctx.configuration.runfiles_enabled() else "0", + "1" if py_internal.runfiles_enabled(ctx) else "0", format = "symlink_runfiles_enabled=%s", ) launch_info.add(python_binary_path, format = "python_bin_path=%s") From 9eccb7943f012b4b20c57107bc40e7cdeccbc145 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 08:36:46 +0900 Subject: [PATCH 35/41] build(deps): bump urllib3 from 1.26.13 to 1.26.17 in /examples/pip_repository_annotations (#1447) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.13 to 1.26.17.
Release notes

Sourced from urllib3's releases.

1.26.17

  • Added the Cookie header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via Retry.remove_headers_on_redirect. (GHSA-v845-jxx5-vc9f)

1.26.16

  • Fixed thread-safety issue where accessing a PoolManager with many distinct origins would cause connection pools to be closed while requests are in progress (#2954)

1.26.15

1.26.14

  • Fixed parsing of port 0 (zero) returning None, instead of 0 (#2850)
  • Removed deprecated HTTPResponse.getheaders() calls in urllib3.contrib module.
Changelog

Sourced from urllib3's changelog.

1.26.17 (2023-10-02)

  • Added the Cookie header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via Retry.remove_headers_on_redirect. ([#3139](https://github.com/urllib3/urllib3/issues/3139) <https://github.com/urllib3/urllib3/pull/3139>_)

1.26.16 (2023-05-23)

  • Fixed thread-safety issue where accessing a PoolManager with many distinct origins would cause connection pools to be closed while requests are in progress ([#2954](https://github.com/urllib3/urllib3/issues/2954) <https://github.com/urllib3/urllib3/pull/2954>_)

1.26.15 (2023-03-10)

  • Fix socket timeout value when HTTPConnection is reused ([#2645](https://github.com/urllib3/urllib3/issues/2645) <https://github.com/urllib3/urllib3/issues/2645>__)
  • Remove "!" character from the unreserved characters in IPv6 Zone ID parsing ([#2899](https://github.com/urllib3/urllib3/issues/2899) <https://github.com/urllib3/urllib3/issues/2899>__)
  • Fix IDNA handling of '\x80' byte ([#2901](https://github.com/urllib3/urllib3/issues/2901) <https://github.com/urllib3/urllib3/issues/2901>__)

1.26.14 (2023-01-11)

  • Fixed parsing of port 0 (zero) returning None, instead of 0. ([#2850](https://github.com/urllib3/urllib3/issues/2850) <https://github.com/urllib3/urllib3/issues/2850>__)
  • Removed deprecated getheaders() calls in contrib module. Fixed the type hint of PoolKey.key_retries by adding bool to the union. ([#2865](https://github.com/urllib3/urllib3/issues/2865) <https://github.com/urllib3/urllib3/issues/2865>__)
Commits
  • c9016bf Release 1.26.17
  • 0122035 Backport GHSA-v845-jxx5-vc9f (#3139)
  • e63989f Fix installing brotli extra on Python 2.7
  • 2e7a24d [1.26] Configure OS for RTD to fix building docs
  • 57181d6 [1.26] Improve error message when calling urllib3.request() (#3058)
  • 3c01480 [1.26] Run coverage even with failed jobs
  • d94029b Release 1.26.16
  • 18e9214 Use trusted publishing for PyPI
  • d25cf83 [1.26] Fix invalid test_ssl_failure_midway_through_conn
  • 25cca38 [1.26] Fix test_ssl_object_attributes
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=urllib3&package-manager=pip&previous-version=1.26.13&new-version=1.26.17)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/bazelbuild/rules_python/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/pip_repository_annotations/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pip_repository_annotations/requirements.txt b/examples/pip_repository_annotations/requirements.txt index 04379ebe24..9063fa7b1c 100644 --- a/examples/pip_repository_annotations/requirements.txt +++ b/examples/pip_repository_annotations/requirements.txt @@ -24,9 +24,9 @@ requests[security]==2.28.1 \ --hash=sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983 \ --hash=sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349 # via -r requirements.in -urllib3==1.26.13 \ - --hash=sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc \ - --hash=sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8 +urllib3==1.26.17 \ + --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ + --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b # via requests wheel==0.38.4 \ --hash=sha256:965f5259b566725405b05e7cf774052044b1ed30119b5d586b2703aafe8719ac \ From 961e233592eeb293b554249ad1f93c4a6559a232 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:25:39 +0900 Subject: [PATCH 36/41] build(deps): bump urllib3 from 1.25.11 to 1.26.17 in /examples/pip_install (#1444) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.25.11 to 1.26.17.
Release notes

Sourced from urllib3's releases.

1.26.17

  • Added the Cookie header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via Retry.remove_headers_on_redirect. (GHSA-v845-jxx5-vc9f)

1.26.16

  • Fixed thread-safety issue where accessing a PoolManager with many distinct origins would cause connection pools to be closed while requests are in progress (#2954)

1.26.15

1.26.14

  • Fixed parsing of port 0 (zero) returning None, instead of 0 (#2850)
  • Removed deprecated HTTPResponse.getheaders() calls in urllib3.contrib module.

1.26.13

  • Deprecated the HTTPResponse.getheaders() and HTTPResponse.getheader() methods.
  • Fixed an issue where parsing a URL with leading zeroes in the port would be rejected even when the port number after removing the zeroes was valid.
  • Fixed a deprecation warning when using cryptography v39.0.0.
  • Removed the <4 in the Requires-Python packaging metadata field.

1.26.12

  • Deprecated the urllib3[secure] extra and the urllib3.contrib.pyopenssl module. Both will be removed in v2.x. See this GitHub issue for justification and info on how to migrate.

1.26.11

If you or your organization rely on urllib3 consider supporting us via GitHub Sponsors.

:warning: urllib3 v2.0 will drop support for Python 2: Read more in the v2.0 Roadmap

  • Fixed an issue where reading more than 2 GiB in a call to HTTPResponse.read would raise an OverflowError on Python 3.9 and earlier.

1.26.10

If you or your organization rely on urllib3 consider supporting us via GitHub Sponsors.

:warning: urllib3 v2.0 will drop support for Python 2: Read more in the v2.0 Roadmap

:closed_lock_with_key: This is the first release to be signed with Sigstore! You can verify the distributables using the .sig and .crt files included on this release.

  • Removed support for Python 3.5
  • Fixed an issue where a ProxyError recommending configuring the proxy as HTTP instead of HTTPS could appear even when an HTTPS proxy wasn't configured.

1.26.9

If you or your organization rely on urllib3 consider supporting us via GitHub Sponsors.

:warning: urllib3 v2.0 will drop support for Python 2: Read more in the v2.0 Roadmap

... (truncated)

Changelog

Sourced from urllib3's changelog.

1.26.17 (2023-10-02)

  • Added the Cookie header to the list of headers to strip from requests when redirecting to a different host. As before, different headers can be set via Retry.remove_headers_on_redirect. ([#3139](https://github.com/urllib3/urllib3/issues/3139) <https://github.com/urllib3/urllib3/pull/3139>_)

1.26.16 (2023-05-23)

  • Fixed thread-safety issue where accessing a PoolManager with many distinct origins would cause connection pools to be closed while requests are in progress ([#2954](https://github.com/urllib3/urllib3/issues/2954) <https://github.com/urllib3/urllib3/pull/2954>_)

1.26.15 (2023-03-10)

  • Fix socket timeout value when HTTPConnection is reused ([#2645](https://github.com/urllib3/urllib3/issues/2645) <https://github.com/urllib3/urllib3/issues/2645>__)
  • Remove "!" character from the unreserved characters in IPv6 Zone ID parsing ([#2899](https://github.com/urllib3/urllib3/issues/2899) <https://github.com/urllib3/urllib3/issues/2899>__)
  • Fix IDNA handling of '\x80' byte ([#2901](https://github.com/urllib3/urllib3/issues/2901) <https://github.com/urllib3/urllib3/issues/2901>__)

1.26.14 (2023-01-11)

  • Fixed parsing of port 0 (zero) returning None, instead of 0. ([#2850](https://github.com/urllib3/urllib3/issues/2850) <https://github.com/urllib3/urllib3/issues/2850>__)
  • Removed deprecated getheaders() calls in contrib module. Fixed the type hint of PoolKey.key_retries by adding bool to the union. ([#2865](https://github.com/urllib3/urllib3/issues/2865) <https://github.com/urllib3/urllib3/issues/2865>__)

1.26.13 (2022-11-23)

  • Deprecated the HTTPResponse.getheaders() and HTTPResponse.getheader() methods.
  • Fixed an issue where parsing a URL with leading zeroes in the port would be rejected even when the port number after removing the zeroes was valid.
  • Fixed a deprecation warning when using cryptography v39.0.0.
  • Removed the <4 in the Requires-Python packaging metadata field.

1.26.12 (2022-08-22)

  • Deprecated the urllib3[secure] extra and the urllib3.contrib.pyopenssl module. Both will be removed in v2.x. See this GitHub issue <https://github.com/urllib3/urllib3/issues/2680>_ for justification and info on how to migrate.

1.26.11 (2022-07-25)

  • Fixed an issue where reading more than 2 GiB in a call to HTTPResponse.read would raise an OverflowError on Python 3.9 and earlier.

1.26.10 (2022-07-07)

... (truncated)

Commits
  • c9016bf Release 1.26.17
  • 0122035 Backport GHSA-v845-jxx5-vc9f (#3139)
  • e63989f Fix installing brotli extra on Python 2.7
  • 2e7a24d [1.26] Configure OS for RTD to fix building docs
  • 57181d6 [1.26] Improve error message when calling urllib3.request() (#3058)
  • 3c01480 [1.26] Run coverage even with failed jobs
  • d94029b Release 1.26.16
  • 18e9214 Use trusted publishing for PyPI
  • d25cf83 [1.26] Fix invalid test_ssl_failure_midway_through_conn
  • 25cca38 [1.26] Fix test_ssl_object_attributes
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=urllib3&package-manager=pip&previous-version=1.25.11&new-version=1.26.17)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/bazelbuild/rules_python/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- examples/pip_install/requirements_windows.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pip_install/requirements_windows.txt b/examples/pip_install/requirements_windows.txt index 298f31f996..c74eeacb06 100644 --- a/examples/pip_install/requirements_windows.txt +++ b/examples/pip_install/requirements_windows.txt @@ -93,9 +93,9 @@ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 # via python-dateutil -urllib3==1.25.11 \ - --hash=sha256:8d7eaa5a82a1cac232164990f04874c594c9453ec55eef02eab885aa02fc17a2 \ - --hash=sha256:f5321fbe4bf3fefa0efd0bfe7fb14e90909eb62a48ccda331726b4319897dd5e +urllib3==1.26.17 \ + --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ + --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b # via botocore yamllint==1.28.0 \ --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \ From 8d7645eb4926810c3bf3926fcf1ac3d3c444419d Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 4 Oct 2023 11:13:38 -0700 Subject: [PATCH 37/41] fix: add missing `@bazel_tools` files to bzl_library dependencies. (#1457) This allows depending on just e.g. `//python:defs_bzl` without having to also depend on our internal //docs targets or manually including the extra bazel_tools files. The missing files were hidden by the doc tests because those tests manually include the extra files. Under the hood, it goes: * defs.bzl -> * py_runtime_pair.bzl -> * @bazel_tools//tools/python:toolchain.bzl -> * @bazel_tools//tools/python:private/defs.bzl -> (Note the relationshps within @bazel_tools are Bazel-version specific) Unfortunately, there isn't a public target for just the subset of files we need from @bazel_tools, so we have to use the larger glob of all of `@bazel_tools//tools`. --- python/private/BUILD.bazel | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index f0eddadc3c..675a763070 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -157,9 +157,10 @@ bzl_library( bzl_library( name = "bazel_tools_bzl", srcs = [ - "@bazel_tools//tools/python:srcs_version.bzl", - "@bazel_tools//tools/python:toolchain.bzl", - "@bazel_tools//tools/python:utils.bzl", + # This set of sources is overly broad, but it's the only public + # target available across Bazel versions that has all the necessary + # sources. + "@bazel_tools//tools:bzl_srcs", ], visibility = ["//python:__pkg__"], ) From 7e07684da8a0db09e1717e362ae3733163c153b9 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 4 Oct 2023 14:15:17 -0700 Subject: [PATCH 38/41] tests(pystar): add analysis tests to cover basic windows building (#1452) The CI currently only runs on Ubuntu, so it assumes that is the target platform. This ends up missing some notable Windows code paths, though. Since its analysis-phase logic, we can force the platform to be Windows for the analysis tests, and then the rules follow the code paths that should be taken under Windows. This allows testing Windows logic under Ubuntu. --- python/private/common/py_executable.bzl | 10 ++++- python/private/common/py_executable_bazel.bzl | 5 +-- tests/base_rules/BUILD.bazel | 14 ------- tests/base_rules/py_executable_base_tests.bzl | 38 ++++++++++++++++++ tests/base_rules/py_test/py_test_tests.bzl | 11 +++--- tests/cc/BUILD.bazel | 27 +++++++++++++ tests/support/BUILD.bazel | 39 +++++++++++++++++++ tests/support/test_platforms.bzl | 20 ++++++++++ 8 files changed, 139 insertions(+), 25 deletions(-) create mode 100644 tests/support/BUILD.bazel create mode 100644 tests/support/test_platforms.bzl diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index 50be673729..bb1f16d61a 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -35,6 +35,7 @@ load( "create_py_info", "csv", "filter_to_py_srcs", + "target_platform_has_any_constraint", "union_attrs", ) load( @@ -48,6 +49,7 @@ load( "ALLOWED_MAIN_EXTENSIONS", "BUILD_DATA_SYMLINK_PATH", "IS_BAZEL", + "PLATFORMS_LOCATION", "PY_RUNTIME_ATTR_NAME", "TOOLS_REPO", ) @@ -93,6 +95,11 @@ filename in `srcs`, `main` must be specified. # NOTE: Some tests care about the order of these values. values = ["PY2", "PY3"], ), + "_windows_constraints": attr.label_list( + default = [ + PLATFORMS_LOCATION + "/os:windows", + ], + ), }, create_srcs_version_attr(values = SRCS_VERSION_ALL_VALUES), create_srcs_attr(mandatory = True), @@ -201,8 +208,7 @@ def _validate_executable(ctx): check_native_allowed(ctx) def _compute_outputs(ctx, output_sources): - # TODO: This should use the configuration instead of the Bazel OS. - if _py_builtins.get_current_os_name() == "windows": + if target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints): executable = ctx.actions.declare_file(ctx.label.name + ".exe") else: executable = ctx.actions.declare_file(ctx.label.name) diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index 3136ab18b7..97712c5e43 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -21,6 +21,7 @@ load( "create_binary_semantics_struct", "create_cc_details_struct", "create_executable_result_struct", + "target_platform_has_any_constraint", "union_attrs", ) load(":common_bazel.bzl", "collect_cc_info", "get_imports", "maybe_precompile") @@ -174,9 +175,7 @@ def _create_executable( runtime_details = runtime_details, ) - # TODO: This should use the configuration instead of the Bazel OS. - # This is just legacy behavior. - is_windows = _py_builtins.get_current_os_name() == "windows" + is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) if is_windows: if not executable.extension == "exe": diff --git a/tests/base_rules/BUILD.bazel b/tests/base_rules/BUILD.bazel index e271850834..aa21042e25 100644 --- a/tests/base_rules/BUILD.bazel +++ b/tests/base_rules/BUILD.bazel @@ -11,17 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -platform( - name = "mac", - constraint_values = [ - "@platforms//os:macos", - ], -) - -platform( - name = "linux", - constraint_values = [ - "@platforms//os:linux", - ], -) diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index 13ec946be5..b5dea172c3 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -13,14 +13,52 @@ # limitations under the License. """Tests common to py_binary and py_test (executable rules).""" +load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//tests/base_rules:base_tests.bzl", "create_base_tests") load("//tests/base_rules:util.bzl", "WINDOWS_ATTR", pt_util = "util") +load("//tests/support:test_platforms.bzl", "WINDOWS") _tests = [] +def _test_basic_windows(name, config): + if rp_config.enable_pystar: + target_compatible_with = [] + else: + target_compatible_with = ["@platforms//:incompatible"] + rt_util.helper_target( + config.rule, + name = name + "_subject", + srcs = ["main.py"], + main = "main.py", + ) + analysis_test( + name = name, + impl = _test_basic_windows_impl, + target = name + "_subject", + config_settings = { + "//command_line_option:cpu": "windows_x86_64", + "//command_line_option:crosstool_top": Label("//tests/cc:cc_toolchain_suite"), + "//command_line_option:extra_toolchains": [str(Label("//tests/cc:all"))], + "//command_line_option:platforms": [WINDOWS], + }, + attr_values = {"target_compatible_with": target_compatible_with}, + ) + +def _test_basic_windows_impl(env, target): + target = env.expect.that_target(target) + target.executable().path().contains(".exe") + target.runfiles().contains_predicate(matching.str_endswith( + target.meta.format_str("/{name}"), + )) + target.runfiles().contains_predicate(matching.str_endswith( + target.meta.format_str("/{name}.exe"), + )) + +_tests.append(_test_basic_windows) + def _test_executable_in_runfiles(name, config): rt_util.helper_target( config.rule, diff --git a/tests/base_rules/py_test/py_test_tests.bzl b/tests/base_rules/py_test/py_test_tests.bzl index 4d0f7d1c3e..f4b704e6ca 100644 --- a/tests/base_rules/py_test/py_test_tests.bzl +++ b/tests/base_rules/py_test/py_test_tests.bzl @@ -21,13 +21,12 @@ load( "create_executable_tests", ) load("//tests/base_rules:util.bzl", pt_util = "util") +load("//tests/support:test_platforms.bzl", "LINUX", "MAC") -# Explicit Label() calls are required so that it resolves in @rules_python context instead of -# @rules_testing context. +# Explicit Label() calls are required so that it resolves in @rules_python +# context instead of @rules_testing context. _FAKE_CC_TOOLCHAIN = Label("//tests/cc:cc_toolchain_suite") _FAKE_CC_TOOLCHAINS = [str(Label("//tests/cc:all"))] -_PLATFORM_MAC = Label("//tests/base_rules:mac") -_PLATFORM_LINUX = Label("//tests/base_rules:linux") _tests = [] @@ -53,7 +52,7 @@ def _test_mac_requires_darwin_for_execution(name, config): "//command_line_option:cpu": "darwin_x86_64", "//command_line_option:crosstool_top": _FAKE_CC_TOOLCHAIN, "//command_line_option:extra_toolchains": _FAKE_CC_TOOLCHAINS, - "//command_line_option:platforms": [_PLATFORM_MAC], + "//command_line_option:platforms": [MAC], }, ) @@ -85,7 +84,7 @@ def _test_non_mac_doesnt_require_darwin_for_execution(name, config): "//command_line_option:cpu": "k8", "//command_line_option:crosstool_top": _FAKE_CC_TOOLCHAIN, "//command_line_option:extra_toolchains": _FAKE_CC_TOOLCHAINS, - "//command_line_option:platforms": [_PLATFORM_LINUX], + "//command_line_option:platforms": [LINUX], }, ) diff --git a/tests/cc/BUILD.bazel b/tests/cc/BUILD.bazel index 3f7925d631..ef64d6dbef 100644 --- a/tests/cc/BUILD.bazel +++ b/tests/cc/BUILD.bazel @@ -50,6 +50,7 @@ cc_toolchain_suite( toolchains = { "darwin_x86_64": ":mac_toolchain", "k8": ":linux_toolchain", + "windows_x86_64": ":windows_toolchain", }, ) @@ -106,3 +107,29 @@ fake_cc_toolchain_config( target_cpu = "k8", toolchain_identifier = "linux-toolchain", ) + +cc_toolchain( + name = "windows_toolchain", + all_files = ":empty", + compiler_files = ":empty", + dwp_files = ":empty", + linker_files = ":empty", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 0, + toolchain_config = ":windows_toolchain_config", + toolchain_identifier = "windows-toolchain", +) + +toolchain( + name = "windows_toolchain_definition", + target_compatible_with = ["@platforms//os:windows"], + toolchain = ":windows_toolchain", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +fake_cc_toolchain_config( + name = "windows_toolchain_config", + target_cpu = "windows_x86_64", + toolchain_identifier = "windows-toolchain", +) diff --git a/tests/support/BUILD.bazel b/tests/support/BUILD.bazel new file mode 100644 index 0000000000..316e9abbf1 --- /dev/null +++ b/tests/support/BUILD.bazel @@ -0,0 +1,39 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ==================== +# NOTE: You probably want to use the constants in test_platforms.bzl +# Otherwise, you'll probably have to manually call Label() on these targets +# to force them to resolve in the proper context. +# ==================== +platform( + name = "mac", + constraint_values = [ + "@platforms//os:macos", + ], +) + +platform( + name = "linux", + constraint_values = [ + "@platforms//os:linux", + ], +) + +platform( + name = "windows", + constraint_values = [ + "@platforms//os:windows", + ], +) diff --git a/tests/support/test_platforms.bzl b/tests/support/test_platforms.bzl new file mode 100644 index 0000000000..3ff3c507fc --- /dev/null +++ b/tests/support/test_platforms.bzl @@ -0,0 +1,20 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Constants for referring to platforms.""" + +# Explicit Label() calls are required so that it resolves in @rules_python +# context instead of e.g. the @rules_testing context. +MAC = Label("//tests/support:mac") +LINUX = Label("//tests/support:linux") +WINDOWS = Label("//tests/support:windows") From 423c1de345c32d67dfe1e8d43510399ab10dc2c4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 4 Oct 2023 15:26:38 -0700 Subject: [PATCH 39/41] docs: move dependency management into respective bzl packages (#1459) This moves the dependency management of what transitive files are needed to generate docs for a .bzl file out of the docs directory and into the respective bzl file's directory. This ensures that the bzl_library targets we expose to users contain all the necessary dependencies. Because there are some projects using the bzl_library targets in //docs, some compatiblity aliases are added to make their migration path easier. Those targets only public for historical reasons and shouldn't be used. Work towards #1458 --- BUILD.bazel | 7 +++ docs/BUILD.bazel | 63 +++++++++------------ python/BUILD.bazel | 34 ++++++++++++ python/pip_install/BUILD.bazel | 67 ++++++++++++++++++++++ python/pip_install/private/BUILD.bazel | 14 +++++ python/private/BUILD.bazel | 77 +++++++++++++++++++++++--- 6 files changed, 217 insertions(+), 45 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 35a3df892f..4d4d3ec26f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load(":version.bzl", "BAZEL_VERSION") package(default_visibility = ["//visibility:public"]) @@ -44,6 +45,12 @@ filegroup( ], ) +bzl_library( + name = "version_bzl", + srcs = ["version.bzl"], + visibility = ["//:__subpackages__"], +) + # Reexport of all bzl files used to allow downstream rules to generate docs # without shipping with a dependency on Skylib filegroup( diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 5e0357f2ee..0959c35ddd 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -18,6 +18,8 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@io_bazel_stardoc//stardoc:stardoc.bzl", "stardoc") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility +# NOTE: Only public visibility for historical reasons. +# This package is only for rules_python to generate its own docs. package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 @@ -32,47 +34,35 @@ _DOCS = { "python": "//docs:core-docs", } -# We define these bzl_library targets here rather than in the //python package -# because they're only used for doc generation. This way, we avoid requiring -# our users to depend on Skylib. - -bzl_library( - name = "bazel_repo_tools", - srcs = [ - "@bazel_tools//tools:bzl_srcs", - ], +# Temporary compatibility aliases for some other projects depending on the old +# bzl_library targets. +alias( + name = "defs", + actual = "//python:defs_bzl", + deprecation = "Use //python:defs_bzl instead; targets under //docs are internal.", ) -bzl_library( - name = "defs", - srcs = [ - "//python:defs.bzl", - "//python/private:reexports.bzl", - ], - deps = [ - ":bazel_repo_tools", - "//python:defs_bzl", - "//python/private:reexports_bzl", - ], +alias( + name = "bazel_repo_tools", + actual = "//python/private:bazel_tools_bzl", + deprecation = "Use @bazel_tools//tools:bzl_srcs instead; targets under //docs are internal.", ) bzl_library( name = "pip_install_bzl", - srcs = [ - "//python:bzl", - "//python/pip_install:bzl", - ], + deprecation = "Use //python:pip_bzl or //python/pip_install:pip_repository_bzl instead; " + + "targets under //docs are internal.", deps = [ - ":defs", - "//:version.bzl", + "//python:pip_bzl", + "//python/pip_install:pip_repository_bzl", ], ) -bzl_library( +alias( name = "requirements_parser_bzl", - srcs = [ - "//python/pip_install:requirements_parser.bzl", - ], + actual = "//python/pip_install:pip_repository_bzl", + deprecation = "Use //python/pip_install:pip_repository_bzl instead; Both the requirements " + + "parser and targets under //docs are internal", ) # Empty list means "compatible with all". @@ -93,7 +83,9 @@ stardoc( out = "python.md_", input = "//python:defs.bzl", target_compatible_with = _TARGET_COMPATIBLE_WITH, - deps = [":defs"], + deps = [ + "//python:defs_bzl", + ], ) stardoc( @@ -102,9 +94,7 @@ stardoc( input = "//python:pip.bzl", target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ - ":bazel_repo_tools", - ":pip_install_bzl", - "@bazel_skylib//lib:versions", + "//python:pip_bzl", ], ) @@ -114,10 +104,7 @@ stardoc( input = "//python/pip_install:pip_repository.bzl", target_compatible_with = _TARGET_COMPATIBLE_WITH, deps = [ - ":bazel_repo_tools", - ":pip_install_bzl", - ":requirements_parser_bzl", - "@bazel_skylib//lib:versions", + "//python/pip_install:pip_repository_bzl", ], ) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 3884349f3b..34b4de3d00 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -82,6 +82,19 @@ bzl_library( ], ) +bzl_library( + name = "pip_bzl", + srcs = ["pip.bzl"], + deps = [ + "//python/pip_install:pip_repository_bzl", + "//python/pip_install:repositories_bzl", + "//python/pip_install:requirements_bzl", + "//python/private:bzlmod_enabled_bzl", + "//python/private:full_version_bzl", + "//python/private:render_pkg_aliases_bzl", + ], +) + bzl_library( name = "proto_bzl", srcs = [ @@ -174,6 +187,27 @@ bzl_library( ], ) +bzl_library( + name = "repositories_bzl", + srcs = ["repositories.bzl"], + deps = [ + ":versions_bzl", + "//python/private:bazel_tools_bzl", + "//python/private:bzlmod_enabled_bzl", + "//python/private:coverage_deps_bzl", + "//python/private:full_version_bzl", + "//python/private:internal_config_repo_bzl", + "//python/private:toolchains_repo_bzl", + "//python/private:which_bzl", + ], +) + +bzl_library( + name = "versions_bzl", + srcs = ["versions.bzl"], + visibility = ["//:__subpackages__"], +) + # NOTE: Remember to add bzl_library targets to //tests:bzl_libraries # ========= bzl_library targets end ========= diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 4e4fbb4a1c..c071033384 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -1,3 +1,70 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = ["//:__subpackages__"], +) + +bzl_library( + name = "pip_repository_bzl", + srcs = ["pip_repository.bzl"], + # Semi-public: What is intended to be public and what is intended to be + # internal is unclear. Some symbols are clearly public (e.g. + # package_annotations), some are clearly internal (e.g. + # pip_hub_repository_bzlmod), and many are unknown. + visibility = ["//visibility:public"], + deps = [ + ":repositories_bzl", + ":requirements_parser_bzl", + "//python:repositories_bzl", + "//python:versions_bzl", + "//python/pip_install/private:generate_whl_library_build_bazel_bzl", + "//python/pip_install/private:srcs_bzl", + "//python/private:bzlmod_enabled_bzl", + "//python/private:normalize_name_bzl", + "//python/private:render_pkg_aliases_bzl", + "//python/private:toolchains_repo_bzl", + "//python/private:which_bzl", + ], +) + +bzl_library( + name = "requirements_bzl", + srcs = ["requirements.bzl"], + deps = [ + ":repositories_bzl", + "//python:defs_bzl", + ], +) + +bzl_library( + name = "requirements_parser_bzl", + srcs = ["requirements_parser.bzl"], +) + +bzl_library( + name = "repositories_bzl", + srcs = ["repositories.bzl"], + deps = [ + "//:version_bzl", + "//python/private:bazel_tools_bzl", + "@bazel_skylib//lib:versions", + ], +) + filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ diff --git a/python/pip_install/private/BUILD.bazel b/python/pip_install/private/BUILD.bazel index 86b4b3d22c..2cc4cbd70d 100644 --- a/python/pip_install/private/BUILD.bazel +++ b/python/pip_install/private/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load(":pip_install_utils.bzl", "srcs_module") package(default_visibility = ["//:__subpackages__"]) @@ -22,3 +23,16 @@ srcs_module( srcs = "//python/pip_install:py_srcs", dest = ":srcs.bzl", ) + +bzl_library( + name = "generate_whl_library_build_bazel_bzl", + srcs = ["generate_whl_library_build_bazel.bzl"], + deps = [ + "//python/private:normalize_name_bzl", + ], +) + +bzl_library( + name = "srcs_bzl", + srcs = ["srcs.bzl"], +) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 675a763070..d1610584c9 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -18,6 +18,10 @@ load("//python:py_library.bzl", "py_library") load("//python:versions.bzl", "print_toolchains_checksums") load(":stamp.bzl", "stamp_build_setting") +package( + default_visibility = ["//:__subpackages__"], +) + licenses(["notice"]) filegroup( @@ -53,13 +57,34 @@ bzl_library( ) bzl_library( - name = "util_bzl", - srcs = ["util.bzl"], - visibility = [ - "//docs:__subpackages__", - "//python:__subpackages__", + name = "bzlmod_enabled_bzl", + srcs = ["bzlmod_enabled.bzl"], +) + +bzl_library( + name = "coverage_deps_bzl", + srcs = ["coverage_deps.bzl"], + deps = [ + ":bazel_tools_bzl", + ":version_label_bzl", ], - deps = ["@bazel_skylib//lib:types"], +) + +bzl_library( + name = "full_version_bzl", + srcs = ["full_version.bzl"], + deps = ["//python:versions_bzl"], +) + +bzl_library( + name = "internal_config_repo_bzl", + srcs = ["internal_config_repo.bzl"], + deps = [":bzlmod_enabled_bzl"], +) + +bzl_library( + name = "normalize_name_bzl", + srcs = ["normalize_name.bzl"], ) bzl_library( @@ -138,12 +163,51 @@ bzl_library( deps = [":bazel_tools_bzl"], ) +bzl_library( + name = "render_pkg_aliases_bzl", + srcs = ["render_pkg_aliases.bzl"], + deps = [ + ":normalize_name_bzl", + ":text_util_bzl", + ":version_label_bzl", + ], +) + bzl_library( name = "stamp_bzl", srcs = ["stamp.bzl"], visibility = ["//:__subpackages__"], ) +bzl_library( + name = "text_util_bzl", + srcs = ["text_util.bzl"], +) + +bzl_library( + name = "toolchains_repo_bzl", + srcs = ["toolchains_repo.bzl"], + deps = [ + ":which_bzl", + "//python:versions_bzl", + ], +) + +bzl_library( + name = "util_bzl", + srcs = ["util.bzl"], + visibility = [ + "//docs:__subpackages__", + "//python:__subpackages__", + ], + deps = ["@bazel_skylib//lib:types"], +) + +bzl_library( + name = "version_label_bzl", + srcs = ["version_label.bzl"], +) + bzl_library( name = "which_bzl", srcs = ["which.bzl"], @@ -162,7 +226,6 @@ bzl_library( # sources. "@bazel_tools//tools:bzl_srcs", ], - visibility = ["//python:__pkg__"], ) # Needed to define bzl_library targets for docgen. (We don't define the From 382b6785a57ee428fc0ec367bcb380c6266cab7b Mon Sep 17 00:00:00 2001 From: Christian von Schultz Date: Thu, 5 Oct 2023 16:04:09 +0200 Subject: [PATCH 40/41] feat(py_wheel): Normalize name and version (#1331) Added the `incompatible_normalize_name` feature flag to normalize the package distribution name according to latest Python packaging standards. Defaults to `False` for the time being. Added the `incompatible_normalize_version` feature flag to normalize the package version according to PEP440 standard. This also adds support for local version specifiers (versions with a `+` in them), in accordance with PEP440. Defaults to `False` for the time being. Instead of following the obsolete PEP 427 escaping procedure for distribution names and versions, we should use the rules specified by https://packaging.python.org/en/latest/specifications (sections "Package name normalization" and "Binary distribution format"). For the versions, this means normalizing them according to PEP 440. Added as feature flags to avoid forcing the user to deal with breaking changes when upgrading `rules_python`: - Distribution names have stronger requirements now: "A valid name consists only of ASCII letters and numbers, period, underscore and hyphen. It must start and end with a letter or number." https://packaging.python.org/en/latest/specifications/name-normalization/ - Versions must be valid PEP 440 version identifiers. Previously versions such as "0.1-2-3" would have been accepted; that is no longer the case. - The file name of generated wheels may have changed, if the distribution name or the version identifier wasn't in normalized form. - The wheelmaker now depends on `packaging.version`, which means the `py_wheel` user now needs to load pip dependencies in their `WORKSPACE.bazel` file: ``` load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies") pip_install_dependencies() ``` Fixes bazelbuild/rules_python#883. Fixes bazelbuild/rules_python#1132. --------- Co-authored-by: Ignas Anikevicius Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 8 + docs/packaging.md | 7 +- examples/wheel/BUILD.bazel | 41 +- examples/wheel/wheel_test.py | 52 +- python/BUILD.bazel | 1 + python/private/BUILD.bazel | 1 + python/private/py_wheel.bzl | 85 ++- python/private/py_wheel_normalize_pep440.bzl | 519 +++++++++++++++++++ tests/py_wheel/py_wheel_tests.bzl | 103 ++++ tools/BUILD.bazel | 1 + tools/wheelmaker.py | 111 +++- 11 files changed, 906 insertions(+), 23 deletions(-) create mode 100644 python/private/py_wheel_normalize_pep440.bzl diff --git a/CHANGELOG.md b/CHANGELOG.md index 59bdac1b06..3c421a9d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,14 @@ A brief description of the categories of changes: authentication against private HTTP hosts serving Python toolchain binaries. * `//python:packaging_bzl` added, a `bzl_library` for the Starlark files `//python:packaging.bzl` requires. +* (py_wheel) Added the `incompatible_normalize_name` feature flag to + normalize the package distribution name according to latest Python + packaging standards. Defaults to `False` for the time being. +* (py_wheel) Added the `incompatible_normalize_version` feature flag + to normalize the package version according to PEP440 standard. This + also adds support for local version specifiers (versions with a `+` + in them), in accordance with PEP440. Defaults to `False` for the + time being. ### Removed diff --git a/docs/packaging.md b/docs/packaging.md index 0e8e110ef5..90c66dc1de 100755 --- a/docs/packaging.md +++ b/docs/packaging.md @@ -59,8 +59,9 @@ This also has the advantage that stamping information is included in the wheel's 
 py_wheel_rule(name, abi, author, author_email, classifiers, console_scripts, deps,
               description_content_type, description_file, distribution, entry_points,
-              extra_distinfo_files, extra_requires, homepage, license, platform, project_urls,
-              python_requires, python_tag, requires, stamp, strip_path_prefixes, summary, version)
+              extra_distinfo_files, extra_requires, homepage, incompatible_normalize_name,
+              incompatible_normalize_version, license, platform, project_urls, python_requires,
+              python_tag, requires, stamp, strip_path_prefixes, summary, version)
Internal rule used by the [py_wheel macro](/docs/packaging.md#py_wheel). @@ -89,6 +90,8 @@ in the way they expect. | extra_distinfo_files | Extra files to add to distinfo directory in the archive. | Dictionary: Label -> String | optional | {} | | extra_requires | List of optional requirements for this package | Dictionary: String -> List of strings | optional | {} | | homepage | A string specifying the URL for the package homepage. | String | optional | "" | +| incompatible_normalize_name | Normalize the package distribution name according to latest Python packaging standards.

See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode and https://packaging.python.org/en/latest/specifications/name-normalization/.

Apart from the valid names according to the above, we also accept '{' and '}', which may be used as placeholders for stamping. | Boolean | optional | False | +| incompatible_normalize_version | Normalize the package version according to PEP440 standard. With this option set to True, if the user wants to pass any stamp variables, they have to be enclosed in '{}', e.g. '{BUILD_TIMESTAMP}'. | Boolean | optional | False | | license | A string specifying the license of the package. | String | optional | "" | | platform | Supported platform. Use 'any' for pure-Python wheel.

If you have included platform-specific data, such as a .pyd or .so extension module, you will need to specify the platform in standard pip format. If you support multiple platforms, you can define platform constraints, then use a select() to specify the appropriate specifier, eg:

platform = select({ "//platforms:windows_x86_64": "win_amd64", "//platforms:macos_x86_64": "macosx_10_7_x86_64", "//platforms:linux_x86_64": "manylinux2014_x86_64", }) | String | optional | "any" | | project_urls | A string dict specifying additional browsable URLs for the project and corresponding labels, where label is the key and url is the value. e.g {{"Bug Tracker": "http://bitbucket.org/tarek/distribute/issues/"}} | Dictionary: String -> String | optional | {} | diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index f56a41b370..81422d37c3 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -54,6 +54,8 @@ py_wheel( testonly = True, # Set this to verify the generated .dist target doesn't break things # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" distribution = "example_minimal_library", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", version = "0.0.1", deps = [ @@ -76,6 +78,8 @@ py_wheel( testonly = True, abi = "$(ABI)", distribution = "example_minimal_library", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "$(PYTHON_TAG)", toolchains = ["//examples/wheel:make_variable_tags"], version = "$(VERSION)", @@ -95,6 +99,8 @@ py_wheel( name = "minimal_with_py_library_with_stamp", # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl" distribution = "example_minimal_library{BUILD_USER}", + incompatible_normalize_name = False, + incompatible_normalize_version = False, python_tag = "py3", stamp = 1, version = "0.1.{BUILD_TIMESTAMP}", @@ -123,6 +129,8 @@ py_wheel( name = "minimal_with_py_package", # Package data. We're building "example_minimal_package-0.0.1-py3-none-any.whl" distribution = "example_minimal_package", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", version = "0.0.1", deps = [":example_pkg"], @@ -156,6 +164,8 @@ py_wheel( "//examples/wheel:README.md": "README", }, homepage = "www.example.com", + incompatible_normalize_name = True, + incompatible_normalize_version = True, license = "Apache 2.0", project_urls = { "Bug Tracker": "www.example.com/issues", @@ -177,6 +187,8 @@ py_wheel( entry_points = { "console_scripts": ["main = foo.bar:baz"], }, + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", strip_path_prefixes = [ "examples", @@ -191,6 +203,8 @@ py_wheel( name = "custom_package_root_multi_prefix", # Package data. We're building "custom_custom_package_root_multi_prefix-0.0.1-py3-none-any.whl" distribution = "example_custom_package_root_multi_prefix", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", strip_path_prefixes = [ "examples/wheel/lib", @@ -206,6 +220,8 @@ py_wheel( name = "custom_package_root_multi_prefix_reverse_order", # Package data. We're building "custom_custom_package_root_multi_prefix_reverse_order-0.0.1-py3-none-any.whl" distribution = "example_custom_package_root_multi_prefix_reverse_order", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", strip_path_prefixes = [ "examples/wheel", @@ -220,6 +236,8 @@ py_wheel( py_wheel( name = "python_requires_in_a_package", distribution = "example_python_requires_in_a_package", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_requires = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", python_tag = "py3", version = "0.0.1", @@ -231,6 +249,8 @@ py_wheel( py_wheel( name = "use_rule_with_dir_in_outs", distribution = "use_rule_with_dir_in_outs", + incompatible_normalize_name = True, + incompatible_normalize_version = True, python_tag = "py3", version = "0.0.1", deps = [ @@ -244,6 +264,8 @@ py_wheel( name = "python_abi3_binary_wheel", abi = "abi3", distribution = "example_python_abi3_binary_wheel", + incompatible_normalize_name = True, + incompatible_normalize_version = True, # these platform strings must line up with test_python_abi3_binary_wheel() in wheel_test.py platform = select({ ":aarch64-apple-darwin": "macosx_11_0_arm64", @@ -258,16 +280,32 @@ py_wheel( ) py_wheel( - name = "filename_escaping", + name = "legacy_filename_escaping", # Per https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode # runs of non-alphanumeric, non-digit symbols should be replaced with a single underscore. # Unicode non-ascii letters should *not* be replaced with underscore. distribution = "file~~name-escaping", + incompatible_normalize_name = False, + incompatible_normalize_version = False, python_tag = "py3", version = "0.0.1-r7", deps = [":example_pkg"], ) +py_wheel( + name = "filename_escaping", + # Per https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode + # runs of "-", "_" and "." should be replaced with a single underscore. + # Unicode non-ascii letters aren't allowed according to + # https://packaging.python.org/en/latest/specifications/name-normalization/. + distribution = "File--Name-Escaping", + incompatible_normalize_name = True, + incompatible_normalize_version = True, + python_tag = "py3", + version = "v0.0.1.RC1+ubuntu-r7", + deps = [":example_pkg"], +) + py_test( name = "wheel_test", srcs = ["wheel_test.py"], @@ -277,6 +315,7 @@ py_test( ":custom_package_root_multi_prefix_reverse_order", ":customized", ":filename_escaping", + ":legacy_filename_escaping", ":minimal_with_py_library", ":minimal_with_py_library_with_stamp", ":minimal_with_py_package", diff --git a/examples/wheel/wheel_test.py b/examples/wheel/wheel_test.py index f51a0ecedc..671bd8ad84 100644 --- a/examples/wheel/wheel_test.py +++ b/examples/wheel/wheel_test.py @@ -153,13 +153,51 @@ def test_customized_wheel(self): second = second.main:s""", ) + def test_legacy_filename_escaping(self): + filename = os.path.join( + os.environ['TEST_SRCDIR'], + 'rules_python', + 'examples', + 'wheel', + 'file_name_escaping-0.0.1_r7-py3-none-any.whl', + ) + with zipfile.ZipFile(filename) as zf: + self.assertEquals( + zf.namelist(), + [ + 'examples/wheel/lib/data.txt', + 'examples/wheel/lib/module_with_data.py', + 'examples/wheel/lib/simple_module.py', + 'examples/wheel/main.py', + # PEP calls for replacing only in the archive filename. + # Alas setuptools also escapes in the dist-info directory + # name, so let's be compatible. + 'file_name_escaping-0.0.1_r7.dist-info/WHEEL', + 'file_name_escaping-0.0.1_r7.dist-info/METADATA', + 'file_name_escaping-0.0.1_r7.dist-info/RECORD', + ], + ) + metadata_contents = zf.read( + 'file_name_escaping-0.0.1_r7.dist-info/METADATA' + ) + self.assertEquals( + metadata_contents, + b"""\ +Metadata-Version: 2.1 +Name: file~~name-escaping +Version: 0.0.1-r7 + +UNKNOWN +""", + ) + def test_filename_escaping(self): filename = os.path.join( os.environ["TEST_SRCDIR"], "rules_python", "examples", "wheel", - "file_name_escaping-0.0.1_r7-py3-none-any.whl", + "file_name_escaping-0.0.1rc1+ubuntu.r7-py3-none-any.whl", ) with zipfile.ZipFile(filename) as zf: self.assertEqual( @@ -172,20 +210,20 @@ def test_filename_escaping(self): # PEP calls for replacing only in the archive filename. # Alas setuptools also escapes in the dist-info directory # name, so let's be compatible. - "file_name_escaping-0.0.1_r7.dist-info/WHEEL", - "file_name_escaping-0.0.1_r7.dist-info/METADATA", - "file_name_escaping-0.0.1_r7.dist-info/RECORD", + "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/WHEEL", + "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/METADATA", + "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/RECORD", ], ) metadata_contents = zf.read( - "file_name_escaping-0.0.1_r7.dist-info/METADATA" + "file_name_escaping-0.0.1rc1+ubuntu.r7.dist-info/METADATA" ) self.assertEqual( metadata_contents, b"""\ Metadata-Version: 2.1 -Name: file~~name-escaping -Version: 0.0.1-r7 +Name: File--Name-Escaping +Version: 0.0.1rc1+ubuntu.r7 UNKNOWN """, diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 34b4de3d00..5ff752e13f 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -77,6 +77,7 @@ bzl_library( ":py_binary_bzl", "//python/private:py_package.bzl", "//python/private:py_wheel_bzl", + "//python/private:py_wheel_normalize_pep440.bzl", "//python/private:stamp_bzl", "//python/private:util_bzl", ], diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index d1610584c9..f6e3012edd 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -236,6 +236,7 @@ exports_files( "coverage.patch", "py_package.bzl", "py_wheel.bzl", + "py_wheel_normalize_pep440.bzl", "reexports.bzl", "stamp.bzl", "util.bzl", diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index d8bceabcb8..4152e08c18 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -16,6 +16,7 @@ load("//python/private:stamp.bzl", "is_stamping_enabled") load(":py_package.bzl", "py_package_lib") +load(":py_wheel_normalize_pep440.bzl", "normalize_pep440") PyWheelInfo = provider( doc = "Information about a wheel produced by `py_wheel`", @@ -117,6 +118,29 @@ See [`py_wheel_dist`](/docs/packaging.md#py_wheel_dist) for more info. ), } +_feature_flags = { + "incompatible_normalize_name": attr.bool( + default = False, + doc = """\ +Normalize the package distribution name according to latest +Python packaging standards. + +See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode +and https://packaging.python.org/en/latest/specifications/name-normalization/. + +Apart from the valid names according to the above, we also accept +'{' and '}', which may be used as placeholders for stamping. +""", + ), + "incompatible_normalize_version": attr.bool( + default = False, + doc = "Normalize the package version according to PEP440 standard. " + + "With this option set to True, if the user wants to pass any " + + "stamp variables, they have to be enclosed in '{}', e.g. " + + "'{BUILD_TIMESTAMP}'.", + ), +} + _requirement_attrs = { "extra_requires": attr.string_list_dict( doc = "List of optional requirements for this package", @@ -203,6 +227,42 @@ _DESCRIPTION_FILE_EXTENSION_TO_TYPE = { } _DEFAULT_DESCRIPTION_FILE_TYPE = "text/plain" +def _escape_filename_distribution_name(name): + """Escape the distribution name component of a filename. + + See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode + and https://packaging.python.org/en/latest/specifications/name-normalization/. + + Apart from the valid names according to the above, we also accept + '{' and '}', which may be used as placeholders for stamping. + """ + escaped = "" + for character in name.elems(): + if character.isalnum() or character in ["{", "}"]: + escaped += character.lower() + elif character in ["-", "_", "."]: + if escaped == "": + fail( + "A valid name must start with a letter or number.", + "Name '%s' does not." % name, + ) + elif escaped.endswith("_"): + pass + else: + escaped += "_" + else: + fail( + "A valid name consists only of ASCII letters ", + "and numbers, period, underscore and hyphen.", + "Name '%s' has bad character '%s'." % (name, character), + ) + if escaped.endswith("_"): + fail( + "A valid name must end with a letter or number.", + "Name '%s' does not." % name, + ) + return escaped + def _escape_filename_segment(segment): """Escape a segment of the wheel filename. @@ -237,13 +297,25 @@ def _py_wheel_impl(ctx): python_tag = _replace_make_variables(ctx.attr.python_tag, ctx) version = _replace_make_variables(ctx.attr.version, ctx) - outfile = ctx.actions.declare_file("-".join([ - _escape_filename_segment(ctx.attr.distribution), - _escape_filename_segment(version), + filename_segments = [] + + if ctx.attr.incompatible_normalize_name: + filename_segments.append(_escape_filename_distribution_name(ctx.attr.distribution)) + else: + filename_segments.append(_escape_filename_segment(ctx.attr.distribution)) + + if ctx.attr.incompatible_normalize_version: + filename_segments.append(normalize_pep440(version)) + else: + filename_segments.append(_escape_filename_segment(version)) + + filename_segments.extend([ _escape_filename_segment(python_tag), _escape_filename_segment(abi), _escape_filename_segment(ctx.attr.platform), - ]) + ".whl") + ]) + + outfile = ctx.actions.declare_file("-".join(filename_segments) + ".whl") name_file = ctx.actions.declare_file(ctx.label.name + ".name") @@ -272,6 +344,10 @@ def _py_wheel_impl(ctx): args.add("--out", outfile) args.add("--name_file", name_file) args.add_all(ctx.attr.strip_path_prefixes, format_each = "--strip_path_prefix=%s") + if ctx.attr.incompatible_normalize_name: + args.add("--incompatible_normalize_name") + if ctx.attr.incompatible_normalize_version: + args.add("--incompatible_normalize_version") # Pass workspace status files if stamping is enabled if is_stamping_enabled(ctx.attr): @@ -423,6 +499,7 @@ tries to locate `.runfiles` directory which is not packaged in the wheel. ), }, _distribution_attrs, + _feature_flags, _requirement_attrs, _entrypoint_attrs, _other_attrs, diff --git a/python/private/py_wheel_normalize_pep440.bzl b/python/private/py_wheel_normalize_pep440.bzl new file mode 100644 index 0000000000..9566348987 --- /dev/null +++ b/python/private/py_wheel_normalize_pep440.bzl @@ -0,0 +1,519 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Implementation of PEP440 version string normalization" + +def mkmethod(self, method): + """Bind a struct as the first arg to a function. + + This is loosely equivalent to creating a bound method of a class. + """ + return lambda *args, **kwargs: method(self, *args, **kwargs) + +def _isdigit(token): + return token.isdigit() + +def _isalnum(token): + return token.isalnum() + +def _lower(token): + # PEP 440: Case sensitivity + return token.lower() + +def _is(reference): + """Predicate testing a token for equality with `reference`.""" + return lambda token: token == reference + +def _is_not(reference): + """Predicate testing a token for inequality with `reference`.""" + return lambda token: token != reference + +def _in(reference): + """Predicate testing if a token is in the list `reference`.""" + return lambda token: token in reference + +def _ctx(start): + return {"norm": "", "start": start} + +def _open_context(self): + """Open an new parsing ctx. + + If the current parsing step succeeds, call self.accept(). + If the current parsing step fails, call self.discard() to + go back to how it was before we opened a new ctx. + + Args: + self: The normalizer. + """ + self.contexts.append(_ctx(_context(self)["start"])) + return self.contexts[-1] + +def _accept(self): + """Close the current ctx successfully and merge the results.""" + finished = self.contexts.pop() + self.contexts[-1]["norm"] += finished["norm"] + self.contexts[-1]["start"] = finished["start"] + return True + +def _context(self): + return self.contexts[-1] + +def _discard(self): + self.contexts.pop() + return False + +def _new(input): + """Create a new normalizer""" + self = struct( + input = input, + contexts = [_ctx(0)], + ) + + public = struct( + # methods: keep sorted + accept = mkmethod(self, _accept), + context = mkmethod(self, _context), + discard = mkmethod(self, _discard), + open_context = mkmethod(self, _open_context), + + # attributes: keep sorted + input = self.input, + ) + return public + +def accept(parser, predicate, value): + """If `predicate` matches the next token, accept the token. + + Accepting the token means adding it (according to `value`) to + the running results maintained in ctx["norm"] and + advancing the cursor in ctx["start"] to the next token in + `version`. + + Args: + parser: The normalizer. + predicate: function taking a token and returning a boolean + saying if we want to accept the token. + value: the string to add if there's a match, or, if `value` + is a function, the function to apply to the current token + to get the string to add. + + Returns: + whether a token was accepted. + """ + + ctx = parser.context() + + if ctx["start"] >= len(parser.input): + return False + + token = parser.input[ctx["start"]] + + if predicate(token): + if type(value) in ["function", "builtin_function_or_method"]: + value = value(token) + + ctx["norm"] += value + ctx["start"] += 1 + return True + + return False + +def accept_placeholder(parser): + """Accept a Bazel placeholder. + + Placeholders aren't actually part of PEP 440, but are used for + stamping purposes. A placeholder might be + ``{BUILD_TIMESTAMP}``, for instance. We'll accept these as + they are, assuming they will expand to something that makes + sense where they appear. Before the stamping has happened, a + resulting wheel file name containing a placeholder will not + actually be valid. + + Args: + parser: The normalizer. + + Returns: + whether a placeholder was accepted. + """ + ctx = parser.open_context() + + if not accept(parser, _is("{"), str): + return parser.discard() + + start = ctx["start"] + for _ in range(start, len(parser.input) + 1): + if not accept(parser, _is_not("}"), str): + break + + if not accept(parser, _is("}"), str): + return parser.discard() + + return parser.accept() + +def accept_digits(parser): + """Accept multiple digits (or placeholders). + + Args: + parser: The normalizer. + + Returns: + whether some digits (or placeholders) were accepted. + """ + + ctx = parser.open_context() + start = ctx["start"] + + for i in range(start, len(parser.input) + 1): + if not accept(parser, _isdigit, str) and not accept_placeholder(parser): + if i - start >= 1: + if ctx["norm"].isdigit(): + # PEP 440: Integer Normalization + ctx["norm"] = str(int(ctx["norm"])) + return parser.accept() + break + + return parser.discard() + +def accept_string(parser, string, replacement): + """Accept a `string` in the input. Output `replacement`. + + Args: + parser: The normalizer. + string: The string to search for in the parser input. + replacement: The normalized string to use if the string was found. + + Returns: + whether the string was accepted. + """ + ctx = parser.open_context() + + for character in string.elems(): + if not accept(parser, _in([character, character.upper()]), ""): + return parser.discard() + + ctx["norm"] = replacement + + return parser.accept() + +def accept_alnum(parser): + """Accept an alphanumeric sequence. + + Args: + parser: The normalizer. + + Returns: + whether an alphanumeric sequence was accepted. + """ + + ctx = parser.open_context() + start = ctx["start"] + + for i in range(start, len(parser.input) + 1): + if not accept(parser, _isalnum, _lower) and not accept_placeholder(parser): + if i - start >= 1: + return parser.accept() + break + + return parser.discard() + +def accept_dot_number(parser): + """Accept a dot followed by digits. + + Args: + parser: The normalizer. + + Returns: + whether a dot+digits pair was accepted. + """ + parser.open_context() + + if accept(parser, _is("."), ".") and accept_digits(parser): + return parser.accept() + else: + return parser.discard() + +def accept_dot_number_sequence(parser): + """Accept a sequence of dot+digits. + + Args: + parser: The normalizer. + + Returns: + whether a sequence of dot+digits pairs was accepted. + """ + ctx = parser.context() + start = ctx["start"] + i = start + + for i in range(start, len(parser.input) + 1): + if not accept_dot_number(parser): + break + return i - start >= 1 + +def accept_separator_alnum(parser): + """Accept a separator followed by an alphanumeric string. + + Args: + parser: The normalizer. + + Returns: + whether a separator and an alphanumeric string were accepted. + """ + parser.open_context() + + # PEP 440: Local version segments + if ( + accept(parser, _in([".", "-", "_"]), ".") and + (accept_digits(parser) or accept_alnum(parser)) + ): + return parser.accept() + + return parser.discard() + +def accept_separator_alnum_sequence(parser): + """Accept a sequence of separator+alphanumeric. + + Args: + parser: The normalizer. + + Returns: + whether a sequence of separator+alphanumerics was accepted. + """ + ctx = parser.context() + start = ctx["start"] + i = start + + for i in range(start, len(parser.input) + 1): + if not accept_separator_alnum(parser): + break + + return i - start >= 1 + +def accept_epoch(parser): + """PEP 440: Version epochs. + + Args: + parser: The normalizer. + + Returns: + whether a PEP 440 epoch identifier was accepted. + """ + ctx = parser.open_context() + if accept_digits(parser) and accept(parser, _is("!"), "!"): + if ctx["norm"] == "0!": + ctx["norm"] = "" + return parser.accept() + else: + return parser.discard() + +def accept_release(parser): + """Accept the release segment, numbers separated by dots. + + Args: + parser: The normalizer. + + Returns: + whether a release segment was accepted. + """ + parser.open_context() + + if not accept_digits(parser): + return parser.discard() + + accept_dot_number_sequence(parser) + return parser.accept() + +def accept_pre_l(parser): + """PEP 440: Pre-release spelling. + + Args: + parser: The normalizer. + + Returns: + whether a prerelease keyword was accepted. + """ + parser.open_context() + + if ( + accept_string(parser, "alpha", "a") or + accept_string(parser, "a", "a") or + accept_string(parser, "beta", "b") or + accept_string(parser, "b", "b") or + accept_string(parser, "c", "rc") or + accept_string(parser, "preview", "rc") or + accept_string(parser, "pre", "rc") or + accept_string(parser, "rc", "rc") + ): + return parser.accept() + else: + return parser.discard() + +def accept_prerelease(parser): + """PEP 440: Pre-releases. + + Args: + parser: The normalizer. + + Returns: + whether a prerelease identifier was accepted. + """ + ctx = parser.open_context() + + # PEP 440: Pre-release separators + accept(parser, _in(["-", "_", "."]), "") + + if not accept_pre_l(parser): + return parser.discard() + + accept(parser, _in(["-", "_", "."]), "") + + if not accept_digits(parser): + # PEP 440: Implicit pre-release number + ctx["norm"] += "0" + + return parser.accept() + +def accept_implicit_postrelease(parser): + """PEP 440: Implicit post releases. + + Args: + parser: The normalizer. + + Returns: + whether an implicit postrelease identifier was accepted. + """ + ctx = parser.open_context() + + if accept(parser, _is("-"), "") and accept_digits(parser): + ctx["norm"] = ".post" + ctx["norm"] + return parser.accept() + + return parser.discard() + +def accept_explicit_postrelease(parser): + """PEP 440: Post-releases. + + Args: + parser: The normalizer. + + Returns: + whether an explicit postrelease identifier was accepted. + """ + ctx = parser.open_context() + + # PEP 440: Post release separators + if not accept(parser, _in(["-", "_", "."]), "."): + ctx["norm"] += "." + + # PEP 440: Post release spelling + if ( + accept_string(parser, "post", "post") or + accept_string(parser, "rev", "post") or + accept_string(parser, "r", "post") + ): + accept(parser, _in(["-", "_", "."]), "") + + if not accept_digits(parser): + # PEP 440: Implicit post release number + ctx["norm"] += "0" + + return parser.accept() + + return parser.discard() + +def accept_postrelease(parser): + """PEP 440: Post-releases. + + Args: + parser: The normalizer. + + Returns: + whether a postrelease identifier was accepted. + """ + parser.open_context() + + if accept_implicit_postrelease(parser) or accept_explicit_postrelease(parser): + return parser.accept() + + return parser.discard() + +def accept_devrelease(parser): + """PEP 440: Developmental releases. + + Args: + parser: The normalizer. + + Returns: + whether a developmental release identifier was accepted. + """ + ctx = parser.open_context() + + # PEP 440: Development release separators + if not accept(parser, _in(["-", "_", "."]), "."): + ctx["norm"] += "." + + if accept_string(parser, "dev", "dev"): + accept(parser, _in(["-", "_", "."]), "") + + if not accept_digits(parser): + # PEP 440: Implicit development release number + ctx["norm"] += "0" + + return parser.accept() + + return parser.discard() + +def accept_local(parser): + """PEP 440: Local version identifiers. + + Args: + parser: The normalizer. + + Returns: + whether a local version identifier was accepted. + """ + parser.open_context() + + if accept(parser, _is("+"), "+") and accept_alnum(parser): + accept_separator_alnum_sequence(parser) + return parser.accept() + + return parser.discard() + +def normalize_pep440(version): + """Escape the version component of a filename. + + See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode + and https://peps.python.org/pep-0440/ + + Args: + version: version string to be normalized according to PEP 440. + + Returns: + string containing the normalized version. + """ + parser = _new(version.strip()) # PEP 440: Leading and Trailing Whitespace + accept(parser, _is("v"), "") # PEP 440: Preceding v character + accept_epoch(parser) + accept_release(parser) + accept_prerelease(parser) + accept_postrelease(parser) + accept_devrelease(parser) + accept_local(parser) + if parser.input[parser.context()["start"]:]: + fail( + "Failed to parse PEP 440 version identifier '%s'." % parser.input, + "Parse error at '%s'" % parser.input[parser.context()["start"]:], + ) + return parser.context()["norm"] diff --git a/tests/py_wheel/py_wheel_tests.bzl b/tests/py_wheel/py_wheel_tests.bzl index e580732aac..3c03a1b8e4 100644 --- a/tests/py_wheel/py_wheel_tests.bzl +++ b/tests/py_wheel/py_wheel_tests.bzl @@ -16,7 +16,9 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:packaging.bzl", "py_wheel") +load("//python/private:py_wheel_normalize_pep440.bzl", "normalize_pep440") # buildifier: disable=bzl-visibility +_basic_tests = [] _tests = [] def _test_metadata(name): @@ -92,8 +94,109 @@ def _test_content_type_from_description_impl(env, target): _tests.append(_test_content_type_from_description) +def _test_pep440_normalization(env): + prefixes = ["v", " v", " \t\r\nv"] + epochs = { + "": ["", "0!", "00!"], + "1!": ["1!", "001!"], + "200!": ["200!", "00200!"], + } + releases = { + "0.1": ["0.1", "0.01"], + "2023.7.19": ["2023.7.19", "2023.07.19"], + } + pres = { + "": [""], + "a0": ["a", ".a", "-ALPHA0", "_alpha0", ".a0"], + "a4": ["alpha4", ".a04"], + "b0": ["b", ".b", "-BETA0", "_beta0", ".b0"], + "b5": ["beta05", ".b5"], + "rc0": ["C", "_c0", "RC", "_rc0", "-preview_0"], + } + explicit_posts = { + "": [""], + ".post0": [], + ".post1": [".post1", "-r1", "_rev1"], + } + implicit_posts = [[".post1", "-1"], [".post2", "-2"]] + devs = { + "": [""], + ".dev0": ["dev", "-DEV", "_Dev-0"], + ".dev9": ["DEV9", ".dev09", ".dev9"], + ".dev{BUILD_TIMESTAMP}": [ + "-DEV{BUILD_TIMESTAMP}", + "_dev_{BUILD_TIMESTAMP}", + ], + } + locals = { + "": [""], + "+ubuntu.7": ["+Ubuntu_7", "+ubuntu-007"], + "+ubuntu.r007": ["+Ubuntu_R007"], + } + epochs = [ + [normalized_epoch, input_epoch] + for normalized_epoch, input_epochs in epochs.items() + for input_epoch in input_epochs + ] + releases = [ + [normalized_release, input_release] + for normalized_release, input_releases in releases.items() + for input_release in input_releases + ] + pres = [ + [normalized_pre, input_pre] + for normalized_pre, input_pres in pres.items() + for input_pre in input_pres + ] + explicit_posts = [ + [normalized_post, input_post] + for normalized_post, input_posts in explicit_posts.items() + for input_post in input_posts + ] + pres_and_posts = [ + [normalized_pre + normalized_post, input_pre + input_post] + for normalized_pre, input_pre in pres + for normalized_post, input_post in explicit_posts + ] + [ + [normalized_pre + normalized_post, input_pre + input_post] + for normalized_pre, input_pre in pres + for normalized_post, input_post in implicit_posts + if input_pre == "" or input_pre[-1].isdigit() + ] + devs = [ + [normalized_dev, input_dev] + for normalized_dev, input_devs in devs.items() + for input_dev in input_devs + ] + locals = [ + [normalized_local, input_local] + for normalized_local, input_locals in locals.items() + for input_local in input_locals + ] + postfixes = ["", " ", " \t\r\n"] + i = 0 + for nepoch, iepoch in epochs: + for nrelease, irelease in releases: + for nprepost, iprepost in pres_and_posts: + for ndev, idev in devs: + for nlocal, ilocal in locals: + prefix = prefixes[i % len(prefixes)] + postfix = postfixes[(i // len(prefixes)) % len(postfixes)] + env.expect.that_str( + normalize_pep440( + prefix + iepoch + irelease + iprepost + + idev + ilocal + postfix, + ), + ).equals( + nepoch + nrelease + nprepost + ndev + nlocal, + ) + i += 1 + +_basic_tests.append(_test_pep440_normalization) + def py_wheel_test_suite(name): test_suite( name = name, + basic_tests = _basic_tests, tests = _tests, ) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index fd951d9086..51bd56df0a 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -21,6 +21,7 @@ licenses(["notice"]) py_binary( name = "wheelmaker", srcs = ["wheelmaker.py"], + deps = ["@pypi__packaging//:lib"], ) filegroup( diff --git a/tools/wheelmaker.py b/tools/wheelmaker.py index 63b833fc5d..dce5406093 100644 --- a/tools/wheelmaker.py +++ b/tools/wheelmaker.py @@ -33,10 +33,67 @@ def commonpath(path1, path2): def escape_filename_segment(segment): - """Escapes a filename segment per https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode""" + """Escapes a filename segment per https://www.python.org/dev/peps/pep-0427/#escaping-and-unicode + + This is a legacy function, kept for backwards compatibility, + and may be removed in the future. See `escape_filename_distribution_name` + and `normalize_pep440` for the modern alternatives. + """ return re.sub(r"[^\w\d.]+", "_", segment, re.UNICODE) +def normalize_package_name(name): + """Normalize a package name according to the Python Packaging User Guide. + + See https://packaging.python.org/en/latest/specifications/name-normalization/ + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def escape_filename_distribution_name(name): + """Escape the distribution name component of a filename. + + See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode + """ + return normalize_package_name(name).replace("-", "_") + + +def normalize_pep440(version): + """Normalize version according to PEP 440, with fallback for placeholders. + + If there's a placeholder in braces, such as {BUILD_TIMESTAMP}, + replace it with 0. Such placeholders can be used with stamping, in + which case they would have been resolved already by now; if they + haven't, we're doing an unstamped build, but we still need to + produce a valid version. If such replacements are made, the + original version string, sanitized to dot-separated alphanumerics, + is appended as a local version segment, so you understand what + placeholder was involved. + + If that still doesn't produce a valid version, use version 0 and + append the original version string, sanitized to dot-separated + alphanumerics, as a local version segment. + + """ + + import packaging.version + + try: + return str(packaging.version.Version(version)) + except packaging.version.InvalidVersion: + pass + + sanitized = re.sub(r'[^a-z0-9]+', '.', version.lower()).strip('.') + substituted = re.sub(r'\{\w+\}', '0', version) + delimiter = '.' if '+' in substituted else '+' + try: + return str( + packaging.version.Version(f'{substituted}{delimiter}{sanitized}') + ) + except packaging.version.InvalidVersion: + return str(packaging.version.Version(f'0+{sanitized}')) + + class WheelMaker(object): def __init__( self, @@ -48,6 +105,8 @@ def __init__( platform, outfile=None, strip_path_prefixes=None, + incompatible_normalize_name=False, + incompatible_normalize_version=False, ): self._name = name self._version = version @@ -60,12 +119,30 @@ def __init__( strip_path_prefixes if strip_path_prefixes is not None else [] ) - self._distinfo_dir = ( - escape_filename_segment(self._name) - + "-" - + escape_filename_segment(self._version) - + ".dist-info/" - ) + if incompatible_normalize_version: + self._version = normalize_pep440(self._version) + self._escaped_version = self._version + else: + self._escaped_version = escape_filename_segment(self._version) + + if incompatible_normalize_name: + escaped_name = escape_filename_distribution_name(self._name) + self._distinfo_dir = ( + escaped_name + "-" + self._escaped_version + ".dist-info/" + ) + self._wheelname_fragment_distribution_name = escaped_name + else: + # The legacy behavior escapes the distinfo dir but not the + # wheel name. Enable incompatible_normalize_name to fix it. + # https://github.com/bazelbuild/rules_python/issues/1132 + self._distinfo_dir = ( + escape_filename_segment(self._name) + + "-" + + self._escaped_version + + ".dist-info/" + ) + self._wheelname_fragment_distribution_name = self._name + self._zipfile = None # Entries for the RECORD file as (filename, hash, size) tuples. self._record = [] @@ -81,7 +158,10 @@ def __exit__(self, type, value, traceback): self._zipfile = None def wheelname(self) -> str: - components = [self._name, self._version] + components = [ + self._wheelname_fragment_distribution_name, + self._version, + ] if self._build_tag: components.append(self._build_tag) components += [self._python_tag, self._abi, self._platform] @@ -330,6 +410,10 @@ def parse_args() -> argparse.Namespace: help="Pass in the stamp info file for stamping", ) + feature_group = parser.add_argument_group("Feature flags") + feature_group.add_argument("--incompatible_normalize_name", action="store_true") + feature_group.add_argument("--incompatible_normalize_version", action="store_true") + return parser.parse_args(sys.argv[1:]) @@ -386,6 +470,8 @@ def main() -> None: platform=arguments.platform, outfile=arguments.out, strip_path_prefixes=strip_prefixes, + incompatible_normalize_name=arguments.incompatible_normalize_name, + incompatible_normalize_version=arguments.incompatible_normalize_version, ) as maker: for package_filename, real_filename in all_files: maker.add_file(package_filename, real_filename) @@ -410,8 +496,15 @@ def main() -> None: with open(arguments.metadata_file, "rt", encoding="utf-8") as metadata_file: metadata = metadata_file.read() + if arguments.incompatible_normalize_version: + version_in_metadata = normalize_pep440(version) + else: + version_in_metadata = version maker.add_metadata( - metadata=metadata, name=name, description=description, version=version + metadata=metadata, + name=name, + description=description, + version=version_in_metadata, ) if arguments.entry_points_file: From fe33a4582c37499f3caeb49a07a78fc7948a8949 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 5 Oct 2023 23:36:02 +0900 Subject: [PATCH 41/41] chore: add new Python toolchains from indygreg (#1461) Updates versions: * 3.8.15 -> 3.8.18 * 3.11.4 -> 3.11.6 Adds versions: 3.8.18, 3.11.6, 3.12.0 Fixes #1396 --- CHANGELOG.md | 14 +++++++-- WORKSPACE | 4 +-- python/versions.bzl | 74 ++++++++++++++++++++++++++++++++++----------- 3 files changed, 69 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c421a9d33..512d820d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,10 +22,10 @@ A brief description of the categories of changes: ### Changed * Python version patch level bumps: - * 3.8.15 -> 3.8.17 + * 3.8.15 -> 3.8.18 * 3.9.17 -> 3.9.18 * 3.10.12 -> 3.10.13 - * 3.11.4 -> 3.11.5 + * 3.11.4 -> 3.11.6 * (deps) Upgrade rules_go 0.39.1 -> 0.41.0; this is so gazelle integration works with upcoming Bazel versions @@ -47,12 +47,16 @@ A brief description of the categories of changes: [`py_console_script_binary`](./docs/py_console_script_binary.md), which allows adding custom dependencies to a package's entry points and customizing the `py_binary` rule used to build it. -* New Python versions available: `3.8.17`, `3.9.18`, `3.10.13`, `3.11.5` using + +* New Python versions available: `3.8.17`, `3.11.5` using https://github.com/indygreg/python-build-standalone/releases/tag/20230826. + * (gazelle) New `# gazelle:python_generation_mode file` directive to support generating one `py_library` per file. + * (python_repository) Support `netrc` and `auth_patterns` attributes to enable authentication against private HTTP hosts serving Python toolchain binaries. + * `//python:packaging_bzl` added, a `bzl_library` for the Starlark files `//python:packaging.bzl` requires. * (py_wheel) Added the `incompatible_normalize_name` feature flag to @@ -64,6 +68,10 @@ A brief description of the categories of changes: in them), in accordance with PEP440. Defaults to `False` for the time being. +* New Python versions available: `3.8.18`, `3.9.18`, `3.10.13`, `3.11.6`, `3.12.0` using + https://github.com/indygreg/python-build-standalone/releases/tag/20231002. + `3.12.0` support is considered beta and may have issues. + ### Removed * (bzlmod) The `entry_point` macro is no longer supported and has been removed diff --git a/WORKSPACE b/WORKSPACE index 6e1e5fc620..ad32013816 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -30,7 +30,7 @@ load("//python:versions.bzl", "MINOR_MAPPING") python_register_multi_toolchains( name = "python", - default_version = MINOR_MAPPING.values()[-1], + default_version = MINOR_MAPPING.values()[-2], python_versions = MINOR_MAPPING.values(), ) @@ -72,7 +72,7 @@ _py_gazelle_deps() # Install twine for our own runfiles wheel publishing. # Eventually we might want to install twine automatically for users too, see: # https://github.com/bazelbuild/rules_python/issues/1016. -load("@python//3.11.5:defs.bzl", "interpreter") +load("@python//3.11.6:defs.bzl", "interpreter") load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( diff --git a/python/versions.bzl b/python/versions.bzl index a79ba91293..6c9bf252bf 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -108,6 +108,17 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.8.18": { + "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "1825b1f7220bc93ff143f2e70b5c6a79c6469e0eeb40824e07a7277f59aabfda", + "aarch64-unknown-linux-gnu": "236a300f386ead02ca98dbddbc026ff4ef4de6701a394106e291ff8b75445ee1", + "x86_64-apple-darwin": "fcf04532e644644213977242cd724fe5e84c0a5ac92ae038e07f1b01b474fca3", + "x86_64-pc-windows-msvc": "a9d203e78caed94de368d154e841610cef6f6b484738573f4ae9059d37e898a5", + "x86_64-unknown-linux-gnu": "1e8a3babd1500111359b0f5675d770984bcbcb2cc8890b117394f0ed342fb9ec", + }, + "strip_prefix": "python", + }, "3.9.10": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", "sha256": { @@ -178,15 +189,15 @@ TOOL_VERSIONS = { "strip_prefix": "python", }, "3.9.18": { - "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", "sha256": { - "aarch64-apple-darwin": "44000d3bd79a6c689f3b6cae846d302d9a4e974c46d078b1bc79cc0c706a0718", - "aarch64-unknown-linux-gnu": "2161e834aa4334cc8bb55335767a073aafff3338cf37392d2a9123b4972276f9", - "ppc64le-unknown-linux-gnu": "1e95c15627cea707156b41d653af994283876162f14ac9280cc1fb8023cf56b3", - "s390x-unknown-linux-gnu": "476d1ba8f85ae8a0e0b5ae7f0e204dd9376fe55afd9c6a7ae7b18bd84a223bf6", - "x86_64-apple-darwin": "ce03b97a41be6d548698baaf5804fff2ce96bf49237fb73f8692aca3f5798454", - "x86_64-pc-windows-msvc": "709c1aabf712aa4553c53c4879a459ebe8575a996d68ccbce492af03db8a6ee0", - "x86_64-unknown-linux-gnu": "377da2aebc3b58c5af901899e8efeb2c91b35b0ea92c8b447036767e529fc5b2", + "aarch64-apple-darwin": "fdc4054837e37b69798c2ef796222a480bc1f80e8ad3a01a95d0168d8282a007", + "aarch64-unknown-linux-gnu": "1e0a3e8ce8e58901a259748c0ab640d2b8294713782d14229e882c6898b2fb36", + "ppc64le-unknown-linux-gnu": "101c38b22fb2f5a0945156da4259c8e9efa0c08de9d7f59afa51e7ce6e22a1cc", + "s390x-unknown-linux-gnu": "eee31e55ffbc1f460d7b17f05dd89e45a2636f374a6f8dc29ea13d0497f7f586", + "x86_64-apple-darwin": "82231cb77d4a5c8081a1a1d5b8ae440abe6993514eb77a926c826e9a69a94fb1", + "x86_64-pc-windows-msvc": "02ea7bb64524886bd2b05d6b6be4401035e4ba4319146f274f0bcd992822cd75", + "x86_64-unknown-linux-gnu": "f3ff38b1ccae7dcebd8bbf2e533c9a984fac881de0ffd1636fbb61842bd924de", }, "strip_prefix": "python", }, @@ -271,15 +282,15 @@ TOOL_VERSIONS = { "strip_prefix": "python", }, "3.10.13": { - "url": "20230826/cpython-{python_version}+20230826-{platform}-{build}.tar.gz", + "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", "sha256": { - "aarch64-apple-darwin": "142332021441ee1ab04eb126baa6c6690dc41699d4af608b72b399a786f6ee71", - "aarch64-unknown-linux-gnu": "0479cf10254adbf7a554453874e91bb526ba62cbac8a758f6865cdcdbef20f2d", - "ppc64le-unknown-linux-gnu": "355ec3d0983e1e454d7175c9c8581221472d4597f6a93d676b60ed4e1655c299", - "s390x-unknown-linux-gnu": "a61ff760d39e2b06794cdcf8b2f62c39d58b97f5a1ddd0e112741f60d6fe712f", - "x86_64-apple-darwin": "3a5d50b98e4981af4fc23cf3fc53a38ef3f9a8f32453849e295e747aa9936b2b", - "x86_64-pc-windows-msvc": "2ae0ee39450d428ce2aa4bea9ad41c96916d4f92fe641a3bf6d6f80d360677c3", - "x86_64-unknown-linux-gnu": "ba512bcca3ac6cb6d834f496cd0a66416f0a53ff20b05c4794fa82ece185b85a", + "aarch64-apple-darwin": "fd027b1dedf1ea034cdaa272e91771bdf75ddef4c8653b05d224a0645aa2ca3c", + "aarch64-unknown-linux-gnu": "8675915ff454ed2f1597e27794bc7df44f5933c26b94aa06af510fe91b58bb97", + "ppc64le-unknown-linux-gnu": "f3f9c43eec1a0c3f72845d0b705da17a336d3906b7df212d2640b8f47e8ff375", + "s390x-unknown-linux-gnu": "859f6cfe9aedb6e8858892fdc124037e83ab05f28d42a7acd314c6a16d6bd66c", + "x86_64-apple-darwin": "be0b19b6af1f7d8c667e5abef5505ad06cf72e5a11bb5844970c395a7e5b1275", + "x86_64-pc-windows-msvc": "b8d930ce0d04bda83037ad3653d7450f8907c88e24bb8255a29b8dab8930d6f1", + "x86_64-unknown-linux-gnu": "5d0429c67c992da19ba3eb58b3acd0b35ec5e915b8cae9a4aa8ca565c423847a", }, "strip_prefix": "python", }, @@ -332,14 +343,41 @@ TOOL_VERSIONS = { }, "strip_prefix": "python", }, + "3.11.6": { + "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "916c35125b5d8323a21526d7a9154ca626453f63d0878e95b9f613a95006c990", + "aarch64-unknown-linux-gnu": "3e26a672df17708c4dc928475a5974c3fb3a34a9b45c65fb4bd1e50504cc84ec", + "ppc64le-unknown-linux-gnu": "7937035f690a624dba4d014ffd20c342e843dd46f89b0b0a1e5726b85deb8eaf", + "s390x-unknown-linux-gnu": "f9f19823dba3209cedc4647b00f46ed0177242917db20fb7fb539970e384531c", + "x86_64-apple-darwin": "178cb1716c2abc25cb56ae915096c1a083e60abeba57af001996e8bc6ce1a371", + "x86_64-pc-windows-msvc": "3933545e6d41462dd6a47e44133ea40995bc6efeed8c2e4cbdf1a699303e95ea", + "x86_64-unknown-linux-gnu": "ee37a7eae6e80148c7e3abc56e48a397c1664f044920463ad0df0fc706eacea8", + }, + "strip_prefix": "python", + }, + "3.12.0": { + "url": "20231002/cpython-{python_version}+20231002-{platform}-{build}.tar.gz", + "sha256": { + "aarch64-apple-darwin": "4734a2be2becb813830112c780c9879ac3aff111a0b0cd590e65ec7465774d02", + "aarch64-unknown-linux-gnu": "bccfe67cf5465a3dfb0336f053966e2613a9bc85a6588c2fcf1366ef930c4f88", + "ppc64le-unknown-linux-gnu": "b5dae075467ace32c594c7877fe6ebe0837681f814601d5d90ba4c0dfd87a1f2", + "s390x-unknown-linux-gnu": "5681621349dd85d9726d1b67c84a9686ce78f72e73a6f9e4cc4119911655759e", + "x86_64-apple-darwin": "5a9e88c8aa52b609d556777b52ebde464ae4b4f77e4aac4eb693af57395c9abf", + "x86_64-pc-windows-msvc": "facfaa1fbc8653f95057f3c4a0f8aa833dab0e0b316e24ee8686bc761d4b4f8d", + "x86_64-unknown-linux-gnu": "e51a5293f214053ddb4645b2c9f84542e2ef86870b8655704367bd4b29d39fe9", + }, + "strip_prefix": "python", + }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { - "3.8": "3.8.17", + "3.8": "3.8.18", "3.9": "3.9.18", "3.10": "3.10.13", - "3.11": "3.11.5", + "3.11": "3.11.6", + "3.12": "3.12.0", } PLATFORMS = {