From 3b9c85e7a466f5f904929ecfcfb11f0c444fef16 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 23 Mar 2023 12:59:58 -0700 Subject: [PATCH 01/12] cleanup: factor reexports.bzl into the respective implementation files (#1137) This helps avoid loading one rule requiring loading everything. Within Google, this makes some Starlark testing frameworks avoid having to mock far away dependencies of dependencies. --- python/BUILD.bazel | 18 +++-- python/defs.bzl | 17 ++--- python/private/BUILD.bazel | 6 ++ python/private/reexports.bzl | 136 ----------------------------------- python/private/util.bzl | 12 ++++ python/py_binary.bzl | 16 ++++- python/py_library.bzl | 14 +++- python/py_runtime.bzl | 14 +++- python/py_runtime_pair.bzl | 72 ++++++++++++++++++- python/py_test.bzl | 16 ++++- 10 files changed, 159 insertions(+), 162 deletions(-) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index a524d2ff94..2582d73751 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -56,9 +56,15 @@ bzl_library( visibility = ["//visibility:public"], deps = [ ":current_py_toolchain_bzl", + ":py_binary_bzl", ":py_import_bzl", + ":py_info_bzl", + ":py_library_bzl", + ":py_runtime_bzl", + ":py_runtime_info_bzl", + ":py_runtime_pair_bzl", + ":py_test_bzl", "//python/private:bazel_tools_bzl", - "//python/private:reexports_bzl", ], ) @@ -76,7 +82,7 @@ bzl_library( bzl_library( name = "py_binary_bzl", srcs = ["py_binary.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = ["//python/private:util_bzl"], ) bzl_library( @@ -99,19 +105,19 @@ bzl_library( bzl_library( name = "py_library_bzl", srcs = ["py_library.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = ["//python/private:util_bzl"], ) bzl_library( name = "py_runtime_bzl", srcs = ["py_runtime.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = ["//python/private:util_bzl"], ) bzl_library( name = "py_runtime_pair_bzl", srcs = ["py_runtime_pair.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = ["//python/private:bazel_tools_bzl"], ) bzl_library( @@ -123,7 +129,7 @@ bzl_library( bzl_library( name = "py_test_bzl", srcs = ["py_test.bzl"], - deps = ["//python/private:reexports_bzl"], + deps = ["//python/private:util_bzl"], ) # NOTE: Remember to add bzl_library targets to //tests:bzl_libraries diff --git a/python/defs.bzl b/python/defs.bzl index ec70c1bf86..6ded66a568 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -14,16 +14,13 @@ """Core rules for building Python projects.""" load("@bazel_tools//tools/python:srcs_version.bzl", _find_requirements = "find_requirements") -load( - "//python/private:reexports.bzl", - "internal_PyInfo", - "internal_PyRuntimeInfo", - _py_binary = "py_binary", - _py_library = "py_library", - _py_runtime = "py_runtime", - _py_runtime_pair = "py_runtime_pair", - _py_test = "py_test", -) +load("//python:py_binary.bzl", _py_binary = "py_binary") +load("//python:py_info.bzl", internal_PyInfo = "PyInfo") +load("//python:py_library.bzl", _py_library = "py_library") +load("//python:py_runtime.bzl", _py_runtime = "py_runtime") +load("//python:py_runtime_info.bzl", internal_PyRuntimeInfo = "PyRuntimeInfo") +load("//python:py_runtime_pair.bzl", _py_runtime_pair = "py_runtime_pair") +load("//python:py_test.bzl", _py_test = "py_test") load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain") load(":py_import.bzl", _py_import = "py_import") diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 21e3c1623f..4068ea480b 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -41,6 +41,12 @@ bzl_library( deps = [":bazel_tools_bzl"], ) +bzl_library( + name = "util_bzl", + srcs = ["util.bzl"], + visibility = ["//python:__subpackages__"], +) + # @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. bzl_library( name = "bazel_tools_bzl", diff --git a/python/private/reexports.bzl b/python/private/reexports.bzl index 987187c155..a300a20365 100644 --- a/python/private/reexports.bzl +++ b/python/private/reexports.bzl @@ -37,20 +37,6 @@ different name. Then we can load it from defs.bzl and export it there under the original name. """ -load("@bazel_tools//tools/python:toolchain.bzl", _py_runtime_pair = "py_runtime_pair") - -# The implementation of the macros and tagging mechanism follows the example -# set by rules_cc and rules_java. - -_MIGRATION_TAG = "__PYTHON_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__" - -def _add_tags(attrs): - if "tags" in attrs and attrs["tags"] != None: - attrs["tags"] = attrs["tags"] + [_MIGRATION_TAG] - else: - attrs["tags"] = [_MIGRATION_TAG] - return attrs - # Don't use underscore prefix, since that would make the symbol local to this # file only. Use a non-conventional name to emphasize that this is not a public # symbol. @@ -59,125 +45,3 @@ internal_PyInfo = PyInfo # buildifier: disable=name-conventions internal_PyRuntimeInfo = PyRuntimeInfo - -def py_library(**attrs): - """See the Bazel core [py_library](https://docs.bazel.build/versions/master/be/python.html#py_library) documentation. - - Args: - **attrs: Rule attributes - """ - if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): - fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - - # buildifier: disable=native-python - native.py_library(**_add_tags(attrs)) - -def py_binary(**attrs): - """See the Bazel core [py_binary](https://docs.bazel.build/versions/master/be/python.html#py_binary) documentation. - - Args: - **attrs: Rule attributes - """ - if attrs.get("python_version") == "PY2": - fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): - fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - - # buildifier: disable=native-python - native.py_binary(**_add_tags(attrs)) - -def py_test(**attrs): - """See the Bazel core [py_test](https://docs.bazel.build/versions/master/be/python.html#py_test) documentation. - - Args: - **attrs: Rule attributes - """ - if attrs.get("python_version") == "PY2": - fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): - fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") - - # buildifier: disable=native-python - native.py_test(**_add_tags(attrs)) - -def py_runtime(**attrs): - """See the Bazel core [py_runtime](https://docs.bazel.build/versions/master/be/python.html#py_runtime) documentation. - - Args: - **attrs: Rule attributes - """ - if attrs.get("python_version") == "PY2": - fail("Python 2 is no longer supported: see https://github.com/bazelbuild/rules_python/issues/886") - - # buildifier: disable=native-python - native.py_runtime(**_add_tags(attrs)) - -# NOTE: This doc is copy/pasted from the builtin py_runtime_pair rule so our -# doc generator gives useful API docs. -def py_runtime_pair(name, py2_runtime = None, py3_runtime = None, **attrs): - """A toolchain rule for Python. - - This used to wrap up to two Python runtimes, one for Python 2 and one for Python 3. - However, Python 2 is no longer supported, so it now only wraps a single Python 3 - runtime. - - Usually the wrapped runtimes are declared using the `py_runtime` rule, but any - rule returning a `PyRuntimeInfo` provider may be used. - - This rule returns a `platform_common.ToolchainInfo` provider with the following - schema: - - ```python - platform_common.ToolchainInfo( - py2_runtime = None, - py3_runtime = , - ) - ``` - - Example usage: - - ```python - # In your BUILD file... - - load("@rules_python//python:defs.bzl", "py_runtime_pair") - - py_runtime( - name = "my_py3_runtime", - interpreter_path = "/system/python3", - python_version = "PY3", - ) - - py_runtime_pair( - name = "my_py_runtime_pair", - py3_runtime = ":my_py3_runtime", - ) - - toolchain( - name = "my_toolchain", - target_compatible_with = <...>, - toolchain = ":my_py_runtime_pair", - toolchain_type = "@rules_python//python:toolchain_type", - ) - ``` - - ```python - # In your WORKSPACE... - - register_toolchains("//my_pkg:my_toolchain") - ``` - - Args: - name: str, the name of the target - py2_runtime: optional Label; must be unset or None; an error is raised - otherwise. - py3_runtime: Label; a target with `PyRuntimeInfo` for Python 3. - **attrs: Extra attrs passed onto the native rule - """ - if attrs.get("py2_runtime"): - fail("PYthon 2 is no longer supported: see https://github.com/bazelbuild/rules_python/issues/886") - _py_runtime_pair( - name = name, - py2_runtime = py2_runtime, - py3_runtime = py3_runtime, - **attrs - ) diff --git a/python/private/util.bzl b/python/private/util.bzl index 8ea1f493f5..25a50aac6a 100644 --- a/python/private/util.bzl +++ b/python/private/util.bzl @@ -29,3 +29,15 @@ def copy_propagating_kwargs(from_kwargs, into_kwargs = None): if attr in from_kwargs and attr not in into_kwargs: into_kwargs[attr] = from_kwargs[attr] return into_kwargs + +# The implementation of the macros and tagging mechanism follows the example +# set by rules_cc and rules_java. + +_MIGRATION_TAG = "__PYTHON_RULES_MIGRATION_DO_NOT_USE_WILL_BREAK__" + +def add_migration_tag(attrs): + if "tags" in attrs and attrs["tags"] != None: + attrs["tags"] = attrs["tags"] + [_MIGRATION_TAG] + else: + attrs["tags"] = [_MIGRATION_TAG] + return attrs diff --git a/python/py_binary.bzl b/python/py_binary.bzl index 9d145d8fa6..6b6f7e0f8a 100644 --- a/python/py_binary.bzl +++ b/python/py_binary.bzl @@ -14,6 +14,18 @@ """Public entry point for py_binary.""" -load("//python/private:reexports.bzl", _py_binary = "py_binary") +load("//python/private:util.bzl", "add_migration_tag") -py_binary = _py_binary +def py_binary(**attrs): + """See the Bazel core [py_binary](https://docs.bazel.build/versions/master/be/python.html#py_binary) documentation. + + Args: + **attrs: Rule attributes + """ + if attrs.get("python_version") == "PY2": + fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") + if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): + fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") + + # buildifier: disable=native-python + native.py_binary(**add_migration_tag(attrs)) diff --git a/python/py_library.bzl b/python/py_library.bzl index 1aff68c100..d54cbb2958 100644 --- a/python/py_library.bzl +++ b/python/py_library.bzl @@ -14,6 +14,16 @@ """Public entry point for py_library.""" -load("//python/private:reexports.bzl", _py_library = "py_library") +load("//python/private:util.bzl", "add_migration_tag") -py_library = _py_library +def py_library(**attrs): + """See the Bazel core [py_library](https://docs.bazel.build/versions/master/be/python.html#py_library) documentation. + + Args: + **attrs: Rule attributes + """ + if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): + fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") + + # buildifier: disable=native-python + native.py_library(**add_migration_tag(attrs)) diff --git a/python/py_runtime.bzl b/python/py_runtime.bzl index 5e80308176..b70f9d4ec4 100644 --- a/python/py_runtime.bzl +++ b/python/py_runtime.bzl @@ -14,6 +14,16 @@ """Public entry point for py_runtime.""" -load("//python/private:reexports.bzl", _py_runtime = "py_runtime") +load("//python/private:util.bzl", "add_migration_tag") -py_runtime = _py_runtime +def py_runtime(**attrs): + """See the Bazel core [py_runtime](https://docs.bazel.build/versions/master/be/python.html#py_runtime) documentation. + + Args: + **attrs: Rule attributes + """ + if attrs.get("python_version") == "PY2": + fail("Python 2 is no longer supported: see https://github.com/bazelbuild/rules_python/issues/886") + + # buildifier: disable=native-python + native.py_runtime(**add_migration_tag(attrs)) diff --git a/python/py_runtime_pair.bzl b/python/py_runtime_pair.bzl index 3f3ecf443b..951c606f4a 100644 --- a/python/py_runtime_pair.bzl +++ b/python/py_runtime_pair.bzl @@ -14,6 +14,74 @@ """Public entry point for py_runtime_pair.""" -load("//python/private:reexports.bzl", _py_runtime_pair = "py_runtime_pair") +load("@bazel_tools//tools/python:toolchain.bzl", _py_runtime_pair = "py_runtime_pair") -py_runtime_pair = _py_runtime_pair +# NOTE: This doc is copy/pasted from the builtin py_runtime_pair rule so our +# doc generator gives useful API docs. +def py_runtime_pair(name, py2_runtime = None, py3_runtime = None, **attrs): + """A toolchain rule for Python. + + This used to wrap up to two Python runtimes, one for Python 2 and one for Python 3. + However, Python 2 is no longer supported, so it now only wraps a single Python 3 + runtime. + + Usually the wrapped runtimes are declared using the `py_runtime` rule, but any + rule returning a `PyRuntimeInfo` provider may be used. + + This rule returns a `platform_common.ToolchainInfo` provider with the following + schema: + + ```python + platform_common.ToolchainInfo( + py2_runtime = None, + py3_runtime = , + ) + ``` + + Example usage: + + ```python + # In your BUILD file... + + load("@rules_python//python:defs.bzl", "py_runtime_pair") + + py_runtime( + name = "my_py3_runtime", + interpreter_path = "/system/python3", + python_version = "PY3", + ) + + py_runtime_pair( + name = "my_py_runtime_pair", + py3_runtime = ":my_py3_runtime", + ) + + toolchain( + name = "my_toolchain", + target_compatible_with = <...>, + toolchain = ":my_py_runtime_pair", + toolchain_type = "@rules_python//python:toolchain_type", + ) + ``` + + ```python + # In your WORKSPACE... + + register_toolchains("//my_pkg:my_toolchain") + ``` + + Args: + name: str, the name of the target + py2_runtime: optional Label; must be unset or None; an error is raised + otherwise. + py3_runtime: Label; a target with `PyRuntimeInfo` for Python 3. + **attrs: Extra attrs passed onto the native rule + """ + if attrs.get("py2_runtime"): + fail("PYthon 2 is no longer supported: see https://github.com/bazelbuild/rules_python/issues/886") + _py_runtime_pair( + name = name, + py2_runtime = py2_runtime, + py3_runtime = py3_runtime, + **attrs + ) diff --git a/python/py_test.bzl b/python/py_test.bzl index 84470bc3af..09580c01c4 100644 --- a/python/py_test.bzl +++ b/python/py_test.bzl @@ -14,6 +14,18 @@ """Public entry point for py_test.""" -load("//python/private:reexports.bzl", _py_test = "py_test") +load("//python/private:util.bzl", "add_migration_tag") -py_test = _py_test +def py_test(**attrs): + """See the Bazel core [py_test](https://docs.bazel.build/versions/master/be/python.html#py_test) documentation. + + Args: + **attrs: Rule attributes + """ + if attrs.get("python_version") == "PY2": + fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") + if attrs.get("srcs_version") in ("PY2", "PY2ONLY"): + fail("Python 2 is no longer supported: https://github.com/bazelbuild/rules_python/issues/886") + + # buildifier: disable=native-python + native.py_test(**add_migration_tag(attrs)) From 260a08b9f4d0572f154f04d9a2add967fc24ecc1 Mon Sep 17 00:00:00 2001 From: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> Date: Thu, 23 Mar 2023 15:34:00 -0700 Subject: [PATCH 02/12] fix: bump installer to handle windows better (#1138) Fixes https://github.com/bazelbuild/rules_python/issues/1121. Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/repositories.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 664556da12..2dd4a3724b 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -37,8 +37,8 @@ _RULE_DEPS = [ ), ( "pypi__installer", - "https://files.pythonhosted.org/packages/bf/42/fe5f10fd0d58d5d8231a0bc39e664de09992f960597e9fbd3753f84423a3/installer-0.6.0-py3-none-any.whl", - "ae7c62d1d6158b5c096419102ad0d01fdccebf857e784cee57f94165635fe038", + "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", ), ( "pypi__packaging", From 64684ae0498576ad2a09fa528fed07afa5e7307d Mon Sep 17 00:00:00 2001 From: Chris Love <335402+chrislovecnm@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:42:53 -0600 Subject: [PATCH 03/12] build: Fixing buildifier (#1148) We have some changes with buildifier 6.1.0, and this commit fixes two files to allow ci to pass. --- python/BUILD.bazel | 2 +- python/pip_install/BUILD.bazel | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 2582d73751..d75889d188 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -33,8 +33,8 @@ licenses(["notice"]) filegroup( name = "distribution", srcs = glob(["**"]) + [ - "//python/constraints:distribution", "//python/config_settings:distribution", + "//python/constraints:distribution", "//python/private:distribution", "//python/runfiles:distribution", ], diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index 281ccba6a9..e8e8633137 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -2,10 +2,10 @@ filegroup( name = "distribution", srcs = glob(["*.bzl"]) + [ "BUILD.bazel", + "//python/pip_install/private:distribution", "//python/pip_install/tools/dependency_resolver:distribution", "//python/pip_install/tools/lib:distribution", "//python/pip_install/tools/wheel_installer:distribution", - "//python/pip_install/private:distribution", ], visibility = ["//:__pkg__"], ) From 03ebeb71e80e6d59dc589d1f7a0242cf913d2861 Mon Sep 17 00:00:00 2001 From: Chris Love <335402+chrislovecnm@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:28:07 -0600 Subject: [PATCH 04/12] docs: Updating documentation for bzlmod (#1149) - Updated primary README.md to include documentation for using bzlmod or a WORKSPACE file. - Updated gazelle/README.md to include documentation for only using bzlmod and provided a link to the older docs. - Included other general updates for the gazelle documentation. --- README.md | 64 +++++++++++++++++++++++++++++++-- gazelle/README.md | 91 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 130 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 07acaf8e19..089837de7d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,47 @@ contribute](CONTRIBUTING.md) page for information on our development workflow. ## Getting started +The next two sections cover using `rules_python` with bzlmod and +the older way of configuring bazel with a `WORKSPACE` file. + +### Using bzlmod + +To import rules_python in your project, you first need to add it to your +`MODULES.bazel` file, using the snippet provided in the +[release you choose](https://github.com/bazelbuild/rules_python/releases). + +#### Toolchain registration with bzlmod + +To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `MODULES.bazel` file: + +```python +# Find the latest version number here: https://github.com/bazelbuild/rules_python/releases +# and change the version number if needed in the line below. +bazel_dep(name = "rules_python", version = "0.20.0") + +# You do not have to use pip for the toolchain, but most people +# will use it for the dependency management. +pip = use_extension("@rules_python//python:extensions.bzl", "pip") + +pip.parse( + name = "pip", + requirements_lock = "//:requirements_lock.txt", +) + +use_repo(pip, "pip") + +# Register a specific python toolchain instead of using the host version +python = use_extension("@rules_python//python:extensions.bzl", "python") + +use_repo(python, "python3_10_toolchains") + +register_toolchains( + "@python3_10_toolchains//:all", +) +``` + +### Using a WORKSPACE file + To import rules_python in your project, you first need to add it to your `WORKSPACE` file, using the snippet provided in the [release you choose](https://github.com/bazelbuild/rules_python/releases) @@ -53,7 +94,7 @@ http_archive( ) ``` -### Toolchain registration +#### Toolchain registration To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `WORKSPACE` file: @@ -118,6 +159,22 @@ target in the appropriate wheel repo. ### Installing third_party packages +#### Using bzlmod + +To add pip dependencies to your `MODULES.bazel` file, use the `pip.parse` extension, and call it to create the +central external repo and individual wheel external repos. + +```python +pip.parse( + name = "my_deps", + requirements_lock = "//:requirements_lock.txt", +) + +use_repo(pip, "my_deps") +``` + +#### Using a WORKSPACE file + To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function, and call it to create the central external repo and individual wheel external repos. @@ -137,14 +194,15 @@ load("@my_deps//:requirements.bzl", "install_deps") install_deps() ``` +#### pip rules + Note that since `pip_parse` is a repository rule and therefore executes pip at WORKSPACE-evaluation time, Bazel has no information about the Python toolchain and cannot enforce that the interpreter used to invoke pip matches the interpreter used to run `py_binary` targets. By default, `pip_parse` uses the system command `"python3"`. This can be overridden by passing the `python_interpreter` attribute or `python_interpreter_target` attribute to `pip_parse`. -You can have multiple `pip_parse`s in the same workspace. This will create multiple external repos that have no relation to -one another, and may result in downloading the same wheels multiple times. +You can have multiple `pip_parse`s in the same workspace. This will create multiple external repos that have no relation to one another, and may result in downloading the same wheels multiple times. As with any repository rule, if you would like to ensure that `pip_parse` is re-executed in order to pick up a non-hermetic change to your environment (e.g., diff --git a/gazelle/README.md b/gazelle/README.md index 0081701241..e9a8052353 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -1,21 +1,48 @@ # Python Gazelle plugin +[Gazelle](https://github.com/bazelbuild/bazel-gazelle) +is a build file generator for Bazel projects. It can create new BUILD.bazel files for a project that follows language conventions, and it can update existing build files to include new sources, dependencies, and options. + +Gazelle may be run by Bazel using the gazelle rule, or it may be installed and run as a command line tool. + This directory contains a plugin for [Gazelle](https://github.com/bazelbuild/bazel-gazelle) -that generates BUILD file content for Python code. +that generates BUILD files content for Python code. + +The following instructions are for when you use [bzlmod](https://docs.bazel.build/versions/5.0.0/bzlmod.html). +Please refer to older documentation that includes instructions on how to use Gazelle +without using bzlmod as your dependency manager. + +## Example -It requires Go 1.16+ to compile. +We have an example of using Gazelle with Python located [here](https://github.com/bazelbuild/rules_python/tree/main/examples/build_file_generation). -## Installation +## Adding Gazelle to your project -First, you'll need to add Gazelle to your `WORKSPACE` file. -Follow the instructions at https://github.com/bazelbuild/bazel-gazelle#running-gazelle-with-bazel +First, you'll need to add Gazelle to your `MODULES.bazel` file. +Get the current version of Gazelle from there releases here: https://github.com/bazelbuild/bazel-gazelle/releases/. -Next, we need to fetch the third-party Go libraries that the python extension -depends on. -See the installation `WORKSPACE` snippet on the Releases page: -https://github.com/bazelbuild/rules_python/releases +See the installation `MODULE.bazel` snippet on the Releases page: +https://github.com/bazelbuild/rules_python/releases in order to configure rules_python. + +You will also need to add the `bazel_dep` for configuration for `rules_python_gazelle_plugin`. + +Here is a snippet of a `MODULE.bazel` file. + +```starlark +# The following stanza defines the dependency rules_python. +bazel_dep(name = "rules_python", version = "0.20.0") + +# The following stanza defines the dependency rules_python. +# For typical setups you set the version. +bazel_dep(name = "rules_python_gazelle_plugin", version = "0.20.0") + +# The following stanza defines the dependency rules_python. +bazel_dep(name = "gazelle", version = "0.30.0", repo_name = "bazel_gazelle") +``` +You will also need to do the other usual configuration for `rules_python` in your +`MODULE.bazel` file. Next, we'll fetch metadata about your Python dependencies, so that gazelle can determine which package a given import statement comes from. This is provided @@ -157,11 +184,29 @@ Next, all source files are collected into the `srcs` of the `py_library`. Finally, the `import` statements in the source files are parsed, and dependencies are added to the `deps` attribute. -### Tests +### Unit Tests + +A `py_test` target is added to the BUILD file when gazelle encounters +a file named `__test__.py`. +Often, Python unit test files are named with the suffix `_test`. +For example, if we had a folder that is a package named "foo" we could have a Python file named `foo_test.py` +and gazelle would create a `py_test` block for the file. -Python test files are those ending in `_test.py`. +The following is an example of a `py_test` target that gazelle would add when +it encounters a file named `__test__.py`. + +```starlark +py_test( + name = "build_file_generation_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [":build_file_generation"], +) +``` -A `py_test` target is added containing all test files as `srcs`. +You can control the naming convention for test targets by adding a gazelle directive named +`# gazelle:python_test_naming_convention`. See the instructions in the section above that +covers directives. ### Binaries @@ -170,16 +215,18 @@ of a Python program. A `py_binary` target will be created, named `[package]_bin`. -## Developing on the extension +## Developer Notes -Gazelle extensions are written in Go. Ours is a hybrid, which also spawns -a Python interpreter as a subprocess to parse python files. +Gazelle extensions are written in Go. This gazelle plugin is a hybrid, as it uses Go to execute a +Python interpreter as a subprocess to parse Python source files. +See the gazelle documentation https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.md +for more information on extending Gazelle. -The Go dependencies are managed by the go.mod file. -After changing that file, run `go mod tidy` to get a `go.sum` file, -then run `bazel run //:update_go_deps` to convert that to the `gazelle/deps.bzl` file. -The latter is loaded in our `/WORKSPACE` to define the external repos -that we can load Go dependencies from. +If you add new Go dependencies to the plugin source code, you need to "tidy" the go.mod file. +After changing that file, run `go mod tidy` or `bazel run @go_sdk//:bin/go -- mod tidy` +to update the go.mod and go.sum files. Then run `bazel run //:update_go_deps` to have gazelle +add the new dependenies to the deps.bzl file. The deps.bzl file is used as defined in our /WORKSPACE +to include the external repos Bazel loads Go dependencies from. -Then after editing Go code, run `bazel run //:gazelle` to generate/update -go_* rules in the BUILD.bazel files in our repo. +Then after editing Go code, run `bazel run //:gazelle` to generate/update the rules in the +BUILD.bazel files in our repo. From 00dd72dd5f8c2416600ecbcca1f4a9498223fa91 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 4 Apr 2023 09:52:20 +0900 Subject: [PATCH 05/12] fix: use a consistent buildifier version for CI and pre-commit (#1151) The CI may be broken because it is using the latest version of buildifier (`6.1.0`) and the pre-commit hooks are using `6.0.0`. `6.1.0` added extra sorting, so it is now not enough to just run `pre-commit run -a buildifier` to fix the errors. I have submitted a PR to update the pre-commit hooks in https://github.com/keith/pre-commit-buildifier/pull/14 and until it is merged and a new version is tagged we should use an older version of the buildifier to ensure the build is green. --------- Co-authored-by: Richard Levasseur --- .bazelci/presubmit.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index a0d9a19047..f10a6487c3 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -14,8 +14,9 @@ --- buildifier: - version: latest - # keep this argument in sync with .pre-commit-config.yaml + # keep these arguments in sync with .pre-commit-config.yaml + # Use a specific version to avoid skew issues when new versions are released. + version: 6.0.0 warnings: "all" .minimum_supported_version: &minimum_supported_version # For testing minimum supported version. From ee8cecfdc210cda71b0d43a5170f47a741721643 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Wed, 5 Apr 2023 00:47:53 +0900 Subject: [PATCH 06/12] chore: bump buildifier to 6.1.0 (#1152) Bump the buildifier to the latest version in pre-commit and CI at the same time. Related to #1148 and #1151. --- .bazelci/presubmit.yml | 2 +- .pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index f10a6487c3..0e9feab093 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -16,7 +16,7 @@ buildifier: # keep these arguments in sync with .pre-commit-config.yaml # Use a specific version to avoid skew issues when new versions are released. - version: 6.0.0 + version: 6.1.0 warnings: "all" .minimum_supported_version: &minimum_supported_version # For testing minimum supported version. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9403dd5338..be5f47fc45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/keith/pre-commit-buildifier - rev: 6.0.0 + rev: 6.1.0 hooks: - id: buildifier args: &args From 52e14b78307a62aedb69694a222decf48d54a09b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Thu, 6 Apr 2023 01:34:11 +0900 Subject: [PATCH 07/12] fix: correct the labels returned by all_requirements lists (#1146) This apparently was not working to begin with, but the CI running the example did not catch it because we did not have an empty `WORKSPACE.bzlmod` file. Tested with the CI and with a clean cache on the local laptop. --- examples/build_file_generation/WORKSPACE.bzlmod | 2 ++ python/pip_install/pip_repository.bzl | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 examples/build_file_generation/WORKSPACE.bzlmod diff --git a/examples/build_file_generation/WORKSPACE.bzlmod b/examples/build_file_generation/WORKSPACE.bzlmod new file mode 100644 index 0000000000..721e065154 --- /dev/null +++ b/examples/build_file_generation/WORKSPACE.bzlmod @@ -0,0 +1,2 @@ +# This file will be used when bzlmod is enabled, keep it empty +# to ensure that all of the setup is done in MODULE.bazel diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 733142ba92..fce0dcdd47 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -370,11 +370,11 @@ def _pip_repository_bzlmod_impl(rctx): rctx.file("BUILD.bazel", build_contents) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@@{}//{}".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) + "@{}//{}".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@@{}//{}:whl".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) + "@{}//{}:whl".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) for p in bzl_packages ]), "%%NAME%%": rctx.attr.name, From 86eadf1ecf89f346476354f42d4d5bc861acab1b Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Fri, 7 Apr 2023 23:05:38 -0700 Subject: [PATCH 08/12] fix: gazelle correctly adds new py_test rules (#1143) Since https://github.com/bazelbuild/rules_python/pull/999, gazelle can generate multiple `py_test` rules in a single package (when it finds multiple `*_test.py` or `test_*.py` files and no `__test__.py` file). In this case, adding new test files to a package with pre-existing `py_test` rules is not handled properly due to the `MatchAny` property on the `py_test` kind - it will match the existing `py_test` rule and edit it instead of adding a new test rule. This PR disables the matching so that new `py_test` rules are properly generated. --- gazelle/python/kinds.go | 2 +- .../python/testdata/multiple_tests/BUILD.in | 12 ++++++++++ .../python/testdata/multiple_tests/BUILD.out | 17 +++++++++++++ .../python/testdata/multiple_tests/README.md | 3 +++ .../python/testdata/multiple_tests/WORKSPACE | 1 + .../testdata/multiple_tests/__init__.py | 0 .../testdata/multiple_tests/bar_test.py | 24 +++++++++++++++++++ .../testdata/multiple_tests/foo_test.py | 24 +++++++++++++++++++ .../python/testdata/multiple_tests/test.yaml | 17 +++++++++++++ 9 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 gazelle/python/testdata/multiple_tests/BUILD.in create mode 100644 gazelle/python/testdata/multiple_tests/BUILD.out create mode 100644 gazelle/python/testdata/multiple_tests/README.md create mode 100644 gazelle/python/testdata/multiple_tests/WORKSPACE create mode 100644 gazelle/python/testdata/multiple_tests/__init__.py create mode 100644 gazelle/python/testdata/multiple_tests/bar_test.py create mode 100644 gazelle/python/testdata/multiple_tests/foo_test.py create mode 100644 gazelle/python/testdata/multiple_tests/test.yaml diff --git a/gazelle/python/kinds.go b/gazelle/python/kinds.go index 0fdc6bc3e9..ab1afb7d55 100644 --- a/gazelle/python/kinds.go +++ b/gazelle/python/kinds.go @@ -65,7 +65,7 @@ var pyKinds = map[string]rule.KindInfo{ }, }, pyTestKind: { - MatchAny: true, + MatchAny: false, NonEmptyAttrs: map[string]bool{ "deps": true, "main": true, diff --git a/gazelle/python/testdata/multiple_tests/BUILD.in b/gazelle/python/testdata/multiple_tests/BUILD.in new file mode 100644 index 0000000000..9e84e5dc32 --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/BUILD.in @@ -0,0 +1,12 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "multiple_tests", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "bar_test", + srcs = ["bar_test.py"], +) diff --git a/gazelle/python/testdata/multiple_tests/BUILD.out b/gazelle/python/testdata/multiple_tests/BUILD.out new file mode 100644 index 0000000000..fd67724e3b --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/BUILD.out @@ -0,0 +1,17 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "multiple_tests", + srcs = ["__init__.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "bar_test", + srcs = ["bar_test.py"], +) + +py_test( + name = "foo_test", + srcs = ["foo_test.py"], +) diff --git a/gazelle/python/testdata/multiple_tests/README.md b/gazelle/python/testdata/multiple_tests/README.md new file mode 100644 index 0000000000..8220f6112d --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/README.md @@ -0,0 +1,3 @@ +# Multiple tests + +This test case asserts that a second `py_test` rule is correctly created when a second `*_test.py` file is added to a package with an existing `py_test` rule. diff --git a/gazelle/python/testdata/multiple_tests/WORKSPACE b/gazelle/python/testdata/multiple_tests/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/multiple_tests/__init__.py b/gazelle/python/testdata/multiple_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/multiple_tests/bar_test.py b/gazelle/python/testdata/multiple_tests/bar_test.py new file mode 100644 index 0000000000..9948f1ccd4 --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/bar_test.py @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +class BarTest(unittest.TestCase): + def test_foo(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/multiple_tests/foo_test.py b/gazelle/python/testdata/multiple_tests/foo_test.py new file mode 100644 index 0000000000..a128adf67f --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/foo_test.py @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +class FooTest(unittest.TestCase): + def test_foo(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/multiple_tests/test.yaml b/gazelle/python/testdata/multiple_tests/test.yaml new file mode 100644 index 0000000000..2410223e59 --- /dev/null +++ b/gazelle/python/testdata/multiple_tests/test.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +expect: + exit_code: 0 From b80b8fde601f3bee9a4174c4aef03a3811912b93 Mon Sep 17 00:00:00 2001 From: Mathieu Sabourin Date: Fri, 7 Apr 2023 23:52:41 -0700 Subject: [PATCH 09/12] fix: respect kind mapping (#1158) When using the kind `gazelle:map_kind` directive, `gazelle` will correctly generate the buildfile on the first pass (or if no target of that type / name are present). However, when running gazelle a second time (or if a target of the mapped kind with the same name is present), `gazelle` will error out saying that it kind create a target of the original kind because a target of mapped kind is present and has the same name. Ex: Given the directive `# gazelle:map_kind py_test py_pytest_test //src/bazel/rules/python:py_pytest_test.bzl`, `gazelle` will correctly generate a `py_pytest_test` target where it would have generated a `py_test` target. But on a second invocation of `gazelle` (and subsequent invocations) it will error our with: ``` gazelle: ERROR: failed to generate target "//test/python/common:common_test" of kind "py_test": a target of kind "py_pytest_test" with the same name already exists. Use the '# gazelle:python_test_naming_convention' directive to change the naming convention. ``` --- gazelle/python/generate.go | 27 +++++++++++++------ .../testdata/respect_kind_mapping/BUILD.in | 15 +++++++++++ .../testdata/respect_kind_mapping/BUILD.out | 20 ++++++++++++++ .../testdata/respect_kind_mapping/README.md | 3 +++ .../testdata/respect_kind_mapping/WORKSPACE | 1 + .../testdata/respect_kind_mapping/__init__.py | 17 ++++++++++++ .../testdata/respect_kind_mapping/__test__.py | 26 ++++++++++++++++++ .../testdata/respect_kind_mapping/foo.py | 16 +++++++++++ .../testdata/respect_kind_mapping/test.yaml | 17 ++++++++++++ 9 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 gazelle/python/testdata/respect_kind_mapping/BUILD.in create mode 100644 gazelle/python/testdata/respect_kind_mapping/BUILD.out create mode 100644 gazelle/python/testdata/respect_kind_mapping/README.md create mode 100644 gazelle/python/testdata/respect_kind_mapping/WORKSPACE create mode 100644 gazelle/python/testdata/respect_kind_mapping/__init__.py create mode 100644 gazelle/python/testdata/respect_kind_mapping/__test__.py create mode 100644 gazelle/python/testdata/respect_kind_mapping/foo.py create mode 100644 gazelle/python/testdata/respect_kind_mapping/test.yaml diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 26ffedaca2..fb41324fd6 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -46,6 +46,13 @@ var ( buildFilenames = []string{"BUILD", "BUILD.bazel"} ) +func GetActualKindName(kind string, args language.GenerateArgs) string { + if kindOverride, ok := args.Config.KindMap[kind]; ok { + return kindOverride.KindName + } + return kind +} + // GenerateRules extracts build metadata from source files in a directory. // GenerateRules is called in each directory where an update is requested // in depth-first post-order. @@ -70,6 +77,10 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes } } + actualPyBinaryKind := GetActualKindName(pyBinaryKind, args) + actualPyLibraryKind := GetActualKindName(pyLibraryKind, args) + actualPyTestKind := GetActualKindName(pyTestKind, args) + pythonProjectRoot := cfg.PythonProjectRoot() packageName := filepath.Base(args.Dir) @@ -217,12 +228,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes // generate it correctly. if args.File != nil { for _, t := range args.File.Rules { - if t.Name() == pyLibraryTargetName && t.Kind() != pyLibraryKind { + if t.Name() == pyLibraryTargetName && t.Kind() != actualPyLibraryKind { fqTarget := label.New("", args.Rel, pyLibraryTargetName) err := fmt.Errorf("failed to generate target %q of kind %q: "+ "a target of kind %q with the same name already exists. "+ "Use the '# gazelle:%s' directive to change the naming convention.", - fqTarget.String(), pyLibraryKind, t.Kind(), pythonconfig.LibraryNamingConvention) + fqTarget.String(), actualPyLibraryKind, t.Kind(), pythonconfig.LibraryNamingConvention) collisionErrors.Add(err) } } @@ -253,12 +264,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes // generate it correctly. if args.File != nil { for _, t := range args.File.Rules { - if t.Name() == pyBinaryTargetName && t.Kind() != pyBinaryKind { + if t.Name() == pyBinaryTargetName && t.Kind() != actualPyBinaryKind { fqTarget := label.New("", args.Rel, pyBinaryTargetName) err := fmt.Errorf("failed to generate target %q of kind %q: "+ "a target of kind %q with the same name already exists. "+ "Use the '# gazelle:%s' directive to change the naming convention.", - fqTarget.String(), pyBinaryKind, t.Kind(), pythonconfig.BinaryNamingConvention) + fqTarget.String(), actualPyBinaryKind, t.Kind(), pythonconfig.BinaryNamingConvention) collisionErrors.Add(err) } } @@ -290,11 +301,11 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes // generate it correctly. if args.File != nil { for _, t := range args.File.Rules { - if t.Name() == conftestTargetname && t.Kind() != pyLibraryKind { + if t.Name() == conftestTargetname && t.Kind() != actualPyLibraryKind { fqTarget := label.New("", args.Rel, conftestTargetname) err := fmt.Errorf("failed to generate target %q of kind %q: "+ "a target of kind %q with the same name already exists.", - fqTarget.String(), pyLibraryKind, t.Kind()) + fqTarget.String(), actualPyLibraryKind, t.Kind()) collisionErrors.Add(err) } } @@ -325,12 +336,12 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes // generate it correctly. if args.File != nil { for _, t := range args.File.Rules { - if t.Name() == pyTestTargetName && t.Kind() != pyTestKind { + if t.Name() == pyTestTargetName && t.Kind() != actualPyTestKind { fqTarget := label.New("", args.Rel, pyTestTargetName) err := fmt.Errorf("failed to generate target %q of kind %q: "+ "a target of kind %q with the same name already exists. "+ "Use the '# gazelle:%s' directive to change the naming convention.", - fqTarget.String(), pyTestKind, t.Kind(), pythonconfig.TestNamingConvention) + fqTarget.String(), actualPyTestKind, t.Kind(), pythonconfig.TestNamingConvention) collisionErrors.Add(err) } } diff --git a/gazelle/python/testdata/respect_kind_mapping/BUILD.in b/gazelle/python/testdata/respect_kind_mapping/BUILD.in new file mode 100644 index 0000000000..6a06737623 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/BUILD.in @@ -0,0 +1,15 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:map_kind py_test my_test :mytest.bzl + +py_library( + name = "respect_kind_mapping", + srcs = ["__init__.py"], +) + +my_test( + name = "respect_kind_mapping_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [":respect_kind_mapping"], +) diff --git a/gazelle/python/testdata/respect_kind_mapping/BUILD.out b/gazelle/python/testdata/respect_kind_mapping/BUILD.out new file mode 100644 index 0000000000..7c5fb0bd20 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/BUILD.out @@ -0,0 +1,20 @@ +load(":mytest.bzl", "my_test") +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:map_kind py_test my_test :mytest.bzl + +py_library( + name = "respect_kind_mapping", + srcs = [ + "__init__.py", + "foo.py", + ], + visibility = ["//:__subpackages__"], +) + +my_test( + name = "respect_kind_mapping_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [":respect_kind_mapping"], +) diff --git a/gazelle/python/testdata/respect_kind_mapping/README.md b/gazelle/python/testdata/respect_kind_mapping/README.md new file mode 100644 index 0000000000..9f0fa6cf39 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/README.md @@ -0,0 +1,3 @@ +# Respect Kind Mapping + +This test case asserts that when using a kind mapping, gazelle will respect that mapping when parsing a BUILD file containing a mapped kind. diff --git a/gazelle/python/testdata/respect_kind_mapping/WORKSPACE b/gazelle/python/testdata/respect_kind_mapping/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/respect_kind_mapping/__init__.py b/gazelle/python/testdata/respect_kind_mapping/__init__.py new file mode 100644 index 0000000000..b274b0d921 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from foo import foo + +_ = foo diff --git a/gazelle/python/testdata/respect_kind_mapping/__test__.py b/gazelle/python/testdata/respect_kind_mapping/__test__.py new file mode 100644 index 0000000000..2b180a5f53 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/__test__.py @@ -0,0 +1,26 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from __init__ import foo + + +class FooTest(unittest.TestCase): + def test_foo(self): + self.assertEqual("foo", foo()) + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/respect_kind_mapping/foo.py b/gazelle/python/testdata/respect_kind_mapping/foo.py new file mode 100644 index 0000000000..932de45b74 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/foo.py @@ -0,0 +1,16 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def foo(): + return "foo" diff --git a/gazelle/python/testdata/respect_kind_mapping/test.yaml b/gazelle/python/testdata/respect_kind_mapping/test.yaml new file mode 100644 index 0000000000..2410223e59 --- /dev/null +++ b/gazelle/python/testdata/respect_kind_mapping/test.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +expect: + exit_code: 0 From 1e869d8d945f0b406da65817cf70b4fa3105a0de Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Tue, 11 Apr 2023 03:45:45 +0900 Subject: [PATCH 10/12] test: cleanup gazelle tests and run them in parallel (#1159) --- gazelle/python/python_test.go | 54 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/gazelle/python/python_test.go b/gazelle/python/python_test.go index 51e0101df1..79450ad584 100644 --- a/gazelle/python/python_test.go +++ b/gazelle/python/python_test.go @@ -74,8 +74,8 @@ func TestGazelleBinary(t *testing.T) { func testPath(t *testing.T, name string, files []bazel.RunfileEntry) { t.Run(name, func(t *testing.T) { - var inputs []testtools.FileSpec - var goldens []testtools.FileSpec + t.Parallel() + var inputs, goldens []testtools.FileSpec var config *testYAML for _, f := range files { @@ -111,43 +111,49 @@ func testPath(t *testing.T, name string, files []bazel.RunfileEntry) { Path: filepath.Join(name, strings.TrimSuffix(shortPath, ".in")), Content: string(content), }) - } else if strings.HasSuffix(shortPath, ".out") { + continue + } + + if strings.HasSuffix(shortPath, ".out") { goldens = append(goldens, testtools.FileSpec{ Path: filepath.Join(name, strings.TrimSuffix(shortPath, ".out")), Content: string(content), }) - } else { - inputs = append(inputs, testtools.FileSpec{ - Path: filepath.Join(name, shortPath), - Content: string(content), - }) - goldens = append(goldens, testtools.FileSpec{ - Path: filepath.Join(name, shortPath), - Content: string(content), - }) + continue } + + inputs = append(inputs, testtools.FileSpec{ + Path: filepath.Join(name, shortPath), + Content: string(content), + }) + goldens = append(goldens, testtools.FileSpec{ + Path: filepath.Join(name, shortPath), + Content: string(content), + }) } testdataDir, cleanup := testtools.CreateFiles(t, inputs) - defer cleanup() - defer func() { - if t.Failed() { - filepath.Walk(testdataDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - t.Logf("%q exists", strings.TrimPrefix(path, testdataDir)) - return nil - }) + t.Cleanup(cleanup) + t.Cleanup(func() { + if !t.Failed() { + return } - }() + + filepath.Walk(testdataDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + t.Logf("%q exists", strings.TrimPrefix(path, testdataDir)) + return nil + }) + }) workspaceRoot := filepath.Join(testdataDir, name) args := []string{"-build_file_name=BUILD,BUILD.bazel"} ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + t.Cleanup(cancel) cmd := exec.CommandContext(ctx, gazellePath, args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout From ebe81b7d1c0f33b62e80d2fc97e2ff1a9219b687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Felipe=20Barco=20Santa?= Date: Mon, 10 Apr 2023 22:11:19 -0500 Subject: [PATCH 11/12] [docs] Fixing rule name in coverage.md docs (#1162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs explains that for activating coverage support we use `register_coverage_tool = True` inside the rule `register_python_toolchains`. There is no such rule, the actual rule name is `python_register_toolchains` Signed-off-by: Andrés Felipe Barco Santa --- docs/coverage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/coverage.md b/docs/coverage.md index bc613f8295..63f25782e0 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -23,7 +23,7 @@ python.toolchain( For WORKSPACE configuration: ```starlark -register_python_toolchains( +python_register_toolchains( register_coverage_tool = True, ) ``` From c72c7bcf4e0899c275042328e8233e3124ccae86 Mon Sep 17 00:00:00 2001 From: yuvalk Date: Tue, 11 Apr 2023 19:07:34 +0300 Subject: [PATCH 12/12] feat: Support specifying multiple download URLs in tool_versions. (#1145) The interface of `repository_ctx.download` and `repository_ctx.download_and_extract` supports string lists as well as strings as the value of the `url` argument. This is the ultimate destination of the `url` attribute in the `tool_versions` dictionary, so it makes sense for it to support lists as well. It is often useful to provide multiple download URLs, e.g. when vendoring deps through a mirror (to guard against issues like [git archive checksums changing](https://github.blog/changelog/2023-01-30-git-archive-checksums-may-change/) while still keeping the canonical download URL) or in an airgapped setting (to support internal URLs alongside external URLs). This is also pretty common around Bazel repository rules that download things, e.g. [http_archive](https://bazel.build/rules/lib/repo/http#http_archive-urls), so it can be expected to work with `tool_versions` too. --- python/repositories.bzl | 14 +++++++++----- python/versions.bzl | 36 ++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index f676610ae2..2429d7e026 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -99,12 +99,14 @@ def is_standalone_interpreter(rctx, python_interpreter_target): def _python_repository_impl(rctx): if rctx.attr.distutils and rctx.attr.distutils_content: fail("Only one of (distutils, distutils_content) should be set.") + if bool(rctx.attr.url) == bool(rctx.attr.urls): + fail("Exactly one of (url, urls) must be set.") platform = rctx.attr.platform python_version = rctx.attr.python_version python_short_version = python_version.rpartition(".")[0] release_filename = rctx.attr.release_filename - url = rctx.attr.url + url = rctx.attr.urls or [rctx.attr.url] if release_filename.endswith(".zst"): rctx.download( @@ -428,8 +430,10 @@ For more information see the official bazel docs doc = "A directory prefix to strip from the extracted files.", ), "url": attr.string( - doc = "The URL of the interpreter to download", - mandatory = True, + doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", + ), + "urls": attr.string_list( + doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), "zstd_sha256": attr.string( default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", @@ -506,7 +510,7 @@ def python_register_toolchains( if not sha256: continue - (release_filename, url, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) + (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions) # allow passing in a tool version coverage_tool = None @@ -536,7 +540,7 @@ def python_register_toolchains( platform = platform, python_version = python_version, release_filename = release_filename, - url = url, + urls = urls, distutils = distutils, distutils_content = distutils_content, strip_prefix = strip_prefix, diff --git a/python/versions.bzl b/python/versions.bzl index 4feeeae58c..662f89d04b 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -41,6 +41,8 @@ DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/ # "strip_prefix": "python", # }, # +# It is possible to provide lists in "url". +# # buildifier: disable=unsorted-dict-items TOOL_VERSIONS = { "3.8.10": { @@ -281,19 +283,28 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U if type(url) == type({}): url = url[platform] + if type(url) != type([]): + url = [url] + strip_prefix = tool_versions[python_version].get("strip_prefix", None) if type(strip_prefix) == type({}): strip_prefix = strip_prefix[platform] - release_filename = url.format( - platform = platform, - python_version = python_version, - build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only", - ) - if "://" in release_filename: # is absolute url? - url = release_filename - else: - url = "/".join([base_url, release_filename]) + release_filename = None + rendered_urls = [] + for u in url: + release_filename = u.format( + platform = platform, + python_version = python_version, + build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only", + ) + if "://" in release_filename: # is absolute url? + rendered_urls.append(release_filename) + else: + rendered_urls.append("/".join([base_url, release_filename])) + + if release_filename == None: + fail("release_filename should be set by now; were any download URLs given?") patches = tool_versions[python_version].get("patches", []) if type(patches) == type({}): @@ -302,7 +313,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U else: patches = [] - return (release_filename, url, strip_prefix, patches) + return (release_filename, rendered_urls, strip_prefix, patches) def print_toolchains_checksums(name): native.genrule( @@ -333,10 +344,11 @@ def _commands_for_version(python_version): "echo \"{python_version}: {platform}: $$(curl --location --fail {release_url_sha256} 2>/dev/null || curl --location --fail {release_url} 2>/dev/null | shasum -a 256 | awk '{{ print $$1 }}')\"".format( python_version = python_version, platform = platform, - release_url = get_release_info(platform, python_version)[1], - release_url_sha256 = get_release_info(platform, python_version)[1] + ".sha256", + release_url = release_url, + release_url_sha256 = release_url + ".sha256", ) for platform in TOOL_VERSIONS[python_version]["sha256"].keys() + for release_url in get_release_info(platform, python_version)[1] ]) def gen_python_config_settings(name = ""):