diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index a1b16bbc66..a0d9a19047 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -73,6 +73,14 @@ tasks: build_targets: ["//..."] test_targets: ["//..."] working_directory: gazelle + gazelle_extension_bzlmod: + <<: *common_bzlmod_flags + name: Test the Gazelle extension under bzlmod + platform: ubuntu2004 + build_targets: ["//..."] + test_targets: ["//..."] + working_directory: gazelle + ubuntu_min: <<: *minimum_supported_version <<: *reusable_config @@ -138,6 +146,32 @@ tasks: working_directory: examples/build_file_generation platform: windows + integration_test_build_file_generation_bzlmod_ubuntu: + <<: *minimum_supported_bzlmod_version + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Ubuntu + working_directory: examples/build_file_generation + platform: ubuntu2004 + integration_test_build_file_generation_bzlmod_debian: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Debian + working_directory: examples/build_file_generation + platform: debian11 + integration_test_build_file_generation_bzlmod_macos: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on macOS + working_directory: examples/build_file_generation + platform: macos + integration_test_build_file_generation_bzlmod_windows: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Windows + working_directory: examples/build_file_generation + platform: windows + integration_test_bzlmod_ubuntu_min: <<: *minimum_supported_bzlmod_version <<: *reusable_build_test_all diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 38e0658e44..0d305b8816 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,44 +1,11 @@ -## PR Checklist - -Please check if your PR fulfills the following requirements: - -- [ ] Tests for the changes have been added (for bug fixes / features) -- [ ] Docs have been added / updated (for bug fixes / features) - - -## PR Type - -What kind of change does this PR introduce? - - - -- [ ] Bugfix -- [ ] Feature (please, look at the "Scope of the project" section in the README.md file) -- [ ] Code style update (formatting, local variables) -- [ ] Refactoring (no functional changes, no api changes) -- [ ] Build related changes -- [ ] CI related changes -- [ ] Documentation content changes -- [ ] Other... Please describe: - - -## What is the current behavior? - - -Issue Number: N/A - - -## What is the new behavior? - - -## Does this PR introduce a breaking change? - -- [ ] Yes -- [ ] No - - - - - -## Other information - +PR Instructions/requirements +* Title uses `type: description` format. See CONTRIBUTING.md for types. +* Common types are: build, docs, feat, fix, refactor, revert, test +* Breaking changes include "!" after the type and a "BREAKING CHANGES:" + section at the bottom. +* Body text describes: + * Why this change is being made, briefly. + * Before and after behavior, as applicable + * References issue number, as applicable +* Update docs and tests, as applicable +* Delete these instructions prior to sending the PR diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index 549af074eb..0c0c4acf41 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -87,5 +87,12 @@ http_archive( strip_prefix = "${PREFIX}/gazelle", url = "https://github.com/bazelbuild/rules_python/releases/download/${TAG}/rules_python-${TAG}.tar.gz", ) + +# To compile the rules_python gazelle extension from source, +# we must fetch some third-party go dependencies that it uses. + +load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps") + +_py_gazelle_deps() \`\`\` EOF diff --git a/README.md b/README.md index a509e28d7e..07acaf8e19 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53" # Latest @ 202 http_archive( name = "rules_python", - sha256 = "3474c5815da4cb003ff22811a36a11894927eda1c2e64bf2dac63e914bfdf30f", + # Bazel will print the proper value to add here during the first build. + # sha256 = "FIXME", strip_prefix = "rules_python-{}".format(rules_python_version), url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version), ) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index d2f0b04b56..e1163d9d0e 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -56,7 +56,11 @@ bzl_library( "//python:defs.bzl", "//python/private:reexports.bzl", ], - deps = [":bazel_python_tools"], + deps = [ + ":bazel_python_tools", + "//python:defs_bzl", + "//python/private:reexports_bzl", + ], ) bzl_library( diff --git a/docs/pip.md b/docs/pip.md index 528abf737d..e4c3f21b79 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -42,8 +42,8 @@ of some other compile_pip_requirements rule that references these requirements It also generates two targets for running pip-compile: -- validate with `bazel test <name>_test` -- update with `bazel run <name>.update` +- validate with `bazel test [name]_test` +- update with `bazel run [name].update` **PARAMETERS** diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 2ccdc64854..c02058e08d 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -8,9 +8,10 @@ 
 pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
-               extra_pip_args, isolated, pip_data_exclude, python_interpreter,
-               python_interpreter_target, quiet, repo_mapping, repo_prefix, requirements_darwin,
-               requirements_linux, requirements_lock, requirements_windows, timeout)
+               extra_pip_args, incompatible_generate_aliases, isolated, pip_data_exclude,
+               python_interpreter, python_interpreter_target, quiet, repo_mapping, repo_prefix,
+               requirements_darwin, requirements_linux, requirements_lock, requirements_windows,
+               timeout)
A rule for importing `requirements.txt` dependencies into Bazel. @@ -64,6 +65,7 @@ py_binary( | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | +| incompatible_generate_aliases | Allow generating aliases '@pip//<pkg>' -> '@pip_<pkg>//:pkg'. | Boolean | optional | False | | isolated | Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to the underlying pip command. Alternatively, the RULES_PYTHON_PIP_ISOLATED enviornment varaible can be used to control this flag. | Boolean | optional | True | | pip_data_exclude | Additional data exclusion parameters to add to the pip packages BUILD file. | List of strings | optional | [] | | python_interpreter | The python interpreter to use. This can either be an absolute path or the name of a binary found on the host's PATH environment variable. If no value is set python3 is defaulted for Unix systems and python.exe for Windows. | String | optional | "" | @@ -83,8 +85,8 @@ py_binary( ## pip_repository_bzlmod 
-pip_repository_bzlmod(name, repo_mapping, requirements_darwin, requirements_linux,
-                      requirements_lock, requirements_windows)
+pip_repository_bzlmod(name, incompatible_generate_aliases, repo_mapping, requirements_darwin,
+                      requirements_linux, requirements_lock, requirements_windows)
A rule for bzlmod pip_repository creation. Intended for private use only. @@ -95,6 +97,7 @@ A rule for bzlmod pip_repository creation. Intended for private use only. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this repository. | Name | required | | +| incompatible_generate_aliases | Allow generating aliases in '@pip//:<pkg>' -> '@pip_<pkg>//:pkg'. This replaces the aliases generated by the bzlmod tooling. | Boolean | optional | False | | repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | | requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | | requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | diff --git a/docs/python.md b/docs/python.md index 6682e48bd1..e42375ad60 100755 --- a/docs/python.md +++ b/docs/python.md @@ -1,9 +1,7 @@ - Core rules for building Python projects. - ## current_py_toolchain diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 6419ef2c70..7c88d9203d 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -43,6 +43,9 @@ gazelle_python_manifest( modules_mapping = ":modules_map", pip_repository_name = "pip", requirements = "//:requirements_lock.txt", + # NOTE: we can use this flag in order to make our setup compatible with + # bzlmod. + use_pip_repository_aliases = True, ) # Our gazelle target points to the python gazelle binary. @@ -65,7 +68,7 @@ py_library( visibility = ["//:__subpackages__"], deps = [ "//random_number_generator", - "@pip_flask//:pkg", + "@pip//flask", ], ) diff --git a/examples/build_file_generation/MODULE.bazel b/examples/build_file_generation/MODULE.bazel new file mode 100644 index 0000000000..5f79fec486 --- /dev/null +++ b/examples/build_file_generation/MODULE.bazel @@ -0,0 +1,43 @@ +module( + name = "example_bzlmod", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.19.0") +bazel_dep(name = "rules_python_gazelle_plugin", version = "0.19.0") +bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") + +# local overrides for the packages for CI purposes. +# for usual setups you should remove this block. +local_path_override( + module_name = "rules_python", + path = "../..", +) + +local_path_override( + module_name = "rules_python_gazelle_plugin", + path = "../../gazelle", +) + +# Register python toolchain +python = use_extension("@rules_python//python:extensions.bzl", "python") +python.toolchain( + name = "python3_9", + python_version = "3.9", +) +use_repo(python, "python3_9_toolchains") + +register_toolchains( + "@python3_9_toolchains//:all", +) + +pip = use_extension("@rules_python//python:extensions.bzl", "pip") +pip.parse( + name = "pip", + # Generate user friendly alias labels for each dependency that we have. + incompatible_generate_aliases = True, + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 674b9eb7ea..65e0a6e5f3 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -55,49 +55,20 @@ gazelle_dependencies() # Remaining setup is for rules_python. -# You do not want to use the following command when you are using a WORKSPACE file -# that is outside of rules_python repository. -# This command allows targets from a local directory to be bound. -# Which allows bazel to use targets defined in base rules_python directory. -# If you are using this example outside of the rules_python git repo, -# use the http_archive command that is commented out below. -# https://bazel.build/reference/be/workspace#local_repository +# DON'T COPY_PASTE THIS. +# Our example uses `local_repository` to point to the HEAD version of rules_python. +# Users should instead use the installation instructions from the release they use. +# See https://github.com/bazelbuild/rules_python/releases local_repository( name = "rules_python", path = "../..", ) -# When not using this example in the rules_python git repo you would load the python -# ruleset using the following StarLark. -# See https://github.com/bazelbuild/rules_python#getting-started for the latest -# ruleset version. -# -# The following StarLark would replace the `local_repository` rule mentioned above. -# -# load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# http_archive( -# name = "rules_python", -# sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a", -# strip_prefix = "rules_python-0.16.1", -# url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz", -# ) - -# We import the repository-local rules_python_gazelle_plugin version in order to -# be able to test development changes to the plugin. local_repository( name = "rules_python_gazelle_plugin", path = "../../gazelle", ) -# When loading the gazelle plugin outside this repo, use the http_archive rule as follows: -# -#http_archive( -# name = "rules_python_gazelle_plugin", -# sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a", -# strip_prefix = "rules_python-0.16.1/gazelle", -# url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz", -#) - # Next we load the toolchain from rules_python. load("@rules_python//python:repositories.bzl", "python_register_toolchains") @@ -119,6 +90,8 @@ load("@rules_python//python:pip.bzl", "pip_parse") # You can instead check this `requirements.bzl` file into your repo. pip_parse( name = "pip", + # Generate user friendly alias labels for each dependency that we have. + incompatible_generate_aliases = True, # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.: # 1. Python interpreter that you compile in the build file. diff --git a/examples/build_file_generation/gazelle_python.yaml b/examples/build_file_generation/gazelle_python.yaml index 847d1ecc55..b57e9f02bc 100644 --- a/examples/build_file_generation/gazelle_python.yaml +++ b/examples/build_file_generation/gazelle_python.yaml @@ -114,4 +114,5 @@ manifest: zipp.py310compat: zipp pip_repository: name: pip -integrity: 2c84a3cabeaff134a1d045e5a173a3178086f236ab20f895ffbd7f3b7a6e5bb0 + use_pip_repository_aliases: true +integrity: 85f073e37e31339508aaaf5e0d5472adae5148fd5f054e9cc586343c026660e1 diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 7b7566bd5a..7ecc035853 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,4 +1,5 @@ load("@pip//:requirements.bzl", "requirement") +load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -35,3 +36,10 @@ py_test( srcs = ["test.py"], deps = [":lib"], ) + +py_test_with_transition( + name = "test_with_transition", + srcs = ["test.py"], + main = "test.py", + deps = [":lib"], +) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 5f984c39df..ce9122810c 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -16,6 +16,7 @@ python.toolchain( configure_coverage_tool = True, python_version = "3.9", ) +use_repo(python, "python3_9") use_repo(python, "python3_9_toolchains") register_toolchains( diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index 4124a826d1..61a43ae6cf 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -13,7 +13,7 @@ # limitations under the License. load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("//examples/wheel/private:wheel_utils.bzl", "directory_writer") +load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags") load("//python:defs.bzl", "py_library", "py_test") load("//python:packaging.bzl", "py_package", "py_wheel") load("//python:versions.bzl", "gen_python_config_settings") @@ -62,6 +62,29 @@ py_wheel( ], ) +# Populate a rule with "Make Variable" arguments for +# abi, python_tag and version. You might want to do this +# for the following use cases: +# - abi, python_tag: introspect a toolchain to map to appropriate cpython tags +# - version: populate given this or a dependent module's version +make_variable_tags( + name = "make_variable_tags", +) + +py_wheel( + name = "minimal_with_py_library_with_make_variables", + testonly = True, + abi = "$(ABI)", + distribution = "example_minimal_library", + python_tag = "$(PYTHON_TAG)", + toolchains = ["//examples/wheel:make_variable_tags"], + version = "$(VERSION)", + deps = [ + "//examples/wheel/lib:module_with_data", + "//examples/wheel/lib:simple_module", + ], +) + build_test( name = "dist_build_tests", targets = [":minimal_with_py_library.dist"], diff --git a/examples/wheel/private/wheel_utils.bzl b/examples/wheel/private/wheel_utils.bzl index af4fa1958b..037fed0175 100644 --- a/examples/wheel/private/wheel_utils.bzl +++ b/examples/wheel/private/wheel_utils.bzl @@ -54,3 +54,20 @@ directory_writer = rule( ), }, ) + +def _make_variable_tags_impl(ctx): # buildifier: disable=unused-variable + # This example is contrived. In a real usage, this rule would + # look at flags or dependencies to determine what values to use. + # If all you're doing is setting constant values, then you can simply + # set them in the py_wheel() call. + vars = {} + vars["ABI"] = "cp38" + vars["PYTHON_TAG"] = "cp38" + vars["VERSION"] = "0.99.0" + return [platform_common.TemplateVariableInfo(vars)] + +make_variable_tags = rule( + attrs = {}, + doc = """Make variable tags to pass to a py_wheel rule.""", + implementation = _make_variable_tags_impl, +) diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel new file mode 100644 index 0000000000..bd634020f3 --- /dev/null +++ b/gazelle/MODULE.bazel @@ -0,0 +1,20 @@ +module( + name = "rules_python_gazelle_plugin", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.18.0") +bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go") +bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") + +go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +use_repo( + go_deps, + "com_github_bazelbuild_buildtools", + "com_github_bmatcuk_doublestar", + "com_github_emirpasic_gods", + "com_github_ghodss_yaml", + "in_gopkg_yaml_v2", +) diff --git a/gazelle/README.md b/gazelle/README.md index a76ac59199..0081701241 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -14,23 +14,8 @@ Follow the instructions at https://github.com/bazelbuild/bazel-gazelle#running-g Next, we need to fetch the third-party Go libraries that the python extension depends on. -Add this to your `WORKSPACE`: - -```starlark -http_archive( - name = "rules_python_gazelle_plugin", - sha256 = "", - strip_prefix = "rules_python-0.17.0/gazelle", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.17.0.tar.gz", -) - -# To compile the rules_python gazelle extension from source, -# we must fetch some third-party go dependencies that it uses. - -load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps") - -_py_gazelle_deps() -``` +See the installation `WORKSPACE` snippet on the Releases page: +https://github.com/bazelbuild/rules_python/releases Next, we'll fetch metadata about your Python dependencies, so that gazelle can determine which package a given import statement comes from. This is provided diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index 357944302c..26f8c66aec 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -28,12 +28,7 @@ def gazelle_deps(): sum = "h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=", version = "v0.0.0-20190523083050-ea95bdfd59fc", ) - go_repository( - name = "com_github_bazelbuild_bazel_gazelle", - importpath = "github.com/bazelbuild/bazel-gazelle", - sum = "h1:+/ZhUxlDy4XnyMIGeKkbRZoIGssy1eO51GijwIvvuwE=", - version = "v0.27.0", - ) + go_repository( name = "com_github_bazelbuild_buildtools", build_naming_convention = "go_default_library", @@ -41,24 +36,14 @@ def gazelle_deps(): sum = "h1:jhiMzJ+8unnLRtV8rpbWBFE9pFNzIqgUTyZU5aA++w8=", version = "v0.0.0-20221004120235-7186f635531b", ) - go_repository( - name = "com_github_bazelbuild_rules_go", - importpath = "github.com/bazelbuild/rules_go", - sum = "h1:ViPR65vOrg74JKntAUFY6qZkheBKGB6to7wFd8gCRU4=", - version = "v0.35.0", - ) + go_repository( name = "com_github_bmatcuk_doublestar", importpath = "github.com/bmatcuk/doublestar", sum = "h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=", version = "v1.3.4", ) - go_repository( - name = "com_github_bmatcuk_doublestar_v4", - importpath = "github.com/bmatcuk/doublestar/v4", - sum = "h1:Qu+u9wR3Vd89LnlLMHvnZ5coJMWKQamqdz9/p5GNthA=", - version = "v4.2.0", - ) + go_repository( name = "com_github_burntsushi_toml", importpath = "github.com/BurntSushi/toml", @@ -113,12 +98,7 @@ def gazelle_deps(): sum = "h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=", version = "v0.1.0", ) - go_repository( - name = "com_github_fsnotify_fsnotify", - importpath = "github.com/fsnotify/fsnotify", - sum = "h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=", - version = "v1.5.4", - ) + go_repository( name = "com_github_ghodss_yaml", importpath = "github.com/ghodss/yaml", @@ -134,14 +114,14 @@ def gazelle_deps(): go_repository( name = "com_github_golang_mock", importpath = "github.com/golang/mock", - sum = "h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=", - version = "v1.6.0", + sum = "h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=", + version = "v1.1.1", ) go_repository( name = "com_github_golang_protobuf", importpath = "github.com/golang/protobuf", - sum = "h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=", - version = "v1.5.2", + sum = "h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=", + version = "v1.4.3", ) go_repository( name = "com_github_google_go_cmp", @@ -149,18 +129,7 @@ def gazelle_deps(): sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", version = "v0.5.9", ) - go_repository( - name = "com_github_pelletier_go_toml", - importpath = "github.com/pelletier/go-toml", - sum = "h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=", - version = "v1.9.5", - ) - go_repository( - name = "com_github_pmezard_go_difflib", - importpath = "github.com/pmezard/go-difflib", - sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=", - version = "v1.0.0", - ) + go_repository( name = "com_github_prometheus_client_model", importpath = "github.com/prometheus/client_model", @@ -218,8 +187,8 @@ def gazelle_deps(): go_repository( name = "org_golang_google_protobuf", importpath = "google.golang.org/protobuf", - sum = "h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=", - version = "v1.28.0", + sum = "h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=", + version = "v1.25.0", ) go_repository( name = "org_golang_x_crypto", @@ -260,8 +229,8 @@ def gazelle_deps(): go_repository( name = "org_golang_x_sync", importpath = "golang.org/x/sync", - sum = "h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=", - version = "v0.0.0-20220907140024-f12130a52804", + sum = "h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=", + version = "v0.0.0-20220722155255-886fb9371eb4", ) go_repository( name = "org_golang_x_sys", diff --git a/gazelle/go.mod b/gazelle/go.mod index 6d6f0332a0..94f19e801f 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -3,9 +3,7 @@ module github.com/bazelbuild/rules_python/gazelle go 1.19 require ( - github.com/bazelbuild/bazel-gazelle v0.27.0 github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b - github.com/bazelbuild/rules_go v0.35.0 github.com/bmatcuk/doublestar v1.3.4 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index 78e0c272ac..05562a1583 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -24,13 +24,16 @@ def gazelle_python_manifest( modules_mapping, pip_repository_name = "", pip_deps_repository_name = "", - manifest = ":gazelle_python.yaml"): + manifest = ":gazelle_python.yaml", + use_pip_repository_aliases = False): """A macro for defining the updating and testing targets for the Gazelle manifest file. Args: name: the name used as a base for the targets. requirements: the target for the requirements.txt file. pip_repository_name: the name of the pip_install or pip_repository target. + use_pip_repository_aliases: boolean flag to enable using user-friendly + python package aliases. pip_deps_repository_name: deprecated - the old pip_install target name. modules_mapping: the target for the generated modules_mapping.json file. manifest: the target for the Gazelle manifest file. @@ -67,6 +70,12 @@ def gazelle_python_manifest( update_target_label, ] + if use_pip_repository_aliases: + update_args += [ + "--use-pip-repository-aliases", + "true", + ] + go_binary( name = update_target, embed = [Label("//manifest/generate:generate_lib")], diff --git a/gazelle/manifest/generate/generate.go b/gazelle/manifest/generate/generate.go index 0f429f8345..1f56e630cc 100644 --- a/gazelle/manifest/generate/generate.go +++ b/gazelle/manifest/generate/generate.go @@ -38,12 +38,15 @@ func init() { } func main() { - var manifestGeneratorHashPath string - var requirementsPath string - var pipRepositoryName string - var modulesMappingPath string - var outputPath string - var updateTarget string + var ( + manifestGeneratorHashPath string + requirementsPath string + pipRepositoryName string + usePipRepositoryAliases bool + modulesMappingPath string + outputPath string + updateTarget string + ) flag.StringVar( &manifestGeneratorHashPath, "manifest-generator-hash", @@ -60,6 +63,11 @@ func main() { "pip-repository-name", "", "The name of the pip_install or pip_repository target.") + flag.BoolVar( + &usePipRepositoryAliases, + "use-pip-repository-aliases", + false, + "Whether to use the pip-repository aliases, which are generated when passing 'incompatible_generate_aliases = True'.") flag.StringVar( &modulesMappingPath, "modules-mapping", @@ -103,7 +111,8 @@ func main() { manifestFile := manifest.NewFile(&manifest.Manifest{ ModulesMapping: modulesMapping, PipRepository: &manifest.PipRepository{ - Name: pipRepositoryName, + Name: pipRepositoryName, + UsePipRepositoryAliases: usePipRepositoryAliases, }, }) if err := writeOutput( diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index bb4826435f..c49951dd3e 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -144,4 +144,7 @@ type Manifest struct { type PipRepository struct { // The name of the pip_install or pip_repository target. Name string + // UsePipRepositoryAliases allows to use aliases generated pip_repository + // when passing incompatible_generate_aliases = True. + UsePipRepositoryAliases bool `yaml:"use_pip_repository_aliases,omitempty"` } diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 3b5ded2139..ddcad2785d 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -61,7 +61,6 @@ go_test( ] + glob(["testdata/**"]), deps = [ "@bazel_gazelle//testtools:go_default_library", - "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_ghodss_yaml//:yaml", "@io_bazel_rules_go//go/tools/bazel:go_default_library", ], diff --git a/gazelle/python/parse.py b/gazelle/python/parse.py index 5cf0b89868..6c0ef69598 100644 --- a/gazelle/python/parse.py +++ b/gazelle/python/parse.py @@ -27,7 +27,7 @@ def parse_import_statements(content, filepath): modules = list() - tree = ast.parse(content) + tree = ast.parse(content, filename=filepath) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: diff --git a/gazelle/pythonconfig/BUILD.bazel b/gazelle/pythonconfig/BUILD.bazel index 79b512163d..d0f1690d94 100644 --- a/gazelle/pythonconfig/BUILD.bazel +++ b/gazelle/pythonconfig/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "pythonconfig", @@ -15,6 +15,12 @@ go_library( ], ) +go_test( + name = "pythonconfig_test", + srcs = ["pythonconfig_test.go"], + deps = [":pythonconfig"], +) + filegroup( name = "distribution", srcs = glob(["**"]), diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index a2fe7d51b2..c7cd7c1a28 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -90,6 +90,14 @@ var defaultIgnoreFiles = map[string]struct{}{ "setup.py": {}, } +func SanitizeDistribution(distributionName string) string { + sanitizedDistribution := strings.ToLower(distributionName) + sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") + sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, ".", "_") + + return sanitizedDistribution +} + // Configs is an extension of map[string]*Config. It provides finding methods // on top of the mapping. type Configs map[string]*Config @@ -218,12 +226,17 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { } else if gazelleManifest.PipRepository != nil { distributionRepositoryName = gazelleManifest.PipRepository.Name } - sanitizedDistribution := strings.ToLower(distributionName) - sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") - var lbl label.Label + sanitizedDistribution := SanitizeDistribution(distributionName) + + if gazelleManifest.PipRepository != nil && gazelleManifest.PipRepository.UsePipRepositoryAliases { + // @// + lbl := label.New(distributionRepositoryName, sanitizedDistribution, sanitizedDistribution) + return lbl.String(), true + } + // @_//:pkg distributionRepositoryName = distributionRepositoryName + "_" + sanitizedDistribution - lbl = label.New(distributionRepositoryName, "", "pkg") + lbl := label.New(distributionRepositoryName, "", "pkg") return lbl.String(), true } } diff --git a/gazelle/pythonconfig/pythonconfig_test.go b/gazelle/pythonconfig/pythonconfig_test.go new file mode 100644 index 0000000000..1512eb97ae --- /dev/null +++ b/gazelle/pythonconfig/pythonconfig_test.go @@ -0,0 +1,28 @@ +package pythonconfig + +import ( + "testing" + + "github.com/bazelbuild/rules_python/gazelle/pythonconfig" +) + +func TestDistributionSanitizing(t *testing.T) { + tests := map[string]struct { + input string + want string + }{ + "upper case": {input: "DistWithUpperCase", want: "distwithuppercase"}, + "dashes": {input: "dist-with-dashes", want: "dist_with_dashes"}, + "dots": {input: "dist.with.dots", want: "dist_with_dots"}, + "mixed": {input: "To-be.sanitized", want: "to_be_sanitized"}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := pythonconfig.SanitizeDistribution(tc.input) + if tc.want != got { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 2e275b6650..a524d2ff94 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -23,11 +23,12 @@ In an ideal renaming, we'd move the packaging rules to a different package so that @rules_python//python is only concerned with the core rules. """ -load(":defs.bzl", "current_py_toolchain") +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load(":current_py_toolchain.bzl", "current_py_toolchain") package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) filegroup( name = "distribution", @@ -40,8 +41,95 @@ filegroup( visibility = ["//:__pkg__"], ) +# ========= bzl_library targets end ========= + +bzl_library( + name = "current_py_toolchain_bzl", + srcs = ["current_py_toolchain.bzl"], +) + +bzl_library( + name = "defs_bzl", + srcs = [ + "defs.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + ":current_py_toolchain_bzl", + ":py_import_bzl", + "//python/private:bazel_tools_bzl", + "//python/private:reexports_bzl", + ], +) + +bzl_library( + name = "proto_bzl", + srcs = [ + "proto.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + "//python/private/proto:py_proto_library_bzl", + ], +) + +bzl_library( + name = "py_binary_bzl", + srcs = ["py_binary.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_cc_link_params_info_bzl", + srcs = ["py_cc_link_params_info.bzl"], +) + +bzl_library( + name = "py_import_bzl", + srcs = ["py_import.bzl"], + deps = [":py_info_bzl"], +) + +bzl_library( + name = "py_info_bzl", + srcs = ["py_info.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_library_bzl", + srcs = ["py_library.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_bzl", + srcs = ["py_runtime.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_pair_bzl", + srcs = ["py_runtime_pair.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_info_bzl", + srcs = ["py_runtime_info.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_test_bzl", + srcs = ["py_test.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +# NOTE: Remember to add bzl_library targets to //tests:bzl_libraries +# ========= bzl_library targets end ========= + # Filegroup of bzl files that can be used by downstream rules for documentation generation -# Using a filegroup rather than bzl_library to not give a transitive dependency on Skylib filegroup( name = "bzl", srcs = [ diff --git a/python/current_py_toolchain.bzl b/python/current_py_toolchain.bzl new file mode 100644 index 0000000000..e3345cb646 --- /dev/null +++ b/python/current_py_toolchain.bzl @@ -0,0 +1,58 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for current_py_toolchain rule.""" + +def _current_py_toolchain_impl(ctx): + toolchain = ctx.toolchains[ctx.attr._toolchain] + + direct = [] + transitive = [] + vars = {} + + if toolchain.py3_runtime and toolchain.py3_runtime.interpreter: + direct.append(toolchain.py3_runtime.interpreter) + transitive.append(toolchain.py3_runtime.files) + vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path + + if toolchain.py2_runtime and toolchain.py2_runtime.interpreter: + direct.append(toolchain.py2_runtime.interpreter) + transitive.append(toolchain.py2_runtime.files) + vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path + + files = depset(direct, transitive = transitive) + return [ + toolchain, + platform_common.TemplateVariableInfo(vars), + DefaultInfo( + runfiles = ctx.runfiles(transitive_files = files), + files = files, + ), + ] + +current_py_toolchain = rule( + doc = """ + This rule exists so that the current python toolchain can be used in the `toolchains` attribute of + other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has + happened, to a rule which expects a concrete implementation of a toolchain, rather than a + toolchain_type which could be resolved to that toolchain. + """, + implementation = _current_py_toolchain_impl, + attrs = { + "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))), + }, + toolchains = [ + str(Label("@bazel_tools//tools/python:toolchain_type")), + ], +) diff --git a/python/defs.bzl b/python/defs.bzl index 7b60c6513b..ec70c1bf86 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -11,10 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" -Core rules for building Python projects. -""" +"""Core rules for building Python projects.""" load("@bazel_tools//tools/python:srcs_version.bzl", _find_requirements = "find_requirements") load( @@ -27,6 +24,8 @@ load( _py_runtime_pair = "py_runtime_pair", _py_test = "py_test", ) +load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain") +load(":py_import.bzl", _py_import = "py_import") # Exports of native-defined providers. @@ -34,98 +33,9 @@ PyInfo = internal_PyInfo PyRuntimeInfo = internal_PyRuntimeInfo -def _current_py_toolchain_impl(ctx): - toolchain = ctx.toolchains[ctx.attr._toolchain] - - direct = [] - transitive = [] - vars = {} - - if toolchain.py3_runtime and toolchain.py3_runtime.interpreter: - direct.append(toolchain.py3_runtime.interpreter) - transitive.append(toolchain.py3_runtime.files) - vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path - - if toolchain.py2_runtime and toolchain.py2_runtime.interpreter: - direct.append(toolchain.py2_runtime.interpreter) - transitive.append(toolchain.py2_runtime.files) - vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path - - files = depset(direct, transitive = transitive) - return [ - toolchain, - platform_common.TemplateVariableInfo(vars), - DefaultInfo( - runfiles = ctx.runfiles(transitive_files = files), - files = files, - ), - ] - -current_py_toolchain = rule( - doc = """ - This rule exists so that the current python toolchain can be used in the `toolchains` attribute of - other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has - happened, to a rule which expects a concrete implementation of a toolchain, rather than a - toolchain_type which could be resolved to that toolchain. - """, - implementation = _current_py_toolchain_impl, - attrs = { - "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))), - }, - toolchains = [ - str(Label("@bazel_tools//tools/python:toolchain_type")), - ], -) +current_py_toolchain = _current_py_toolchain -def _py_import_impl(ctx): - # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 . - import_paths = [ - "/".join([ctx.workspace_name, x.short_path]) - for x in ctx.files.srcs - ] - - return [ - DefaultInfo( - default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True), - ), - PyInfo( - transitive_sources = depset(transitive = [ - dep[PyInfo].transitive_sources - for dep in ctx.attr.deps - ]), - imports = depset(direct = import_paths, transitive = [ - dep[PyInfo].imports - for dep in ctx.attr.deps - ]), - ), - ] - -py_import = rule( - doc = """This rule allows the use of Python packages as dependencies. - - It imports the given `.egg` file(s), which might be checked in source files, - fetched externally as with `http_file`, or produced as outputs of other rules. - - It may be used like a `py_library`, in the `deps` of other Python rules. - - This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import). - """, - implementation = _py_import_impl, - attrs = { - "deps": attr.label_list( - doc = "The list of other libraries to be linked in to the " + - "binary target.", - providers = [PyInfo], - ), - "srcs": attr.label_list( - doc = "The list of Python package files provided to Python targets " + - "that depend on this target. Note that currently only the .egg " + - "format is accepted. For .whl files, try the whl_library rule. " + - "We accept contributions to extend py_import to handle .whl.", - allow_files = [".egg"], - ), - }, -) +py_import = _py_import # Re-exports of Starlark-defined symbols in @bazel_tools//tools/python. diff --git a/python/extensions.bzl b/python/extensions.bzl index 01f731f14f..2b0c188554 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -30,6 +30,7 @@ def _python_impl(module_ctx): # Toolchain registration in bzlmod is done in MODULE file register_toolchains = False, register_coverage_tool = attr.configure_coverage_tool, + ignore_root_user_error = attr.ignore_root_user_error, ) python = module_extension( @@ -41,6 +42,11 @@ python = module_extension( mandatory = False, doc = "Whether or not to configure the default coverage tool for the toolchains.", ), + "ignore_root_user_error": attr.bool( + default = False, + doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", + mandatory = False, + ), "name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True), }, @@ -78,6 +84,7 @@ def _pip_impl(module_ctx): pip_repository_bzlmod( name = attr.name, requirements_lock = attr.requirements_lock, + incompatible_generate_aliases = attr.incompatible_generate_aliases, ) for name, requirement_line in requirements: diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 982d8536ba..733142ba92 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -261,6 +261,52 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile def _clean_pkg_name(name): return name.replace("-", "_").replace(".", "_").lower() +def _pkg_aliases(rctx, repo_name, bzl_packages): + """Create alias declarations for each python dependency. + + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases + allow users to use requirement() without needed a corresponding `use_repo()` for each dep + when using bzlmod. + + Args: + rctx: the repository context. + repo_name: the repository name of the parent that is visible to the users. + bzl_packages: the list of packages to setup. + """ + for name in bzl_packages: + build_content = """package(default_visibility = ["//visibility:public"]) + +alias( + name = "{name}", + actual = "@{repo_name}_{dep}//:pkg", +) + +alias( + name = "pkg", + actual = "@{repo_name}_{dep}//:pkg", +) + +alias( + name = "whl", + actual = "@{repo_name}_{dep}//:whl", +) + +alias( + name = "data", + actual = "@{repo_name}_{dep}//:data", +) + +alias( + name = "dist_info", + actual = "@{repo_name}_{dep}//:dist_info", +) +""".format( + name = name, + repo_name = repo_name, + dep = name, + ) + rctx.file("{}/BUILD.bazel".format(name), build_content) + def _bzlmod_pkg_aliases(repo_name, bzl_packages): """Create alias declarations for each python dependency. @@ -314,16 +360,21 @@ def _pip_repository_bzlmod_impl(rctx): repo_name = rctx.attr.name.split("~")[-1] - build_contents = _BUILD_FILE_CONTENTS + _bzlmod_pkg_aliases(repo_name, bzl_packages) + build_contents = _BUILD_FILE_CONTENTS + + if rctx.attr.incompatible_generate_aliases: + _pkg_aliases(rctx, repo_name, bzl_packages) + else: + build_contents += _bzlmod_pkg_aliases(repo_name, bzl_packages) rctx.file("BUILD.bazel", build_contents) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}//:{}_pkg".format(repo_name, p) + "@@{}//{}".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}//:{}_whl".format(repo_name, p) + "@@{}//{}:whl".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) for p in bzl_packages ]), "%%NAME%%": rctx.attr.name, @@ -331,6 +382,10 @@ def _pip_repository_bzlmod_impl(rctx): }) pip_repository_bzlmod_attrs = { + "incompatible_generate_aliases": attr.bool( + default = False, + doc = "Allow generating aliases in '@pip//:' -> '@pip_//:pkg'. This replaces the aliases generated by the `bzlmod` tooling.", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", @@ -405,14 +460,17 @@ def _pip_repository_impl(rctx): if rctx.attr.python_interpreter_target: config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.incompatible_generate_aliases: + _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}_{}//:pkg".format(rctx.attr.name, p) + "@{}//{}".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}_{}//:whl".format(rctx.attr.name, p) + "@{}//{}:whl".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), @@ -520,6 +578,10 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), + "incompatible_generate_aliases": attr.bool( + default = False, + doc = "Allow generating aliases '@pip//' -> '@pip_//:pkg'.", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index af3c194d18..dd38c9df5b 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -39,8 +39,8 @@ def compile_pip_requirements( It also generates two targets for running pip-compile: - - validate with `bazel test _test` - - update with `bazel run .update` + - validate with `bazel test [name]_test` + - update with `bazel run [name].update` Args: name: base name for generated targets, typically "requirements". diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 7d321ebbe7..21e3c1623f 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python:versions.bzl", "print_toolchains_checksums") load(":stamp.bzl", "stamp_build_setting") @@ -24,13 +25,33 @@ filegroup( ) # Filegroup of bzl files that can be used by downstream rules for documentation generation -# Using a filegroup rather than bzl_library to not give a transitive dependency on Skylib filegroup( name = "bzl", srcs = glob(["**/*.bzl"]), visibility = ["//python:__pkg__"], ) +bzl_library( + name = "reexports_bzl", + srcs = ["reexports.bzl"], + visibility = [ + "//docs:__pkg__", + "//python:__pkg__", + ], + deps = [":bazel_tools_bzl"], +) + +# @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. +bzl_library( + name = "bazel_tools_bzl", + srcs = [ + "@bazel_tools//tools/python:srcs_version.bzl", + "@bazel_tools//tools/python:toolchain.bzl", + "@bazel_tools//tools/python:utils.bzl", + ], + visibility = ["//python:__pkg__"], +) + # Needed to define bzl_library targets for docgen. (We don't define the # bzl_library target here because it'd give our users a transitive dependency # on Skylib.) diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD.bazel similarity index 64% rename from python/private/proto/BUILD rename to python/private/proto/BUILD.bazel index 8483d19c2f..65c09444f7 100644 --- a/python/private/proto/BUILD +++ b/python/private/proto/BUILD.bazel @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -package(default_visibility = ["//visibility:public"]) +package(default_visibility = ["//visibility:private"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) filegroup( name = "distribution", @@ -24,9 +25,22 @@ filegroup( visibility = ["//python/private:__pkg__"], ) +bzl_library( + name = "py_proto_library_bzl", + srcs = ["py_proto_library.bzl"], + visibility = ["//python:__pkg__"], + deps = [ + "//python:defs_bzl", + "@rules_proto//proto:defs", + ], +) + proto_lang_toolchain( name = "python_toolchain", command_line = "--python_out=%s", progress_message = "Generating Python proto_library %{label}", runtime = "@com_google_protobuf//:protobuf_python", + # NOTE: This isn't *actually* public. It's an implicit dependency of py_proto_library, + # so must be public so user usages of the rule can reference it. + visibility = ["//visibility:public"], ) diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index 77690edc65..b6f2bfae56 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -207,12 +207,15 @@ def _input_file_to_arg(input_file): return "%s;%s" % (py_package_lib.path_inside_wheel(input_file), input_file.path) def _py_wheel_impl(ctx): + abi = _replace_make_variables(ctx.attr.abi, ctx) + python_tag = _replace_make_variables(ctx.attr.python_tag, ctx) version = _replace_make_variables(ctx.attr.version, ctx) + outfile = ctx.actions.declare_file("-".join([ _escape_filename_segment(ctx.attr.distribution), _escape_filename_segment(version), - _escape_filename_segment(ctx.attr.python_tag), - _escape_filename_segment(ctx.attr.abi), + _escape_filename_segment(python_tag), + _escape_filename_segment(abi), _escape_filename_segment(ctx.attr.platform), ]) + ".whl") @@ -237,8 +240,8 @@ def _py_wheel_impl(ctx): args = ctx.actions.args() args.add("--name", ctx.attr.distribution) args.add("--version", version) - args.add("--python_tag", ctx.attr.python_tag) - args.add("--abi", ctx.attr.abi) + args.add("--python_tag", python_tag) + args.add("--abi", abi) args.add("--platform", ctx.attr.platform) args.add("--out", outfile) args.add("--name_file", name_file) diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 4b832d941a..9bed73e55c 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -31,10 +31,13 @@ load( "WINDOWS_NAME", ) +def get_repository_name(repository_workspace): + dummy_label = "//:_" + return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@" + def _toolchains_repo_impl(rctx): - rules_python_repository_name = rctx.attr._rules_python_workspace.workspace_name - python_version_constraint = "@{rules_python}//python/config_settings:is_python_{python_version}".format( - rules_python = rules_python_repository_name, + python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( + rules_python = get_repository_name(rctx.attr._rules_python_workspace), python_version = rctx.attr.python_version, ) @@ -90,8 +93,6 @@ def _toolchain_aliases_impl(rctx): is_windows = (os_name == WINDOWS_NAME) python3_binary_path = "python.exe" if is_windows else "bin/python3" - rules_python_repository_name = rctx.attr._rules_python_workspace.workspace_name - # Base BUILD file for this repository. build_contents = """\ # Generated by python/private/toolchains_repo.bzl @@ -123,8 +124,8 @@ alias(name = "pip", actual = select({{":" + item: "@{py_repository}_ rctx.file("defs.bzl", content = """\ # Generated by python/private/toolchains_repo.bzl -load("@{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") -load("@{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") +load("{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") +load("{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") host_platform = "{host_platform}" interpreter = "@{py_repository}_{host_platform}//:{python3_binary_path}" @@ -156,7 +157,7 @@ def compile_pip_requirements(name, **kwargs): py_repository = rctx.attr.user_repository_name, python_version = rctx.attr.python_version, python3_binary_path = python3_binary_path, - rules_python = rules_python_repository_name, + rules_python = get_repository_name(rctx.attr._rules_python_workspace), )) toolchain_aliases = repository_rule( diff --git a/python/py_binary.bzl b/python/py_binary.bzl new file mode 100644 index 0000000000..9d145d8fa6 --- /dev/null +++ b/python/py_binary.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_binary.""" + +load("//python/private:reexports.bzl", _py_binary = "py_binary") + +py_binary = _py_binary diff --git a/python/py_cc_link_params_info.bzl b/python/py_cc_link_params_info.bzl new file mode 100644 index 0000000000..0ebd64b208 --- /dev/null +++ b/python/py_cc_link_params_info.bzl @@ -0,0 +1,3 @@ +"""Public entry point for PyCcLinkParamsInfo.""" + +PyCcLinkParamsInfo = PyCcLinkParamsProvider diff --git a/python/py_import.bzl b/python/py_import.bzl new file mode 100644 index 0000000000..c9284121d6 --- /dev/null +++ b/python/py_import.bzl @@ -0,0 +1,67 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_import rule.""" + +load(":py_info.bzl", "PyInfo") + +def _py_import_impl(ctx): + # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 . + import_paths = [ + "/".join([ctx.workspace_name, x.short_path]) + for x in ctx.files.srcs + ] + + return [ + DefaultInfo( + default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True), + ), + PyInfo( + transitive_sources = depset(transitive = [ + dep[PyInfo].transitive_sources + for dep in ctx.attr.deps + ]), + imports = depset(direct = import_paths, transitive = [ + dep[PyInfo].imports + for dep in ctx.attr.deps + ]), + ), + ] + +py_import = rule( + doc = """This rule allows the use of Python packages as dependencies. + + It imports the given `.egg` file(s), which might be checked in source files, + fetched externally as with `http_file`, or produced as outputs of other rules. + + It may be used like a `py_library`, in the `deps` of other Python rules. + + This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import). + """, + implementation = _py_import_impl, + attrs = { + "deps": attr.label_list( + doc = "The list of other libraries to be linked in to the " + + "binary target.", + providers = [PyInfo], + ), + "srcs": attr.label_list( + doc = "The list of Python package files provided to Python targets " + + "that depend on this target. Note that currently only the .egg " + + "format is accepted. For .whl files, try the whl_library rule. " + + "We accept contributions to extend py_import to handle .whl.", + allow_files = [".egg"], + ), + }, +) diff --git a/python/py_info.bzl b/python/py_info.bzl new file mode 100644 index 0000000000..2c3997dee2 --- /dev/null +++ b/python/py_info.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for PyInfo.""" + +load("//python/private:reexports.bzl", "internal_PyInfo") + +PyInfo = internal_PyInfo diff --git a/python/py_library.bzl b/python/py_library.bzl new file mode 100644 index 0000000000..1aff68c100 --- /dev/null +++ b/python/py_library.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_library.""" + +load("//python/private:reexports.bzl", _py_library = "py_library") + +py_library = _py_library diff --git a/python/py_runtime.bzl b/python/py_runtime.bzl new file mode 100644 index 0000000000..5e80308176 --- /dev/null +++ b/python/py_runtime.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_runtime.""" + +load("//python/private:reexports.bzl", _py_runtime = "py_runtime") + +py_runtime = _py_runtime diff --git a/python/py_runtime_info.bzl b/python/py_runtime_info.bzl new file mode 100644 index 0000000000..15598ee903 --- /dev/null +++ b/python/py_runtime_info.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for PyRuntimeInfo.""" + +load("//python/private:reexports.bzl", "internal_PyRuntimeInfo") + +PyRuntimeInfo = internal_PyRuntimeInfo diff --git a/python/py_runtime_pair.bzl b/python/py_runtime_pair.bzl new file mode 100644 index 0000000000..3f3ecf443b --- /dev/null +++ b/python/py_runtime_pair.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_runtime_pair.""" + +load("//python/private:reexports.bzl", _py_runtime_pair = "py_runtime_pair") + +py_runtime_pair = _py_runtime_pair diff --git a/python/py_test.bzl b/python/py_test.bzl new file mode 100644 index 0000000000..84470bc3af --- /dev/null +++ b/python/py_test.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_test.""" + +load("//python/private:reexports.bzl", _py_test = "py_test") + +py_test = _py_test diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index ee9c5550e8..abbe62ddff 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,8 +1,9 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//tools/bazel_integration_test:bazel_integration_test.bzl", "bazel_integration_test") package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) bazel_integration_test( name = "pip_repository_entry_points_example", @@ -10,3 +11,22 @@ bazel_integration_test( # The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260 tags = ["fix-windows"], ) + +build_test( + name = "bzl_libraries_build_test", + targets = [ + # keep sorted + "//python:current_py_toolchain_bzl", + "//python:defs_bzl", + "//python:proto_bzl", + "//python:py_binary_bzl", + "//python:py_cc_link_params_info_bzl", + "//python:py_import_bzl", + "//python:py_info_bzl", + "//python:py_library_bzl", + "//python:py_runtime_bzl", + "//python:py_runtime_info_bzl", + "//python:py_runtime_pair_bzl", + "//python:py_test_bzl", + ], +)