From 51458e88f7223d8d7db282018f085828e9a5a312 Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Sat, 4 Mar 2023 19:35:15 -0800 Subject: [PATCH 01/19] chore: fix some lingering GH archive URLs (#1108) This was pointed out in a thread on bazel-discuss@googlegroups.com --- .github/workflows/create_archive_and_notes.sh | 7 ++++ README.md | 3 +- examples/build_file_generation/WORKSPACE | 37 ++----------------- gazelle/README.md | 19 +--------- 4 files changed, 15 insertions(+), 51 deletions(-) diff --git a/.github/workflows/create_archive_and_notes.sh b/.github/workflows/create_archive_and_notes.sh index 549af074eb..0c0c4acf41 100755 --- a/.github/workflows/create_archive_and_notes.sh +++ b/.github/workflows/create_archive_and_notes.sh @@ -87,5 +87,12 @@ http_archive( strip_prefix = "${PREFIX}/gazelle", url = "https://github.com/bazelbuild/rules_python/releases/download/${TAG}/rules_python-${TAG}.tar.gz", ) + +# To compile the rules_python gazelle extension from source, +# we must fetch some third-party go dependencies that it uses. + +load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps") + +_py_gazelle_deps() \`\`\` EOF diff --git a/README.md b/README.md index a509e28d7e..07acaf8e19 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53" # Latest @ 202 http_archive( name = "rules_python", - sha256 = "3474c5815da4cb003ff22811a36a11894927eda1c2e64bf2dac63e914bfdf30f", + # Bazel will print the proper value to add here during the first build. + # sha256 = "FIXME", strip_prefix = "rules_python-{}".format(rules_python_version), url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version), ) diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 674b9eb7ea..9f1dae8aaf 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -55,49 +55,20 @@ gazelle_dependencies() # Remaining setup is for rules_python. -# You do not want to use the following command when you are using a WORKSPACE file -# that is outside of rules_python repository. -# This command allows targets from a local directory to be bound. -# Which allows bazel to use targets defined in base rules_python directory. -# If you are using this example outside of the rules_python git repo, -# use the http_archive command that is commented out below. -# https://bazel.build/reference/be/workspace#local_repository +# DON'T COPY_PASTE THIS. +# Our example uses `local_repository` to point to the HEAD version of rules_python. +# Users should instead use the installation instructions from the release they use. +# See https://github.com/bazelbuild/rules_python/releases local_repository( name = "rules_python", path = "../..", ) -# When not using this example in the rules_python git repo you would load the python -# ruleset using the following StarLark. -# See https://github.com/bazelbuild/rules_python#getting-started for the latest -# ruleset version. -# -# The following StarLark would replace the `local_repository` rule mentioned above. -# -# load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# http_archive( -# name = "rules_python", -# sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a", -# strip_prefix = "rules_python-0.16.1", -# url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz", -# ) - -# We import the repository-local rules_python_gazelle_plugin version in order to -# be able to test development changes to the plugin. local_repository( name = "rules_python_gazelle_plugin", path = "../../gazelle", ) -# When loading the gazelle plugin outside this repo, use the http_archive rule as follows: -# -#http_archive( -# name = "rules_python_gazelle_plugin", -# sha256 = "497ca47374f48c8b067d786b512ac10a276211810f4a580178ee9b9ad139323a", -# strip_prefix = "rules_python-0.16.1/gazelle", -# url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.16.1.tar.gz", -#) - # Next we load the toolchain from rules_python. load("@rules_python//python:repositories.bzl", "python_register_toolchains") diff --git a/gazelle/README.md b/gazelle/README.md index a76ac59199..0081701241 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -14,23 +14,8 @@ Follow the instructions at https://github.com/bazelbuild/bazel-gazelle#running-g Next, we need to fetch the third-party Go libraries that the python extension depends on. -Add this to your `WORKSPACE`: - -```starlark -http_archive( - name = "rules_python_gazelle_plugin", - sha256 = "", - strip_prefix = "rules_python-0.17.0/gazelle", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.17.0.tar.gz", -) - -# To compile the rules_python gazelle extension from source, -# we must fetch some third-party go dependencies that it uses. - -load("@rules_python_gazelle_plugin//:deps.bzl", _py_gazelle_deps = "gazelle_deps") - -_py_gazelle_deps() -``` +See the installation `WORKSPACE` snippet on the Releases page: +https://github.com/bazelbuild/rules_python/releases Next, we'll fetch metadata about your Python dependencies, so that gazelle can determine which package a given import statement comes from. This is provided From 7d82f06e7ba1cf217c6d855cd5ffbb3b6598587c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius Date: Mon, 6 Mar 2023 10:33:30 +0900 Subject: [PATCH 02/19] feat: add bzlmod support for gazelle plugin (#1077) * feat: add optional pkg alias generation to pip_repository * feat: support using aliases in gazelle * doc: migrate gazelle example to use user friendly aliases * feat: gazelle supports bzlmod * chore: update gazelle plugin deps * chore: bazel run //:gazelle * fix: remove deps that are imported as bazel deps See: bazelbuild/bazel-gazelle#1403 * ci: add build_file_generation to show that we support bzlmod --- .bazelci/presubmit.yml | 34 +++++++++ docs/pip_repository.md | 13 ++-- examples/build_file_generation/BUILD.bazel | 5 +- examples/build_file_generation/MODULE.bazel | 43 +++++++++++ examples/build_file_generation/WORKSPACE | 2 + .../build_file_generation/gazelle_python.yaml | 3 +- gazelle/MODULE.bazel | 20 ++++++ gazelle/deps.bzl | 57 ++++----------- gazelle/go.mod | 2 - gazelle/manifest/defs.bzl | 11 ++- gazelle/manifest/generate/generate.go | 23 ++++-- gazelle/manifest/manifest.go | 3 + gazelle/python/BUILD.bazel | 1 - gazelle/pythonconfig/pythonconfig.go | 10 ++- python/extensions.bzl | 1 + python/pip_install/pip_repository.bzl | 72 +++++++++++++++++-- 16 files changed, 231 insertions(+), 69 deletions(-) create mode 100644 examples/build_file_generation/MODULE.bazel create mode 100644 gazelle/MODULE.bazel diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index a1b16bbc66..a0d9a19047 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -73,6 +73,14 @@ tasks: build_targets: ["//..."] test_targets: ["//..."] working_directory: gazelle + gazelle_extension_bzlmod: + <<: *common_bzlmod_flags + name: Test the Gazelle extension under bzlmod + platform: ubuntu2004 + build_targets: ["//..."] + test_targets: ["//..."] + working_directory: gazelle + ubuntu_min: <<: *minimum_supported_version <<: *reusable_config @@ -138,6 +146,32 @@ tasks: working_directory: examples/build_file_generation platform: windows + integration_test_build_file_generation_bzlmod_ubuntu: + <<: *minimum_supported_bzlmod_version + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Ubuntu + working_directory: examples/build_file_generation + platform: ubuntu2004 + integration_test_build_file_generation_bzlmod_debian: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Debian + working_directory: examples/build_file_generation + platform: debian11 + integration_test_build_file_generation_bzlmod_macos: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on macOS + working_directory: examples/build_file_generation + platform: macos + integration_test_build_file_generation_bzlmod_windows: + <<: *common_bzlmod_flags + <<: *reusable_build_test_all + name: build_file_generation_bzlmod integration tests on Windows + working_directory: examples/build_file_generation + platform: windows + integration_test_bzlmod_ubuntu_min: <<: *minimum_supported_bzlmod_version <<: *reusable_build_test_all diff --git a/docs/pip_repository.md b/docs/pip_repository.md index 2ccdc64854..c02058e08d 100644 --- a/docs/pip_repository.md +++ b/docs/pip_repository.md @@ -8,9 +8,10 @@ 
 pip_repository(name, annotations, download_only, enable_implicit_namespace_pkgs, environment,
-               extra_pip_args, isolated, pip_data_exclude, python_interpreter,
-               python_interpreter_target, quiet, repo_mapping, repo_prefix, requirements_darwin,
-               requirements_linux, requirements_lock, requirements_windows, timeout)
+               extra_pip_args, incompatible_generate_aliases, isolated, pip_data_exclude,
+               python_interpreter, python_interpreter_target, quiet, repo_mapping, repo_prefix,
+               requirements_darwin, requirements_linux, requirements_lock, requirements_windows,
+               timeout)
A rule for importing `requirements.txt` dependencies into Bazel. @@ -64,6 +65,7 @@ py_binary( | enable_implicit_namespace_pkgs | If true, disables conversion of native namespace packages into pkg-util style namespace packages. When set all py_binary and py_test targets must specify either legacy_create_init=False or the global Bazel option --incompatible_default_to_explicit_init_py to prevent __init__.py being automatically generated in every directory.

This option is required to support some packages which cannot handle the conversion to pkg-util style. | Boolean | optional | False | | environment | Environment variables to set in the pip subprocess. Can be used to set common variables such as http_proxy, https_proxy and no_proxy Note that pip is run with "--isolated" on the CLI so PIP_<VAR>_<NAME> style env vars are ignored, but env vars that control requests and urllib3 can be passed. | Dictionary: String -> String | optional | {} | | extra_pip_args | Extra arguments to pass on to pip. Must not contain spaces. | List of strings | optional | [] | +| incompatible_generate_aliases | Allow generating aliases '@pip//<pkg>' -> '@pip_<pkg>//:pkg'. | Boolean | optional | False | | isolated | Whether or not to pass the [--isolated](https://pip.pypa.io/en/stable/cli/pip/#cmdoption-isolated) flag to the underlying pip command. Alternatively, the RULES_PYTHON_PIP_ISOLATED enviornment varaible can be used to control this flag. | Boolean | optional | True | | pip_data_exclude | Additional data exclusion parameters to add to the pip packages BUILD file. | List of strings | optional | [] | | python_interpreter | The python interpreter to use. This can either be an absolute path or the name of a binary found on the host's PATH environment variable. If no value is set python3 is defaulted for Unix systems and python.exe for Windows. | String | optional | "" | @@ -83,8 +85,8 @@ py_binary( ## pip_repository_bzlmod 
-pip_repository_bzlmod(name, repo_mapping, requirements_darwin, requirements_linux,
-                      requirements_lock, requirements_windows)
+pip_repository_bzlmod(name, incompatible_generate_aliases, repo_mapping, requirements_darwin,
+                      requirements_linux, requirements_lock, requirements_windows)
A rule for bzlmod pip_repository creation. Intended for private use only. @@ -95,6 +97,7 @@ A rule for bzlmod pip_repository creation. Intended for private use only. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this repository. | Name | required | | +| incompatible_generate_aliases | Allow generating aliases in '@pip//:<pkg>' -> '@pip_<pkg>//:pkg'. This replaces the aliases generated by the bzlmod tooling. | Boolean | optional | False | | repo_mapping | A dictionary from local repository name to global repository name. This allows controls over workspace dependency resolution for dependencies of this repository.<p>For example, an entry "@foo": "@bar" declares that, for any time this repository depends on @foo (such as a dependency on @foo//some:target, it should actually resolve that dependency within globally-declared @bar (@bar//some:target). | Dictionary: String -> String | required | | | requirements_darwin | Override the requirements_lock attribute when the host platform is Mac OS | Label | optional | None | | requirements_linux | Override the requirements_lock attribute when the host platform is Linux | Label | optional | None | diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 6419ef2c70..7c88d9203d 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -43,6 +43,9 @@ gazelle_python_manifest( modules_mapping = ":modules_map", pip_repository_name = "pip", requirements = "//:requirements_lock.txt", + # NOTE: we can use this flag in order to make our setup compatible with + # bzlmod. + use_pip_repository_aliases = True, ) # Our gazelle target points to the python gazelle binary. @@ -65,7 +68,7 @@ py_library( visibility = ["//:__subpackages__"], deps = [ "//random_number_generator", - "@pip_flask//:pkg", + "@pip//flask", ], ) diff --git a/examples/build_file_generation/MODULE.bazel b/examples/build_file_generation/MODULE.bazel new file mode 100644 index 0000000000..5f79fec486 --- /dev/null +++ b/examples/build_file_generation/MODULE.bazel @@ -0,0 +1,43 @@ +module( + name = "example_bzlmod", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.19.0") +bazel_dep(name = "rules_python_gazelle_plugin", version = "0.19.0") +bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") + +# local overrides for the packages for CI purposes. +# for usual setups you should remove this block. +local_path_override( + module_name = "rules_python", + path = "../..", +) + +local_path_override( + module_name = "rules_python_gazelle_plugin", + path = "../../gazelle", +) + +# Register python toolchain +python = use_extension("@rules_python//python:extensions.bzl", "python") +python.toolchain( + name = "python3_9", + python_version = "3.9", +) +use_repo(python, "python3_9_toolchains") + +register_toolchains( + "@python3_9_toolchains//:all", +) + +pip = use_extension("@rules_python//python:extensions.bzl", "pip") +pip.parse( + name = "pip", + # Generate user friendly alias labels for each dependency that we have. + incompatible_generate_aliases = True, + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") diff --git a/examples/build_file_generation/WORKSPACE b/examples/build_file_generation/WORKSPACE index 9f1dae8aaf..65e0a6e5f3 100644 --- a/examples/build_file_generation/WORKSPACE +++ b/examples/build_file_generation/WORKSPACE @@ -90,6 +90,8 @@ load("@rules_python//python:pip.bzl", "pip_parse") # You can instead check this `requirements.bzl` file into your repo. pip_parse( name = "pip", + # Generate user friendly alias labels for each dependency that we have. + incompatible_generate_aliases = True, # (Optional) You can provide a python_interpreter (path) or a python_interpreter_target (a Bazel target, that # acts as an executable). The latter can be anything that could be used as Python interpreter. E.g.: # 1. Python interpreter that you compile in the build file. diff --git a/examples/build_file_generation/gazelle_python.yaml b/examples/build_file_generation/gazelle_python.yaml index 847d1ecc55..b57e9f02bc 100644 --- a/examples/build_file_generation/gazelle_python.yaml +++ b/examples/build_file_generation/gazelle_python.yaml @@ -114,4 +114,5 @@ manifest: zipp.py310compat: zipp pip_repository: name: pip -integrity: 2c84a3cabeaff134a1d045e5a173a3178086f236ab20f895ffbd7f3b7a6e5bb0 + use_pip_repository_aliases: true +integrity: 85f073e37e31339508aaaf5e0d5472adae5148fd5f054e9cc586343c026660e1 diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel new file mode 100644 index 0000000000..bd634020f3 --- /dev/null +++ b/gazelle/MODULE.bazel @@ -0,0 +1,20 @@ +module( + name = "rules_python_gazelle_plugin", + version = "0.0.0", + compatibility_level = 1, +) + +bazel_dep(name = "rules_python", version = "0.18.0") +bazel_dep(name = "rules_go", version = "0.38.1", repo_name = "io_bazel_rules_go") +bazel_dep(name = "gazelle", version = "0.29.0", repo_name = "bazel_gazelle") + +go_deps = use_extension("@bazel_gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +use_repo( + go_deps, + "com_github_bazelbuild_buildtools", + "com_github_bmatcuk_doublestar", + "com_github_emirpasic_gods", + "com_github_ghodss_yaml", + "in_gopkg_yaml_v2", +) diff --git a/gazelle/deps.bzl b/gazelle/deps.bzl index 357944302c..26f8c66aec 100644 --- a/gazelle/deps.bzl +++ b/gazelle/deps.bzl @@ -28,12 +28,7 @@ def gazelle_deps(): sum = "h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=", version = "v0.0.0-20190523083050-ea95bdfd59fc", ) - go_repository( - name = "com_github_bazelbuild_bazel_gazelle", - importpath = "github.com/bazelbuild/bazel-gazelle", - sum = "h1:+/ZhUxlDy4XnyMIGeKkbRZoIGssy1eO51GijwIvvuwE=", - version = "v0.27.0", - ) + go_repository( name = "com_github_bazelbuild_buildtools", build_naming_convention = "go_default_library", @@ -41,24 +36,14 @@ def gazelle_deps(): sum = "h1:jhiMzJ+8unnLRtV8rpbWBFE9pFNzIqgUTyZU5aA++w8=", version = "v0.0.0-20221004120235-7186f635531b", ) - go_repository( - name = "com_github_bazelbuild_rules_go", - importpath = "github.com/bazelbuild/rules_go", - sum = "h1:ViPR65vOrg74JKntAUFY6qZkheBKGB6to7wFd8gCRU4=", - version = "v0.35.0", - ) + go_repository( name = "com_github_bmatcuk_doublestar", importpath = "github.com/bmatcuk/doublestar", sum = "h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=", version = "v1.3.4", ) - go_repository( - name = "com_github_bmatcuk_doublestar_v4", - importpath = "github.com/bmatcuk/doublestar/v4", - sum = "h1:Qu+u9wR3Vd89LnlLMHvnZ5coJMWKQamqdz9/p5GNthA=", - version = "v4.2.0", - ) + go_repository( name = "com_github_burntsushi_toml", importpath = "github.com/BurntSushi/toml", @@ -113,12 +98,7 @@ def gazelle_deps(): sum = "h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=", version = "v0.1.0", ) - go_repository( - name = "com_github_fsnotify_fsnotify", - importpath = "github.com/fsnotify/fsnotify", - sum = "h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=", - version = "v1.5.4", - ) + go_repository( name = "com_github_ghodss_yaml", importpath = "github.com/ghodss/yaml", @@ -134,14 +114,14 @@ def gazelle_deps(): go_repository( name = "com_github_golang_mock", importpath = "github.com/golang/mock", - sum = "h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=", - version = "v1.6.0", + sum = "h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=", + version = "v1.1.1", ) go_repository( name = "com_github_golang_protobuf", importpath = "github.com/golang/protobuf", - sum = "h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=", - version = "v1.5.2", + sum = "h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=", + version = "v1.4.3", ) go_repository( name = "com_github_google_go_cmp", @@ -149,18 +129,7 @@ def gazelle_deps(): sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", version = "v0.5.9", ) - go_repository( - name = "com_github_pelletier_go_toml", - importpath = "github.com/pelletier/go-toml", - sum = "h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=", - version = "v1.9.5", - ) - go_repository( - name = "com_github_pmezard_go_difflib", - importpath = "github.com/pmezard/go-difflib", - sum = "h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=", - version = "v1.0.0", - ) + go_repository( name = "com_github_prometheus_client_model", importpath = "github.com/prometheus/client_model", @@ -218,8 +187,8 @@ def gazelle_deps(): go_repository( name = "org_golang_google_protobuf", importpath = "google.golang.org/protobuf", - sum = "h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=", - version = "v1.28.0", + sum = "h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=", + version = "v1.25.0", ) go_repository( name = "org_golang_x_crypto", @@ -260,8 +229,8 @@ def gazelle_deps(): go_repository( name = "org_golang_x_sync", importpath = "golang.org/x/sync", - sum = "h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A=", - version = "v0.0.0-20220907140024-f12130a52804", + sum = "h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=", + version = "v0.0.0-20220722155255-886fb9371eb4", ) go_repository( name = "org_golang_x_sys", diff --git a/gazelle/go.mod b/gazelle/go.mod index 6d6f0332a0..94f19e801f 100644 --- a/gazelle/go.mod +++ b/gazelle/go.mod @@ -3,9 +3,7 @@ module github.com/bazelbuild/rules_python/gazelle go 1.19 require ( - github.com/bazelbuild/bazel-gazelle v0.27.0 github.com/bazelbuild/buildtools v0.0.0-20221004120235-7186f635531b - github.com/bazelbuild/rules_go v0.35.0 github.com/bmatcuk/doublestar v1.3.4 github.com/emirpasic/gods v1.18.1 github.com/ghodss/yaml v1.0.0 diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index 78e0c272ac..05562a1583 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -24,13 +24,16 @@ def gazelle_python_manifest( modules_mapping, pip_repository_name = "", pip_deps_repository_name = "", - manifest = ":gazelle_python.yaml"): + manifest = ":gazelle_python.yaml", + use_pip_repository_aliases = False): """A macro for defining the updating and testing targets for the Gazelle manifest file. Args: name: the name used as a base for the targets. requirements: the target for the requirements.txt file. pip_repository_name: the name of the pip_install or pip_repository target. + use_pip_repository_aliases: boolean flag to enable using user-friendly + python package aliases. pip_deps_repository_name: deprecated - the old pip_install target name. modules_mapping: the target for the generated modules_mapping.json file. manifest: the target for the Gazelle manifest file. @@ -67,6 +70,12 @@ def gazelle_python_manifest( update_target_label, ] + if use_pip_repository_aliases: + update_args += [ + "--use-pip-repository-aliases", + "true", + ] + go_binary( name = update_target, embed = [Label("//manifest/generate:generate_lib")], diff --git a/gazelle/manifest/generate/generate.go b/gazelle/manifest/generate/generate.go index 0f429f8345..1f56e630cc 100644 --- a/gazelle/manifest/generate/generate.go +++ b/gazelle/manifest/generate/generate.go @@ -38,12 +38,15 @@ func init() { } func main() { - var manifestGeneratorHashPath string - var requirementsPath string - var pipRepositoryName string - var modulesMappingPath string - var outputPath string - var updateTarget string + var ( + manifestGeneratorHashPath string + requirementsPath string + pipRepositoryName string + usePipRepositoryAliases bool + modulesMappingPath string + outputPath string + updateTarget string + ) flag.StringVar( &manifestGeneratorHashPath, "manifest-generator-hash", @@ -60,6 +63,11 @@ func main() { "pip-repository-name", "", "The name of the pip_install or pip_repository target.") + flag.BoolVar( + &usePipRepositoryAliases, + "use-pip-repository-aliases", + false, + "Whether to use the pip-repository aliases, which are generated when passing 'incompatible_generate_aliases = True'.") flag.StringVar( &modulesMappingPath, "modules-mapping", @@ -103,7 +111,8 @@ func main() { manifestFile := manifest.NewFile(&manifest.Manifest{ ModulesMapping: modulesMapping, PipRepository: &manifest.PipRepository{ - Name: pipRepositoryName, + Name: pipRepositoryName, + UsePipRepositoryAliases: usePipRepositoryAliases, }, }) if err := writeOutput( diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index bb4826435f..c49951dd3e 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -144,4 +144,7 @@ type Manifest struct { type PipRepository struct { // The name of the pip_install or pip_repository target. Name string + // UsePipRepositoryAliases allows to use aliases generated pip_repository + // when passing incompatible_generate_aliases = True. + UsePipRepositoryAliases bool `yaml:"use_pip_repository_aliases,omitempty"` } diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 3b5ded2139..ddcad2785d 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -61,7 +61,6 @@ go_test( ] + glob(["testdata/**"]), deps = [ "@bazel_gazelle//testtools:go_default_library", - "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_ghodss_yaml//:yaml", "@io_bazel_rules_go//go/tools/bazel:go_default_library", ], diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index a2fe7d51b2..ea2ae65c4c 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -220,10 +220,16 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { } sanitizedDistribution := strings.ToLower(distributionName) sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") - var lbl label.Label + + if gazelleManifest.PipRepository != nil && gazelleManifest.PipRepository.UsePipRepositoryAliases { + // @// + lbl := label.New(distributionRepositoryName, sanitizedDistribution, sanitizedDistribution) + return lbl.String(), true + } + // @_//:pkg distributionRepositoryName = distributionRepositoryName + "_" + sanitizedDistribution - lbl = label.New(distributionRepositoryName, "", "pkg") + lbl := label.New(distributionRepositoryName, "", "pkg") return lbl.String(), true } } diff --git a/python/extensions.bzl b/python/extensions.bzl index 01f731f14f..75de4157bb 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -78,6 +78,7 @@ def _pip_impl(module_ctx): pip_repository_bzlmod( name = attr.name, requirements_lock = attr.requirements_lock, + incompatible_generate_aliases = attr.incompatible_generate_aliases, ) for name, requirement_line in requirements: diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index 982d8536ba..733142ba92 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -261,6 +261,52 @@ A requirements_lock attribute must be specified, or a platform-specific lockfile def _clean_pkg_name(name): return name.replace("-", "_").replace(".", "_").lower() +def _pkg_aliases(rctx, repo_name, bzl_packages): + """Create alias declarations for each python dependency. + + The aliases should be appended to the pip_repository BUILD.bazel file. These aliases + allow users to use requirement() without needed a corresponding `use_repo()` for each dep + when using bzlmod. + + Args: + rctx: the repository context. + repo_name: the repository name of the parent that is visible to the users. + bzl_packages: the list of packages to setup. + """ + for name in bzl_packages: + build_content = """package(default_visibility = ["//visibility:public"]) + +alias( + name = "{name}", + actual = "@{repo_name}_{dep}//:pkg", +) + +alias( + name = "pkg", + actual = "@{repo_name}_{dep}//:pkg", +) + +alias( + name = "whl", + actual = "@{repo_name}_{dep}//:whl", +) + +alias( + name = "data", + actual = "@{repo_name}_{dep}//:data", +) + +alias( + name = "dist_info", + actual = "@{repo_name}_{dep}//:dist_info", +) +""".format( + name = name, + repo_name = repo_name, + dep = name, + ) + rctx.file("{}/BUILD.bazel".format(name), build_content) + def _bzlmod_pkg_aliases(repo_name, bzl_packages): """Create alias declarations for each python dependency. @@ -314,16 +360,21 @@ def _pip_repository_bzlmod_impl(rctx): repo_name = rctx.attr.name.split("~")[-1] - build_contents = _BUILD_FILE_CONTENTS + _bzlmod_pkg_aliases(repo_name, bzl_packages) + build_contents = _BUILD_FILE_CONTENTS + + if rctx.attr.incompatible_generate_aliases: + _pkg_aliases(rctx, repo_name, bzl_packages) + else: + build_contents += _bzlmod_pkg_aliases(repo_name, bzl_packages) rctx.file("BUILD.bazel", build_contents) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}//:{}_pkg".format(repo_name, p) + "@@{}//{}".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}//:{}_whl".format(repo_name, p) + "@@{}//{}:whl".format(repo_name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) for p in bzl_packages ]), "%%NAME%%": rctx.attr.name, @@ -331,6 +382,10 @@ def _pip_repository_bzlmod_impl(rctx): }) pip_repository_bzlmod_attrs = { + "incompatible_generate_aliases": attr.bool( + default = False, + doc = "Allow generating aliases in '@pip//:' -> '@pip_//:pkg'. This replaces the aliases generated by the `bzlmod` tooling.", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", @@ -405,14 +460,17 @@ def _pip_repository_impl(rctx): if rctx.attr.python_interpreter_target: config["python_interpreter_target"] = str(rctx.attr.python_interpreter_target) + if rctx.attr.incompatible_generate_aliases: + _pkg_aliases(rctx, rctx.attr.name, bzl_packages) + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) rctx.template("requirements.bzl", rctx.attr._template, substitutions = { "%%ALL_REQUIREMENTS%%": _format_repr_list([ - "@{}_{}//:pkg".format(rctx.attr.name, p) + "@{}//{}".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:pkg".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ALL_WHL_REQUIREMENTS%%": _format_repr_list([ - "@{}_{}//:whl".format(rctx.attr.name, p) + "@{}//{}:whl".format(rctx.attr.name, p) if rctx.attr.incompatible_generate_aliases else "@{}_{}//:whl".format(rctx.attr.name, p) for p in bzl_packages ]), "%%ANNOTATIONS%%": _format_dict(_repr_dict(annotations)), @@ -520,6 +578,10 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), + "incompatible_generate_aliases": attr.bool( + default = False, + doc = "Allow generating aliases '@pip//' -> '@pip_//:pkg'.", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", From de8f428f4a73f39c875d1e03ba5d0be07790f534 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 5 Mar 2023 17:34:53 -0800 Subject: [PATCH 03/19] docs: Simplify pull request template (#1100) The existing template is very verbose with many of the lines not being applicable to any given PR. This also makes the PR description poorly suitable for a commit description. By having a cleaner PR description that is commit message friendly, whoever merges the PR can more easily create a meaningful commit message. It also allows other maintainer's to clean up the description prior to it being merged. --- .github/PULL_REQUEST_TEMPLATE.md | 55 +++++++------------------------- 1 file changed, 11 insertions(+), 44 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 38e0658e44..0d305b8816 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,44 +1,11 @@ -## PR Checklist - -Please check if your PR fulfills the following requirements: - -- [ ] Tests for the changes have been added (for bug fixes / features) -- [ ] Docs have been added / updated (for bug fixes / features) - - -## PR Type - -What kind of change does this PR introduce? - - - -- [ ] Bugfix -- [ ] Feature (please, look at the "Scope of the project" section in the README.md file) -- [ ] Code style update (formatting, local variables) -- [ ] Refactoring (no functional changes, no api changes) -- [ ] Build related changes -- [ ] CI related changes -- [ ] Documentation content changes -- [ ] Other... Please describe: - - -## What is the current behavior? - - -Issue Number: N/A - - -## What is the new behavior? - - -## Does this PR introduce a breaking change? - -- [ ] Yes -- [ ] No - - - - - -## Other information - +PR Instructions/requirements +* Title uses `type: description` format. See CONTRIBUTING.md for types. +* Common types are: build, docs, feat, fix, refactor, revert, test +* Breaking changes include "!" after the type and a "BREAKING CHANGES:" + section at the bottom. +* Body text describes: + * Why this change is being made, briefly. + * Before and after behavior, as applicable + * References issue number, as applicable +* Update docs and tests, as applicable +* Delete these instructions prior to sending the PR From c73dc0c4d92d9e794dc197cbc66158476e1b528a Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Mon, 6 Mar 2023 14:42:39 -0800 Subject: [PATCH 04/19] chore: fix syntax that stardoc misunderstands as HTML (#1110) Update requirements.bzl --- docs/pip.md | 4 ++-- python/pip_install/requirements.bzl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pip.md b/docs/pip.md index 528abf737d..e4c3f21b79 100644 --- a/docs/pip.md +++ b/docs/pip.md @@ -42,8 +42,8 @@ of some other compile_pip_requirements rule that references these requirements It also generates two targets for running pip-compile: -- validate with `bazel test <name>_test` -- update with `bazel run <name>.update` +- validate with `bazel test [name]_test` +- update with `bazel run [name].update` **PARAMETERS** diff --git a/python/pip_install/requirements.bzl b/python/pip_install/requirements.bzl index af3c194d18..dd38c9df5b 100644 --- a/python/pip_install/requirements.bzl +++ b/python/pip_install/requirements.bzl @@ -39,8 +39,8 @@ def compile_pip_requirements( It also generates two targets for running pip-compile: - - validate with `bazel test _test` - - update with `bazel run .update` + - validate with `bazel test [name]_test` + - update with `bazel run [name].update` Args: name: base name for generated targets, typically "requirements". From 244c6064f5f4e72432ec056c2ad1a0b068468a93 Mon Sep 17 00:00:00 2001 From: Mathieu Sabourin Date: Mon, 6 Mar 2023 16:22:00 -0800 Subject: [PATCH 05/19] fix: update gazelle to properly handle dot in package name. (#1083) --- gazelle/pythonconfig/BUILD.bazel | 8 ++++++- gazelle/pythonconfig/pythonconfig.go | 11 +++++++-- gazelle/pythonconfig/pythonconfig_test.go | 28 +++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 gazelle/pythonconfig/pythonconfig_test.go diff --git a/gazelle/pythonconfig/BUILD.bazel b/gazelle/pythonconfig/BUILD.bazel index 79b512163d..d0f1690d94 100644 --- a/gazelle/pythonconfig/BUILD.bazel +++ b/gazelle/pythonconfig/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "pythonconfig", @@ -15,6 +15,12 @@ go_library( ], ) +go_test( + name = "pythonconfig_test", + srcs = ["pythonconfig_test.go"], + deps = [":pythonconfig"], +) + filegroup( name = "distribution", srcs = glob(["**"]), diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index ea2ae65c4c..c7cd7c1a28 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -90,6 +90,14 @@ var defaultIgnoreFiles = map[string]struct{}{ "setup.py": {}, } +func SanitizeDistribution(distributionName string) string { + sanitizedDistribution := strings.ToLower(distributionName) + sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") + sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, ".", "_") + + return sanitizedDistribution +} + // Configs is an extension of map[string]*Config. It provides finding methods // on top of the mapping. type Configs map[string]*Config @@ -218,8 +226,7 @@ func (c *Config) FindThirdPartyDependency(modName string) (string, bool) { } else if gazelleManifest.PipRepository != nil { distributionRepositoryName = gazelleManifest.PipRepository.Name } - sanitizedDistribution := strings.ToLower(distributionName) - sanitizedDistribution = strings.ReplaceAll(sanitizedDistribution, "-", "_") + sanitizedDistribution := SanitizeDistribution(distributionName) if gazelleManifest.PipRepository != nil && gazelleManifest.PipRepository.UsePipRepositoryAliases { // @// diff --git a/gazelle/pythonconfig/pythonconfig_test.go b/gazelle/pythonconfig/pythonconfig_test.go new file mode 100644 index 0000000000..1512eb97ae --- /dev/null +++ b/gazelle/pythonconfig/pythonconfig_test.go @@ -0,0 +1,28 @@ +package pythonconfig + +import ( + "testing" + + "github.com/bazelbuild/rules_python/gazelle/pythonconfig" +) + +func TestDistributionSanitizing(t *testing.T) { + tests := map[string]struct { + input string + want string + }{ + "upper case": {input: "DistWithUpperCase", want: "distwithuppercase"}, + "dashes": {input: "dist-with-dashes", want: "dist_with_dashes"}, + "dots": {input: "dist.with.dots", want: "dist_with_dots"}, + "mixed": {input: "To-be.sanitized", want: "to_be_sanitized"}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := pythonconfig.SanitizeDistribution(tc.input) + if tc.want != got { + t.Fatalf("expected %q, got %q", tc.want, got) + } + }) + } +} From 0ba98a6e3bc529af57d871e8b7e3e9a14c5d707c Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Tue, 7 Mar 2023 11:11:08 -0800 Subject: [PATCH 06/19] fix(bzlmod): expose ignore_root_user_error attribute from python_register_toolchains (#1114) --- python/extensions.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/extensions.bzl b/python/extensions.bzl index 75de4157bb..42f0b58226 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -30,6 +30,7 @@ def _python_impl(module_ctx): # Toolchain registration in bzlmod is done in MODULE file register_toolchains = False, register_coverage_tool = attr.configure_coverage_tool, + ignore_root_user_error = attr.ignore_root_user_error, ) python = module_extension( @@ -41,6 +42,7 @@ python = module_extension( mandatory = False, doc = "Whether or not to configure the default coverage tool for the toolchains.", ), + "ignore_root_user_error": attr.bool(), "name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True), }, From 7ffe2f7f03737448e3dc7564e80b256bc54e653c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 7 Mar 2023 19:55:52 -0800 Subject: [PATCH 07/19] feat: add bzl_library for defs.bzl and its dependencies (#1115) This is so that the transitive dependencies of defs.bzl can be easily found and validated; some Google internal tooling does this validation. The old comment indicated bzl_library wasn't used to avoid a dependency on skylib, however, we've since added a dependency on skylib. Work towards #1069 --- python/BUILD.bazel | 14 +++++++++++++- python/private/BUILD.bazel | 20 +++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 2e275b6650..e5be3e8a53 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -23,6 +23,7 @@ In an ideal renaming, we'd move the packaging rules to a different package so that @rules_python//python is only concerned with the core rules. """ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load(":defs.bzl", "current_py_toolchain") package(default_visibility = ["//visibility:public"]) @@ -40,8 +41,19 @@ filegroup( visibility = ["//:__pkg__"], ) +bzl_library( + name = "defs_bzl", + srcs = [ + "defs.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + "//python/private:bazel_tools_bzl", + "//python/private:reexports_bzl", + ], +) + # Filegroup of bzl files that can be used by downstream rules for documentation generation -# Using a filegroup rather than bzl_library to not give a transitive dependency on Skylib filegroup( name = "bzl", srcs = [ diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 7d321ebbe7..f3278478b8 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python:versions.bzl", "print_toolchains_checksums") load(":stamp.bzl", "stamp_build_setting") @@ -24,13 +25,30 @@ filegroup( ) # Filegroup of bzl files that can be used by downstream rules for documentation generation -# Using a filegroup rather than bzl_library to not give a transitive dependency on Skylib filegroup( name = "bzl", srcs = glob(["**/*.bzl"]), visibility = ["//python:__pkg__"], ) +bzl_library( + name = "reexports_bzl", + srcs = ["reexports.bzl"], + visibility = ["//python:__pkg__"], + deps = [":bazel_tools_bzl"], +) + +# @bazel_tools can't define bzl_library itself, so we just put a wrapper around it. +bzl_library( + name = "bazel_tools_bzl", + srcs = [ + "@bazel_tools//tools/python:srcs_version.bzl", + "@bazel_tools//tools/python:toolchain.bzl", + "@bazel_tools//tools/python:utils.bzl", + ], + visibility = ["//python:__pkg__"], +) + # Needed to define bzl_library targets for docgen. (We don't define the # bzl_library target here because it'd give our users a transitive dependency # on Skylib.) From 8400610298056e9ab69cf927424a39ae93dfb31f Mon Sep 17 00:00:00 2001 From: Daniel Stonier Date: Tue, 7 Mar 2023 21:51:51 -0800 Subject: [PATCH 08/19] fix: docs for ignore_root_user_error at the module level (#1112) --- python/extensions.bzl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/extensions.bzl b/python/extensions.bzl index 42f0b58226..2b0c188554 100644 --- a/python/extensions.bzl +++ b/python/extensions.bzl @@ -42,7 +42,11 @@ python = module_extension( mandatory = False, doc = "Whether or not to configure the default coverage tool for the toolchains.", ), - "ignore_root_user_error": attr.bool(), + "ignore_root_user_error": attr.bool( + default = False, + doc = "Whether the check for root should be ignored or not. This causes cache misses with .pyc files.", + mandatory = False, + ), "name": attr.string(mandatory = True), "python_version": attr.string(mandatory = True), }, From 5ff514ab84c0486b432340e6a04de5beb9f2fd1f Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Wed, 8 Mar 2023 20:16:02 +0100 Subject: [PATCH 09/19] fix: generation of toolchain aliases //:defs.bzl file. (#1088) ## PR Checklist Please check if your PR fulfills the following requirements: - [ ] Tests for the changes have been added (for bug fixes / features) - [ ] Docs have been added / updated (for bug fixes / features) ## PR Type What kind of change does this PR introduce? - [X] Bugfix - [ ] Feature (please, look at the "Scope of the project" section in the README.md file) - [ ] Code style update (formatting, local variables) - [ ] Refactoring (no functional changes, no api changes) - [ ] Build related changes - [ ] CI related changes - [ ] Documentation content changes - [ ] Other... Please describe: ## What is the current behavior? With `common --experimental_enable_bzlmod` option defs.bzl is generated as ``` load("@rules_python~override//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") load("@rules_python~override//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") ``` and these lines cause a problem at ``` load("@python3_9//:defs.bzl", "interpreter") ``` as ``` ERROR: .../BUILD:25:11: error loading package 'src': at .../external/rules_python~override~python~python3_9/defs.bzl:4:6: Unable to find package for @[unknown repo 'rules_python~override' requested from @rules_python~override~python~python3_9]//python:pip.bzl: The repository '@[unknown repo 'rules_python~override' requested from @rules_python~override~python~python3_9]' could not be resolved: No repository visible as '@rules_python~override' from repository '@rules_python~override~python~python3_9'. and referenced by '...' ``` Issue Number: N/A ## What is the new behavior? Generated load statements ``` load("@@rules_python~override//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") load("@@rules_python~override//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") ``` ## Does this PR introduce a breaking change? - [ ] Yes - [X] No ## Other information --- examples/bzlmod/BUILD.bazel | 8 ++++++++ examples/bzlmod/MODULE.bazel | 1 + python/private/toolchains_repo.bzl | 17 +++++++++-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/examples/bzlmod/BUILD.bazel b/examples/bzlmod/BUILD.bazel index 7b7566bd5a..7ecc035853 100644 --- a/examples/bzlmod/BUILD.bazel +++ b/examples/bzlmod/BUILD.bazel @@ -1,4 +1,5 @@ load("@pip//:requirements.bzl", "requirement") +load("@python3_9//:defs.bzl", py_test_with_transition = "py_test") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") @@ -35,3 +36,10 @@ py_test( srcs = ["test.py"], deps = [":lib"], ) + +py_test_with_transition( + name = "test_with_transition", + srcs = ["test.py"], + main = "test.py", + deps = [":lib"], +) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 5f984c39df..ce9122810c 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -16,6 +16,7 @@ python.toolchain( configure_coverage_tool = True, python_version = "3.9", ) +use_repo(python, "python3_9") use_repo(python, "python3_9_toolchains") register_toolchains( diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 4b832d941a..9bed73e55c 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -31,10 +31,13 @@ load( "WINDOWS_NAME", ) +def get_repository_name(repository_workspace): + dummy_label = "//:_" + return str(repository_workspace.relative(dummy_label))[:-len(dummy_label)] or "@" + def _toolchains_repo_impl(rctx): - rules_python_repository_name = rctx.attr._rules_python_workspace.workspace_name - python_version_constraint = "@{rules_python}//python/config_settings:is_python_{python_version}".format( - rules_python = rules_python_repository_name, + python_version_constraint = "{rules_python}//python/config_settings:is_python_{python_version}".format( + rules_python = get_repository_name(rctx.attr._rules_python_workspace), python_version = rctx.attr.python_version, ) @@ -90,8 +93,6 @@ def _toolchain_aliases_impl(rctx): is_windows = (os_name == WINDOWS_NAME) python3_binary_path = "python.exe" if is_windows else "bin/python3" - rules_python_repository_name = rctx.attr._rules_python_workspace.workspace_name - # Base BUILD file for this repository. build_contents = """\ # Generated by python/private/toolchains_repo.bzl @@ -123,8 +124,8 @@ alias(name = "pip", actual = select({{":" + item: "@{py_repository}_ rctx.file("defs.bzl", content = """\ # Generated by python/private/toolchains_repo.bzl -load("@{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") -load("@{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") +load("{rules_python}//python/config_settings:transition.bzl", _py_binary = "py_binary", _py_test = "py_test") +load("{rules_python}//python:pip.bzl", _compile_pip_requirements = "compile_pip_requirements") host_platform = "{host_platform}" interpreter = "@{py_repository}_{host_platform}//:{python3_binary_path}" @@ -156,7 +157,7 @@ def compile_pip_requirements(name, **kwargs): py_repository = rctx.attr.user_repository_name, python_version = rctx.attr.python_version, python3_binary_path = python3_binary_path, - rules_python = rules_python_repository_name, + rules_python = get_repository_name(rctx.attr._rules_python_workspace), )) toolchain_aliases = repository_rule( From 1c5b92b279481ddcdfbd71058857e5eb3ebac724 Mon Sep 17 00:00:00 2001 From: Daniel Stonier Date: Wed, 8 Mar 2023 16:46:55 -0800 Subject: [PATCH 10/19] feat: make variable substitution for py_wheel abi, python_tag args (#1113) Expands make variables in to `abi` and `python_tag` attributes --------- Co-authored-by: Richard Levasseur --- examples/wheel/BUILD.bazel | 25 ++++++++++++++++++++++++- examples/wheel/private/wheel_utils.bzl | 17 +++++++++++++++++ python/private/py_wheel.bzl | 11 +++++++---- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/examples/wheel/BUILD.bazel b/examples/wheel/BUILD.bazel index 4124a826d1..61a43ae6cf 100644 --- a/examples/wheel/BUILD.bazel +++ b/examples/wheel/BUILD.bazel @@ -13,7 +13,7 @@ # limitations under the License. load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("//examples/wheel/private:wheel_utils.bzl", "directory_writer") +load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags") load("//python:defs.bzl", "py_library", "py_test") load("//python:packaging.bzl", "py_package", "py_wheel") load("//python:versions.bzl", "gen_python_config_settings") @@ -62,6 +62,29 @@ py_wheel( ], ) +# Populate a rule with "Make Variable" arguments for +# abi, python_tag and version. You might want to do this +# for the following use cases: +# - abi, python_tag: introspect a toolchain to map to appropriate cpython tags +# - version: populate given this or a dependent module's version +make_variable_tags( + name = "make_variable_tags", +) + +py_wheel( + name = "minimal_with_py_library_with_make_variables", + testonly = True, + abi = "$(ABI)", + distribution = "example_minimal_library", + python_tag = "$(PYTHON_TAG)", + toolchains = ["//examples/wheel:make_variable_tags"], + version = "$(VERSION)", + deps = [ + "//examples/wheel/lib:module_with_data", + "//examples/wheel/lib:simple_module", + ], +) + build_test( name = "dist_build_tests", targets = [":minimal_with_py_library.dist"], diff --git a/examples/wheel/private/wheel_utils.bzl b/examples/wheel/private/wheel_utils.bzl index af4fa1958b..037fed0175 100644 --- a/examples/wheel/private/wheel_utils.bzl +++ b/examples/wheel/private/wheel_utils.bzl @@ -54,3 +54,20 @@ directory_writer = rule( ), }, ) + +def _make_variable_tags_impl(ctx): # buildifier: disable=unused-variable + # This example is contrived. In a real usage, this rule would + # look at flags or dependencies to determine what values to use. + # If all you're doing is setting constant values, then you can simply + # set them in the py_wheel() call. + vars = {} + vars["ABI"] = "cp38" + vars["PYTHON_TAG"] = "cp38" + vars["VERSION"] = "0.99.0" + return [platform_common.TemplateVariableInfo(vars)] + +make_variable_tags = rule( + attrs = {}, + doc = """Make variable tags to pass to a py_wheel rule.""", + implementation = _make_variable_tags_impl, +) diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index 77690edc65..b6f2bfae56 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -207,12 +207,15 @@ def _input_file_to_arg(input_file): return "%s;%s" % (py_package_lib.path_inside_wheel(input_file), input_file.path) def _py_wheel_impl(ctx): + abi = _replace_make_variables(ctx.attr.abi, ctx) + python_tag = _replace_make_variables(ctx.attr.python_tag, ctx) version = _replace_make_variables(ctx.attr.version, ctx) + outfile = ctx.actions.declare_file("-".join([ _escape_filename_segment(ctx.attr.distribution), _escape_filename_segment(version), - _escape_filename_segment(ctx.attr.python_tag), - _escape_filename_segment(ctx.attr.abi), + _escape_filename_segment(python_tag), + _escape_filename_segment(abi), _escape_filename_segment(ctx.attr.platform), ]) + ".whl") @@ -237,8 +240,8 @@ def _py_wheel_impl(ctx): args = ctx.actions.args() args.add("--name", ctx.attr.distribution) args.add("--version", version) - args.add("--python_tag", ctx.attr.python_tag) - args.add("--abi", ctx.attr.abi) + args.add("--python_tag", python_tag) + args.add("--abi", abi) args.add("--platform", ctx.attr.platform) args.add("--out", outfile) args.add("--name_file", name_file) From 3aa221f82edec3ea2c7b80462e6e609b7aa1276a Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 8 Mar 2023 16:50:55 -0800 Subject: [PATCH 11/19] feat: add bzl_library for proto.bzl (#1116) This is mostly so Google internal tooling can find the complete deps, but also as a best practice for consumption by other rules and tools. --- python/BUILD.bazel | 11 +++++++++++ python/private/proto/BUILD | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/python/BUILD.bazel b/python/BUILD.bazel index e5be3e8a53..d30f0db24a 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -53,6 +53,17 @@ bzl_library( ], ) +bzl_library( + name = "proto_bzl", + srcs = [ + "proto.bzl", + ], + visibility = ["//visibility:public"], + deps = [ + "//python/private/proto:py_proto_library_bzl", + ], +) + # Filegroup of bzl files that can be used by downstream rules for documentation generation filegroup( name = "bzl", diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD index 8483d19c2f..139696cde2 100644 --- a/python/private/proto/BUILD +++ b/python/private/proto/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") package(default_visibility = ["//visibility:public"]) @@ -24,6 +25,16 @@ filegroup( visibility = ["//python/private:__pkg__"], ) +bzl_library( + name = "py_proto_library_bzl", + srcs = ["py_proto_library.bzl"], + visibility = ["//python:__pkg__"], + deps = [ + "//python:defs_bzl", + "@rules_proto//proto:defs", + ], +) + proto_lang_toolchain( name = "python_toolchain", command_line = "--python_out=%s", From 25e4175abc994d8942a41d5ac60a0603dc033f3c Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 9 Mar 2023 13:06:23 -0800 Subject: [PATCH 12/19] cleanup: Remove license comment in proto build file (#1118) The license comments aren't necessary anymore. --- python/private/proto/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD index 139696cde2..971b354e34 100644 --- a/python/private/proto/BUILD +++ b/python/private/proto/BUILD @@ -17,7 +17,7 @@ load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) filegroup( name = "distribution", From 9ef11b9fabea5fb2d2068141eaa7513486965f75 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 9 Mar 2023 13:06:43 -0800 Subject: [PATCH 13/19] fix: restrict proto package visibility to private (#1117) This is to prevent accidentally leaking targets that shouldn't be accessible. The `:python_toolchain` target is public because it's an implicit dependency of `py_proto_library`. --- python/private/proto/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD index 971b354e34..65c09444f7 100644 --- a/python/private/proto/BUILD +++ b/python/private/proto/BUILD @@ -15,7 +15,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") -package(default_visibility = ["//visibility:public"]) +package(default_visibility = ["//visibility:private"]) licenses(["notice"]) @@ -40,4 +40,7 @@ proto_lang_toolchain( command_line = "--python_out=%s", progress_message = "Generating Python proto_library %{label}", runtime = "@com_google_protobuf//:protobuf_python", + # NOTE: This isn't *actually* public. It's an implicit dependency of py_proto_library, + # so must be public so user usages of the rule can reference it. + visibility = ["//visibility:public"], ) From 31d0efd69e27c93d57f8b2b5664867e1e21828c6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 9 Mar 2023 13:40:32 -0800 Subject: [PATCH 14/19] cleanup: rename proto BUILD -> BUILD.bazel (#1119) The rest of the project uses BUILD.bazel --- python/private/proto/{BUILD => BUILD.bazel} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/private/proto/{BUILD => BUILD.bazel} (100%) diff --git a/python/private/proto/BUILD b/python/private/proto/BUILD.bazel similarity index 100% rename from python/private/proto/BUILD rename to python/private/proto/BUILD.bazel From 756264aa526eba8eeb4badfa8109050a1f6e0966 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 9 Mar 2023 14:38:05 -0800 Subject: [PATCH 15/19] feat: bzl file per rule/provider (#1122) This is basically a performance optimization. Bazel can only see that a bzl file changed, not the particular contents. This means that any downstream bzl file loading it is invalidated, even if it doesn't load any of the affected code. As an example, if a package only loads `py_library.bzl`, then changing `py_test.bzl` doesn't need to invalidate all libraries. * Also removes some more extraneous license comments Work towards #1069 --- docs/BUILD.bazel | 6 ++- python/BUILD.bazel | 64 +++++++++++++++++++++- python/current_py_toolchain.bzl | 58 ++++++++++++++++++++ python/defs.bzl | 95 ++------------------------------- python/private/BUILD.bazel | 5 +- python/py_binary.bzl | 19 +++++++ python/py_import.bzl | 67 +++++++++++++++++++++++ python/py_info.bzl | 19 +++++++ python/py_library.bzl | 19 +++++++ python/py_runtime.bzl | 19 +++++++ python/py_runtime_info.bzl | 19 +++++++ python/py_runtime_pair.bzl | 19 +++++++ python/py_test.bzl | 19 +++++++ tests/BUILD.bazel | 21 +++++++- 14 files changed, 353 insertions(+), 96 deletions(-) create mode 100644 python/current_py_toolchain.bzl create mode 100644 python/py_binary.bzl create mode 100644 python/py_import.bzl create mode 100644 python/py_info.bzl create mode 100644 python/py_library.bzl create mode 100644 python/py_runtime.bzl create mode 100644 python/py_runtime_info.bzl create mode 100644 python/py_runtime_pair.bzl create mode 100644 python/py_test.bzl diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index d2f0b04b56..e1163d9d0e 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -56,7 +56,11 @@ bzl_library( "//python:defs.bzl", "//python/private:reexports.bzl", ], - deps = [":bazel_python_tools"], + deps = [ + ":bazel_python_tools", + "//python:defs_bzl", + "//python/private:reexports_bzl", + ], ) bzl_library( diff --git a/python/BUILD.bazel b/python/BUILD.bazel index d30f0db24a..4d75b781ba 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -24,11 +24,11 @@ that @rules_python//python is only concerned with the core rules. """ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load(":defs.bzl", "current_py_toolchain") +load(":current_py_toolchain.bzl", "current_py_toolchain") package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) filegroup( name = "distribution", @@ -41,6 +41,13 @@ filegroup( visibility = ["//:__pkg__"], ) +# ========= bzl_library targets end ========= + +bzl_library( + name = "current_py_toolchain_bzl", + srcs = ["current_py_toolchain.bzl"], +) + bzl_library( name = "defs_bzl", srcs = [ @@ -48,6 +55,8 @@ bzl_library( ], visibility = ["//visibility:public"], deps = [ + ":current_py_toolchain_bzl", + ":py_import_bzl", "//python/private:bazel_tools_bzl", "//python/private:reexports_bzl", ], @@ -64,6 +73,57 @@ bzl_library( ], ) +bzl_library( + name = "py_binary_bzl", + srcs = ["py_binary.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_import_bzl", + srcs = ["py_import.bzl"], + deps = [":py_info_bzl"], +) + +bzl_library( + name = "py_info_bzl", + srcs = ["py_info.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_library_bzl", + srcs = ["py_library.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_bzl", + srcs = ["py_runtime.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_pair_bzl", + srcs = ["py_runtime_pair.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_runtime_info_bzl", + srcs = ["py_runtime_info.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +bzl_library( + name = "py_test_bzl", + srcs = ["py_test.bzl"], + deps = ["//python/private:reexports_bzl"], +) + +# NOTE: Remember to add bzl_library targets to //tests:bzl_libraries +# ========= bzl_library targets end ========= + # Filegroup of bzl files that can be used by downstream rules for documentation generation filegroup( name = "bzl", diff --git a/python/current_py_toolchain.bzl b/python/current_py_toolchain.bzl new file mode 100644 index 0000000000..e3345cb646 --- /dev/null +++ b/python/current_py_toolchain.bzl @@ -0,0 +1,58 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for current_py_toolchain rule.""" + +def _current_py_toolchain_impl(ctx): + toolchain = ctx.toolchains[ctx.attr._toolchain] + + direct = [] + transitive = [] + vars = {} + + if toolchain.py3_runtime and toolchain.py3_runtime.interpreter: + direct.append(toolchain.py3_runtime.interpreter) + transitive.append(toolchain.py3_runtime.files) + vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path + + if toolchain.py2_runtime and toolchain.py2_runtime.interpreter: + direct.append(toolchain.py2_runtime.interpreter) + transitive.append(toolchain.py2_runtime.files) + vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path + + files = depset(direct, transitive = transitive) + return [ + toolchain, + platform_common.TemplateVariableInfo(vars), + DefaultInfo( + runfiles = ctx.runfiles(transitive_files = files), + files = files, + ), + ] + +current_py_toolchain = rule( + doc = """ + This rule exists so that the current python toolchain can be used in the `toolchains` attribute of + other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has + happened, to a rule which expects a concrete implementation of a toolchain, rather than a + toolchain_type which could be resolved to that toolchain. + """, + implementation = _current_py_toolchain_impl, + attrs = { + "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))), + }, + toolchains = [ + str(Label("@bazel_tools//tools/python:toolchain_type")), + ], +) diff --git a/python/defs.bzl b/python/defs.bzl index 7b60c6513b..e106166f03 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -27,6 +27,8 @@ load( _py_runtime_pair = "py_runtime_pair", _py_test = "py_test", ) +load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain") +load(":py_import.bzl", _py_import = "py_import") # Exports of native-defined providers. @@ -34,98 +36,9 @@ PyInfo = internal_PyInfo PyRuntimeInfo = internal_PyRuntimeInfo -def _current_py_toolchain_impl(ctx): - toolchain = ctx.toolchains[ctx.attr._toolchain] +current_py_toolchain = _current_py_toolchain - direct = [] - transitive = [] - vars = {} - - if toolchain.py3_runtime and toolchain.py3_runtime.interpreter: - direct.append(toolchain.py3_runtime.interpreter) - transitive.append(toolchain.py3_runtime.files) - vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path - - if toolchain.py2_runtime and toolchain.py2_runtime.interpreter: - direct.append(toolchain.py2_runtime.interpreter) - transitive.append(toolchain.py2_runtime.files) - vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path - - files = depset(direct, transitive = transitive) - return [ - toolchain, - platform_common.TemplateVariableInfo(vars), - DefaultInfo( - runfiles = ctx.runfiles(transitive_files = files), - files = files, - ), - ] - -current_py_toolchain = rule( - doc = """ - This rule exists so that the current python toolchain can be used in the `toolchains` attribute of - other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has - happened, to a rule which expects a concrete implementation of a toolchain, rather than a - toolchain_type which could be resolved to that toolchain. - """, - implementation = _current_py_toolchain_impl, - attrs = { - "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))), - }, - toolchains = [ - str(Label("@bazel_tools//tools/python:toolchain_type")), - ], -) - -def _py_import_impl(ctx): - # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 . - import_paths = [ - "/".join([ctx.workspace_name, x.short_path]) - for x in ctx.files.srcs - ] - - return [ - DefaultInfo( - default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True), - ), - PyInfo( - transitive_sources = depset(transitive = [ - dep[PyInfo].transitive_sources - for dep in ctx.attr.deps - ]), - imports = depset(direct = import_paths, transitive = [ - dep[PyInfo].imports - for dep in ctx.attr.deps - ]), - ), - ] - -py_import = rule( - doc = """This rule allows the use of Python packages as dependencies. - - It imports the given `.egg` file(s), which might be checked in source files, - fetched externally as with `http_file`, or produced as outputs of other rules. - - It may be used like a `py_library`, in the `deps` of other Python rules. - - This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import). - """, - implementation = _py_import_impl, - attrs = { - "deps": attr.label_list( - doc = "The list of other libraries to be linked in to the " + - "binary target.", - providers = [PyInfo], - ), - "srcs": attr.label_list( - doc = "The list of Python package files provided to Python targets " + - "that depend on this target. Note that currently only the .egg " + - "format is accepted. For .whl files, try the whl_library rule. " + - "We accept contributions to extend py_import to handle .whl.", - allow_files = [".egg"], - ), - }, -) +py_import = _py_import # Re-exports of Starlark-defined symbols in @bazel_tools//tools/python. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index f3278478b8..21e3c1623f 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -34,7 +34,10 @@ filegroup( bzl_library( name = "reexports_bzl", srcs = ["reexports.bzl"], - visibility = ["//python:__pkg__"], + visibility = [ + "//docs:__pkg__", + "//python:__pkg__", + ], deps = [":bazel_tools_bzl"], ) diff --git a/python/py_binary.bzl b/python/py_binary.bzl new file mode 100644 index 0000000000..9d145d8fa6 --- /dev/null +++ b/python/py_binary.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_binary.""" + +load("//python/private:reexports.bzl", _py_binary = "py_binary") + +py_binary = _py_binary diff --git a/python/py_import.bzl b/python/py_import.bzl new file mode 100644 index 0000000000..c9284121d6 --- /dev/null +++ b/python/py_import.bzl @@ -0,0 +1,67 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_import rule.""" + +load(":py_info.bzl", "PyInfo") + +def _py_import_impl(ctx): + # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 . + import_paths = [ + "/".join([ctx.workspace_name, x.short_path]) + for x in ctx.files.srcs + ] + + return [ + DefaultInfo( + default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True), + ), + PyInfo( + transitive_sources = depset(transitive = [ + dep[PyInfo].transitive_sources + for dep in ctx.attr.deps + ]), + imports = depset(direct = import_paths, transitive = [ + dep[PyInfo].imports + for dep in ctx.attr.deps + ]), + ), + ] + +py_import = rule( + doc = """This rule allows the use of Python packages as dependencies. + + It imports the given `.egg` file(s), which might be checked in source files, + fetched externally as with `http_file`, or produced as outputs of other rules. + + It may be used like a `py_library`, in the `deps` of other Python rules. + + This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import). + """, + implementation = _py_import_impl, + attrs = { + "deps": attr.label_list( + doc = "The list of other libraries to be linked in to the " + + "binary target.", + providers = [PyInfo], + ), + "srcs": attr.label_list( + doc = "The list of Python package files provided to Python targets " + + "that depend on this target. Note that currently only the .egg " + + "format is accepted. For .whl files, try the whl_library rule. " + + "We accept contributions to extend py_import to handle .whl.", + allow_files = [".egg"], + ), + }, +) diff --git a/python/py_info.bzl b/python/py_info.bzl new file mode 100644 index 0000000000..2c3997dee2 --- /dev/null +++ b/python/py_info.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for PyInfo.""" + +load("//python/private:reexports.bzl", "internal_PyInfo") + +PyInfo = internal_PyInfo diff --git a/python/py_library.bzl b/python/py_library.bzl new file mode 100644 index 0000000000..1aff68c100 --- /dev/null +++ b/python/py_library.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_library.""" + +load("//python/private:reexports.bzl", _py_library = "py_library") + +py_library = _py_library diff --git a/python/py_runtime.bzl b/python/py_runtime.bzl new file mode 100644 index 0000000000..5e80308176 --- /dev/null +++ b/python/py_runtime.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_runtime.""" + +load("//python/private:reexports.bzl", _py_runtime = "py_runtime") + +py_runtime = _py_runtime diff --git a/python/py_runtime_info.bzl b/python/py_runtime_info.bzl new file mode 100644 index 0000000000..15598ee903 --- /dev/null +++ b/python/py_runtime_info.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for PyRuntimeInfo.""" + +load("//python/private:reexports.bzl", "internal_PyRuntimeInfo") + +PyRuntimeInfo = internal_PyRuntimeInfo diff --git a/python/py_runtime_pair.bzl b/python/py_runtime_pair.bzl new file mode 100644 index 0000000000..3f3ecf443b --- /dev/null +++ b/python/py_runtime_pair.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_runtime_pair.""" + +load("//python/private:reexports.bzl", _py_runtime_pair = "py_runtime_pair") + +py_runtime_pair = _py_runtime_pair diff --git a/python/py_test.bzl b/python/py_test.bzl new file mode 100644 index 0000000000..84470bc3af --- /dev/null +++ b/python/py_test.bzl @@ -0,0 +1,19 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Public entry point for py_test.""" + +load("//python/private:reexports.bzl", _py_test = "py_test") + +py_test = _py_test diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index ee9c5550e8..f5133c4e65 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,8 +1,9 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//tools/bazel_integration_test:bazel_integration_test.bzl", "bazel_integration_test") package(default_visibility = ["//visibility:public"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) bazel_integration_test( name = "pip_repository_entry_points_example", @@ -10,3 +11,21 @@ bazel_integration_test( # The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260 tags = ["fix-windows"], ) + +build_test( + name = "bzl_libaries_build_test", + targets = [ + # keep sorted + "//python:current_py_toolchain_bzl", + "//python:defs_bzl", + "//python:proto_bzl", + "//python:py_binary_bzl", + "//python:py_import_bzl", + "//python:py_info_bzl", + "//python:py_library_bzl", + "//python:py_runtime_bzl", + "//python:py_runtime_info_bzl", + "//python:py_runtime_pair_bzl", + "//python:py_test_bzl", + ], +) From c0c08a33a1c5d63cb4454fe430ade3a4eb7f04df Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 10 Mar 2023 16:21:24 -0800 Subject: [PATCH 16/19] cleanup: fix typo: libraries, not libaries (#1127) Silent r's are tricky. --- tests/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index f5133c4e65..a196d2af7f 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -13,7 +13,7 @@ bazel_integration_test( ) build_test( - name = "bzl_libaries_build_test", + name = "bzl_libraries_build_test", targets = [ # keep sorted "//python:current_py_toolchain_bzl", From 4e3d01c9f6a6cda0fb34b7c2961d6b1ef5313218 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 10 Mar 2023 18:13:12 -0800 Subject: [PATCH 17/19] feat: add public entry point for PyCcLinkParamsInfo (#1128) This provides a public entry point for loading the underlying `PyCcLinkParamsProvider` provider that is built into Bazel. This provider isn't yet usable from Bazel, but adding a loadable way for it to migrate off the built-in rules is the first step. Work towards #1069 --- python/BUILD.bazel | 5 +++++ python/py_cc_link_params_info.bzl | 3 +++ tests/BUILD.bazel | 1 + 3 files changed, 9 insertions(+) create mode 100644 python/py_cc_link_params_info.bzl diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 4d75b781ba..a524d2ff94 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -79,6 +79,11 @@ bzl_library( deps = ["//python/private:reexports_bzl"], ) +bzl_library( + name = "py_cc_link_params_info_bzl", + srcs = ["py_cc_link_params_info.bzl"], +) + bzl_library( name = "py_import_bzl", srcs = ["py_import.bzl"], diff --git a/python/py_cc_link_params_info.bzl b/python/py_cc_link_params_info.bzl new file mode 100644 index 0000000000..0ebd64b208 --- /dev/null +++ b/python/py_cc_link_params_info.bzl @@ -0,0 +1,3 @@ +"""Public entry point for PyCcLinkParamsInfo.""" + +PyCcLinkParamsInfo = PyCcLinkParamsProvider diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index a196d2af7f..abbe62ddff 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -20,6 +20,7 @@ build_test( "//python:defs_bzl", "//python:proto_bzl", "//python:py_binary_bzl", + "//python:py_cc_link_params_info_bzl", "//python:py_import_bzl", "//python:py_info_bzl", "//python:py_library_bzl", From 31bc04bf93d555f8a9842c0ba2d3d243c4ac6cd4 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 13 Mar 2023 13:29:30 -0700 Subject: [PATCH 18/19] cleanup: reformat defs.bzl doc string. (#1126) The extraneous newlines were bothering me. --- docs/python.md | 2 -- python/defs.bzl | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/python.md b/docs/python.md index 6682e48bd1..e42375ad60 100755 --- a/docs/python.md +++ b/docs/python.md @@ -1,9 +1,7 @@ - Core rules for building Python projects. - ## current_py_toolchain diff --git a/python/defs.bzl b/python/defs.bzl index e106166f03..ec70c1bf86 100644 --- a/python/defs.bzl +++ b/python/defs.bzl @@ -11,10 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -""" -Core rules for building Python projects. -""" +"""Core rules for building Python projects.""" load("@bazel_tools//tools/python:srcs_version.bzl", _find_requirements = "find_requirements") load( From c394c46fc1b21853bc68a7a47c1fe2db828d1dd0 Mon Sep 17 00:00:00 2001 From: John Laxson Date: Thu, 16 Mar 2023 12:34:27 -0700 Subject: [PATCH 19/19] fix: Include filename when parsing imports for gazelle (#1133) --- gazelle/python/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gazelle/python/parse.py b/gazelle/python/parse.py index 5cf0b89868..6c0ef69598 100644 --- a/gazelle/python/parse.py +++ b/gazelle/python/parse.py @@ -27,7 +27,7 @@ def parse_import_statements(content, filepath): modules = list() - tree = ast.parse(content) + tree = ast.parse(content, filename=filepath) for node in ast.walk(tree): if isinstance(node, ast.Import): for subnode in node.names: