Skip to content

Refactor and separate concerns of external python package handling code #953

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jan 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/keith/pre-commit-buildifier
rev: 5.1.0.1
rev: 6.0.0
hooks:
- id: buildifier
args: &args
Expand Down
4 changes: 1 addition & 3 deletions MODULE.bazel
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
module(
name = "rules_python",
compatibility_level = 1,
version = "0.0.0",
compatibility_level = 1,
)

bazel_dep(name = "platforms", version = "0.0.4")

internal_deps = use_extension("@rules_python//python:extensions.bzl", "internal_deps")

internal_deps.install()

use_repo(
internal_deps,
"pypi__build",
Expand Down
7 changes: 1 addition & 6 deletions examples/bzlmod/MODULE.bazel
Original file line number Diff line number Diff line change
@@ -1,35 +1,30 @@
module(
name = "example_bzlmod",
compatibility_level = 1,
version = "0.0.0",
compatibility_level = 1,
)

bazel_dep(name = "rules_python", version = "0.0.0")

local_path_override(
module_name = "rules_python",
path = "../..",
)

python = use_extension("@rules_python//python:extensions.bzl", "python")

python.toolchain(
name = "python3_9",
python_version = "3.9",
)

use_repo(python, "python3_9_toolchains")

register_toolchains(
"@python3_9_toolchains//:all",
)

pip = use_extension("@rules_python//python:extensions.bzl", "pip")

pip.parse(
name = "pip",
requirements_lock = "//:requirements_lock.txt",
requirements_windows = "//:requirements_windows.txt",
)

use_repo(pip, "pip")
2 changes: 1 addition & 1 deletion python/extensions.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ def _pip_impl(module_ctx):
environment = attr.environment,
)

# Keep in sync with python/pip_install/extract_wheels/bazel.py
# Keep in sync with python/pip_install/tools/bazel.py
def _sanitize_name(name):
return name.replace("-", "_").replace(".", "_").lower()

Expand Down
13 changes: 8 additions & 5 deletions python/pip_install/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
exports_files(["pip_compile.py"])

filegroup(
name = "distribution",
srcs = glob(["*.bzl"]) + [
"BUILD.bazel",
"pip_compile.py",
"//python/pip_install/extract_wheels:distribution",
"//python/pip_install/tools/dependency_resolver:distribution",
"//python/pip_install/tools/lib:distribution",
"//python/pip_install/tools/lock_file_generator:distribution",
"//python/pip_install/tools/wheel_installer:distribution",
"//python/pip_install/private:distribution",
],
visibility = ["//:__pkg__"],
Expand All @@ -22,7 +22,10 @@ filegroup(
filegroup(
name = "py_srcs",
srcs = [
"//python/pip_install/extract_wheels:py_srcs",
"//python/pip_install/tools/dependency_resolver:py_srcs",
"//python/pip_install/tools/lib:py_srcs",
"//python/pip_install/tools/lock_file_generator:py_srcs",
"//python/pip_install/tools/wheel_installer:py_srcs",
],
visibility = ["//python/pip_install/private:__pkg__"],
)
Expand Down
4 changes: 2 additions & 2 deletions python/pip_install/pip_repository.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ def _pip_repository_impl(rctx):
args = [
python_interpreter,
"-m",
"python.pip_install.extract_wheels.parse_requirements_to_bzl",
"python.pip_install.tools.lock_file_generator.lock_file_generator",
"--requirements_lock",
rctx.path(requirements_txt),
"--requirements_lock_label",
Expand Down Expand Up @@ -522,7 +522,7 @@ def _whl_library_impl(rctx):
args = [
python_interpreter,
"-m",
"python.pip_install.extract_wheels.wheel_installer",
"python.pip_install.tools.wheel_installer.wheel_installer",
"--requirement",
rctx.attr.requirement,
"--repo",
Expand Down
4 changes: 2 additions & 2 deletions python/pip_install/private/pip_install_utils.bzl
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
"""Utilities for `rules_python` pip rules"""

_SRCS_TEMPLATE = """\
\"\"\"A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
\"\"\"A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
"\"\"
Expand Down
23 changes: 13 additions & 10 deletions python/pip_install/private/srcs.bzl
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
"""A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
"""A generated file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules

This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.update` target. Please
`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
"""

# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
# sources changed.
PIP_INSTALL_PY_SRCS = [
"@rules_python//python/pip_install/extract_wheels:__init__.py",
"@rules_python//python/pip_install/extract_wheels:annotation.py",
"@rules_python//python/pip_install/extract_wheels:arguments.py",
"@rules_python//python/pip_install/extract_wheels:bazel.py",
"@rules_python//python/pip_install/extract_wheels:namespace_pkgs.py",
"@rules_python//python/pip_install/extract_wheels:parse_requirements_to_bzl.py",
"@rules_python//python/pip_install/extract_wheels:wheel.py",
"@rules_python//python/pip_install/extract_wheels:wheel_installer.py",
"@rules_python//python/pip_install/tools/dependency_resolver:__init__.py",
"@rules_python//python/pip_install/tools/dependency_resolver:dependency_resolver.py",
"@rules_python//python/pip_install/tools/lib:__init__.py",
"@rules_python//python/pip_install/tools/lib:annotation.py",
"@rules_python//python/pip_install/tools/lib:arguments.py",
"@rules_python//python/pip_install/tools/lib:bazel.py",
"@rules_python//python/pip_install/tools/lock_file_generator:__init__.py",
"@rules_python//python/pip_install/tools/lock_file_generator:lock_file_generator.py",
"@rules_python//python/pip_install/tools/wheel_installer:namespace_pkgs.py",
"@rules_python//python/pip_install/tools/wheel_installer:wheel.py",
"@rules_python//python/pip_install/tools/wheel_installer:wheel_installer.py",
]
2 changes: 1 addition & 1 deletion python/pip_install/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ py_library(
srcs = glob(["**/*.py"]),
data = glob(["**/*"], exclude=[
# These entries include those put into user-installed dependencies by
# data_exclude in /python/pip_install/extract_wheels/bazel.py
# data_exclude in /python/pip_install/tools/bazel.py
# to avoid non-determinism following pip install's behavior.
"**/*.py",
"**/*.pyc",
Expand Down
2 changes: 1 addition & 1 deletion python/pip_install/requirements.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def compile_pip_requirements(

# Use the Label constructor so this is expanded in the context of the file
# where it appears, which is to say, in @rules_python
pip_compile = Label("//python/pip_install:pip_compile.py")
pip_compile = Label("//python/pip_install/tools/dependency_resolver:dependency_resolver.py")

loc = "$(rootpath {})"

Expand Down
19 changes: 19 additions & 0 deletions python/pip_install/tools/dependency_resolver/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
exports_files(["dependency_resolver.py"])

filegroup(
name = "distribution",
srcs = glob(
["*"],
exclude = ["*_test.py"],
),
visibility = ["//python/pip_install:__subpackages__"],
)

filegroup(
name = "py_srcs",
srcs = glob(
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
visibility = ["//python/pip_install:__subpackages__"],
)
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("//python/pip_install:repositories.bzl", "requirement")
load("//python:defs.bzl", "py_library", "py_test")
load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file")

py_library(
Expand All @@ -8,31 +7,8 @@ py_library(
"annotation.py",
"arguments.py",
"bazel.py",
"namespace_pkgs.py",
"parse_requirements_to_bzl.py",
"wheel.py",
"wheel_installer.py",
],
deps = [
requirement("installer"),
requirement("setuptools"),
],
)

py_binary(
name = "wheel_installer",
srcs = [
"wheel_installer.py",
],
deps = [":lib"],
)

py_binary(
name = "parse_requirements_to_bzl",
srcs = [
"parse_requirements_to_bzl.py",
],
deps = [":lib"],
visibility = ["//python/pip_install:__subpackages__"],
)

package_annotations_file(
Expand Down Expand Up @@ -70,57 +46,18 @@ py_test(
srcs = ["annotations_test.py"],
data = [":mock_annotations"],
env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"},
tags = ["unit"],
deps = [
":lib",
"//python/runfiles",
],
)

py_test(
name = "namespace_pkgs_test",
size = "small",
srcs = [
"namespace_pkgs_test.py",
],
tags = ["unit"],
deps = [
":lib",
],
)

py_test(
name = "wheel_installer_test",
size = "small",
srcs = [
"wheel_installer_test.py",
],
data = ["//examples/wheel:minimal_with_py_package"],
tags = ["unit"],
deps = [
":lib",
],
)

py_test(
name = "arguments_test",
size = "small",
srcs = [
"arguments_test.py",
],
tags = ["unit"],
deps = [
":lib",
],
)

py_test(
name = "parse_requirements_to_bzl_test",
size = "small",
srcs = [
"parse_requirements_to_bzl_test.py",
],
tags = ["unit"],
deps = [
":lib",
],
Expand Down
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import unittest
from pathlib import Path

from python.pip_install.extract_wheels.annotation import Annotation, AnnotationsMap
from python.pip_install.tools.lib.annotation import Annotation, AnnotationsMap
from python.runfiles import runfiles


Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Helper macros and rules for testing the `annotations` module of `extract_wheels`"""
"""Helper macros and rules for testing the `annotations` module of `tools`"""

load("//python:pip.bzl", _package_annotation = "package_annotation")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import json
import unittest

from python.pip_install.extract_wheels import arguments
from python.pip_install.tools.lib import arguments


class ArgumentsTestCase(unittest.TestCase):
Expand Down
50 changes: 50 additions & 0 deletions python/pip_install/tools/lock_file_generator/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
load("//python:defs.bzl", "py_binary", "py_library", "py_test")
load("//python/pip_install:repositories.bzl", "requirement")

py_library(
name = "lib",
srcs = [
"lock_file_generator.py",
],
deps = [
"//python/pip_install/tools/lib",
requirement("pip"),
],
)

py_binary(
name = "lock_file_generator",
srcs = [
"lock_file_generator.py",
],
deps = [":lib"],
)

py_test(
name = "lock_file_generator_test",
size = "small",
srcs = [
"lock_file_generator_test.py",
],
deps = [
":lib",
],
)

filegroup(
name = "distribution",
srcs = glob(
["*"],
exclude = ["*_test.py"],
),
visibility = ["//python/pip_install:__subpackages__"],
)

filegroup(
name = "py_srcs",
srcs = glob(
include = ["**/*.py"],
exclude = ["**/*_test.py"],
),
visibility = ["//python/pip_install:__subpackages__"],
)
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
)
from pip._internal.req.req_install import InstallRequirement

from python.pip_install.extract_wheels import annotation, arguments, bazel
from python.pip_install.tools.lib import annotation, arguments, bazel


def parse_install_requirements(
Expand Down
Loading