Skip to content

Commit aef17ad

Browse files
BoleynSualexeagle
BoleynSu
and
alexeagle
authored
pip_compile: remove external/workspace_name prefix from generated requirements.txt (#690)
* pip_compile: remove external/workspace_name prefix from generated requirements.txt * add some tests and a demo impl of pip_deps * update pinned requirement Co-authored-by: Alex Eagle <alex@aspect.dev>
1 parent dae610b commit aef17ad

File tree

10 files changed

+256
-4
lines changed

10 files changed

+256
-4
lines changed

‎.bazelci/presubmit.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,6 @@ platforms:
3636
- "-//gazelle/..."
3737
# The dependencies needed for this test are not cross-platform: https://github.com/bazelbuild/rules_python/issues/260
3838
- "-//tests:pip_repository_entry_points_example"
39+
- "-//tests:pip_deps_example"
3940
test_flags:
4041
- "--test_tag_filters=-fix-windows"

‎.bazelrc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
# This lets us glob() up all the files inside the examples to make them inputs to tests
44
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
55
# To update these lines, run tools/bazel_integration_test/update_deleted_packages.sh
6-
build --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points
7-
query --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points
6+
build --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
7+
query --deleted_packages=examples/build_file_generation,examples/pip_install,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_import,examples/relative_requirements,tests/pip_repository_entry_points,tests/pip_deps
88

99
test --test_output=errors
1010

‎python/pip_install/pip_compile.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,12 @@
1414
sys.exit(1)
1515

1616
requirements_in = os.path.relpath(sys.argv.pop(1))
17-
requirements_txt = sys.argv.pop(1)
17+
requirements_txt = os.path.relpath(sys.argv.pop(1))
18+
parts = requirements_in.split(os.path.sep, 2)
19+
if parts[0] == "external":
20+
requirements_in = parts[2]
21+
requirements_txt = requirements_txt if "BUILD_WORKSPACE_DIRECTORY" in os.environ else os.path.join("..", "..", requirements_txt)
22+
os.chdir(os.path.join(parts[0], parts[1]))
1823
update_target_label = sys.argv.pop(1)
1924

2025
# Before loading click, set the locale for its parser.
@@ -49,7 +54,7 @@
4954
#
5055
# Changing to the WORKSPACE root avoids 'file not found' errors when the `.update` target is run
5156
# from different directories within the WORKSPACE.
52-
os.chdir(os.environ["BUILD_WORKSPACE_DIRECTORY"])
57+
requirements_txt = os.path.join(os.environ["BUILD_WORKSPACE_DIRECTORY"], requirements_txt)
5358
else:
5459
err_msg = (
5560
"Expected to find BUILD_WORKSPACE_DIRECTORY (running under `bazel run`) or "

‎tests/BUILD

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,8 @@ bazel_integration_test(
88
name = "pip_repository_entry_points_example",
99
timeout = "long",
1010
)
11+
12+
bazel_integration_test(
13+
name = "pip_deps_example",
14+
timeout = "long",
15+
)

‎tests/pip_deps/BUILD

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
filegroup(
2+
name = "requirements_txt",
3+
srcs = ["requirements.txt"],
4+
visibility = ["//visibility:public"],
5+
)
6+
7+
test_suite(
8+
name = "external_tests",
9+
tests = [
10+
"@unpinned_pip//:pin_test",
11+
],
12+
)
13+
14+
sh_test(
15+
name = "no_external_test",
16+
srcs = ["test.sh"],
17+
args = ["$(rootpath :requirements_txt)"],
18+
data = [":requirements_txt"],
19+
)

‎tests/pip_deps/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Run `bazel run @unpinned_pip//:pin.update` to keep `PIP_PACKAGES` in `//:WORKSPACE` in sync with `//:requirements.txt`.

‎tests/pip_deps/WORKSPACE

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
workspace(name = "pip_deps")
2+
3+
local_repository(
4+
name = "rules_python",
5+
path = "../..",
6+
)
7+
8+
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
9+
10+
# This toolchain is explicitly 3.10 while `rules_python` is 3.9 to act as
11+
# a regression test, ensuring 3.10 is functional
12+
python_register_toolchains(
13+
name = "python310",
14+
python_version = "3.10",
15+
)
16+
17+
load("@python310//:defs.bzl", "interpreter")
18+
load("//:pip_deps.bzl", "pip_deps")
19+
20+
PIP_PACKAGES = {"sphinx": "4.5.0"}
21+
22+
pip_deps(
23+
name = "pip",
24+
packages = PIP_PACKAGES,
25+
python_interpreter_target = interpreter,
26+
)

‎tests/pip_deps/pip_deps.bzl

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
""" A demo implementation for pip_deps which provides @unpinned_pip//:pin. """
2+
3+
load("@rules_python//python:pip.bzl", "pip_parse")
4+
5+
def _requirements_in_impl(repository_ctx):
6+
repository_ctx.file(
7+
"requirements.in",
8+
content = "".join(["{package} == {version}\n".format(
9+
package = package,
10+
version = version,
11+
) for (package, version) in repository_ctx.attr.packages.items()]),
12+
)
13+
repository_ctx.file("WORKSPACE", content = "")
14+
repository_ctx.file("BUILD", content = """
15+
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
16+
17+
compile_pip_requirements(
18+
name = "pin",
19+
extra_args = ["--allow-unsafe"],
20+
requirements_in = "requirements.in",
21+
requirements_txt = "@{workspace_name}//{package}:{name}",
22+
)
23+
""".format(
24+
workspace_name = repository_ctx.attr.requirements_lock.workspace_name,
25+
package = repository_ctx.attr.requirements_lock.package,
26+
name = repository_ctx.attr.requirements_lock.name,
27+
))
28+
29+
_requirements_in = repository_rule(
30+
implementation = _requirements_in_impl,
31+
attrs = {
32+
"packages": attr.string_dict(),
33+
"requirements_lock": attr.label(allow_single_file = True),
34+
},
35+
)
36+
37+
def pip_deps(
38+
*,
39+
name = "pip",
40+
packages = {},
41+
requirements_lock_target = Label("//:requirements_txt"),
42+
requirements_lock_file = Label("//:requirements.txt"),
43+
python_interpreter_target = None,
44+
**kwargs):
45+
_requirements_in(
46+
name = "unpinned_" + name,
47+
packages = packages,
48+
requirements_lock = requirements_lock_target,
49+
)
50+
pip_parse(
51+
name = name,
52+
requirements_lock = requirements_lock_file,
53+
python_interpreter_target = python_interpreter_target,
54+
**kwargs
55+
)

‎tests/pip_deps/requirements.txt

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#
2+
# This file is autogenerated by pip-compile with python 3.10
3+
# To update, run:
4+
#
5+
# bazel run //:pin.update
6+
#
7+
alabaster==0.7.12 \
8+
--hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
9+
--hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02
10+
# via sphinx
11+
babel==2.10.1 \
12+
--hash=sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2 \
13+
--hash=sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13
14+
# via sphinx
15+
certifi==2021.10.8 \
16+
--hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
17+
--hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
18+
# via requests
19+
charset-normalizer==2.0.12 \
20+
--hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \
21+
--hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df
22+
# via requests
23+
docutils==0.17.1 \
24+
--hash=sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125 \
25+
--hash=sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61
26+
# via sphinx
27+
idna==3.3 \
28+
--hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \
29+
--hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d
30+
# via requests
31+
imagesize==1.3.0 \
32+
--hash=sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c \
33+
--hash=sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d
34+
# via sphinx
35+
jinja2==3.1.2 \
36+
--hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
37+
--hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
38+
# via sphinx
39+
markupsafe==2.1.1 \
40+
--hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \
41+
--hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \
42+
--hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \
43+
--hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \
44+
--hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \
45+
--hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \
46+
--hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \
47+
--hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \
48+
--hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \
49+
--hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \
50+
--hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \
51+
--hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \
52+
--hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \
53+
--hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \
54+
--hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \
55+
--hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \
56+
--hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \
57+
--hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \
58+
--hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \
59+
--hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \
60+
--hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \
61+
--hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \
62+
--hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \
63+
--hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \
64+
--hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \
65+
--hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \
66+
--hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \
67+
--hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \
68+
--hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \
69+
--hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \
70+
--hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \
71+
--hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \
72+
--hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \
73+
--hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \
74+
--hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \
75+
--hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \
76+
--hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \
77+
--hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \
78+
--hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \
79+
--hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7
80+
# via jinja2
81+
packaging==21.3 \
82+
--hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \
83+
--hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522
84+
# via sphinx
85+
pygments==2.12.0 \
86+
--hash=sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb \
87+
--hash=sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519
88+
# via sphinx
89+
pyparsing==3.0.8 \
90+
--hash=sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954 \
91+
--hash=sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06
92+
# via packaging
93+
pytz==2022.1 \
94+
--hash=sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7 \
95+
--hash=sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c
96+
# via babel
97+
requests==2.27.1 \
98+
--hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \
99+
--hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d
100+
# via sphinx
101+
snowballstemmer==2.2.0 \
102+
--hash=sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1 \
103+
--hash=sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a
104+
# via sphinx
105+
sphinx==4.5.0 \
106+
--hash=sha256:7bf8ca9637a4ee15af412d1a1d9689fec70523a68ca9bb9127c2f3eeb344e2e6 \
107+
--hash=sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226
108+
# via -r requirements.in
109+
sphinxcontrib-applehelp==1.0.2 \
110+
--hash=sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a \
111+
--hash=sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58
112+
# via sphinx
113+
sphinxcontrib-devhelp==1.0.2 \
114+
--hash=sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e \
115+
--hash=sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4
116+
# via sphinx
117+
sphinxcontrib-htmlhelp==2.0.0 \
118+
--hash=sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07 \
119+
--hash=sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2
120+
# via sphinx
121+
sphinxcontrib-jsmath==1.0.1 \
122+
--hash=sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178 \
123+
--hash=sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8
124+
# via sphinx
125+
sphinxcontrib-qthelp==1.0.3 \
126+
--hash=sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72 \
127+
--hash=sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6
128+
# via sphinx
129+
sphinxcontrib-serializinghtml==1.1.5 \
130+
--hash=sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd \
131+
--hash=sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952
132+
# via sphinx
133+
urllib3==1.26.9 \
134+
--hash=sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14 \
135+
--hash=sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e
136+
# via requests

‎tests/pip_deps/test.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
! grep external "$1"

0 commit comments

Comments
 (0)