Skip to content

Commit c99aaec

Browse files
aignasaignas
authored
feat: add a tool to update internal dependencies (#1321)
Before this change the updates to the dependencies would happen very seldomly, with this script, I propose we do it before each minor version release. Adding a shell script and adding a reminder to the release process may help with that.
1 parent 608ddb7 commit c99aaec

13 files changed

+595
-77
lines changed

‎DEVELOPING.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
# For Developers
22

3+
## Updating internal dependencies
4+
5+
1. Modify the `./python/pip_install/tools/requirements.txt` file and run:
6+
```
7+
bazel run //tools/private/update_deps:update_pip_deps
8+
```
9+
1. Bump the coverage dependencies using the script using:
10+
```
11+
bazel run //tools/private/update_deps:update_coverage_deps <VERSION>
12+
```
13+
314
## Releasing
415

516
Start from a clean checkout at `main`.

‎MODULE.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ internal_deps = use_extension("@rules_python//python/extensions/private:internal
1515
internal_deps.install()
1616
use_repo(
1717
internal_deps,
18+
# START: maintained by 'bazel run //tools/private:update_pip_deps'
1819
"pypi__build",
1920
"pypi__click",
2021
"pypi__colorama",
@@ -29,6 +30,7 @@ use_repo(
2930
"pypi__tomli",
3031
"pypi__wheel",
3132
"pypi__zipp",
33+
# END: maintained by 'bazel run //tools/private:update_pip_deps'
3234
)
3335

3436
# We need to do another use_extension call to expose the "pythons_hub"

‎python/pip_install/BUILD.bazel

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,18 @@ filegroup(
99
visibility = ["//:__pkg__"],
1010
)
1111

12+
filegroup(
13+
name = "repositories",
14+
srcs = ["repositories.bzl"],
15+
visibility = ["//tools/private/update_deps:__pkg__"],
16+
)
17+
18+
filegroup(
19+
name = "requirements_txt",
20+
srcs = ["tools/requirements.txt"],
21+
visibility = ["//tools/private/update_deps:__pkg__"],
22+
)
23+
1224
filegroup(
1325
name = "bzl",
1426
srcs = glob(["*.bzl"]) + [

‎python/pip_install/repositories.bzl

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
2020
load("//:version.bzl", "MINIMUM_BAZEL_VERSION")
2121

2222
_RULE_DEPS = [
23+
# START: maintained by 'bazel run //tools/private:update_pip_deps'
2324
(
2425
"pypi__build",
2526
"https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl",
@@ -35,11 +36,21 @@ _RULE_DEPS = [
3536
"https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
3637
"4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
3738
),
39+
(
40+
"pypi__importlib_metadata",
41+
"https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
42+
"bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
43+
),
3844
(
3945
"pypi__installer",
4046
"https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl",
4147
"05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53",
4248
),
49+
(
50+
"pypi__more_itertools",
51+
"https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
52+
"c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
53+
),
4354
(
4455
"pypi__packaging",
4556
"https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl",
@@ -75,21 +86,12 @@ _RULE_DEPS = [
7586
"https://files.pythonhosted.org/packages/bd/7c/d38a0b30ce22fc26ed7dbc087c6d00851fb3395e9d0dac40bec1f905030c/wheel-0.38.4-py3-none-any.whl",
7687
"b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8",
7788
),
78-
(
79-
"pypi__importlib_metadata",
80-
"https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
81-
"bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
82-
),
8389
(
8490
"pypi__zipp",
8591
"https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl",
8692
"8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
8793
),
88-
(
89-
"pypi__more_itertools",
90-
"https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
91-
"c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
92-
),
94+
# END: maintained by 'bazel run //tools/private:update_pip_deps'
9395
]
9496

9597
_GENERIC_WHEEL = """\

‎python/pip_install/tools/requirements.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
build==0.9
2+
click==8.0.1
3+
colorama
4+
importlib_metadata==1.4.0
5+
installer
6+
more_itertools==8.13.0
7+
packaging==22.0
8+
pep517
9+
pip==22.3.1
10+
pip_tools==6.12.1
11+
setuptools==60.10
12+
tomli
13+
wheel==0.38.4
14+
zipp==1.0.0

‎python/private/BUILD.bazel

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ filegroup(
2424
visibility = ["//python:__pkg__"],
2525
)
2626

27+
filegroup(
28+
name = "coverage_deps",
29+
srcs = ["coverage_deps.bzl"],
30+
visibility = ["//tools/private/update_deps:__pkg__"],
31+
)
32+
2733
# Filegroup of bzl files that can be used by downstream rules for documentation generation
2834
filegroup(
2935
name = "bzl",

‎python/private/coverage_deps.bzl

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
1919
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
2020
load("//python/private:version_label.bzl", "version_label")
2121

22-
# Update with './tools/update_coverage_deps.py <version>'
23-
#START: managed by update_coverage_deps.py script
22+
# START: maintained by 'bazel run //tools/private:update_coverage_deps'
2423
_coverage_deps = {
2524
"cp310": {
2625
"aarch64-apple-darwin": (
@@ -95,7 +94,7 @@ _coverage_deps = {
9594
),
9695
},
9796
}
98-
#END: managed by update_coverage_deps.py script
97+
# END: maintained by 'bazel run //tools/private:update_coverage_deps'
9998

10099
_coverage_patch = Label("//python/private:coverage.patch")
101100

‎tools/private/update_deps/BUILD.bazel

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
load("//python:py_binary.bzl", "py_binary")
15+
load("//python:py_library.bzl", "py_library")
16+
load("//python:py_test.bzl", "py_test")
17+
18+
licenses(["notice"])
19+
20+
py_library(
21+
name = "args",
22+
srcs = ["args.py"],
23+
imports = ["../../.."],
24+
deps = ["//python/runfiles"],
25+
)
26+
27+
py_library(
28+
name = "update_file",
29+
srcs = ["update_file.py"],
30+
imports = ["../../.."],
31+
)
32+
33+
py_binary(
34+
name = "update_coverage_deps",
35+
srcs = ["update_coverage_deps.py"],
36+
data = [
37+
"//python/private:coverage_deps",
38+
],
39+
env = {
40+
"UPDATE_FILE": "$(rlocationpath //python/private:coverage_deps)",
41+
},
42+
imports = ["../../.."],
43+
deps = [
44+
":args",
45+
":update_file",
46+
],
47+
)
48+
49+
py_binary(
50+
name = "update_pip_deps",
51+
srcs = ["update_pip_deps.py"],
52+
data = [
53+
"//:MODULE.bazel",
54+
"//python/pip_install:repositories",
55+
"//python/pip_install:requirements_txt",
56+
],
57+
env = {
58+
"MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)",
59+
"REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)",
60+
"REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)",
61+
},
62+
imports = ["../../.."],
63+
deps = [
64+
":args",
65+
":update_file",
66+
],
67+
)
68+
69+
py_test(
70+
name = "update_file_test",
71+
srcs = ["update_file_test.py"],
72+
imports = ["../../.."],
73+
deps = [
74+
":update_file",
75+
],
76+
)

‎tools/private/update_deps/args.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""A small library for common arguments when updating files."""
16+
17+
import pathlib
18+
19+
from python.runfiles import runfiles
20+
21+
22+
def path_from_runfiles(input: str) -> pathlib.Path:
23+
"""A helper to create a path from runfiles.
24+
25+
Args:
26+
input: the string input to construct a path.
27+
28+
Returns:
29+
the pathlib.Path path to a file which is verified to exist.
30+
"""
31+
path = pathlib.Path(runfiles.Create().Rlocation(input))
32+
if not path.exists():
33+
raise ValueError(f"Path '{path}' does not exist")
34+
35+
return path

‎tools/update_coverage_deps.py renamed to ‎tools/private/update_deps/update_coverage_deps.py

Lines changed: 14 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import argparse
2323
import difflib
2424
import json
25+
import os
2526
import pathlib
2627
import sys
2728
import textwrap
@@ -30,6 +31,9 @@
3031
from typing import Any
3132
from urllib import request
3233

34+
from tools.private.update_deps.args import path_from_runfiles
35+
from tools.private.update_deps.update_file import update_file
36+
3337
# This should be kept in sync with //python:versions.bzl
3438
_supported_platforms = {
3539
# Windows is unsupported right now
@@ -110,64 +114,6 @@ def _map(
110114
)
111115

112116

113-
def _writelines(path: pathlib.Path, lines: list[str]):
114-
with open(path, "w") as f:
115-
f.writelines(lines)
116-
117-
118-
def _difflines(path: pathlib.Path, lines: list[str]):
119-
with open(path) as f:
120-
input = f.readlines()
121-
122-
rules_python = pathlib.Path(__file__).parent.parent
123-
p = path.relative_to(rules_python)
124-
125-
print(f"Diff of the changes that would be made to '{p}':")
126-
for line in difflib.unified_diff(
127-
input,
128-
lines,
129-
fromfile=f"a/{p}",
130-
tofile=f"b/{p}",
131-
):
132-
print(line, end="")
133-
134-
# Add an empty line at the end of the diff
135-
print()
136-
137-
138-
def _update_file(
139-
path: pathlib.Path,
140-
snippet: str,
141-
start_marker: str,
142-
end_marker: str,
143-
dry_run: bool = True,
144-
):
145-
with open(path) as f:
146-
input = f.readlines()
147-
148-
out = []
149-
skip = False
150-
for line in input:
151-
if skip:
152-
if not line.startswith(end_marker):
153-
continue
154-
155-
skip = False
156-
157-
out.append(line)
158-
159-
if not line.startswith(start_marker):
160-
continue
161-
162-
skip = True
163-
out.extend([f"{line}\n" for line in snippet.splitlines()])
164-
165-
if dry_run:
166-
_difflines(path, out)
167-
else:
168-
_writelines(path, out)
169-
170-
171117
def _parse_args() -> argparse.Namespace:
172118
parser = argparse.ArgumentParser(__doc__)
173119
parser.add_argument(
@@ -193,6 +139,12 @@ def _parse_args() -> argparse.Namespace:
193139
action="store_true",
194140
help="Wether to write to files",
195141
)
142+
parser.add_argument(
143+
"--update-file",
144+
type=path_from_runfiles,
145+
default=os.environ.get("UPDATE_FILE"),
146+
help="The path for the file to be updated, defaults to the value taken from UPDATE_FILE",
147+
)
196148
return parser.parse_args()
197149

198150

@@ -230,14 +182,12 @@ def main():
230182

231183
urls.sort(key=lambda x: f"{x.python}_{x.platform}")
232184

233-
rules_python = pathlib.Path(__file__).parent.parent
234-
235185
# Update the coverage_deps, which are used to register deps
236-
_update_file(
237-
path=rules_python / "python" / "private" / "coverage_deps.bzl",
186+
update_file(
187+
path=args.update_file,
238188
snippet=f"_coverage_deps = {repr(Deps(urls))}\n",
239-
start_marker="#START: managed by update_coverage_deps.py script",
240-
end_marker="#END: managed by update_coverage_deps.py script",
189+
start_marker="# START: maintained by 'bazel run //tools/private:update_coverage_deps'",
190+
end_marker="# END: maintained by 'bazel run //tools/private:update_coverage_deps'",
241191
dry_run=args.dry_run,
242192
)
243193

0 commit comments

Comments
 (0)