Skip to content

Commit 2af0020

Browse files
aignasaignas
authored
feat(toolchain): support freethreaded toolchains (#2372)
Before this PR freethreaded toolchains were not possible to be used, this adds the minimum plumbing to get the things working. Coverage support is also added. Whilst at it: - Add plumbing to print checksums only for a particular python version. - Bump the remaining toolchain versions that used to use the 20241008 release - Pass around the loaded platform list so that we are only defining toolchains for the platforms that we have loaded the hermetic toolchain for. Tested: ``` $ bazel run --//python/config_settings:python_version=3.13.0 --//python/config_settings:py_freethreaded="yes" //python/private:current_interpreter_executable ... Python 3.13.0 experimental free-threading build (main, Oct 16 2024, 03:26:14) [Clang 18.1.8 ] on linux Type "help", "copyright", "credits" or "license" for more information. >>> ``` Closes #2129. Work towards #2386.
1 parent 7010148 commit 2af0020

15 files changed

+429
-197
lines changed

‎.pre-commit-config.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
# See https://pre-commit.com for more information
1717
# See https://pre-commit.com/hooks.html for more hooks
1818
repos:
19+
- repo: https://github.com/pre-commit/pre-commit-hooks
20+
rev: v5.0.0 # Use the ref you want to point at
21+
hooks:
22+
- id: check-merge-conflict
1923
- repo: https://github.com/keith/pre-commit-buildifier
2024
rev: 6.1.0
2125
hooks:

‎CHANGELOG.md

+11-1
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,24 @@ A brief description of the categories of changes:
2828
{#v0-0-0-changed}
2929
### Changed
3030
* (deps) bazel_skylib 1.6.1 -> 1.7.1
31+
* (toolchains) Use the latest indygreg toolchain release [20241016] for Python versions:
32+
* 3.9.20
33+
* 3.10.15
34+
* 3.11.10
35+
* 3.12.7
36+
* 3.13.0
37+
38+
[20241016]: https://github.com/indygreg/python-build-standalone/releases/tag/20241016
3139

3240
{#v0-0-0-fixed}
3341
### Fixed
3442
* Nothing yet
3543

3644
{#v0-0-0-added}
3745
### Added
38-
* Nothing yet
46+
* (toolchain) Support for freethreaded Python toolchains is now available. Use
47+
the config flag `//python/config_settings:py_freethreaded` to toggle the
48+
selection of the free-threaded toolchains.
3949

4050
{#v0-0-0-removed}
4151
### Removed

‎docs/api/rules_python/python/config_settings/index.md

+10
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,16 @@ Values:
149149
:::
150150
::::
151151

152+
::::{bzl:flag} py_freethreaded
153+
Set whether to use an interpreter with the experimental freethreaded option set to true.
154+
155+
Values:
156+
* `no`: Use regular Python toolchains, default.
157+
* `yes`: Use the experimental Python toolchain with freethreaded compile option enabled.
158+
:::{versionadded} 0.38.0
159+
:::
160+
::::
161+
152162
::::{bzl:flag} pip_whl
153163
Set what distributions are used in the `pip` integration.
154164

‎python/config_settings/BUILD.bazel

+14
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ load(
55
"AddSrcsToRunfilesFlag",
66
"BootstrapImplFlag",
77
"ExecToolsToolchainFlag",
8+
"FreeThreadedFlag",
89
"PrecompileFlag",
910
"PrecompileSourceRetentionFlag",
1011
)
@@ -92,6 +93,19 @@ string_flag(
9293
visibility = ["//visibility:public"],
9394
)
9495

96+
string_flag(
97+
name = "py_freethreaded",
98+
build_setting_default = FreeThreadedFlag.NO,
99+
values = sorted(FreeThreadedFlag.__members__.values()),
100+
visibility = ["//visibility:public"],
101+
)
102+
103+
config_setting(
104+
name = "is_py_freethreaded",
105+
flag_values = {":py_freethreaded": FreeThreadedFlag.YES},
106+
visibility = ["//visibility:public"],
107+
)
108+
95109
# pip.parse related flags
96110

97111
string_flag(

‎python/private/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@ bzl_library(
256256
deps = [
257257
":py_toolchain_suite_bzl",
258258
":text_util_bzl",
259+
"//python:versions_bzl",
259260
],
260261
)
261262

‎python/private/coverage_deps.bzl

+12
Original file line numberDiff line numberDiff line change
@@ -80,11 +80,23 @@ _coverage_deps = {
8080
"https://files.pythonhosted.org/packages/b9/67/e1413d5a8591622a46dd04ff80873b04c849268831ed5c304c16433e7e30/coverage-7.6.1-cp313-cp313-macosx_11_0_arm64.whl",
8181
"a6d3adcf24b624a7b778533480e32434a39ad8fa30c315208f6d3e5542aeb6e9",
8282
),
83+
"aarch64-apple-darwin-freethreaded": (
84+
"https://files.pythonhosted.org/packages/c4/ae/b5d58dff26cade02ada6ca612a76447acd69dccdbb3a478e9e088eb3d4b9/coverage-7.6.1-cp313-cp313t-macosx_11_0_arm64.whl",
85+
"502753043567491d3ff6d08629270127e0c31d4184c4c8d98f92c26f65019962",
86+
),
8387
"aarch64-unknown-linux-gnu": (
88+
"https://files.pythonhosted.org/packages/14/5b/9dec847b305e44a5634d0fb8498d135ab1d88330482b74065fcec0622224/coverage-7.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
89+
"d0c212c49b6c10e6951362f7c6df3329f04c2b1c28499563d4035d964ab8e08c",
90+
),
91+
"aarch64-unknown-linux-gnu-freethreaded": (
8492
"https://files.pythonhosted.org/packages/b8/d7/62095e355ec0613b08dfb19206ce3033a0eedb6f4a67af5ed267a8800642/coverage-7.6.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
8593
"6a89ecca80709d4076b95f89f308544ec8f7b4727e8a547913a35f16717856cb",
8694
),
8795
"x86_64-unknown-linux-gnu": (
96+
"https://files.pythonhosted.org/packages/f7/95/d2fd31f1d638df806cae59d7daea5abf2b15b5234016a5ebb502c2f3f7ee/coverage-7.6.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
97+
"78b260de9790fd81e69401c2dc8b17da47c8038176a79092a89cb2b7d945d060",
98+
),
99+
"x86_64-unknown-linux-gnu-freethreaded": (
88100
"https://files.pythonhosted.org/packages/8b/61/a7a6a55dd266007ed3b1df7a3386a0d760d014542d72f7c2c6938483b7bd/coverage-7.6.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
89101
"13b0a73a0896988f053e4fbb7de6d93388e6dd292b0d87ee51d106f2c11b465b",
90102
),

‎python/private/flags.bzl

+10
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,13 @@ PrecompileSourceRetentionFlag = enum(
122122
OMIT_SOURCE = "omit_source",
123123
get_effective_value = _precompile_source_retention_flag_get_effective_value,
124124
)
125+
126+
# Used for matching freethreaded toolchains and would have to be used in wheels
127+
# as well.
128+
# buildifier: disable=name-conventions
129+
FreeThreadedFlag = enum(
130+
# Use freethreaded python toolchain and wheels.
131+
YES = "yes",
132+
# Do not use freethreaded python toolchain and wheels.
133+
NO = "no",
134+
)

‎python/private/hermetic_runtime_repo_setup.bzl

+71-11
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ load(":glob_excludes.bzl", "glob_excludes")
2121
load(":py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain")
2222
load(":semver.bzl", "semver")
2323

24+
_IS_FREETHREADED = Label("//python/config_settings:is_py_freethreaded")
25+
2426
def define_hermetic_runtime_toolchain_impl(
2527
*,
2628
name,
@@ -45,7 +47,7 @@ def define_hermetic_runtime_toolchain_impl(
4547
python_version: {type}`str` The Python version, in `major.minor.micro`
4648
format.
4749
python_bin: {type}`str` The path to the Python binary within the
48-
repositoroy.
50+
repository.
4951
coverage_tool: {type}`str` optional target to the coverage tool to
5052
use.
5153
"""
@@ -67,19 +69,23 @@ def define_hermetic_runtime_toolchain_impl(
6769
exclude = [
6870
# Unused shared libraries. `python` executable and the `:libpython` target
6971
# depend on `libpython{python_version}.so.1.0`.
70-
"lib/libpython{major}.{minor}.so".format(**version_dict),
72+
"lib/libpython{major}.{minor}*.so".format(**version_dict),
7173
# static libraries
7274
"lib/**/*.a",
7375
# tests for the standard libraries.
74-
"lib/python{major}.{minor}/**/test/**".format(**version_dict),
75-
"lib/python{major}.{minor}/**/tests/**".format(**version_dict),
76-
"**/__pycache__/*.pyc.*", # During pyc creation, temp files named *.pyc.NNN are created
76+
"lib/python{major}.{minor}*/**/test/**".format(**version_dict),
77+
"lib/python{major}.{minor}*/**/tests/**".format(**version_dict),
78+
# During pyc creation, temp files named *.pyc.NNN are created
79+
"**/__pycache__/*.pyc.*",
7780
] + glob_excludes.version_dependent_exclusions() + extra_files_glob_exclude,
7881
),
7982
)
8083
cc_import(
8184
name = "interface",
82-
interface_library = "libs/python{major}{minor}.lib".format(**version_dict),
85+
interface_library = select({
86+
_IS_FREETHREADED: "libs/python{major}{minor}t.lib".format(**version_dict),
87+
"//conditions:default": "libs/python{major}{minor}.lib".format(**version_dict),
88+
}),
8389
system_provided = True,
8490
)
8591

@@ -96,14 +102,62 @@ def define_hermetic_runtime_toolchain_impl(
96102
hdrs = [":includes"],
97103
includes = [
98104
"include",
99-
"include/python{major}.{minor}".format(**version_dict),
100-
"include/python{major}.{minor}m".format(**version_dict),
105+
] + select({
106+
_IS_FREETHREADED: [
107+
"include/python{major}.{minor}t".format(**version_dict),
108+
],
109+
"//conditions:default": [
110+
"include/python{major}.{minor}".format(**version_dict),
111+
"include/python{major}.{minor}m".format(**version_dict),
112+
],
113+
}),
114+
)
115+
native.config_setting(
116+
name = "is_freethreaded_linux",
117+
flag_values = {
118+
Label("//python/config_settings:py_freethreaded"): "yes",
119+
},
120+
constraint_values = [
121+
"@platforms//os:linux",
101122
],
123+
visibility = ["//visibility:private"],
102124
)
125+
native.config_setting(
126+
name = "is_freethreaded_osx",
127+
flag_values = {
128+
Label("//python/config_settings:py_freethreaded"): "yes",
129+
},
130+
constraint_values = [
131+
"@platforms//os:osx",
132+
],
133+
visibility = ["//visibility:private"],
134+
)
135+
native.config_setting(
136+
name = "is_freethreaded_windows",
137+
flag_values = {
138+
Label("//python/config_settings:py_freethreaded"): "yes",
139+
},
140+
constraint_values = [
141+
"@platforms//os:windows",
142+
],
143+
visibility = ["//visibility:private"],
144+
)
145+
103146
cc_library(
104147
name = "libpython",
105148
hdrs = [":includes"],
106149
srcs = select({
150+
":is_freethreaded_linux": [
151+
"lib/libpython{major}.{minor}t.so".format(**version_dict),
152+
"lib/libpython{major}.{minor}t.so.1.0".format(**version_dict),
153+
],
154+
":is_freethreaded_osx": [
155+
"lib/libpython{major}.{minor}t.dylib".format(**version_dict),
156+
],
157+
":is_freethreaded_windows": [
158+
"python3.dll",
159+
"libs/python{major}{minor}t.lib".format(**version_dict),
160+
],
107161
"@platforms//os:linux": [
108162
"lib/libpython{major}.{minor}.so".format(**version_dict),
109163
"lib/libpython{major}.{minor}.so.1.0".format(**version_dict),
@@ -132,12 +186,18 @@ def define_hermetic_runtime_toolchain_impl(
132186
"micro": str(version_info.patch),
133187
"minor": str(version_info.minor),
134188
},
135-
# Convert empty string to None
136-
coverage_tool = coverage_tool or None,
189+
coverage_tool = select({
190+
# Convert empty string to None
191+
":coverage_enabled": coverage_tool or None,
192+
"//conditions:default": None,
193+
}),
137194
python_version = "PY3",
138195
implementation_name = "cpython",
139196
# See https://peps.python.org/pep-3147/ for pyc tag infix format
140-
pyc_tag = "cpython-{major}{minor}".format(**version_dict),
197+
pyc_tag = select({
198+
_IS_FREETHREADED: "cpython-{major}{minor}t".format(**version_dict),
199+
"//conditions:default": "cpython-{major}{minor}".format(**version_dict),
200+
}),
141201
)
142202

143203
py_runtime_pair(

‎python/private/python.bzl

+4-2
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ def parse_modules(*, module_ctx, _fail = fail):
213213
def _python_impl(module_ctx):
214214
py = parse_modules(module_ctx = module_ctx)
215215

216+
loaded_platforms = {}
216217
for toolchain_info in py.toolchains:
217218
# Ensure that we pass the full version here.
218219
full_python_version = full_version(
@@ -228,7 +229,7 @@ def _python_impl(module_ctx):
228229
kwargs.update(py.config.kwargs.get(toolchain_info.python_version, {}))
229230
kwargs.update(py.config.kwargs.get(full_python_version, {}))
230231
kwargs.update(py.config.default)
231-
python_register_toolchains(
232+
loaded_platforms[full_python_version] = python_register_toolchains(
232233
name = toolchain_info.name,
233234
_internal_bzlmod_toolchain_call = True,
234235
**kwargs
@@ -257,6 +258,7 @@ def _python_impl(module_ctx):
257258
for i in range(len(py.toolchains))
258259
],
259260
toolchain_user_repository_names = [t.name for t in py.toolchains],
261+
loaded_platforms = loaded_platforms,
260262
)
261263

262264
# This is require in order to support multiple version py_test
@@ -464,7 +466,7 @@ def _get_toolchain_config(*, modules, _fail = fail):
464466
"url": {
465467
platform: [item["url"]]
466468
for platform in item["sha256"]
467-
},
469+
} if type(item["url"]) == type("") else item["url"],
468470
}
469471
for version, item in TOOL_VERSIONS.items()
470472
}

‎python/private/python_register_toolchains.bzl

+6-1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,9 @@ def python_register_toolchains(
7373
minor_mapping: {type}`dict[str, str]` contains a mapping from `X.Y` to `X.Y.Z`
7474
version.
7575
**kwargs: passed to each {obj}`python_repository` call.
76+
77+
Returns:
78+
On bzlmod this returns the loaded platform labels. Otherwise None.
7679
"""
7780
bzlmod_toolchain_call = kwargs.pop("_internal_bzlmod_toolchain_call", False)
7881
if bzlmod_toolchain_call:
@@ -168,11 +171,13 @@ def python_register_toolchains(
168171

169172
# in bzlmod we write out our own toolchain repos
170173
if bzlmod_toolchain_call:
171-
return
174+
return loaded_platforms
172175

173176
toolchains_repo(
174177
name = toolchain_repo_name,
175178
python_version = python_version,
176179
set_python_version_constraint = set_python_version_constraint,
177180
user_repository_name = name,
181+
platforms = loaded_platforms,
178182
)
183+
return None

‎python/private/python_repository.bzl

+6-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"""This file contains repository rules and macros to support toolchain registration.
1616
"""
1717

18-
load("//python:versions.bzl", "PLATFORMS")
18+
load("//python:versions.bzl", "FREETHREADED", "PLATFORMS")
1919
load(":auth.bzl", "get_auth")
2020
load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
2121
load(":text_util.bzl", "render")
@@ -63,8 +63,12 @@ def _python_repository_impl(rctx):
6363
platform = rctx.attr.platform
6464
python_version = rctx.attr.python_version
6565
python_version_info = python_version.split(".")
66-
python_short_version = "{0}.{1}".format(*python_version_info)
6766
release_filename = rctx.attr.release_filename
67+
version_suffix = "t" if FREETHREADED in release_filename else ""
68+
python_short_version = "{0}.{1}{suffix}".format(
69+
suffix = version_suffix,
70+
*python_version_info
71+
)
6872
urls = rctx.attr.urls or [rctx.attr.url]
6973
auth = get_auth(rctx, urls)
7074

‎python/private/pythons_hub.bzl

+12-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels"
1616

17+
load("//python:versions.bzl", "PLATFORMS")
1718
load(":text_util.bzl", "render")
1819
load(":toolchains_repo.bzl", "python_toolchain_build_file_content")
1920

@@ -46,7 +47,8 @@ def _hub_build_file_content(
4647
python_versions,
4748
set_python_version_constraints,
4849
user_repository_names,
49-
workspace_location):
50+
workspace_location,
51+
loaded_platforms):
5052
"""This macro iterates over each of the lists and returns the toolchain content.
5153
5254
python_toolchain_build_file_content is called to generate each of the toolchain
@@ -65,6 +67,11 @@ def _hub_build_file_content(
6567
python_version = python_versions[i],
6668
set_python_version_constraint = set_python_version_constraints[i],
6769
user_repository_name = user_repository_names[i],
70+
loaded_platforms = {
71+
k: v
72+
for k, v in PLATFORMS.items()
73+
if k in loaded_platforms[python_versions[i]]
74+
},
6875
)
6976
for i in range(len(python_versions))
7077
],
@@ -103,6 +110,7 @@ def _hub_repo_impl(rctx):
103110
rctx.attr.toolchain_set_python_version_constraints,
104111
rctx.attr.toolchain_user_repository_names,
105112
rctx.attr._rules_python_workspace,
113+
rctx.attr.loaded_platforms,
106114
),
107115
executable = False,
108116
)
@@ -149,6 +157,9 @@ This rule also writes out the various toolchains for the different Python versio
149157
doc = "Default Python version for the build in `X.Y` or `X.Y.Z` format.",
150158
mandatory = True,
151159
),
160+
"loaded_platforms": attr.string_list_dict(
161+
doc = "The list of loaded platforms keyed by the toolchain full python version",
162+
),
152163
"minor_mapping": attr.string_dict(
153164
doc = "The minor mapping of the `X.Y` to `X.Y.Z` format that is used in config settings.",
154165
mandatory = True,

0 commit comments

Comments
 (0)