Skip to content

Added windows support to hermetic toolchains #628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 2, 2022
Merged

Added windows support to hermetic toolchains #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
227aa3d
Added windows support to hermetic toolchains
UebelAndre Feb 24, 2022
09c57bd
Update python/repositories.bzl
UebelAndre Mar 2, 2022
8a53ef4
Update python/repositories.bzl
UebelAndre Mar 2, 2022
3ce5c91
Update python/repositories.bzl
UebelAndre Mar 2, 2022
e063c20
Update python/repositories.bzl
UebelAndre Mar 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions python/private/tests/toolchains/python_version_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest
import platform
import unittest


class TestPythonVersion(unittest.TestCase):
def test_match_toolchain(self):
self.assertEqual(platform.python_version(), "3.9.7")
self.assertEqual(platform.python_version(), "3.9.10")


if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()
1 change: 1 addition & 0 deletions python/private/tests/toolchains/versions_test.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ load("//python/private:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS")

required_platforms = [
"x86_64-apple-darwin",
"x86_64-pc-windows-msvc",
"x86_64-unknown-linux-gnu",
]

Expand Down
6 changes: 6 additions & 0 deletions python/private/toolchains_repo.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ PLATFORMS = {
"@platforms//cpu:x86_64",
],
),
"x86_64-pc-windows-msvc": struct(
compatible_with = [
"@platforms//os:windows",
"@platforms//cpu:x86_64",
],
),
"x86_64-unknown-linux-gnu": struct(
compatible_with = [
"@platforms//os:linux",
Expand Down
33 changes: 17 additions & 16 deletions python/private/versions.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,33 @@
"""The Python versions we use for the toolchains.
"""

RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20211017"
RELEASE_DATE = "20211017T1616"
RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20220227"
RELEASE_DATE = "20220227"

# The integrity hashes can be computed with:
# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64
# buildifier: disable=unsorted-dict-items
TOOL_VERSIONS = {
"3.8.12": {
"x86_64-apple-darwin": "sha384-es0kCVBb4q5xSC09lOw83TKXtR6qdt0NeU56JtK7Y5M5V784k9MM2q8leE3QWGH6",
"x86_64-unknown-linux-gnu": "sha384-61BjosOUm+Uo2yDtkLcRqbSIAXIPiHKmvySGO2J/GSOHXkj5QBf4cwzrDuOWR0H2",
"x86_64-apple-darwin": "sha256-8yP7xVgDXBOoXOImfQ+tnokoImjsuBDjZP/x0KB51SU=",
"x86_64-pc-windows-msvc": "sha256-kk+f1R/2zMUz7Y6WxUYXaNpXges9/BHYRvnjAPq0Tto=",
"x86_64-unknown-linux-gnu": "sha256-W+nG1h4ji5Df2UdVBRwNOi2AI+v/20sPpOj+3QmmyrY=",
},
"3.9.7": {
"x86_64-apple-darwin": "sha384-Ix7lxb+niRWcDCOsK1q53dAkOp0CIw40kM8wHv//WQs2O/il8SRJ60cP3R67fEnm",
"aarch64-apple-darwin": "sha384-JSi3XZdHVZ+6DMybPYKgEYSQuBPpjXgBlj1uGqB9f/r3Wi6P0+CnYRG12TEzgcs6",
"x86_64-unknown-linux-gnu": "sha384-+PNcmKXJ+ZiyKyZ2WR1XedDXJ05ujC2w9TdXl80vloYMqfIOpcVPOWUgre+btI+3",
"3.9.10": {
"aarch64-apple-darwin": "sha256-rWbCo+cmMUfgRqMmlN57iXpG+wEkQJ0p06k+3mMciu4=",
"x86_64-apple-darwin": "sha256-/a9ZQUJEYCnjFKm+uR8ax1r4ZjILULi5aBgeWSVQzWg=",
"x86_64-pc-windows-msvc": "sha256-W8Zc4CNhS/SWpnSOQdypNLcPxfrG36zEaqjbytdyr8I=",
"x86_64-unknown-linux-gnu": "sha256-RVCJzFdr2aWNtF6RnR/IZ+zbsCCAZ9/8hFzJu/BwG3A=",
},
"3.10.0": {
"x86_64-apple-darwin": "sha384-eVTq704heZyKW6SB/DzarWB9S5bRH1LTQ4rFHzlKTE9jDjHDCBKyQhSgYy8a62lt",
"aarch64-apple-darwin": "sha384-NbhxnZL0pBTKpzEfoCYWl6s2GYdfiI9HOSSHn1iCMZnIY6htt/KhzjwIgCP+Nj2u",
"x86_64-unknown-linux-gnu": "sha384-iYJF9Y9o2Ain3YkuuF7ZGrGuJ+MyiD/xnrjJSap0TF2DR+I9XDx4stunNgx17gSn",
"3.10.2": {
"aarch64-apple-darwin": "sha256-FAms2aUG4tHTtlwUiNtOQNjxnQmn3wmWZ8h6UG9xwO8=",
"x86_64-apple-darwin": "sha256-gUatQ5BxDsabMWpWSZEt8CR9NfSkLiqpYVv/2Hs+I1o=",
"x86_64-pc-windows-msvc": "sha256-opPFg43ZyEOKhDcvuV3al1LfY5KKiirlFkOPGH+JVn0=",
"x86_64-unknown-linux-gnu": "sha256-m2TsoqlPev+UCa1wvap/u/gUhpJmLnZEAYg5V5Q2IN0=",
},
}

# buildifier: disable=unsorted-dict-items
MINOR_MAPPING = {
"3.8": "3.8.12",
"3.9": "3.9.7",
"3.10": "3.10.0",
"3.9": "3.9.10",
"3.10": "3.10.2",
}
135 changes: 66 additions & 69 deletions python/repositories.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,89 +27,79 @@ def py_repositories():
########
# Remaining content of the file is only used to support toolchains.
########
_DOC = "Fetch external tools needed for python toolchain."
_ATTRS = {
"integrity": attr.string(mandatory = True),
"platform": attr.string(mandatory = True, values = PLATFORMS.keys()),
"python_version": attr.string(mandatory = True, values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys()),
}

def _python_repository_impl(rctx):
python_version = rctx.attr.python_version
platform = rctx.attr.platform
release_filename = "cpython-{version}-{platform}-pgo+lto-{release_date}.tar.zst".format(
is_host_windows = "win" in rctx.os.name
is_target_windows = "windows" in rctx.attr.platform
python_version = rctx.attr.python_version
build = "static-install_only" if is_target_windows else "install_only"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not a Windows user. Would a static binary for Windows have the same limitation as a fully static binary for Linux that a shared object (i.e. a native Python extension) cannot be loaded at runtime (not able to do dlopen)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure myself, but this was the only artifact that worked. My thought was to hopefully see something merged that works in some cases and fix other issues as they're reported.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, this could possibly be a blocker for merging my PR into main. I'll take this for now, but will have to investigate before releasing this feature.

release_filename = "cpython-{version}+{release_date}-{platform}-{build}.tar.gz".format(
build = build,
platform = platform,
release_date = RELEASE_DATE,
version = python_version,
platform = platform,
)
url = "{release_url}/{release_filename}".format(
release_url = RELEASE_URL,
release_filename = release_filename,
)
integrity = rctx.attr.integrity
rctx.download(
download_result = rctx.download_and_extract(
url = url,
integrity = integrity,
output = release_filename,
integrity = rctx.attr.integrity,
stripPrefix = "python",
)
unzstd = rctx.which("unzstd")
if not unzstd:
url = rctx.attr._zstd_url.format(version = rctx.attr._zstd_version)
rctx.download_and_extract(
url = url,
sha256 = rctx.attr._zstd_sha256,
)
working_directory = "zstd-{version}".format(version = rctx.attr._zstd_version)
rctx.execute(
["make", "--jobs=4"],
timeout = 600,
quiet = True,
working_directory = working_directory,
)
zstd = "{working_directory}/zstd".format(working_directory = working_directory)
unzstd = "./unzstd"
rctx.symlink(zstd, unzstd)

exec_result = rctx.execute([
"tar",
"--extract",
"--strip-components=2",
"--use-compress-program={unzstd}".format(unzstd = unzstd),
"--file={}".format(release_filename),
])
if exec_result.return_code:
fail(exec_result.stderr)

# Remove files with spaces.
exec_result = rctx.execute([
"find",
".",
"-type",
"f",
"-name",
"*[[:space:]]*",
"-delete",
])
if is_host_windows:
arguments = [
"powershell.exe",
"-c",
"""Get-ChildItem -File -Path "$(Get-Location)" -Include "* *" -Recurse | Remove-Item -Force -Verbose""",
]
else:
arguments = [
"find",
".",
"-type",
"f",
"-name",
"*[[:space:]]*",
"-delete",
]
exec_result = rctx.execute(arguments)
if exec_result.return_code:
fail(exec_result.stderr)

python_bin = "python.exe" if is_target_windows else "bin/python3"

build_content = """\
# Generated by python/repositories.bzl

load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")

package(default_visibility = ["//visibility:public"])

filegroup(
name = "files",
srcs = glob([
"bin/**",
"include/**",
"lib/**",
"share/**",
]),
visibility = ["//visibility:public"],
srcs = glob(
include = [
"bin/**",
"DLLs/**",
"extensions/**",
"include/**",
"lib/**",
"libs/**",
"Scripts/**",
"share/**",
],
Comment on lines +86 to +95
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A select with the platforms would be nicer here.

exclude = [
"**/__pycache__/**",
],
),
)

exports_files(["bin/python3"])
exports_files(["{python_path}"])

py_runtime(
name = "py2_runtime",
Expand All @@ -120,7 +110,7 @@ py_runtime(
py_runtime(
name = "py3_runtime",
files = [":files"],
interpreter = "bin/python3",
interpreter = "{python_path}",
python_version = "PY3",
)

Expand All @@ -129,30 +119,37 @@ py_runtime_pair(
py2_runtime = ":py2_runtime",
py3_runtime = ":py3_runtime",
)
"""
""".format(
python_path = python_bin,
)
rctx.file("BUILD.bazel", build_content)

return {
"integrity": integrity,
"integrity": download_result.integrity,
"name": rctx.attr.name,
"platform": platform,
"python_version": python_version,
}

python_repository = repository_rule(
_python_repository_impl,
doc = _DOC,
attrs = dict({
"_zstd_sha256": attr.string(
default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
doc = "Fetches the external tools needed for the Python toolchain.",
attrs = {
"integrity": attr.string(
doc = "The integrity hash for the Python interpreter tarball.",
mandatory = True,
),
"_zstd_url": attr.string(
default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz",
"platform": attr.string(
doc = "The platform name for the Python interpreter tarball.",
mandatory = True,
values = PLATFORMS.keys(),
),
"_zstd_version": attr.string(
default = "1.5.2",
"python_version": attr.string(
doc = "The Python version.",
mandatory = True,
values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys(),
),
}, **_ATTRS),
},
)

# Wrapper macro around everything above, this is the primary API.
Expand Down