From 227aa3dc0975b4ebcfae27a0f10c5d341f3aa4b3 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Thu, 24 Feb 2022 08:57:52 -0800 Subject: [PATCH 1/5] Added windows support to hermetic toolchains --- .../tests/toolchains/python_version_test.py | 8 +- .../tests/toolchains/versions_test.bzl | 1 + python/private/toolchains_repo.bzl | 6 + python/private/versions.bzl | 33 ++--- python/repositories.bzl | 135 +++++++++--------- 5 files changed, 95 insertions(+), 88 deletions(-) diff --git a/python/private/tests/toolchains/python_version_test.py b/python/private/tests/toolchains/python_version_test.py index ad3596f467..700b6ec5e2 100644 --- a/python/private/tests/toolchains/python_version_test.py +++ b/python/private/tests/toolchains/python_version_test.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import platform +import unittest + class TestPythonVersion(unittest.TestCase): def test_match_toolchain(self): - self.assertEqual(platform.python_version(), "3.9.7") + self.assertEqual(platform.python_version(), "3.9.10") + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/python/private/tests/toolchains/versions_test.bzl b/python/private/tests/toolchains/versions_test.bzl index 8650c89565..866249b5c0 100644 --- a/python/private/tests/toolchains/versions_test.bzl +++ b/python/private/tests/toolchains/versions_test.bzl @@ -21,6 +21,7 @@ load("//python/private:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") required_platforms = [ "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu", ] diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 297024f291..c9ad35c6e5 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -36,6 +36,12 @@ PLATFORMS = { "@platforms//cpu:x86_64", ], ), + "x86_64-pc-windows-msvc": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), "x86_64-unknown-linux-gnu": struct( compatible_with = [ "@platforms//os:linux", diff --git a/python/private/versions.bzl b/python/private/versions.bzl index 2bd32f3da1..7d136f4aa6 100644 --- a/python/private/versions.bzl +++ b/python/private/versions.bzl @@ -15,32 +15,33 @@ """The Python versions we use for the toolchains. """ -RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20211017" -RELEASE_DATE = "20211017T1616" +RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20220227" +RELEASE_DATE = "20220227" -# The integrity hashes can be computed with: -# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64 # buildifier: disable=unsorted-dict-items TOOL_VERSIONS = { "3.8.12": { - "x86_64-apple-darwin": "sha384-es0kCVBb4q5xSC09lOw83TKXtR6qdt0NeU56JtK7Y5M5V784k9MM2q8leE3QWGH6", - "x86_64-unknown-linux-gnu": "sha384-61BjosOUm+Uo2yDtkLcRqbSIAXIPiHKmvySGO2J/GSOHXkj5QBf4cwzrDuOWR0H2", + "x86_64-apple-darwin": "sha256-8yP7xVgDXBOoXOImfQ+tnokoImjsuBDjZP/x0KB51SU=", + "x86_64-pc-windows-msvc": "sha256-kk+f1R/2zMUz7Y6WxUYXaNpXges9/BHYRvnjAPq0Tto=", + "x86_64-unknown-linux-gnu": "sha256-W+nG1h4ji5Df2UdVBRwNOi2AI+v/20sPpOj+3QmmyrY=", }, - "3.9.7": { - "x86_64-apple-darwin": "sha384-Ix7lxb+niRWcDCOsK1q53dAkOp0CIw40kM8wHv//WQs2O/il8SRJ60cP3R67fEnm", - "aarch64-apple-darwin": "sha384-JSi3XZdHVZ+6DMybPYKgEYSQuBPpjXgBlj1uGqB9f/r3Wi6P0+CnYRG12TEzgcs6", - "x86_64-unknown-linux-gnu": "sha384-+PNcmKXJ+ZiyKyZ2WR1XedDXJ05ujC2w9TdXl80vloYMqfIOpcVPOWUgre+btI+3", + "3.9.10": { + "aarch64-apple-darwin": "sha256-rWbCo+cmMUfgRqMmlN57iXpG+wEkQJ0p06k+3mMciu4=", + "x86_64-apple-darwin": "sha256-/a9ZQUJEYCnjFKm+uR8ax1r4ZjILULi5aBgeWSVQzWg=", + "x86_64-pc-windows-msvc": "sha256-W8Zc4CNhS/SWpnSOQdypNLcPxfrG36zEaqjbytdyr8I=", + "x86_64-unknown-linux-gnu": "sha256-RVCJzFdr2aWNtF6RnR/IZ+zbsCCAZ9/8hFzJu/BwG3A=", }, - "3.10.0": { - "x86_64-apple-darwin": "sha384-eVTq704heZyKW6SB/DzarWB9S5bRH1LTQ4rFHzlKTE9jDjHDCBKyQhSgYy8a62lt", - "aarch64-apple-darwin": "sha384-NbhxnZL0pBTKpzEfoCYWl6s2GYdfiI9HOSSHn1iCMZnIY6htt/KhzjwIgCP+Nj2u", - "x86_64-unknown-linux-gnu": "sha384-iYJF9Y9o2Ain3YkuuF7ZGrGuJ+MyiD/xnrjJSap0TF2DR+I9XDx4stunNgx17gSn", + "3.10.2": { + "aarch64-apple-darwin": "sha256-FAms2aUG4tHTtlwUiNtOQNjxnQmn3wmWZ8h6UG9xwO8=", + "x86_64-apple-darwin": "sha256-gUatQ5BxDsabMWpWSZEt8CR9NfSkLiqpYVv/2Hs+I1o=", + "x86_64-pc-windows-msvc": "sha256-opPFg43ZyEOKhDcvuV3al1LfY5KKiirlFkOPGH+JVn0=", + "x86_64-unknown-linux-gnu": "sha256-m2TsoqlPev+UCa1wvap/u/gUhpJmLnZEAYg5V5Q2IN0=", }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { "3.8": "3.8.12", - "3.9": "3.9.7", - "3.10": "3.10.0", + "3.9": "3.9.10", + "3.10": "3.10.2", } diff --git a/python/repositories.bzl b/python/repositories.bzl index 2ec6233cec..a9e6f523ce 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -27,89 +27,79 @@ def py_repositories(): ######## # Remaining content of the file is only used to support toolchains. ######## -_DOC = "Fetch external tools needed for python toolchain." -_ATTRS = { - "integrity": attr.string(mandatory = True), - "platform": attr.string(mandatory = True, values = PLATFORMS.keys()), - "python_version": attr.string(mandatory = True, values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys()), -} def _python_repository_impl(rctx): - python_version = rctx.attr.python_version platform = rctx.attr.platform - release_filename = "cpython-{version}-{platform}-pgo+lto-{release_date}.tar.zst".format( + is_host_windows = "win" in rctx.os.name + is_target_windows = "windows" in rctx.attr.platform + python_version = rctx.attr.python_version + build = "static-install_only" if is_target_windows else "install_only" + release_filename = "cpython-{version}+{release_date}-{platform}-{build}.tar.gz".format( + build = build, + platform = platform, release_date = RELEASE_DATE, version = python_version, - platform = platform, ) url = "{release_url}/{release_filename}".format( release_url = RELEASE_URL, release_filename = release_filename, ) - integrity = rctx.attr.integrity - rctx.download( + download_result = rctx.download_and_extract( url = url, - integrity = integrity, - output = release_filename, + integrity = rctx.attr.integrity, + stripPrefix = "python", ) - unzstd = rctx.which("unzstd") - if not unzstd: - url = rctx.attr._zstd_url.format(version = rctx.attr._zstd_version) - rctx.download_and_extract( - url = url, - sha256 = rctx.attr._zstd_sha256, - ) - working_directory = "zstd-{version}".format(version = rctx.attr._zstd_version) - rctx.execute( - ["make", "--jobs=4"], - timeout = 600, - quiet = True, - working_directory = working_directory, - ) - zstd = "{working_directory}/zstd".format(working_directory = working_directory) - unzstd = "./unzstd" - rctx.symlink(zstd, unzstd) - - exec_result = rctx.execute([ - "tar", - "--extract", - "--strip-components=2", - "--use-compress-program={unzstd}".format(unzstd = unzstd), - "--file={}".format(release_filename), - ]) - if exec_result.return_code: - fail(exec_result.stderr) # Remove files with spaces. - exec_result = rctx.execute([ - "find", - ".", - "-type", - "f", - "-name", - "*[[:space:]]*", - "-delete", - ]) + if is_host_windows: + arguments = [ + "powershell.exe", + "-c", + """Get-ChildItem -File -Path "$(Get-Location)" -Include "* *" -Recurse | Remove-Item -Force -Verbose""", + ] + else: + arguments = [ + "find", + ".", + "-type", + "f", + "-name", + "*[[:space:]]*", + "-delete", + ] + exec_result = rctx.execute(arguments) if exec_result.return_code: fail(exec_result.stderr) + python_bin = "python.exe" if is_target_windows else "bin/python3" + build_content = """\ # Generated by python/repositories.bzl load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") +package(default_visibility = ["//visibility:public"]) + filegroup( name = "files", - srcs = glob([ - "bin/**", - "include/**", - "lib/**", - "share/**", - ]), - visibility = ["//visibility:public"], + srcs = glob( + include = [ + "bin/**", + "DLLs/**", + "extensions/**", + "include/**", + "lib/**", + "libs/**", + "Scripts/**", + "share/**", + ], + exclude = [ + "**/__pycache__/**", + ], + ), ) -exports_files(["bin/python3"]) +exports_files(["{python_path}"]) py_runtime( name = "py2_runtime", @@ -120,7 +110,7 @@ py_runtime( py_runtime( name = "py3_runtime", files = [":files"], - interpreter = "bin/python3", + interpreter = "{python_path}", python_version = "PY3", ) @@ -129,11 +119,13 @@ py_runtime_pair( py2_runtime = ":py2_runtime", py3_runtime = ":py3_runtime", ) -""" +""".format( + python_path = python_bin, + ) rctx.file("BUILD.bazel", build_content) return { - "integrity": integrity, + "integrity": download_result.integrity, "name": rctx.attr.name, "platform": platform, "python_version": python_version, @@ -141,18 +133,23 @@ py_runtime_pair( python_repository = repository_rule( _python_repository_impl, - doc = _DOC, - attrs = dict({ - "_zstd_sha256": attr.string( - default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", + doc = "Fetch external tools needed for python toolchain.", + attrs = { + "integrity": attr.string( + doc = "TODO", + mandatory = True, ), - "_zstd_url": attr.string( - default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", + "platform": attr.string( + doc = "TODO", + mandatory = True, + values = PLATFORMS.keys(), ), - "_zstd_version": attr.string( - default = "1.5.2", + "python_version": attr.string( + doc = "TODO", + mandatory = True, + values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys(), ), - }, **_ATTRS), + }, ) # Wrapper macro around everything above, this is the primary API. From 09c57bd2709f67f123d3c4c86ae76d8117ed613d Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 2 Mar 2022 12:07:42 -0800 Subject: [PATCH 2/5] Update python/repositories.bzl Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index a9e6f523ce..b4b75a2c41 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -145,7 +145,7 @@ python_repository = repository_rule( values = PLATFORMS.keys(), ), "python_version": attr.string( - doc = "TODO", + doc = "The Python version.", mandatory = True, values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys(), ), From 8a53ef46ab84fd1aeb13eb453b4bb83561f60470 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 2 Mar 2022 12:07:49 -0800 Subject: [PATCH 3/5] Update python/repositories.bzl Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index b4b75a2c41..fc09ebfafa 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -133,7 +133,7 @@ py_runtime_pair( python_repository = repository_rule( _python_repository_impl, - doc = "Fetch external tools needed for python toolchain.", + doc = "Fetches the external tools needed for the Python toolchain.", attrs = { "integrity": attr.string( doc = "TODO", From 3ce5c91dcb314e4f78f0adf37970cf56bfce5795 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 2 Mar 2022 12:08:03 -0800 Subject: [PATCH 4/5] Update python/repositories.bzl Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index fc09ebfafa..ae115a23de 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -136,7 +136,7 @@ python_repository = repository_rule( doc = "Fetches the external tools needed for the Python toolchain.", attrs = { "integrity": attr.string( - doc = "TODO", + doc = "The integrity hash for the Python interpreter tarball.", mandatory = True, ), "platform": attr.string( From e063c204efbbe7128eaf8de559399cd8767e87f2 Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Wed, 2 Mar 2022 12:08:12 -0800 Subject: [PATCH 5/5] Update python/repositories.bzl Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/repositories.bzl b/python/repositories.bzl index ae115a23de..9a4b5e394d 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -140,7 +140,7 @@ python_repository = repository_rule( mandatory = True, ), "platform": attr.string( - doc = "TODO", + doc = "The platform name for the Python interpreter tarball.", mandatory = True, values = PLATFORMS.keys(), ),