diff --git a/.bazelrc b/.bazelrc index d2e0721526..4e6f2fa187 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f67c8a5ec..aa7fc9d415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,9 +94,9 @@ END_UNRELEASED_TEMPLATE (the default), the subprocess's stdout/stderr will be logged. * (toolchains) Local toolchains can be activated with custom flags. See [Conditionally using local toolchains] docs for how to configure. -* (pypi) `RULES_PYTHON_ENABLE_PIPSTAR` environment variable: when `1`, the Starlark - implementation of wheel METADATA parsing is used (which has improved multi-platform - build support). +* (pypi) Starlark-based evaluation of environment markers (requirements.txt conditionals) + available (not enabled by default) for improved multi-platform build support. + Set the `RULES_PYTHON_ENABLE_PIPSTAR=1` environment variable to enable it. {#v0-0-0-removed} ### Removed diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 3389a0dacc..867c43478a 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -93,9 +93,9 @@ bzl_library( "//python/private:bzlmod_enabled_bzl", "//python/private:py_package.bzl", "//python/private:py_wheel_bzl", - "//python/private:py_wheel_normalize_pep440.bzl", "//python/private:stamp_bzl", "//python/private:util_bzl", + "//python/private:version.bzl", "@bazel_skylib//rules:native_binary", ], ) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 9cc8ffc62c..e72a8fcaa7 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -658,6 +658,11 @@ bzl_library( ], ) +bzl_library( + name = "version_bzl", + srcs = ["version.bzl"], +) + bzl_library( name = "version_label_bzl", srcs = ["version_label.bzl"], @@ -701,7 +706,7 @@ exports_files( "repack_whl.py", "py_package.bzl", "py_wheel.bzl", - "py_wheel_normalize_pep440.bzl", + "version.bzl", "reexports.bzl", "stamp.bzl", "util.bzl", diff --git a/python/private/py_wheel.bzl b/python/private/py_wheel.bzl index c196ca6ad0..ffc24f6846 100644 --- a/python/private/py_wheel.bzl +++ b/python/private/py_wheel.bzl @@ -16,8 +16,8 @@ load(":py_info.bzl", "PyInfo") load(":py_package.bzl", "py_package_lib") -load(":py_wheel_normalize_pep440.bzl", "normalize_pep440") load(":stamp.bzl", "is_stamping_enabled") +load(":version.bzl", "version") PyWheelInfo = provider( doc = "Information about a wheel produced by `py_wheel`", @@ -306,11 +306,11 @@ def _input_file_to_arg(input_file): def _py_wheel_impl(ctx): abi = _replace_make_variables(ctx.attr.abi, ctx) python_tag = _replace_make_variables(ctx.attr.python_tag, ctx) - version = _replace_make_variables(ctx.attr.version, ctx) + version_str = _replace_make_variables(ctx.attr.version, ctx) filename_segments = [ _escape_filename_distribution_name(ctx.attr.distribution), - normalize_pep440(version), + version.normalize(version_str), _escape_filename_segment(python_tag), _escape_filename_segment(abi), _escape_filename_segment(ctx.attr.platform), @@ -343,7 +343,7 @@ def _py_wheel_impl(ctx): args = ctx.actions.args() args.add("--name", ctx.attr.distribution) - args.add("--version", version) + args.add("--version", version_str) args.add("--python_tag", python_tag) args.add("--abi", abi) args.add("--platform", ctx.attr.platform) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index d5d897ef8c..f541cbe98b 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -251,6 +251,7 @@ bzl_library( srcs = ["pep508_env.bzl"], deps = [ ":pep508_platform_bzl", + "//python/private:version_bzl", ], ) diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl index 70840c76c6..61a5b19999 100644 --- a/python/private/pypi/pep508_evaluate.bzl +++ b/python/private/pypi/pep508_evaluate.bzl @@ -16,23 +16,11 @@ """ load("//python/private:enum.bzl", "enum") -load("//python/private:semver.bzl", "semver") +load("//python/private:version.bzl", "version") # The expression parsing and resolution for the PEP508 is below # -# Taken from -# https://peps.python.org/pep-0508/#grammar -# -# version_cmp = wsp* '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' -_VERSION_CMP = sorted( - [ - i.strip(" '") - for i in "'<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '==='".split(" | ") - ], - key = lambda x: (-len(x), x), -) - _STATE = enum( STRING = "string", VAR = "var", @@ -353,36 +341,34 @@ def _env_expr(left, op, right): elif op == ">=": return left >= right else: - return fail("TODO: op unsupported: '{}'".format(op)) + return fail("unsupported op: '{}' {} '{}'".format(left, op, right)) def _version_expr(left, op, right): """Evaluate a version comparison expression""" - left = semver(left) - right = semver(right) - _left = left.key() - _right = right.key() - if op == "<": - return _left < _right + _left = version.parse(left) + _right = version.parse(right) + if _left == None or _right == None: + # Per spec, if either can't be normalized to a version, then + # fallback to simple string comparison. Usually this is `platform_version` + # or `platform_release`, which vary depending on platform. + return _env_expr(left, op, right) + + if op == "===": + return version.is_eeq(_left, _right) + elif op == "!=": + return version.is_ne(_left, _right) + elif op == "==": + return version.is_eq(_left, _right) + elif op == "<": + return version.is_lt(_left, _right) elif op == ">": - return _left > _right + return version.is_gt(_left, _right) elif op == "<=": - return _left <= _right + return version.is_le(_left, _right) elif op == ">=": - return _left >= _right - elif op == "!=": - return _left != _right - elif op == "==": - # Matching of major, minor, patch only - return _left[:3] == _right[:3] + return version.is_ge(_left, _right) elif op == "~=": - right_plus = right.upper() - _right_plus = right_plus.key() - return _left >= _right and _left < _right_plus - elif op == "===": - # Strict matching - return _left == _right - elif op in _VERSION_CMP: - fail("TODO: op unsupported: '{}'".format(op)) + return version.is_compatible(_left, _right) else: return False # Let's just ignore the invalid ops diff --git a/python/private/semver.bzl b/python/private/semver.bzl index cc9ae6ecb6..0cbd172348 100644 --- a/python/private/semver.bzl +++ b/python/private/semver.bzl @@ -43,32 +43,6 @@ def _to_dict(self): "pre_release": self.pre_release, } -def _upper(self): - major = self.major - minor = self.minor - patch = self.patch - build = "" - pre_release = "" - version = self.str() - - if patch != None: - minor = minor + 1 - patch = 0 - elif minor != None: - major = major + 1 - minor = 0 - elif minor == None: - major = major + 1 - - return _new( - major = major, - minor = minor, - patch = patch, - build = build, - pre_release = pre_release, - version = "~" + version, - ) - def _new(*, major, minor, patch, pre_release, build, version = None): # buildifier: disable=uninitialized self = struct( @@ -82,7 +56,6 @@ def _new(*, major, minor, patch, pre_release, build, version = None): key = lambda: _key(self), str = lambda: version, to_dict = lambda: _to_dict(self), - upper = lambda: _upper(self), ) return self diff --git a/python/private/py_wheel_normalize_pep440.bzl b/python/private/version.bzl similarity index 52% rename from python/private/py_wheel_normalize_pep440.bzl rename to python/private/version.bzl index 9566348987..4425cc7661 100644 --- a/python/private/py_wheel_normalize_pep440.bzl +++ b/python/private/version.bzl @@ -59,18 +59,23 @@ def _open_context(self): self.contexts.append(_ctx(_context(self)["start"])) return self.contexts[-1] -def _accept(self): +def _accept(self, key = None): """Close the current ctx successfully and merge the results.""" finished = self.contexts.pop() self.contexts[-1]["norm"] += finished["norm"] + if key: + self.contexts[-1][key] = finished["norm"] + self.contexts[-1]["start"] = finished["start"] return True def _context(self): return self.contexts[-1] -def _discard(self): +def _discard(self, key = None): self.contexts.pop() + if key: + self.contexts[-1][key] = "" return False def _new(input): @@ -313,9 +318,9 @@ def accept_epoch(parser): if accept_digits(parser) and accept(parser, _is("!"), "!"): if ctx["norm"] == "0!": ctx["norm"] = "" - return parser.accept() + return parser.accept("epoch") else: - return parser.discard() + return parser.discard("epoch") def accept_release(parser): """Accept the release segment, numbers separated by dots. @@ -329,10 +334,10 @@ def accept_release(parser): parser.open_context() if not accept_digits(parser): - return parser.discard() + return parser.discard("release") accept_dot_number_sequence(parser) - return parser.accept() + return parser.accept("release") def accept_pre_l(parser): """PEP 440: Pre-release spelling. @@ -374,7 +379,7 @@ def accept_prerelease(parser): accept(parser, _in(["-", "_", "."]), "") if not accept_pre_l(parser): - return parser.discard() + return parser.discard("pre") accept(parser, _in(["-", "_", "."]), "") @@ -382,7 +387,7 @@ def accept_prerelease(parser): # PEP 440: Implicit pre-release number ctx["norm"] += "0" - return parser.accept() + return parser.accept("pre") def accept_implicit_postrelease(parser): """PEP 440: Implicit post releases. @@ -444,9 +449,9 @@ def accept_postrelease(parser): parser.open_context() if accept_implicit_postrelease(parser) or accept_explicit_postrelease(parser): - return parser.accept() + return parser.accept("post") - return parser.discard() + return parser.discard("post") def accept_devrelease(parser): """PEP 440: Developmental releases. @@ -470,9 +475,9 @@ def accept_devrelease(parser): # PEP 440: Implicit development release number ctx["norm"] += "0" - return parser.accept() + return parser.accept("dev") - return parser.discard() + return parser.discard("dev") def accept_local(parser): """PEP 440: Local version identifiers. @@ -487,9 +492,9 @@ def accept_local(parser): if accept(parser, _is("+"), "+") and accept_alnum(parser): accept_separator_alnum_sequence(parser) - return parser.accept() + return parser.accept("local") - return parser.discard() + return parser.discard("local") def normalize_pep440(version): """Escape the version component of a filename. @@ -503,7 +508,31 @@ def normalize_pep440(version): Returns: string containing the normalized version. """ - parser = _new(version.strip()) # PEP 440: Leading and Trailing Whitespace + return _parse(version, strict = True)["norm"] + +def _parse(version_str, strict = True): + """Escape the version component of a filename. + + See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode + and https://peps.python.org/pep-0440/ + + Args: + version_str: version string to be normalized according to PEP 440. + strict: fail if the version is invalid, defaults to True. + + Returns: + string containing the normalized version. + """ + + # https://packaging.python.org/en/latest/specifications/version-specifiers/#leading-and-trailing-whitespace + version = version_str.strip() + is_prefix = False + + if not strict: + is_prefix = version.endswith(".*") + version = version.strip(" .*") # PEP 440: Leading and Trailing Whitespace and ".*" + + parser = _new(version) accept(parser, _is("v"), "") # PEP 440: Preceding v character accept_epoch(parser) accept_release(parser) @@ -511,9 +540,317 @@ def normalize_pep440(version): accept_postrelease(parser) accept_devrelease(parser) accept_local(parser) - if parser.input[parser.context()["start"]:]: - fail( - "Failed to parse PEP 440 version identifier '%s'." % parser.input, - "Parse error at '%s'" % parser.input[parser.context()["start"]:], - ) - return parser.context()["norm"] + + parser_ctx = parser.context() + if parser.input[parser_ctx["start"]:]: + if strict: + fail( + "Failed to parse PEP 440 version identifier '%s'." % parser.input, + "Parse error at '%s'" % parser.input[parser_ctx["start"]:], + ) + + return None + + parser_ctx["is_prefix"] = is_prefix + return parser_ctx + +def parse(version_str, strict = False): + """Parse a PEP4408 compliant version. + + This is similar to `normalize_pep440`, but it parses individual components to + comparable types. + + Args: + version_str: version string to be normalized according to PEP 440. + strict: fail if the version is invalid. + + Returns: + a struct with individual components of a version: + * `epoch` {type}`int`, defaults to `0` + * `release` {type}`tuple[int]` an n-tuple of ints + * `pre` {type}`tuple[str, int] | None` a tuple of a string and an int, + e.g. ("a", 1) + * `post` {type}`tuple[str, int] | None` a tuple of a string and an int, + e.g. ("~", 1) + * `dev` {type}`tuple[str, int] | None` a tuple of a string and an int, + e.g. ("", 1) + * `local` {type}`tuple[str, int] | None` a tuple of components in the local + version, e.g. ("abc", 123). + * `is_prefix` {type}`bool` whether the version_str ends with `.*`. + * `string` {type}`str` normalized value of the input. + """ + + parts = _parse(version_str, strict = strict) + if not parts: + return None + + if parts["is_prefix"] and (parts["local"] or parts["post"] or parts["dev"] or parts["pre"]): + if strict: + fail("local version part has been obtained, but only public segments can have prefix matches") + + # https://peps.python.org/pep-0440/#public-version-identifiers + return None + + return struct( + epoch = _parse_epoch(parts["epoch"]), + release = _parse_release(parts["release"]), + pre = _parse_pre(parts["pre"]), + post = _parse_post(parts["post"]), + dev = _parse_dev(parts["dev"]), + local = _parse_local(parts["local"]), + string = parts["norm"], + is_prefix = parts["is_prefix"], + ) + +def _parse_epoch(value): + if not value: + return 0 + + if not value.endswith("!"): + fail("epoch string segment needs to end with '!', got: {}".format(value)) + + return int(value[:-1]) + +def _parse_release(value): + return tuple([int(d) for d in value.split(".")]) + +def _parse_local(value): + if not value: + return None + + if not value.startswith("+"): + fail("local release identifier must start with '+', got: {}".format(value)) + + # If the part is numerical, handle it as a number + return tuple([int(part) if part.isdigit() else part for part in value[1:].split(".")]) + +def _parse_dev(value): + if not value: + return None + + if not value.startswith(".dev"): + fail("dev release identifier must start with '.dev', got: {}".format(value)) + dev = int(value[len(".dev"):]) + + # Empty string goes first when comparing + return ("", dev) + +def _parse_pre(value): + if not value: + return None + + if value.startswith("rc"): + prefix = "rc" + else: + prefix = value[0] + + return (prefix, int(value[len(prefix):])) + +def _parse_post(value): + if not value: + return None + + if not value.startswith(".post"): + fail("post release identifier must start with '.post', got: {}".format(value)) + post = int(value[len(".post"):]) + + # We choose `~` since almost all of the ASCII characters will be before + # it. Use `ord` and `chr` functions to find a good value. + return ("~", post) + +def _pad_zeros(release, n): + padding = n - len(release) + if padding <= 0: + return release + + release = list(release) + [0] * padding + return tuple(release) + +def _prefix_err(left, op, right): + if left.is_prefix or right.is_prefix: + fail("PEP440: only '==' and '!=' operators can use prefix matching: {} {} {}".format( + left.string, + op, + right.string, + )) + +def _version_eeq(left, right): + """=== operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, "===", right)) + + # https://peps.python.org/pep-0440/#arbitrary-equality + # > simple string equality operations + return left.string == right.string + +def _version_eq(left, right): + """== operator""" + if left.is_prefix and right.is_prefix: + fail("Invalid comparison: both versions cannot be prefix matching") + if left.is_prefix: + return right.string.startswith("{}.".format(left.string)) + if right.is_prefix: + return left.string.startswith("{}.".format(right.string)) + + if left.epoch != right.epoch: + return False + + release_len = max(len(left.release), len(right.release)) + left_release = _pad_zeros(left.release, release_len) + right_release = _pad_zeros(right.release, release_len) + + if left_release != right_release: + return False + + return ( + left.pre == right.pre and + left.post == right.post and + left.dev == right.dev + # local is ignored for == checks + ) + +def _version_compatible(left, right): + """~= operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, "~=", right)) + + # https://peps.python.org/pep-0440/#compatible-release + # Note, the ~= operator can be also expressed as: + # >= V.N, == V.* + + right_star = ".".join([str(d) for d in right.release[:-1]]) + if right.epoch: + right_star = "{}!{}.".format(right.epoch, right_star) + else: + right_star = "{}.".format(right_star) + + return _version_ge(left, right) and left.string.startswith(right_star) + +def _version_ne(left, right): + """!= operator""" + return not _version_eq(left, right) + +def _version_lt(left, right): + """< operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, "<", right)) + + if left.epoch > right.epoch: + return False + elif left.epoch < right.epoch: + return True + + release_len = max(len(left.release), len(right.release)) + left_release = _pad_zeros(left.release, release_len) + right_release = _pad_zeros(right.release, release_len) + + if left_release > right_release: + return False + elif left_release < right_release: + return True + + # From PEP440, this is not a simple ordering check and we need to check the version + # semantically: + # * The exclusive ordered comparison operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, ">", right)) + + if left.epoch > right.epoch: + return True + elif left.epoch < right.epoch: + return False + + release_len = max(len(left.release), len(right.release)) + left_release = _pad_zeros(left.release, release_len) + right_release = _pad_zeros(right.release, release_len) + + if left_release > right_release: + return True + elif left_release < right_release: + return False + + # From PEP440, this is not a simple ordering check and we need to check the version + # semantically: + # * The exclusive ordered comparison >V MUST NOT allow a post-release of the given version + # unless V itself is a post release. + # + # * The exclusive ordered comparison >V MUST NOT match a local version of the specified + # version. + + if left.post and right.post: + return left.post > right.post + else: + # ignore the left.post if right is not a post if right is a post, then this evaluates to + # False anyway. + return False + +def _version_le(left, right): + """<= operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, "<=", right)) + + # PEP440: simple order check + # https://peps.python.org/pep-0440/#inclusive-ordered-comparison + _left = _version_key(left, local = False) + _right = _version_key(right, local = False) + return _left < _right or _version_eq(left, right) + +def _version_ge(left, right): + """>= operator""" + if left.is_prefix or right.is_prefix: + fail(_prefix_err(left, ">=", right)) + + # PEP440: simple order check + # https://peps.python.org/pep-0440/#inclusive-ordered-comparison + _left = _version_key(left, local = False) + _right = _version_key(right, local = False) + return _left > _right or _version_eq(left, right) + +def _version_key(self, *, local = True): + """This function returns a tuple that can be used in 'sorted' calls. + + This implements the PEP440 version sorting. + """ + release_key = ("z",) + local = self.local if local else [] + local = local or [] + + return ( + self.epoch, + self.release, + # PEP440 Within a pre-release, post-release or development release segment with + # a shared prefix, ordering MUST be by the value of the numeric component. + # PEP440 release ordering: .devN, aN, bN, rcN, , .postN + # We choose to first match the pre-release, then post release, then dev and + # then stable + self.pre or self.post or self.dev or release_key, + # PEP440 local versions go before post versions + tuple([(type(item) == "int", item) for item in local]), + # PEP440 - pre-release ordering: .devN, , .postN + self.post or self.dev or release_key, + # PEP440 - post release ordering: .devN, + self.dev or release_key, + ) + +version = struct( + normalize = normalize_pep440, + parse = parse, + # methods, keep sorted + key = _version_key, + is_compatible = _version_compatible, + is_eq = _version_eq, + is_eeq = _version_eeq, + is_ge = _version_ge, + is_gt = _version_gt, + is_le = _version_le, + is_lt = _version_lt, + is_ne = _version_ne, +) diff --git a/tests/py_wheel/py_wheel_tests.bzl b/tests/py_wheel/py_wheel_tests.bzl index 091e01c37d..43c068e597 100644 --- a/tests/py_wheel/py_wheel_tests.bzl +++ b/tests/py_wheel/py_wheel_tests.bzl @@ -17,7 +17,6 @@ load("@rules_testing//lib:analysis_test.bzl", "analysis_test", "test_suite") load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:packaging.bzl", "py_wheel") -load("//python/private:py_wheel_normalize_pep440.bzl", "normalize_pep440") # buildifier: disable=bzl-visibility _basic_tests = [] _tests = [] @@ -168,106 +167,6 @@ def _test_content_type_from_description_impl(env, target): _tests.append(_test_content_type_from_description) -def _test_pep440_normalization(env): - prefixes = ["v", " v", " \t\r\nv"] - epochs = { - "": ["", "0!", "00!"], - "1!": ["1!", "001!"], - "200!": ["200!", "00200!"], - } - releases = { - "0.1": ["0.1", "0.01"], - "2023.7.19": ["2023.7.19", "2023.07.19"], - } - pres = { - "": [""], - "a0": ["a", ".a", "-ALPHA0", "_alpha0", ".a0"], - "a4": ["alpha4", ".a04"], - "b0": ["b", ".b", "-BETA0", "_beta0", ".b0"], - "b5": ["beta05", ".b5"], - "rc0": ["C", "_c0", "RC", "_rc0", "-preview_0"], - } - explicit_posts = { - "": [""], - ".post0": [], - ".post1": [".post1", "-r1", "_rev1"], - } - implicit_posts = [[".post1", "-1"], [".post2", "-2"]] - devs = { - "": [""], - ".dev0": ["dev", "-DEV", "_Dev-0"], - ".dev9": ["DEV9", ".dev09", ".dev9"], - ".dev{BUILD_TIMESTAMP}": [ - "-DEV{BUILD_TIMESTAMP}", - "_dev_{BUILD_TIMESTAMP}", - ], - } - locals = { - "": [""], - "+ubuntu.7": ["+Ubuntu_7", "+ubuntu-007"], - "+ubuntu.r007": ["+Ubuntu_R007"], - } - epochs = [ - [normalized_epoch, input_epoch] - for normalized_epoch, input_epochs in epochs.items() - for input_epoch in input_epochs - ] - releases = [ - [normalized_release, input_release] - for normalized_release, input_releases in releases.items() - for input_release in input_releases - ] - pres = [ - [normalized_pre, input_pre] - for normalized_pre, input_pres in pres.items() - for input_pre in input_pres - ] - explicit_posts = [ - [normalized_post, input_post] - for normalized_post, input_posts in explicit_posts.items() - for input_post in input_posts - ] - pres_and_posts = [ - [normalized_pre + normalized_post, input_pre + input_post] - for normalized_pre, input_pre in pres - for normalized_post, input_post in explicit_posts - ] + [ - [normalized_pre + normalized_post, input_pre + input_post] - for normalized_pre, input_pre in pres - for normalized_post, input_post in implicit_posts - if input_pre == "" or input_pre[-1].isdigit() - ] - devs = [ - [normalized_dev, input_dev] - for normalized_dev, input_devs in devs.items() - for input_dev in input_devs - ] - locals = [ - [normalized_local, input_local] - for normalized_local, input_locals in locals.items() - for input_local in input_locals - ] - postfixes = ["", " ", " \t\r\n"] - i = 0 - for nepoch, iepoch in epochs: - for nrelease, irelease in releases: - for nprepost, iprepost in pres_and_posts: - for ndev, idev in devs: - for nlocal, ilocal in locals: - prefix = prefixes[i % len(prefixes)] - postfix = postfixes[(i // len(prefixes)) % len(postfixes)] - env.expect.that_str( - normalize_pep440( - prefix + iepoch + irelease + iprepost + - idev + ilocal + postfix, - ), - ).equals( - nepoch + nrelease + nprepost + ndev + nlocal, - ) - i += 1 - -_basic_tests.append(_test_pep440_normalization) - def py_wheel_test_suite(name): test_suite( name = name, diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index 303c167900..7b6c064b94 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -19,6 +19,12 @@ load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # bui _tests = [] +def _check_evaluate(env, expr, expected, values, strict = True): + env.expect.where( + expression = expr, + values = values, + ).that_bool(evaluate(expr, env = values, strict = strict)).equals(expected) + def _tokenize_tests(env): for input, want in { "": [], @@ -82,23 +88,11 @@ def _evaluate_non_version_env_tests(env): "{} > 'osx'".format(var_name): False, "{} >= 'osx'".format(var_name): True, }.items(): - got = evaluate( - input, - env = marker_env, - ) - env.expect.where( - expr = input, - env = marker_env, - ).that_bool(got).equals(want) + _check_evaluate(env, input, want, marker_env) # Check that the non-strict eval gives us back the input when no # env is supplied. - got = evaluate( - input, - env = {}, - strict = False, - ) - env.expect.that_bool(got).equals(input.replace("'", '"')) + _check_evaluate(env, input, input.replace("'", '"'), {}, strict = False) _tests.append(_evaluate_non_version_env_tests) @@ -123,6 +117,7 @@ def _evaluate_version_env_tests(env): "{} <= '3.7.10'".format(var_name): True, "{} <= '3.7.8'".format(var_name): False, "{} == '3.7.9'".format(var_name): True, + "{} == '3.7.*'".format(var_name): True, "{} != '3.7.9'".format(var_name): False, "{} ~= '3.7.1'".format(var_name): True, "{} ~= '3.7.10'".format(var_name): False, @@ -131,23 +126,32 @@ def _evaluate_version_env_tests(env): "{} === '3.7.9'".format(var_name): True, "{} == '3.7.9+rc2'".format(var_name): True, }.items(): # buildifier: @unsorted-dict-items - got = evaluate( - input, - env = marker_env, - ) - env.expect.that_collection((input, got)).contains_exactly((input, want)) + _check_evaluate(env, input, want, marker_env) # Check that the non-strict eval gives us back the input when no # env is supplied. - got = evaluate( - input, - env = {}, - strict = False, - ) - env.expect.that_bool(got).equals(input.replace("'", '"')) + _check_evaluate(env, input, input.replace("'", '"'), {}, strict = False) _tests.append(_evaluate_version_env_tests) +def _evaluate_platform_version_is_special(env): + # Given + marker_env = {"platform_version": "FooBar Linux v1.2.3"} + + # When the platform version is not + input = "platform_version == '0'" + _check_evaluate(env, input, False, marker_env) + + # And when I compare it as string + input = "'FooBar' in platform_version" + _check_evaluate(env, input, True, marker_env) + + # Check that the non-strict eval gives us back the input when no + # env is supplied. + _check_evaluate(env, input, input.replace("'", '"'), {}, strict = False) + +_tests.append(_evaluate_platform_version_is_special) + def _logical_expression_tests(env): for input, want in { # Basic @@ -195,13 +199,7 @@ def _logical_expression_tests(env): "not not os_name == 'foo'": True, "not not not os_name == 'foo'": False, }.items(): # buildifier: @unsorted-dict-items - got = evaluate( - input, - env = { - "os_name": "foo", - }, - ) - env.expect.that_collection((input, got)).contains_exactly((input, want)) + _check_evaluate(env, input, want, {"os_name": "foo"}) if not input.strip("()"): # These cases will just return True, because they will be evaluated @@ -210,12 +208,7 @@ def _logical_expression_tests(env): # Check that the non-strict eval gives us back the input when no env # is supplied. - got = evaluate( - input, - env = {}, - strict = False, - ) - env.expect.that_bool(got).equals(input.replace("'", '"')) + _check_evaluate(env, input, input.replace("'", '"'), {}, strict = False) _tests.append(_logical_expression_tests) @@ -244,6 +237,7 @@ def _evaluate_partial_only_extra(env): strict = False, ) env.expect.that_bool(got).equals(want) + _check_evaluate(env, input, want, {"extra": extra}, strict = False) _tests.append(_evaluate_partial_only_extra) @@ -268,14 +262,61 @@ def _evaluate_with_aliases(env): }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - got = evaluate( - input, - env = pep508_env(target_platform), - ) - env.expect.that_bool(got).equals(want) + _check_evaluate(env, input, want, pep508_env(target_platform)) _tests.append(_evaluate_with_aliases) +def _expr_case(expr, want, env): + return struct(expr = expr.strip(), want = want, env = env) + +_MISC_EXPRESSIONS = [ + _expr_case('python_version == "3.*"', True, {"python_version": "3.10.1"}), + _expr_case('python_version != "3.10.*"', False, {"python_version": "3.10.1"}), + _expr_case('python_version != "3.11.*"', True, {"python_version": "3.10.1"}), + _expr_case('python_version != "3.10"', False, {"python_version": "3.10.0"}), + _expr_case('python_version == "3.10"', True, {"python_version": "3.10.0"}), + # Cases for the '>' operator + # Taken from spec: https://peps.python.org/pep-0440/#exclusive-ordered-comparison + _expr_case('python_version > "1.7"', True, {"python_version": "1.7.1"}), + _expr_case('python_version > "1.7"', False, {"python_version": "1.7.0.post0"}), + _expr_case('python_version > "1.7"', True, {"python_version": "1.7.1"}), + _expr_case('python_version > "1.7.post2"', True, {"python_version": "1.7.1"}), + _expr_case('python_version > "1.7.post2"', True, {"python_version": "1.7.post3"}), + _expr_case('python_version > "1.7.post2"', False, {"python_version": "1.7.0"}), + _expr_case('python_version > "1.7.1+local"', False, {"python_version": "1.7.1"}), + _expr_case('python_version > "1.7.1+local"', True, {"python_version": "1.7.2"}), + # Extra cases for the '<' operator + _expr_case('python_version < "1.7.1"', False, {"python_version": "1.7.2"}), + _expr_case('python_version < "1.7.3"', True, {"python_version": "1.7.2"}), + _expr_case('python_version < "1.7.1"', True, {"python_version": "1.7"}), + _expr_case('python_version < "1.7.1"', False, {"python_version": "1.7.1-rc2"}), + _expr_case('python_version < "1.7.1-rc3"', True, {"python_version": "1.7.1-rc2"}), + _expr_case('python_version < "1.7.1-rc1"', False, {"python_version": "1.7.1-rc2"}), + # Extra tests + _expr_case('python_version <= "1.7.1"', True, {"python_version": "1.7.1"}), + _expr_case('python_version <= "1.7.2"', True, {"python_version": "1.7.1"}), + _expr_case('python_version >= "1.7.1"', True, {"python_version": "1.7.1"}), + _expr_case('python_version >= "1.7.0"', True, {"python_version": "1.7.1"}), + # Compatible version tests: + # https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release + _expr_case('python_version ~= "2.2"', True, {"python_version": "2.3"}), + _expr_case('python_version ~= "2.2"', False, {"python_version": "2.1"}), + _expr_case('python_version ~= "2.2.post3"', False, {"python_version": "2.2"}), + _expr_case('python_version ~= "2.2.post3"', True, {"python_version": "2.3"}), + _expr_case('python_version ~= "2.2.post3"', False, {"python_version": "3.0"}), + _expr_case('python_version ~= "1!2.2"', False, {"python_version": "2.7"}), + _expr_case('python_version ~= "0!2.2"', True, {"python_version": "2.7"}), + _expr_case('python_version ~= "1!2.2"', True, {"python_version": "1!2.7"}), + _expr_case('python_version ~= "1.2.3"', True, {"python_version": "1.2.4"}), + _expr_case('python_version ~= "1.2.3"', False, {"python_version": "1.3.2"}), +] + +def _misc_expressions(env): + for case in _MISC_EXPRESSIONS: + _check_evaluate(env, case.expr, case.want, case.env) + +_tests.append(_misc_expressions) + def evaluate_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/semver/semver_test.bzl b/tests/semver/semver_test.bzl index aef3deca82..9d13402c92 100644 --- a/tests/semver/semver_test.bzl +++ b/tests/semver/semver_test.bzl @@ -104,24 +104,6 @@ def _test_semver_sort(env): _tests.append(_test_semver_sort) -def _test_upper(env): - for input, want in { - # Depending on how many version numbers are specified we will increase - # the upper bound differently. See https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release for docs - "0.0.1": "0.1.0", - "0.1": "1.0", - "0.1.0": "0.2.0", - "1": "2", - "1.0.0-pre": "1.1.0", # pre-release info is dropped - "1.2.0": "1.3.0", - "2.0.0+build0": "2.1.0", # build info is dropped - }.items(): - actual = semver(input).upper().key() - want = semver(want).key() - env.expect.that_collection(actual).contains_exactly(want).in_order() - -_tests.append(_test_upper) - def semver_test_suite(name): """Create the test suite. diff --git a/tests/version/BUILD.bazel b/tests/version/BUILD.bazel new file mode 100644 index 0000000000..d6fdecd4cf --- /dev/null +++ b/tests/version/BUILD.bazel @@ -0,0 +1,3 @@ +load(":version_test.bzl", "version_test_suite") + +version_test_suite(name = "version_tests") diff --git a/tests/version/version_test.bzl b/tests/version/version_test.bzl new file mode 100644 index 0000000000..589f9ac05d --- /dev/null +++ b/tests/version/version_test.bzl @@ -0,0 +1,157 @@ +"" + +load("@rules_testing//lib:analysis_test.bzl", "test_suite") +load("//python/private:version.bzl", "version") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_normalization(env): + prefixes = ["v", " v", " \t\r\nv"] + epochs = { + "": ["", "0!", "00!"], + "1!": ["1!", "001!"], + "200!": ["200!", "00200!"], + } + releases = { + "0.1": ["0.1", "0.01"], + "2023.7.19": ["2023.7.19", "2023.07.19"], + } + pres = { + "": [""], + "a0": ["a", ".a", "-ALPHA0", "_alpha0", ".a0"], + "a4": ["alpha4", ".a04"], + "b0": ["b", ".b", "-BETA0", "_beta0", ".b0"], + "b5": ["beta05", ".b5"], + "rc0": ["C", "_c0", "RC", "_rc0", "-preview_0"], + } + explicit_posts = { + "": [""], + ".post0": [], + ".post1": [".post1", "-r1", "_rev1"], + } + implicit_posts = [[".post1", "-1"], [".post2", "-2"]] + devs = { + "": [""], + ".dev0": ["dev", "-DEV", "_Dev-0"], + ".dev9": ["DEV9", ".dev09", ".dev9"], + ".dev{BUILD_TIMESTAMP}": [ + "-DEV{BUILD_TIMESTAMP}", + "_dev_{BUILD_TIMESTAMP}", + ], + } + locals = { + "": [""], + "+ubuntu.7": ["+Ubuntu_7", "+ubuntu-007"], + "+ubuntu.r007": ["+Ubuntu_R007"], + } + epochs = [ + [normalized_epoch, input_epoch] + for normalized_epoch, input_epochs in epochs.items() + for input_epoch in input_epochs + ] + releases = [ + [normalized_release, input_release] + for normalized_release, input_releases in releases.items() + for input_release in input_releases + ] + pres = [ + [normalized_pre, input_pre] + for normalized_pre, input_pres in pres.items() + for input_pre in input_pres + ] + explicit_posts = [ + [normalized_post, input_post] + for normalized_post, input_posts in explicit_posts.items() + for input_post in input_posts + ] + pres_and_posts = [ + [normalized_pre + normalized_post, input_pre + input_post] + for normalized_pre, input_pre in pres + for normalized_post, input_post in explicit_posts + ] + [ + [normalized_pre + normalized_post, input_pre + input_post] + for normalized_pre, input_pre in pres + for normalized_post, input_post in implicit_posts + if input_pre == "" or input_pre[-1].isdigit() + ] + devs = [ + [normalized_dev, input_dev] + for normalized_dev, input_devs in devs.items() + for input_dev in input_devs + ] + locals = [ + [normalized_local, input_local] + for normalized_local, input_locals in locals.items() + for input_local in input_locals + ] + postfixes = ["", " ", " \t\r\n"] + i = 0 + for nepoch, iepoch in epochs: + for nrelease, irelease in releases: + for nprepost, iprepost in pres_and_posts: + for ndev, idev in devs: + for nlocal, ilocal in locals: + prefix = prefixes[i % len(prefixes)] + postfix = postfixes[(i // len(prefixes)) % len(postfixes)] + env.expect.that_str( + version.normalize( + prefix + iepoch + irelease + iprepost + + idev + ilocal + postfix, + ), + ).equals( + nepoch + nrelease + nprepost + ndev + nlocal, + ) + i += 1 + +_tests.append(_test_normalization) + +def _test_ordering(env): + want = [ + # Taken from https://peps.python.org/pep-0440/#summary-of-permitted-suffixes-and-relative-ordering + "1.dev0", + "1.0.dev456", + "1.0a1", + "1.0a2.dev456", + "1.0a12.dev456", + "1.0a12", + "1.0b1.dev456", + "1.0b1.dev457", + "1.0b2", + "1.0b2.post345.dev456", + "1.0b2.post345.dev457", + "1.0b2.post345", + "1.0rc1.dev456", + "1.0rc1", + "1.0", + "1.0+abc.5", + "1.0+abc.7", + "1.0+5", + "1.0.post456.dev34", + "1.0.post456", + "1.0.15", + "1.1.dev1", + "1!0.1", + ] + + for lower, higher in zip(want[:-1], want[1:]): + lower = version.parse(lower, strict = True) + higher = version.parse(higher, strict = True) + + lower_key = version.key(lower) + higher_key = version.key(higher) + + if not lower_key < higher_key: + env.fail("Expected '{}'.key() to be smaller than '{}'.key(), but got otherwise: {} > {}".format( + lower.string, + higher.string, + lower_key, + higher_key, + )) + +_tests.append(_test_ordering) + +def version_test_suite(name): + test_suite( + name = name, + basic_tests = _tests, + )