From 620acaad2cdb403bba57fb337418dbf3a7d72fc0 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Thu, 2 May 2024 00:33:19 -0500 Subject: [PATCH 01/10] Initial stab. --- Lib/_pydecimal.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index de4561a5ee050b..b4cd222ace2cb8 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -423,6 +423,39 @@ def sin(x): # (because Decimals are not interoperable with floats). See the notes in # numbers.py for more detail. +import re +_is_power_of_10 = re.compile(r"10*").fullmatch +del re + +# In Decimal._power_exact(), there are two blocks of the form +# if xc > 10**p: +# return None +# ... +# code using str(zc) +# where p is the maximum exponent. Which can be truly gigantic. So Python +# can appear to hang if those blocke are hit, or, if you wait long enough, +# run out of RAM. For example, +# https://github.com/python/cpython/issues/118027 +# where this happens during decimal.Decimal(2) ** 117 after +# ctx.Emax is set to decimal.MAX_EMAX. +# +# This function instead converts to string first, then sees whether the +# string is compatible with the claim that it's a value <= 10**p. If so, +# it returns the string; else None. So the code above becomes instead: +# str_xc = _convert_to_str(zc, p) +# if str_xc is None: +# return None +# use `str_xc` instead of str(xc) +# The code in _power_exact() is very involved, and I'm not certain xc can't +# be very much larger than 10**p. I _doubt_ it can. But, if I'm wrong, a +# different approach would be better. +def _convert_to_str(ec, p): + sc = str(ec) + if len(sc) <= p or _is_power_of_10(sc): + return sc + else: + return None + class Decimal(object): """Floating point class for decimal arithmetic.""" @@ -2131,10 +2164,11 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + strxc = _convert_to_str(xc, p) + if strxc is None: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2184,13 +2218,13 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + str_xc = _convert_to_str(xc, p) + if str_xc is None: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) From ecf1394c6b4e0517cf1d6fb3c5ca533eda263040 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 3 May 2024 15:54:34 -0500 Subject: [PATCH 02/10] Test the tentative fix. Hangs "forever" without this change. --- Lib/test/test_decimal.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7010c34792e093..114c846de63356 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -5705,6 +5705,20 @@ def test_format_fallback_rounding(self): with C.localcontext(rounding=C.ROUND_DOWN): self.assertEqual(format(y, '#.1f'), '6.0') + def test_python_hang_in_exact_power(self): + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**MAX_EMAX. + # Fixed via https://github.com/python/cpython/pull/118503. + with P.localcontext() as ctx: + ctx.prec = P.MAX_PREC + ctx.Emax = P.MAX_EMAX + ctx.Emin = P.MIN_EMIN + ctx.traps[P.Inexact] = 1 + D2 = P.Decimal(2) + # If the bug is still present, the next statement won't complete + res = D2 ** 117 + self.assertEqual(res, 1 << 117) @requires_docstrings @requires_cdecimal From 3e9417d88643582c7bb6363366cd3de8290c1d06 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 3 May 2024 16:17:14 -0500 Subject: [PATCH 03/10] Move the new test to a better spot. --- Lib/test/test_decimal.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 114c846de63356..4a331070d037d9 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4719,6 +4719,19 @@ def test_py_exact_power(self): c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**MAX_EMAX. + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5705,21 +5718,6 @@ def test_format_fallback_rounding(self): with C.localcontext(rounding=C.ROUND_DOWN): self.assertEqual(format(y, '#.1f'), '6.0') - def test_python_hang_in_exact_power(self): - # See https://github.com/python/cpython/issues/118027 - # Testing for an exact power could appear to hang, in the Python - # version, as it attempted to compute 10**MAX_EMAX. - # Fixed via https://github.com/python/cpython/pull/118503. - with P.localcontext() as ctx: - ctx.prec = P.MAX_PREC - ctx.Emax = P.MAX_EMAX - ctx.Emin = P.MIN_EMIN - ctx.traps[P.Inexact] = 1 - D2 = P.Decimal(2) - # If the bug is still present, the next statement won't complete - res = D2 ** 117 - self.assertEqual(res, 1 << 117) - @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): From bb8f4edb9dc82a6160c7c09cf4dddeb0e7d8bde6 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 3 May 2024 22:11:05 -0500 Subject: [PATCH 04/10] New comment to explain why _convert_to_str allows any poewr of 10. --- Lib/_pydecimal.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index b4cd222ace2cb8..67c3f365b98dcc 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -449,6 +449,16 @@ def sin(x): # The code in _power_exact() is very involved, and I'm not certain xc can't # be very much larger than 10**p. I _doubt_ it can. But, if I'm wrong, a # different approach would be better. +# BUG ALERT: the code in the first block above returns None if xc ix sn +# integer power of 10 > 10**p. But the replacement accepts any such power of +# 10. This was intentional, but may be wrong. I (Tim) am taking the comments +# at their word: that the purpose is to identify cases that are exactly +# representable in decimal floating point with p digits of precision. Every +# integer power of 10 is, and even if p is just 1. If that's wrong, the +# original behavior can be gotten by changing the test below to: +# len(sc) <= p or (len(sc) == p+1 and _is_power_of_10(sc)) +# Alas, I don't know how to contrive inputs to trigger these paths. + def _convert_to_str(ec, p): sc = str(ec) if len(sc) <= p or _is_power_of_10(sc): From a73b81138f2e5c3b89d887f757f7d9f6797aed26 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 3 May 2024 23:45:17 -0500 Subject: [PATCH 05/10] Fixed a comment, and fleshed out an existing test that appeared unfinished. --- Lib/test/test_decimal.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 4a331070d037d9..e927e24b582a5d 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4716,12 +4716,23 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") # See https://github.com/python/cpython/issues/118027 # Testing for an exact power could appear to hang, in the Python - # version, as it attempted to compute 10**MAX_EMAX. + # version, as it attempted to compute 10**(MAX_EMAX + 1). # Fixed via https://github.com/python/cpython/pull/118503. c.prec = P.MAX_PREC c.Emax = P.MAX_EMAX From e45a0b06db07ea9d03344803151892690d6f67b2 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 4 May 2024 02:14:47 -0500 Subject: [PATCH 06/10] Added temporary asserts. Or maybe permanent ;-) --- Lib/_pydecimal.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 67c3f365b98dcc..2d56a258e21021 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -449,6 +449,7 @@ def sin(x): # The code in _power_exact() is very involved, and I'm not certain xc can't # be very much larger than 10**p. I _doubt_ it can. But, if I'm wrong, a # different approach would be better. +# # BUG ALERT: the code in the first block above returns None if xc ix sn # integer power of 10 > 10**p. But the replacement accepts any such power of # 10. This was intentional, but may be wrong. I (Tim) am taking the comments @@ -458,6 +459,12 @@ def sin(x): # original behavior can be gotten by changing the test below to: # len(sc) <= p or (len(sc) == p+1 and _is_power_of_10(sc)) # Alas, I don't know how to contrive inputs to trigger these paths. +# +# LATER: I believe it's impossible for _is_power_of_10(sc) to be true in +# either of the code blocks using _convert_to_str() at this time. But I'll +# leave it in anyway, "just in case", and so this function is more +# bulletproof if it gets used in other contexts. It's a cheap test and rarely +# executed. def _convert_to_str(ec, p): sc = str(ec) @@ -2224,6 +2231,8 @@ def _power_exact(self, other, p): # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > # 10**p and the result is not representable. + assert xc != 1, self + assert not _is_power_of_10(str(xc)), self if xc > 1 and m > p*100//_log10_lb(xc): return None xc = xc**m From 8f95bae196bd1446d474b5a96d5ae156e3c0f70c Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 4 May 2024 13:39:35 -0500 Subject: [PATCH 07/10] Update Lib/_pydecimal.py Co-authored-by: Serhiy Storchaka --- Lib/_pydecimal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 2d56a258e21021..5911432c798454 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -2232,7 +2232,7 @@ def _power_exact(self, other, p): # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > # 10**p and the result is not representable. assert xc != 1, self - assert not _is_power_of_10(str(xc)), self + assert xc % 10 != 0, self if xc > 1 and m > p*100//_log10_lb(xc): return None xc = xc**m From 65bced33a879e246d92d500070ac041c0e6629c7 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 4 May 2024 14:21:57 -0500 Subject: [PATCH 08/10] Remove the new _convert_to_str(). Serhiy and I independently concluded that exact powers of 10 aren't possible in these contexts, so just checking the string length is sufficient. --- Lib/_pydecimal.py | 65 +++++++---------------------------------------- 1 file changed, 9 insertions(+), 56 deletions(-) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 5911432c798454..0cc2826ab10da1 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -423,56 +423,6 @@ def sin(x): # (because Decimals are not interoperable with floats). See the notes in # numbers.py for more detail. -import re -_is_power_of_10 = re.compile(r"10*").fullmatch -del re - -# In Decimal._power_exact(), there are two blocks of the form -# if xc > 10**p: -# return None -# ... -# code using str(zc) -# where p is the maximum exponent. Which can be truly gigantic. So Python -# can appear to hang if those blocke are hit, or, if you wait long enough, -# run out of RAM. For example, -# https://github.com/python/cpython/issues/118027 -# where this happens during decimal.Decimal(2) ** 117 after -# ctx.Emax is set to decimal.MAX_EMAX. -# -# This function instead converts to string first, then sees whether the -# string is compatible with the claim that it's a value <= 10**p. If so, -# it returns the string; else None. So the code above becomes instead: -# str_xc = _convert_to_str(zc, p) -# if str_xc is None: -# return None -# use `str_xc` instead of str(xc) -# The code in _power_exact() is very involved, and I'm not certain xc can't -# be very much larger than 10**p. I _doubt_ it can. But, if I'm wrong, a -# different approach would be better. -# -# BUG ALERT: the code in the first block above returns None if xc ix sn -# integer power of 10 > 10**p. But the replacement accepts any such power of -# 10. This was intentional, but may be wrong. I (Tim) am taking the comments -# at their word: that the purpose is to identify cases that are exactly -# representable in decimal floating point with p digits of precision. Every -# integer power of 10 is, and even if p is just 1. If that's wrong, the -# original behavior can be gotten by changing the test below to: -# len(sc) <= p or (len(sc) == p+1 and _is_power_of_10(sc)) -# Alas, I don't know how to contrive inputs to trigger these paths. -# -# LATER: I believe it's impossible for _is_power_of_10(sc) to be true in -# either of the code blocks using _convert_to_str() at this time. But I'll -# leave it in anyway, "just in case", and so this function is more -# bulletproof if it gets used in other contexts. It's a cheap test and rarely -# executed. - -def _convert_to_str(ec, p): - sc = str(ec) - if len(sc) <= p or _is_power_of_10(sc): - return sc - else: - return None - class Decimal(object): """Floating point class for decimal arithmetic.""" @@ -2181,8 +2131,8 @@ def _power_exact(self, other, p): else: return None - strxc = _convert_to_str(xc, p) - if strxc is None: + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe return _dec_from_triple(0, strxc, xe) @@ -2231,14 +2181,17 @@ def _power_exact(self, other, p): # if m > p*100//_log10_lb(xc) then m > p/log10(xc), hence xc**m > # 10**p and the result is not representable. - assert xc != 1, self - assert xc % 10 != 0, self if xc > 1 and m > p*100//_log10_lb(xc): return None xc = xc**m xe *= m - str_xc = _convert_to_str(xc, p) - if str_xc is None: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable From 2b8f377644b02454161b87a82e6914d1e60421a2 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 4 May 2024 14:28:11 -0500 Subject: [PATCH 09/10] At least for now, add the asserts to the other block too. --- Lib/_pydecimal.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 0cc2826ab10da1..613123ec7b4329 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -2131,6 +2131,11 @@ def _power_exact(self, other, p): else: return None + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self strxc = str(xc) if len(strxc) > p: return None From 1b584eba4a97e69af8c3d76521418dffead1f903 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 20:23:02 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst b/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst new file mode 100644 index 00000000000000..80dc868540418f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst @@ -0,0 +1 @@ +The Python implementation of the ``decimal`` module could appear to hang in relatively small power cases (like ``2**117``) if context precision was set to a very high value. A different method to check for exactly representable results is used now that doesn't rely on computing ``10**precision`` (which could be effectively too large to compute).