From 5958a24ed703e8ed731aeba1aeb7061f4fc1271c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 8 May 2025 13:43:57 +0300 Subject: [PATCH 1/4] gh-133489: Remove size restrictions on getrandbits() and randbytes() random.getrandbits() can now generate more that 2**31 bits. random.randbytes() can now generate more that 256 MiB. --- Lib/test/test_random.py | 14 +++++++++++++ ...-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst | 2 ++ Modules/_randommodule.c | 20 +++++++++---------- Modules/clinic/_randommodule.c.h | 10 +++++----- 4 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 43957f525f10c0..5c55672a76a5f1 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -806,6 +806,20 @@ def test_getrandbits(self): self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + def test_getrandbits_2G_bits(self): + self.gen.seed(1234567) + x = self.gen.getrandbits(2**31) + self.assertEqual(x.bit_length(), 2**31) + self.assertEqual(x >> (2**31-100), 1226514312032729439655761284440) + self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + + def test_getrandbits_4G_bits(self): + self.gen.seed(1234568) + x = self.gen.getrandbits(2**32) + self.assertEqual(x.bit_length(), 2**32) + self.assertEqual(x >> (2**32-100), 739728759900339699429794460738) + self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange # Use same seed as in the cross-platform repeatability test diff --git a/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst b/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst new file mode 100644 index 00000000000000..0c07beb76938f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-08-13-43-19.gh-issue-133489.9eGS1Z.rst @@ -0,0 +1,2 @@ +:func:`random.getrandbits` can now generate more that 2\ :sup:`31` bits. +:func:`random.randbytes` can now generate more that 256 MiB. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index d5bac2f5b78120..2f4f388ce1161a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -497,34 +497,32 @@ _random_Random_setstate_impl(RandomObject *self, PyObject *state) _random.Random.getrandbits self: self(type="RandomObject *") - k: int + k: uint64 / getrandbits(k) -> x. Generates an int with k random bits. [clinic start generated code]*/ static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k) +/*[clinic end generated code: output=c30ef8435f3433cf input=64226ac13bb4d2a3]*/ { - int i, words; + Py_ssize_t i, words; uint32_t r; uint32_t *wordarray; PyObject *result; - if (k < 0) { - PyErr_SetString(PyExc_ValueError, - "number of bits must be non-negative"); - return NULL; - } - if (k == 0) return PyLong_FromLong(0); if (k <= 32) /* Fast path */ return PyLong_FromUnsignedLong(genrand_uint32(self) >> (32 - k)); - words = (k - 1) / 32 + 1; + if ((k - 1u) / 32u + 1u > PY_SSIZE_T_MAX / 4u) { + PyErr_NoMemory(); + return NULL; + } + words = (k - 1u) / 32u + 1u; wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 1e989e970c9de5..2563a16aea0b6f 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_UInt64_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -124,16 +125,15 @@ PyDoc_STRVAR(_random_Random_getrandbits__doc__, {"getrandbits", (PyCFunction)_random_Random_getrandbits, METH_O, _random_Random_getrandbits__doc__}, static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k); +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k); static PyObject * _random_Random_getrandbits(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - int k; + uint64_t k; - k = PyLong_AsInt(arg); - if (k == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt64_Converter(arg, &k)) { goto exit; } Py_BEGIN_CRITICAL_SECTION(self); @@ -143,4 +143,4 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4458b5a69201ebea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ From 24cbd8d5020e45f6fb1b54e651076099eab051b2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 8 May 2025 14:13:53 +0300 Subject: [PATCH 2/4] Add more tests. --- Lib/test/test_random.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 5c55672a76a5f1..746a6fb6f1a6c5 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -392,6 +392,8 @@ def test_getrandbits(self): self.assertRaises(TypeError, self.gen.getrandbits) self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) self.assertRaises(ValueError, self.gen.getrandbits, -1) + self.assertRaises(OverflowError, self.gen.getrandbits, 1<<1000) + self.assertRaises(ValueError, self.gen.getrandbits, -1<<1000) self.assertRaises(TypeError, self.gen.getrandbits, 10.1) def test_pickling(self): @@ -435,6 +437,8 @@ def test_randbytes(self): self.assertRaises(TypeError, self.gen.randbytes) self.assertRaises(TypeError, self.gen.randbytes, 1, 2) self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) self.assertRaises(TypeError, self.gen.randbytes, 1.0) def test_mu_sigma_default_args(self): @@ -976,6 +980,13 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + def test_randbytes_256M(self): + self.gen.seed(2849427419) + x = self.gen.randbytes(2**29) + self.assertEqual(len(x), 2**29) + self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') + self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc') + def test_sample_counts_equivalence(self): # Test the documented strong equivalence to a sample with repeated elements. # We run this test on random.Random() which makes deterministic selections From 5967c0ef2d45348aadca546ffa03cfbd58ad19da Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 8 May 2025 15:38:23 +0300 Subject: [PATCH 3/4] Fix OverflowError in _PyLong_FromByteArray. --- Objects/longobject.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 0b2dfa003fac53..2b533312fee673 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -971,16 +971,9 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, ++numsignificantbytes; } - /* How many Python int digits do we need? We have - 8*numsignificantbytes bits, and each Python int digit has - PyLong_SHIFT bits, so it's the ceiling of the quotient. */ - /* catch overflow before it happens */ - if (numsignificantbytes > (PY_SSIZE_T_MAX - PyLong_SHIFT) / 8) { - PyErr_SetString(PyExc_OverflowError, - "byte array too long to convert to int"); - return NULL; - } - ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; + /* avoid integer overflow */ + ndigits = numsignificantbytes / PyLong_SHIFT * 8 + + (numsignificantbytes % PyLong_SHIFT * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; v = long_alloc(ndigits); if (v == NULL) return NULL; From fb24a5459e849d14daa7ecdaf79993cc08a3430f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 8 May 2025 17:19:37 +0300 Subject: [PATCH 4/4] Use bigmemtest. --- Lib/test/test_random.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 746a6fb6f1a6c5..5b7fef6ef8ece4 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -811,18 +811,20 @@ def test_getrandbits(self): 97904845777343510404718956115) def test_getrandbits_2G_bits(self): + size = 2**31 self.gen.seed(1234567) - x = self.gen.getrandbits(2**31) - self.assertEqual(x.bit_length(), 2**31) - self.assertEqual(x >> (2**31-100), 1226514312032729439655761284440) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + self.assertEqual(x >> (size-100), 1226514312032729439655761284440) - def test_getrandbits_4G_bits(self): + @support.bigmemtest(size=2**32, memuse=1/8+2/15, dry_run=False) + def test_getrandbits_4G_bits(self, size): self.gen.seed(1234568) - x = self.gen.getrandbits(2**32) - self.assertEqual(x.bit_length(), 2**32) - self.assertEqual(x >> (2**32-100), 739728759900339699429794460738) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + self.assertEqual(x >> (size-100), 739728759900339699429794460738) def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange @@ -980,10 +982,11 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) - def test_randbytes_256M(self): + @support.bigmemtest(size=2**29, memuse=1+16/15, dry_run=False) + def test_randbytes_256M(self, size): self.gen.seed(2849427419) - x = self.gen.randbytes(2**29) - self.assertEqual(len(x), 2**29) + x = self.gen.randbytes(size) + self.assertEqual(len(x), size) self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc')