From dbf3d61fedd9af3077158cbb920a02cf147c8867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:43:23 +0100 Subject: [PATCH 01/22] fix overflow in frame's stacksizes --- Lib/test/test_code.py | 12 ++++++++++++ Objects/codeobject.c | 7 ++++++- Objects/frameobject.c | 19 +++++++++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 93f3a5833cb01e..7f5ff03b874a7e 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -523,6 +523,18 @@ def test_code_equal_with_instrumentation(self): self.assertNotEqual(code1, code2) sys.settrace(None) + def test_co_stacksize_overflow(self): + # See: https://github.com/python/cpython/issues/126119. + + # Since co_framesize = nlocalsplus + co_stacksize + FRAME_SPECIALS_SIZE, + # we need to check that co_stacksize is not too large. We could fortify + # the test by explicitly checking that bound, but this needs to expose + # FRAME_SPECIALS_SIZE and a getter for 'nlocalsplus'. + + c = (lambda: ...).__code__ + with self.assertRaisesRegex(OverflowError, "co_stacksize"): + c.__replace__(co_stacksize=2147483647) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 775ea7aca824c4..87c1834777f67d 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -436,7 +436,12 @@ _PyCode_Validate(struct _PyCodeConstructor *con) PyErr_SetString(PyExc_ValueError, "code: co_varnames is too small"); return -1; } - + /* Ensure that the framesize will not overflow */ + int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); + if (con->stacksize >= INT_MAX - nlocalsplus - FRAME_SPECIALS_SIZE) { + PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); + return -1; + } return 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index af2a2ef18e627a..a4e100167b304b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1804,11 +1804,22 @@ PyDoc_STRVAR(clear__doc__, static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res; - res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus); + Py_ssize_t res = offsetof(PyFrameObject, _f_frame_data) + + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); - res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); - return PyLong_FromSsize_t(res); + int nslots = _PyFrame_NumSlotsForCodeObject(code); + assert(nslots >= 0); + if ((size_t)nslots >= (PY_SSIZE_T_MAX - res) / sizeof(PyObject *)) { + // This could happen if the underlying code object has a + // very large stacksize (but at most INT_MAX) yet that the + // above offsets make the result overflow. + // + // This should normally only happen if PY_SSIZE_T_MAX == INT_MAX, + // but we anyway raise an exception on other systems for safety. + PyErr_SetString(PyExc_OverflowError, "size exceeds PY_SSIZE_T_MAX"); + return NULL; + } + return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); } PyDoc_STRVAR(sizeof__doc__, From a5664693c1e22ddffdc28dab2f925048b6089ac2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:47:22 +0100 Subject: [PATCH 02/22] blurb --- .../2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst new file mode 100644 index 00000000000000..02c523d85a3d55 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst @@ -0,0 +1,3 @@ +Fix a crash due to an overflow when the :attr:`co_stacksize +` field of a :ref:`code object ` is +set to an absurdely large integer. Patch by Bénédikt Tran. From 222de28942da56c3e0d34ef2cc00764538b7bdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:35:35 +0100 Subject: [PATCH 03/22] blurb v2 --- .../2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst index 02c523d85a3d55..ce12f5509095b1 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst @@ -1,3 +1,4 @@ -Fix a crash due to an overflow when the :attr:`co_stacksize +Fix a crash in DEBUG builds due to an overflow when the :attr:`co_stacksize ` field of a :ref:`code object ` is -set to an absurdely large integer. Patch by Bénédikt Tran. +set to an absurdely large integer. +Reported by Valery Fedorenko. Patch by Bénédikt Tran. From 303109bc3d451023383ed65c040acf6257e67db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:39:43 +0100 Subject: [PATCH 04/22] fix more cases --- Objects/frameobject.c | 6 ------ Objects/genobject.c | 13 +++++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index a4e100167b304b..195e07b81ffdbb 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1810,12 +1810,6 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); if ((size_t)nslots >= (PY_SSIZE_T_MAX - res) / sizeof(PyObject *)) { - // This could happen if the underlying code object has a - // very large stacksize (but at most INT_MAX) yet that the - // above offsets make the result overflow. - // - // This should normally only happen if PY_SSIZE_T_MAX == INT_MAX, - // but we anyway raise an exception on other systems for safety. PyErr_SetString(PyExc_OverflowError, "size exceeds PY_SSIZE_T_MAX"); return NULL; } diff --git a/Objects/genobject.c b/Objects/genobject.c index 19c2c4e3331a89..cd25c56de1119f 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -817,11 +817,16 @@ static PyMemberDef gen_memberlist[] = { static PyObject * gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res; - res = offsetof(PyGenObject, gi_iframe) + offsetof(_PyInterpreterFrame, localsplus); + Py_ssize_t res = offsetof(PyGenObject, gi_iframe) + + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyGen_GetCode(gen); - res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); - return PyLong_FromSsize_t(res); + int nslots = _PyFrame_NumSlotsForCodeObject(code); + assert(nslots >= 0); + if ((size_t)nslots >= (PY_SSIZE_T_MAX - res) / sizeof(PyObject *)) { + PyErr_SetString(PyExc_OverflowError, "size exceeds PY_SSIZE_T_MAX"); + return NULL; + } + return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); } PyDoc_STRVAR(sizeof__doc__, From d743a3d74a83357a60d2949f0c30371043ae5f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:56:33 +0100 Subject: [PATCH 05/22] improve test coverage! --- Lib/test/test_code.py | 44 ++++++++++++++++++++++++++++++------- Lib/test/test_frame.py | 9 ++++++++ Lib/test/test_generators.py | 12 ++++++++++ Objects/codeobject.c | 15 ++++++++++--- Objects/frameobject.c | 9 ++++---- Objects/genobject.c | 9 ++++---- 6 files changed, 79 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 7f5ff03b874a7e..611f8e9d5e00fd 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -139,6 +139,18 @@ import ctypes except ImportError: ctypes = None + +try: + import _testcapi +except ImportError: + _testcapi = None + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +from test import support from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, gc_collect, Py_GIL_DISABLED) @@ -523,17 +535,33 @@ def test_code_equal_with_instrumentation(self): self.assertNotEqual(code1, code2) sys.settrace(None) - def test_co_stacksize_overflow(self): + @unittest.skipUnless(_testcapi, "requires _testcapi") + @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") + def test_co_framesize_overflow(self): # See: https://github.com/python/cpython/issues/126119. - # Since co_framesize = nlocalsplus + co_stacksize + FRAME_SPECIALS_SIZE, - # we need to check that co_stacksize is not too large. We could fortify - # the test by explicitly checking that bound, but this needs to expose - # FRAME_SPECIALS_SIZE and a getter for 'nlocalsplus'. + def foo(a, b): + x = a * b + return x - c = (lambda: ...).__code__ - with self.assertRaisesRegex(OverflowError, "co_stacksize"): - c.__replace__(co_stacksize=2147483647) + c = foo.__code__ + co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) + # co_framesize = co_stacksize + co_nlocalsplus + FRAME_SPECIALS_SIZE + co_framesize = _testinternalcapi.get_co_framesize(c) + FRAME_SPECIALS_SIZE = co_framesize - c.co_stacksize - co_nlocalsplus + + ptr_sizeof = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) + + for evil_co_stacksize in [ + _testcapi.INT_MAX, + _testcapi.INT_MAX // ptr_sizeof, + (_testcapi.INT_MAX - co_nlocalsplus - FRAME_SPECIALS_SIZE) // ptr_sizeof, + ]: + with ( + self.subTest(evil_co_stacksize), + self.assertRaisesRegex(OverflowError, "co_stacksize") + ): + foo.__code__.__replace__(co_stacksize=evil_co_stacksize) def isinterned(s): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 11f191700ccef0..735ebe8fc4638e 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -222,6 +222,15 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno + @unittest.skipUnless(_testcapi, "requires _testcapi") + def test_sizeof_overflow(self): + # See: https://github.com/python/cpython/issues/126119 + evil_co_stacksize = _testcapi.INT_MAX // support.calcobjsize('P') + f, _, _ = self.make_frames() + evil_code = f.f_code.replace(co_stacksize=evil_co_stacksize) + frame = _testcapi.frame_new(evil_code, globals(), locals()) + self.assertGreaterEqual(frame.__sizeof__(), evil_co_stacksize) + class ReprTest(unittest.TestCase): """ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index bf2cb1160723b0..66a43348acdb54 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1,6 +1,7 @@ import copy import gc import pickle +import re import sys import doctest import unittest @@ -268,6 +269,17 @@ def loop(): #This should not raise loop() + @unittest.skipUnless(_testcapi, "requires _testcapi.INT_MAX") + def test_gi_frame_f_code_overflow(self): + # See: https://github.com/python/cpython/issues/126119 + + def f(): yield + + evil_co_stacksize = _testcapi.INT_MAX // support.calcobjsize("P") + evil = f().gi_frame.f_code.__replace__(co_stacksize=evil_co_stacksize) + evil_gi = types.FunctionType(evil, {})() + self.assertGreaterEqual(evil_gi.__sizeof__(), evil_co_stacksize) + class ModifyUnderlyingIterableTest(unittest.TestCase): iterables = [ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 87c1834777f67d..49715120e86124 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -436,9 +436,18 @@ _PyCode_Validate(struct _PyCodeConstructor *con) PyErr_SetString(PyExc_ValueError, "code: co_varnames is too small"); return -1; } - /* Ensure that the framesize will not overflow */ - int nlocalsplus = (int)PyTuple_GET_SIZE(con->localsplusnames); - if (con->stacksize >= INT_MAX - nlocalsplus - FRAME_SPECIALS_SIZE) { + /* + * Ensure that the framesize will not overflow. + * + * There are various places in the code where the size of the object + * is assumed to be at most INT_MAX / sizeof(PyObject *). Since this + * size is the framesize, we need to guarantee that there will not + * be an overflow on "framesize * sizeof(PyObject *) + CONSTANT". + */ + int nlocalsplus = PyTuple_GET_SIZE(con->localsplusnames); + if ((size_t)con->stacksize + >= (INT_MAX - FRAME_SPECIALS_SIZE - nlocalsplus) / sizeof(PyObject *)) + { PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); return -1; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 195e07b81ffdbb..39a29445002224 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1809,10 +1809,11 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) PyCodeObject *code = _PyFrame_GetCode(f->f_frame); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); - if ((size_t)nslots >= (PY_SSIZE_T_MAX - res) / sizeof(PyObject *)) { - PyErr_SetString(PyExc_OverflowError, "size exceeds PY_SSIZE_T_MAX"); - return NULL; - } + // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. + // It should not be possible to have nslots >= PY_SSIZE_T_MAX + // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize + // is checked in _PyCode_Validate(). + assert((size_t)nslots < (INT_MAX - res) / sizeof(PyObject *)); return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); } diff --git a/Objects/genobject.c b/Objects/genobject.c index cd25c56de1119f..1897ea807ca36d 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -822,10 +822,11 @@ gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) PyCodeObject *code = _PyGen_GetCode(gen); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); - if ((size_t)nslots >= (PY_SSIZE_T_MAX - res) / sizeof(PyObject *)) { - PyErr_SetString(PyExc_OverflowError, "size exceeds PY_SSIZE_T_MAX"); - return NULL; - } + // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. + // It should not be possible to have nslots >= PY_SSIZE_T_MAX + // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize + // is checked in _PyCode_Validate(). + assert((size_t)nslots < (INT_MAX - res) / sizeof(PyObject *)); return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); } From 6b34c22038f287891f082afc125d8c587030320c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:00:00 +0100 Subject: [PATCH 06/22] improve test coverage! --- Lib/test/test_code.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 611f8e9d5e00fd..65a79c341bac1c 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -550,18 +550,23 @@ def foo(a, b): co_framesize = _testinternalcapi.get_co_framesize(c) FRAME_SPECIALS_SIZE = co_framesize - c.co_stacksize - co_nlocalsplus - ptr_sizeof = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) + ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) + smallest_evil_co_stacksize = ( + (_testcapi.INT_MAX - co_nlocalsplus - FRAME_SPECIALS_SIZE) // ps + ) for evil_co_stacksize in [ _testcapi.INT_MAX, - _testcapi.INT_MAX // ptr_sizeof, - (_testcapi.INT_MAX - co_nlocalsplus - FRAME_SPECIALS_SIZE) // ptr_sizeof, + _testcapi.INT_MAX // ps, + smallest_evil_co_stacksize, ]: with ( self.subTest(evil_co_stacksize), self.assertRaisesRegex(OverflowError, "co_stacksize") ): - foo.__code__.__replace__(co_stacksize=evil_co_stacksize) + c.__replace__(co_stacksize=evil_co_stacksize) + + c.__replace__(co_stacksize=smallest_evil_co_stacksize - 1) def isinterned(s): From 044d1a584d852cfce568ddf2ddf5e8bbe8232142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:35:19 +0100 Subject: [PATCH 07/22] fix logic --- Lib/test/support/__init__.py | 15 +++++++++++++++ Lib/test/test_code.py | 28 +++++++++------------------- Lib/test/test_frame.py | 15 ++++++++++++--- Lib/test/test_generators.py | 17 +++++++++++++---- Objects/codeobject.c | 17 ++++++++--------- Objects/frameobject.c | 17 ++++++++++++----- Objects/genobject.c | 17 ++++++++++++----- 7 files changed, 81 insertions(+), 45 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 7c1ef42a4970d7..5e1444a92635a7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -901,6 +901,21 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) + +def get_frame_specials_size(): + """Compute the C defined constant FRAME_SPECIALS_SIZE in codeobject.c.""" + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("_testinternalcapi required") + + c = (lambda: ...).__code__ + # co_framesize = co_stacksize + co_nlocalsplus + FRAME_SPECIALS_SIZE + co_framesize = _testinternalcapi.get_co_framesize(c) + co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) + return co_framesize - c.co_stacksize - co_nlocalsplus + + #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards. diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 65a79c341bac1c..92e05cef5da2f6 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -535,6 +535,7 @@ def test_code_equal_with_instrumentation(self): self.assertNotEqual(code1, code2) sys.settrace(None) + @unittest.skipUnless(ctypes, "requires ctypes") @unittest.skipUnless(_testcapi, "requires _testcapi") @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") def test_co_framesize_overflow(self): @@ -545,28 +546,17 @@ def foo(a, b): return x c = foo.__code__ - co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) - # co_framesize = co_stacksize + co_nlocalsplus + FRAME_SPECIALS_SIZE - co_framesize = _testinternalcapi.get_co_framesize(c) - FRAME_SPECIALS_SIZE = co_framesize - c.co_stacksize - co_nlocalsplus + fss = support.get_frame_specials_size() ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) - smallest_evil_co_stacksize = ( - (_testcapi.INT_MAX - co_nlocalsplus - FRAME_SPECIALS_SIZE) // ps - ) + co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) + # anything below that limit is a valid co_stacksize + evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) + self.assertLessEqual(evil_stacksize, _testcapi.INT_MAX // ps) - for evil_co_stacksize in [ - _testcapi.INT_MAX, - _testcapi.INT_MAX // ps, - smallest_evil_co_stacksize, - ]: - with ( - self.subTest(evil_co_stacksize), - self.assertRaisesRegex(OverflowError, "co_stacksize") - ): - c.__replace__(co_stacksize=evil_co_stacksize) - - c.__replace__(co_stacksize=smallest_evil_co_stacksize - 1) + with self.assertRaisesRegex(OverflowError, "co_stacksize"): + c.__replace__(co_stacksize=evil_stacksize) + c.__replace__(co_stacksize=evil_stacksize - 1) def isinterned(s): diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 735ebe8fc4638e..3046deea7e4766 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -225,11 +225,20 @@ def test_f_lineno_del_segfault(self): @unittest.skipUnless(_testcapi, "requires _testcapi") def test_sizeof_overflow(self): # See: https://github.com/python/cpython/issues/126119 - evil_co_stacksize = _testcapi.INT_MAX // support.calcobjsize('P') + ctypes = import_helper.import_module('ctypes') + f, _, _ = self.make_frames() - evil_code = f.f_code.replace(co_stacksize=evil_co_stacksize) + c = f.f_code + co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) + + fss = support.get_frame_specials_size() + ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) + evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) + # an evil code with a valid (but very large) stack size + evil_code = f.f_code.replace(co_stacksize=evil_stacksize - 1) frame = _testcapi.frame_new(evil_code, globals(), locals()) - self.assertGreaterEqual(frame.__sizeof__(), evil_co_stacksize) + message = re.escape("size exceeds INT_MAX") + self.assertRaisesRegex(OverflowError, message, frame.__sizeof__) class ReprTest(unittest.TestCase): diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 66a43348acdb54..d812b5353ccac2 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -11,6 +11,7 @@ import types from test import support +from test.support import import_helper try: import _testcapi @@ -269,16 +270,24 @@ def loop(): #This should not raise loop() - @unittest.skipUnless(_testcapi, "requires _testcapi.INT_MAX") + @unittest.skipUnless(_testcapi, "requires _testcapi") def test_gi_frame_f_code_overflow(self): # See: https://github.com/python/cpython/issues/126119 + ctypes = import_helper.import_module('ctypes') def f(): yield + c = f().gi_frame.f_code + co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) - evil_co_stacksize = _testcapi.INT_MAX // support.calcobjsize("P") - evil = f().gi_frame.f_code.__replace__(co_stacksize=evil_co_stacksize) + ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) + fss = support.get_frame_specials_size() + # anything below that limit is a valid co_stacksize + evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) + + evil = c.__replace__(co_stacksize=evil_stacksize - 1) evil_gi = types.FunctionType(evil, {})() - self.assertGreaterEqual(evil_gi.__sizeof__(), evil_co_stacksize) + message = re.escape("size exceeds INT_MAX") + self.assertRaisesRegex(OverflowError, message, evil_gi.__sizeof__) class ModifyUnderlyingIterableTest(unittest.TestCase): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 49715120e86124..07e72aecb2e79f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -437,17 +437,16 @@ _PyCode_Validate(struct _PyCodeConstructor *con) return -1; } /* - * Ensure that the framesize will not overflow. + * The framesize = stacksize + nlocalsplus + FRAME_SPECIALS_SIZE is used + * as framesize * sizeof(PyObject *) and assumed to be < INT_MAX. Thus, + * we need to dynamically limit the value of stacksize. * - * There are various places in the code where the size of the object - * is assumed to be at most INT_MAX / sizeof(PyObject *). Since this - * size is the framesize, we need to guarantee that there will not - * be an overflow on "framesize * sizeof(PyObject *) + CONSTANT". + * See https://github.com/python/cpython/issues/126119 for details. */ - int nlocalsplus = PyTuple_GET_SIZE(con->localsplusnames); - if ((size_t)con->stacksize - >= (INT_MAX - FRAME_SPECIALS_SIZE - nlocalsplus) / sizeof(PyObject *)) - { + int max_stacksize = (int)(INT_MAX / sizeof(PyObject *)) + - FRAME_SPECIALS_SIZE + - PyTuple_GET_SIZE(con->localsplusnames); + if (con->stacksize >= max_stacksize) { PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); return -1; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 39a29445002224..64d36c087b9f60 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1804,17 +1804,24 @@ PyDoc_STRVAR(clear__doc__, static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res = offsetof(PyFrameObject, _f_frame_data) - + offsetof(_PyInterpreterFrame, localsplus); + Py_ssize_t base = offsetof(PyFrameObject, _f_frame_data) + + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. // It should not be possible to have nslots >= PY_SSIZE_T_MAX // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize - // is checked in _PyCode_Validate(). - assert((size_t)nslots < (INT_MAX - res) / sizeof(PyObject *)); - return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); + // is checked in _PyCode_Validate(). However, it is possible + // to make base + nslots * sizeof(PyObject *) >= INT_MAX since + // 'base' is not yet known when creating code objects. + // + // See https://github.com/python/cpython/issues/126119 for details. + if (nslots >= (int)((INT_MAX - base) / sizeof(PyObject *))) { + PyErr_SetString(PyExc_OverflowError, "size exceeds INT_MAX"); + return NULL; + } + return PyLong_FromSsize_t(base + nslots * sizeof(PyObject *)); } PyDoc_STRVAR(sizeof__doc__, diff --git a/Objects/genobject.c b/Objects/genobject.c index 1897ea807ca36d..bd4060cb4e42f2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -817,17 +817,24 @@ static PyMemberDef gen_memberlist[] = { static PyObject * gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res = offsetof(PyGenObject, gi_iframe) - + offsetof(_PyInterpreterFrame, localsplus); + Py_ssize_t base = offsetof(PyGenObject, gi_iframe) + + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyGen_GetCode(gen); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. // It should not be possible to have nslots >= PY_SSIZE_T_MAX // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize - // is checked in _PyCode_Validate(). - assert((size_t)nslots < (INT_MAX - res) / sizeof(PyObject *)); - return PyLong_FromSsize_t(res + nslots * sizeof(PyObject *)); + // is checked in _PyCode_Validate(). However, it is possible + // to make base + nslots * sizeof(PyObject *) >= INT_MAX since + // 'base' is not yet known when creating code objects. + // + // See https://github.com/python/cpython/issues/126119 for details. + if (nslots >= (int)((INT_MAX - base) / sizeof(PyObject *))) { + PyErr_SetString(PyExc_OverflowError, "size exceeds INT_MAX"); + return NULL; + } + return PyLong_FromSsize_t(base + nslots * sizeof(PyObject *)); } PyDoc_STRVAR(sizeof__doc__, From d6f3bc49f3bf255dfe776e10b3e036895d7c3c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:50:48 +0100 Subject: [PATCH 08/22] remove un-necessary assertion --- Lib/test/test_code.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 92e05cef5da2f6..a3d3945e8867a2 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -552,7 +552,6 @@ def foo(a, b): co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) # anything below that limit is a valid co_stacksize evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) - self.assertLessEqual(evil_stacksize, _testcapi.INT_MAX // ps) with self.assertRaisesRegex(OverflowError, "co_stacksize"): c.__replace__(co_stacksize=evil_stacksize) From 31f36dee429e16b101628c333d3c8b0d1b392dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:05:34 +0100 Subject: [PATCH 09/22] skip a test on free-threaded builds to avoid crash --- Lib/test/test_generators.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d812b5353ccac2..65187b4c3bcc3a 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -285,7 +285,11 @@ def f(): yield evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) evil = c.__replace__(co_stacksize=evil_stacksize - 1) - evil_gi = types.FunctionType(evil, {})() + + if support.Py_GIL_DISABLED: + self.skipTest("segmentation fault on free-threaded builds") + # the following crashes on free-threaded builds for now + evil_gi_func = types.FunctionType(evil, {})() message = re.escape("size exceeds INT_MAX") self.assertRaisesRegex(OverflowError, message, evil_gi.__sizeof__) From b26dd7240f14fd77301e9e103e6b91a34e4ff8a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:21:28 +0100 Subject: [PATCH 10/22] fix tests on 32-bit platforms --- Lib/test/test_frame.py | 12 +++++++++--- Lib/test/test_generators.py | 12 ++++++++---- Objects/codeobject.c | 4 +++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 3046deea7e4766..c3d1e7b43b3e9b 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,5 +1,6 @@ import copy import operator +import platform import re import sys import textwrap @@ -236,9 +237,14 @@ def test_sizeof_overflow(self): evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) # an evil code with a valid (but very large) stack size evil_code = f.f_code.replace(co_stacksize=evil_stacksize - 1) - frame = _testcapi.frame_new(evil_code, globals(), locals()) - message = re.escape("size exceeds INT_MAX") - self.assertRaisesRegex(OverflowError, message, frame.__sizeof__) + + if sys.maxsize == 2147483647: # 32-bit machine + with self.assertRaises(MemoryError): + frame = _testcapi.frame_new(evil_code, globals(), locals()) + else: + frame = _testcapi.frame_new(evil_code, globals(), locals()) + message = re.escape("size exceeds INT_MAX") + self.assertRaisesRegex(OverflowError, message, frame.__sizeof__) class ReprTest(unittest.TestCase): diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 65187b4c3bcc3a..c2f311376fbc49 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -288,10 +288,14 @@ def f(): yield if support.Py_GIL_DISABLED: self.skipTest("segmentation fault on free-threaded builds") - # the following crashes on free-threaded builds for now - evil_gi_func = types.FunctionType(evil, {})() - message = re.escape("size exceeds INT_MAX") - self.assertRaisesRegex(OverflowError, message, evil_gi.__sizeof__) + elif sys.maxsize == 2147483647: # 32-bit machine + with self.assertRaises(MemoryError): + evil_gi = types.FunctionType(evil, {})() + else: + # the following crashes on free-threaded builds for now + evil_gi = types.FunctionType(evil, {})() + message = re.escape("size exceeds INT_MAX") + self.assertRaisesRegex(OverflowError, message, evil_gi.__sizeof__) class ModifyUnderlyingIterableTest(unittest.TestCase): diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 07e72aecb2e79f..5de77ad309b9e2 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -439,7 +439,9 @@ _PyCode_Validate(struct _PyCodeConstructor *con) /* * The framesize = stacksize + nlocalsplus + FRAME_SPECIALS_SIZE is used * as framesize * sizeof(PyObject *) and assumed to be < INT_MAX. Thus, - * we need to dynamically limit the value of stacksize. + * we need to dynamically limit the value of stacksize. Note that this + * usually prevents crashes due to assertions but a MemoryError may still + * be triggered later. * * See https://github.com/python/cpython/issues/126119 for details. */ From fe0b04ee2fa6d4b0149bb7e3c41b5409fae840c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:38:40 +0100 Subject: [PATCH 11/22] fix casts --- Objects/codeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 5de77ad309b9e2..b509d30161d95f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -447,7 +447,7 @@ _PyCode_Validate(struct _PyCodeConstructor *con) */ int max_stacksize = (int)(INT_MAX / sizeof(PyObject *)) - FRAME_SPECIALS_SIZE - - PyTuple_GET_SIZE(con->localsplusnames); + - (int)PyTuple_GET_SIZE(con->localsplusnames); if (con->stacksize >= max_stacksize) { PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); return -1; From 1fe8e284957b0ecde8accc57c06388267f4b0198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:46:50 +0100 Subject: [PATCH 12/22] fix boundary conditions --- Objects/codeobject.c | 7 +++---- Objects/frameobject.c | 1 + Objects/genobject.c | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index b509d30161d95f..3c0a942cc909db 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -445,10 +445,9 @@ _PyCode_Validate(struct _PyCodeConstructor *con) * * See https://github.com/python/cpython/issues/126119 for details. */ - int max_stacksize = (int)(INT_MAX / sizeof(PyObject *)) - - FRAME_SPECIALS_SIZE - - (int)PyTuple_GET_SIZE(con->localsplusnames); - if (con->stacksize >= max_stacksize) { + int ub = (int)(INT_MAX / sizeof(PyObject *)) - FRAME_SPECIALS_SIZE; + Py_ssize_t nlocalsplus = PyTuple_GET_SIZE(con->localsplusnames); + if (nlocalsplus >= (Py_ssize_t)ub || con->stacksize >= (int)ub - nlocalsplus) { PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); return -1; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 64d36c087b9f60..2f246eef7791f9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1806,6 +1806,7 @@ frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { Py_ssize_t base = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus); + assert(base <= INT_MAX); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); diff --git a/Objects/genobject.c b/Objects/genobject.c index bd4060cb4e42f2..33eb880cd0dc2a 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -819,6 +819,7 @@ gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) { Py_ssize_t base = offsetof(PyGenObject, gi_iframe) + offsetof(_PyInterpreterFrame, localsplus); + assert(base <= INT_MAX); PyCodeObject *code = _PyGen_GetCode(gen); int nslots = _PyFrame_NumSlotsForCodeObject(code); assert(nslots >= 0); From 8c7ce9c461c4aadf0783d2663a3a343b961089b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 7 Nov 2024 20:06:52 +0100 Subject: [PATCH 13/22] Update Lib/test/test_frame.py --- Lib/test/test_frame.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index c3d1e7b43b3e9b..d4b3695563a668 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,6 +1,5 @@ import copy import operator -import platform import re import sys import textwrap From f8a0eef528b45968a5b9f53c72c120260358276c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:28:02 +0100 Subject: [PATCH 14/22] Update Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst --- .../2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst index ce12f5509095b1..c177a81d657005 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-29-11-47-19.gh-issue-126119.xbAvxt.rst @@ -1,4 +1,4 @@ Fix a crash in DEBUG builds due to an overflow when the :attr:`co_stacksize ` field of a :ref:`code object ` is -set to an absurdely large integer. +set to an absurdly large integer. Reported by Valery Fedorenko. Patch by Bénédikt Tran. From 3130f94965a84d125ea08267c41474cfa7d6dc0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:34:35 +0100 Subject: [PATCH 15/22] change co_stacksize upper limit --- Objects/codeobject.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 3c0a942cc909db..64dbe5cd30cb6b 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -443,12 +443,14 @@ _PyCode_Validate(struct _PyCodeConstructor *con) * usually prevents crashes due to assertions but a MemoryError may still * be triggered later. * - * See https://github.com/python/cpython/issues/126119 for details. + * See https://github.com/python/cpython/issues/126119 for details + * and corresponding PR for the rationale on the upper limit value. */ - int ub = (int)(INT_MAX / sizeof(PyObject *)) - FRAME_SPECIALS_SIZE; + Py_ssize_t limit = (Py_ssize_t)(INT_MAX / 16) - FRAME_SPECIALS_SIZE; Py_ssize_t nlocalsplus = PyTuple_GET_SIZE(con->localsplusnames); - if (nlocalsplus >= (Py_ssize_t)ub || con->stacksize >= (int)ub - nlocalsplus) { - PyErr_SetString(PyExc_OverflowError, "code: co_stacksize is too large"); + if (nlocalsplus >= limit || con->stacksize >= limit - nlocalsplus) { + PyErr_SetString(PyExc_OverflowError, + "code: locals + stack size is too large"); return -1; } return 0; From 91f95de5b3bb4e561f676a4a436b35fc4af4245a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:34:53 +0100 Subject: [PATCH 16/22] remove test that cannot overflow now --- Lib/test/test_frame.py | 23 ----------------------- Lib/test/test_generators.py | 29 ----------------------------- 2 files changed, 52 deletions(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index d4b3695563a668..11f191700ccef0 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -222,29 +222,6 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno - @unittest.skipUnless(_testcapi, "requires _testcapi") - def test_sizeof_overflow(self): - # See: https://github.com/python/cpython/issues/126119 - ctypes = import_helper.import_module('ctypes') - - f, _, _ = self.make_frames() - c = f.f_code - co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) - - fss = support.get_frame_specials_size() - ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) - evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) - # an evil code with a valid (but very large) stack size - evil_code = f.f_code.replace(co_stacksize=evil_stacksize - 1) - - if sys.maxsize == 2147483647: # 32-bit machine - with self.assertRaises(MemoryError): - frame = _testcapi.frame_new(evil_code, globals(), locals()) - else: - frame = _testcapi.frame_new(evil_code, globals(), locals()) - message = re.escape("size exceeds INT_MAX") - self.assertRaisesRegex(OverflowError, message, frame.__sizeof__) - class ReprTest(unittest.TestCase): """ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index c2f311376fbc49..bf2cb1160723b0 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1,7 +1,6 @@ import copy import gc import pickle -import re import sys import doctest import unittest @@ -11,7 +10,6 @@ import types from test import support -from test.support import import_helper try: import _testcapi @@ -270,33 +268,6 @@ def loop(): #This should not raise loop() - @unittest.skipUnless(_testcapi, "requires _testcapi") - def test_gi_frame_f_code_overflow(self): - # See: https://github.com/python/cpython/issues/126119 - ctypes = import_helper.import_module('ctypes') - - def f(): yield - c = f().gi_frame.f_code - co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) - - ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) - fss = support.get_frame_specials_size() - # anything below that limit is a valid co_stacksize - evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) - - evil = c.__replace__(co_stacksize=evil_stacksize - 1) - - if support.Py_GIL_DISABLED: - self.skipTest("segmentation fault on free-threaded builds") - elif sys.maxsize == 2147483647: # 32-bit machine - with self.assertRaises(MemoryError): - evil_gi = types.FunctionType(evil, {})() - else: - # the following crashes on free-threaded builds for now - evil_gi = types.FunctionType(evil, {})() - message = re.escape("size exceeds INT_MAX") - self.assertRaisesRegex(OverflowError, message, evil_gi.__sizeof__) - class ModifyUnderlyingIterableTest(unittest.TestCase): iterables = [ From c5d75967720524fe75b9469b02d75de86c9c4351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:38:58 +0100 Subject: [PATCH 17/22] fix tests --- Lib/test/test_code.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 5981b8a075b88c..1014f1feb66cbe 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -604,12 +604,11 @@ def foo(a, b): c = foo.__code__ fss = support.get_frame_specials_size() - ps = ctypes.sizeof(ctypes.c_void_p) # sizeof(PyObject *) co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) # anything below that limit is a valid co_stacksize - evil_stacksize = int(_testcapi.INT_MAX / ps - fss - co_nlocalsplus) + evil_stacksize = int(_testcapi.INT_MAX / 16 - fss - co_nlocalsplus) - with self.assertRaisesRegex(OverflowError, "co_stacksize"): + with self.assertRaisesRegex(OverflowError, "stack size is too large"): c.__replace__(co_stacksize=evil_stacksize) c.__replace__(co_stacksize=evil_stacksize - 1) From a0b85d4f014847c86c3621fdfb8424c0b52dead7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 8 Nov 2024 14:39:32 +0100 Subject: [PATCH 18/22] remove unused imports --- Lib/test/test_code.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 1014f1feb66cbe..8c0a5a57a331fd 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -200,11 +200,6 @@ except ImportError: _testcapi = None -try: - import _testinternalcapi -except ImportError: - _testinternalcapi = None - from test import support from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, @@ -593,7 +588,6 @@ def test_code_equal_with_instrumentation(self): @unittest.skipUnless(ctypes, "requires ctypes") @unittest.skipUnless(_testcapi, "requires _testcapi") - @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi") def test_co_framesize_overflow(self): # See: https://github.com/python/cpython/issues/126119. From 04abc46ec6b8da3a8ee7dd917ad8aec9e2bc6d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:37:19 +0100 Subject: [PATCH 19/22] update comment --- Objects/codeobject.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 64dbe5cd30cb6b..1ef6a07a7fc0b1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -437,16 +437,15 @@ _PyCode_Validate(struct _PyCodeConstructor *con) return -1; } /* - * The framesize = stacksize + nlocalsplus + FRAME_SPECIALS_SIZE is used - * as framesize * sizeof(PyObject *) and assumed to be < INT_MAX. Thus, - * we need to dynamically limit the value of stacksize. Note that this - * usually prevents crashes due to assertions but a MemoryError may still - * be triggered later. + * Since framesize = stacksize + nlocalsplus + FRAME_SPECIALS_SIZE is used + * as framesize * sizeof(PyObject *) and assumed to be < INT_MAX in many + * other places, we need to limit stacksize + nlocalsplus in order to + * avoid overflows. * * See https://github.com/python/cpython/issues/126119 for details * and corresponding PR for the rationale on the upper limit value. */ - Py_ssize_t limit = (Py_ssize_t)(INT_MAX / 16) - FRAME_SPECIALS_SIZE; + Py_ssize_t limit = (Py_ssize_t)(INT_MAX / 16); Py_ssize_t nlocalsplus = PyTuple_GET_SIZE(con->localsplusnames); if (nlocalsplus >= limit || con->stacksize >= limit - nlocalsplus) { PyErr_SetString(PyExc_OverflowError, From 0187b72ccd3e4251042aeec9a53f3ea25010a16c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:41:46 +0100 Subject: [PATCH 20/22] remove assertions from `gen_sizeof` and `frame_sizeof` --- Objects/frameobject.c | 22 ++++------------------ Objects/genobject.c | 22 ++++------------------ 2 files changed, 8 insertions(+), 36 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 9a9097643e5ba6..55394afa523213 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1802,25 +1802,11 @@ PyDoc_STRVAR(clear__doc__, static PyObject * frame_sizeof(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t base = offsetof(PyFrameObject, _f_frame_data) - + offsetof(_PyInterpreterFrame, localsplus); - assert(base <= INT_MAX); + Py_ssize_t res; + res = offsetof(PyFrameObject, _f_frame_data) + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); - int nslots = _PyFrame_NumSlotsForCodeObject(code); - assert(nslots >= 0); - // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. - // It should not be possible to have nslots >= PY_SSIZE_T_MAX - // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize - // is checked in _PyCode_Validate(). However, it is possible - // to make base + nslots * sizeof(PyObject *) >= INT_MAX since - // 'base' is not yet known when creating code objects. - // - // See https://github.com/python/cpython/issues/126119 for details. - if (nslots >= (int)((INT_MAX - base) / sizeof(PyObject *))) { - PyErr_SetString(PyExc_OverflowError, "size exceeds INT_MAX"); - return NULL; - } - return PyLong_FromSsize_t(base + nslots * sizeof(PyObject *)); + res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); + return PyLong_FromSsize_t(res); } PyDoc_STRVAR(sizeof__doc__, diff --git a/Objects/genobject.c b/Objects/genobject.c index 33eb880cd0dc2a..19c2c4e3331a89 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -817,25 +817,11 @@ static PyMemberDef gen_memberlist[] = { static PyObject * gen_sizeof(PyGenObject *gen, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t base = offsetof(PyGenObject, gi_iframe) - + offsetof(_PyInterpreterFrame, localsplus); - assert(base <= INT_MAX); + Py_ssize_t res; + res = offsetof(PyGenObject, gi_iframe) + offsetof(_PyInterpreterFrame, localsplus); PyCodeObject *code = _PyGen_GetCode(gen); - int nslots = _PyFrame_NumSlotsForCodeObject(code); - assert(nslots >= 0); - // By construction, 0 <= nslots < code->co_framesize <= INT_MAX. - // It should not be possible to have nslots >= PY_SSIZE_T_MAX - // even if PY_SSIZE_T_MAX < INT_MAX because code->co_framesize - // is checked in _PyCode_Validate(). However, it is possible - // to make base + nslots * sizeof(PyObject *) >= INT_MAX since - // 'base' is not yet known when creating code objects. - // - // See https://github.com/python/cpython/issues/126119 for details. - if (nslots >= (int)((INT_MAX - base) / sizeof(PyObject *))) { - PyErr_SetString(PyExc_OverflowError, "size exceeds INT_MAX"); - return NULL; - } - return PyLong_FromSsize_t(base + nslots * sizeof(PyObject *)); + res += _PyFrame_NumSlotsForCodeObject(code) * sizeof(PyObject *); + return PyLong_FromSsize_t(res); } PyDoc_STRVAR(sizeof__doc__, From c9969a469e7926154c1969d6cc12b6f6e06f5f6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:44:09 +0100 Subject: [PATCH 21/22] update test --- Lib/test/support/__init__.py | 14 -------------- Lib/test/test_code.py | 4 +--- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 5e1444a92635a7..971cfb94f96499 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -902,20 +902,6 @@ def check_sizeof(test, o, size): test.assertEqual(result, size, msg) -def get_frame_specials_size(): - """Compute the C defined constant FRAME_SPECIALS_SIZE in codeobject.c.""" - try: - import _testinternalcapi - except ImportError: - raise unittest.SkipTest("_testinternalcapi required") - - c = (lambda: ...).__code__ - # co_framesize = co_stacksize + co_nlocalsplus + FRAME_SPECIALS_SIZE - co_framesize = _testinternalcapi.get_co_framesize(c) - co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) - return co_framesize - c.co_stacksize - co_nlocalsplus - - #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards. diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 8c0a5a57a331fd..a7b9e8a37ef31b 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -200,7 +200,6 @@ except ImportError: _testcapi = None -from test import support from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, gc_collect, Py_GIL_DISABLED) @@ -597,10 +596,9 @@ def foo(a, b): c = foo.__code__ - fss = support.get_frame_specials_size() co_nlocalsplus = len({*c.co_varnames, *c.co_cellvars, *c.co_freevars}) # anything below that limit is a valid co_stacksize - evil_stacksize = int(_testcapi.INT_MAX / 16 - fss - co_nlocalsplus) + evil_stacksize = int(_testcapi.INT_MAX / 16 - co_nlocalsplus) with self.assertRaisesRegex(OverflowError, "stack size is too large"): c.__replace__(co_stacksize=evil_stacksize) From 6c0e1a62150cad7896c3f968046dea14a3ceb3b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 9 Nov 2024 10:44:23 +0100 Subject: [PATCH 22/22] update test --- Lib/test/support/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 971cfb94f96499..7c1ef42a4970d7 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -901,7 +901,6 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) - #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards.