diff --git a/Include/listobject.h b/Include/listobject.h index 6057279d51c3a4..69dc9b7463fb14 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -62,6 +62,7 @@ PyAPI_FUNC(int) PyList_Reverse(PyObject *); PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyList_Copy(PyObject *); PyAPI_FUNC(int) PyList_ClearFreeList(void); PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 2c64c55e7e3476..cc41d1aab5ad8f 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -105,14 +105,16 @@ future_schedule_callbacks(FutureObj *fut) return 0; } - callbacks = PyList_GetSlice(fut->fut_callbacks, 0, len); + callbacks = _PyList_Copy(fut->fut_callbacks); if (callbacks == NULL) { return -1; } + len = PyList_GET_SIZE(fut->fut_callbacks); if (PyList_SetSlice(fut->fut_callbacks, 0, len, NULL) < 0) { Py_DECREF(callbacks); return -1; } + len = PyList_GET_SIZE(callbacks); for (i = 0; i < len; i++) { PyObject *handle; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index bddac851d9c708..63389a5b77ef40 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -912,9 +912,17 @@ _elementtree_Element___getstate___impl(ElementObject *self) PyObject *instancedict = NULL, *children; /* Build a list of children. */ + again: children = PyList_New(self->extra ? self->extra->length : 0); if (!children) return NULL; + if (PyList_GET_SIZE(children) != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(children); + goto again; + } for (i = 0; i < PyList_GET_SIZE(children); i++) { PyObject *child = self->extra->children[i]; Py_INCREF(child); @@ -1375,12 +1383,20 @@ _elementtree_Element_getchildren_impl(ElementObject *self) return NULL; } + again: if (!self->extra) return PyList_New(0); list = PyList_New(self->extra->length); if (!list) return NULL; + if (PyList_GET_SIZE(list) != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(list); + goto again; + } for (i = 0; i < self->extra->length; i++) { PyObject* item = self->extra->children[i]; @@ -1739,6 +1755,7 @@ element_subscr(PyObject* self_, PyObject* item) Py_ssize_t start, stop, step, slicelen, cur, i; PyObject* list; + again: if (!self->extra) return PyList_New(0); @@ -1754,6 +1771,13 @@ element_subscr(PyObject* self_, PyObject* item) list = PyList_New(slicelen); if (!list) return NULL; + if (slicelen != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(list); + goto again; + } for (cur = start, i = 0; i < slicelen; cur += step, i++) { @@ -1788,11 +1812,12 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) return element_setitem(self_, i, value); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelen, newlen, cur, i; - - PyObject* recycle = NULL; + Py_ssize_t start, stop, step, slicelen, oldlen, newlen, cur, i; + PyObject* recycle; PyObject* seq; + again: + recycle = NULL; if (!self->extra) { if (create_extra(self, NULL) < 0) return -1; @@ -1801,6 +1826,7 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } + oldlen = self->extra->length; slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, step); @@ -1827,9 +1853,15 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) * scheduled for removal. */ if (!(recycle = PyList_New(slicelen))) { - PyErr_NoMemory(); return -1; } + if (oldlen != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(recycle); + goto again; + } /* This loop walks over all the children that have to be deleted, * with cur pointing at them. num_moved is the amount of children @@ -1899,6 +1931,14 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) Py_DECREF(seq); return -1; } + oldlen += newlen - slicelen; + if (oldlen != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(seq); + goto again; + } } if (slicelen > 0) { @@ -1910,6 +1950,14 @@ element_ass_subscr(PyObject* self_, PyObject* item, PyObject* value) Py_DECREF(seq); return -1; } + if (oldlen != (self->extra ? self->extra->length : 0)) { + /* Durnit. The allocations caused the extra to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(seq); + Py_DECREF(recycle); + goto again; + } for (cur = start, i = 0; i < slicelen; cur += step, i++) PyList_SET_ITEM(recycle, i, self->extra->children[cur]); diff --git a/Modules/_weakref.c b/Modules/_weakref.c index f9c68d6a640f34..9aaaa48e573f9b 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -86,10 +86,19 @@ weakref_getweakrefs(PyObject *self, PyObject *object) if (PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); + Py_ssize_t count; + again: + count = _PyWeakref_GetWeakrefCount(*list); result = PyList_New(count); if (result != NULL) { + if (count != _PyWeakref_GetWeakrefCount(*list)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(result); + goto again; + } PyWeakReference *current = *list; Py_ssize_t i; for (i = 0; i < count; ++i) { diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 4f778a2dea3ec3..3ef745ca154229 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1565,15 +1565,32 @@ static PyObject * array_array_tolist_impl(arrayobject *self) /*[clinic end generated code: output=00b60cc9eab8ef89 input=a8d7784a94f86b53]*/ { - PyObject *list = PyList_New(Py_SIZE(self)); + PyObject *list; Py_ssize_t i; + again: + list = PyList_New(Py_SIZE(self)); if (list == NULL) return NULL; + if (PyList_GET_SIZE(list) != Py_SIZE(self)) { + /* Durnit. The allocations caused the array to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(list); + goto again; + } for (i = 0; i < Py_SIZE(self); i++) { PyObject *v = getarrayitem((PyObject *)self, i); if (v == NULL) goto error; + if (PyList_GET_SIZE(list) != Py_SIZE(self)) { + /* Durnit. The allocations caused the array to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(v); + Py_DECREF(list); + goto again; + } if (PyList_SetItem(list, i, v) < 0) goto error; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 91902363649882..2f79064ca5a443 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -507,11 +507,20 @@ static PyObject * teedataobject_reduce(teedataobject *tdo) { int i; + PyObject *values; /* create a temporary list of already iterated values */ - PyObject *values = PyList_New(tdo->numread); + again: + values = PyList_New(tdo->numread); if (!values) return NULL; + if (PyList_GET_SIZE(values) != tdo->numread) { + /* Durnit. The allocations caused the number of already iterated values to change. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(values); + goto again; + } for (i=0 ; inumread ; i++) { Py_INCREF(tdo->values[i]); PyList_SET_ITEM(values, i, tdo->values[i]); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5f30b202ec6167..b62504cd783319 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -414,7 +414,7 @@ run_at_forkers(PyObject *lst, int reverse) /* Use a list copy in case register_at_fork() is called from * one of the callbacks. */ - cpy = PyList_GetSlice(lst, 0, PyList_GET_SIZE(lst)); + cpy = _PyList_Copy(lst); if (cpy == NULL) PyErr_WriteUnraisable(lst); else { diff --git a/Objects/listobject.c b/Objects/listobject.c index 858532252e92a7..f845bd93b61c57 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -17,6 +17,8 @@ class list "PyListObject *" "&PyList_Type" #include "clinic/listobject.c.h" +static PyObject *list_copy_impl(PyListObject *); + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -431,24 +433,42 @@ list_item(PyListObject *a, Py_ssize_t i) return a->ob_item[i]; } -static PyObject * -list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +PyObject * +PyList_GetSlice(PyObject *o, Py_ssize_t low, Py_ssize_t high) { - PyListObject *np; + PyListObject *a, *np; PyObject **src, **dest; - Py_ssize_t i, len; + Py_ssize_t i, llen, len, ilow, ihigh; + + if (!PyList_Check(o)) { + PyErr_BadInternalCall(); + return NULL; + } + a = (PyListObject *)o; + + again: + llen = Py_SIZE(a); + ilow = low; + ihigh = high; if (ilow < 0) ilow = 0; - else if (ilow > Py_SIZE(a)) - ilow = Py_SIZE(a); + else if (ilow > llen) + ilow = llen; if (ihigh < ilow) ihigh = ilow; - else if (ihigh > Py_SIZE(a)) - ihigh = Py_SIZE(a); + else if (ihigh > llen) + ihigh = llen; len = ihigh - ilow; np = (PyListObject *) PyList_New(len); if (np == NULL) return NULL; + if (llen != Py_SIZE(a)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(np); + goto again; + } src = a->ob_item + ilow; dest = np->ob_item; @@ -460,16 +480,6 @@ list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) return (PyObject *)np; } -PyObject * -PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) -{ - if (!PyList_Check(a)) { - PyErr_BadInternalCall(); - return NULL; - } - return list_slice((PyListObject *)a, ilow, ihigh); -} - static PyObject * list_concat(PyListObject *a, PyObject *bb) { @@ -484,6 +494,7 @@ list_concat(PyListObject *a, PyObject *bb) return NULL; } #define b ((PyListObject *)bb) + again: if (Py_SIZE(a) > PY_SSIZE_T_MAX - Py_SIZE(b)) return PyErr_NoMemory(); size = Py_SIZE(a) + Py_SIZE(b); @@ -491,6 +502,13 @@ list_concat(PyListObject *a, PyObject *bb) if (np == NULL) { return NULL; } + if (size != Py_SIZE(a) + Py_SIZE(b)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(np); + goto again; + } src = a->ob_item; dest = np->ob_item; for (i = 0; i < Py_SIZE(a); i++) { @@ -513,20 +531,29 @@ static PyObject * list_repeat(PyListObject *a, Py_ssize_t n) { Py_ssize_t i, j; - Py_ssize_t size; + Py_ssize_t llen, size; PyListObject *np; PyObject **p, **items; PyObject *elem; if (n < 0) n = 0; - if (n > 0 && Py_SIZE(a) > PY_SSIZE_T_MAX / n) + again: + llen = Py_SIZE(a); + if (n > 0 && llen > PY_SSIZE_T_MAX / n) return PyErr_NoMemory(); - size = Py_SIZE(a) * n; + size = llen * n; if (size == 0) return PyList_New(0); np = (PyListObject *) PyList_New(size); if (np == NULL) return NULL; + if (llen != Py_SIZE(a)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(np); + goto again; + } items = np->ob_item; if (Py_SIZE(a) == 1) { @@ -604,7 +631,7 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) else { if (a == b) { /* Special case "a[i:j] = a" -- copy b first */ - v = list_slice(b, 0, Py_SIZE(b)); + v = list_copy_impl(b); if (v == NULL) return result; result = list_ass_slice(a, ilow, ihigh, v); @@ -791,7 +818,37 @@ static PyObject * list_copy_impl(PyListObject *self) /*[clinic end generated code: output=ec6b72d6209d418e input=6453ab159e84771f]*/ { - return list_slice(self, 0, Py_SIZE(self)); + PyListObject *np; + PyObject **src, **dest; + Py_ssize_t i, len; + + again: + len = Py_SIZE(self); + np = (PyListObject *) PyList_New(len); + if (np == NULL) + return NULL; + if (len != Py_SIZE(self)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(np); + goto again; + } + + src = self->ob_item; + dest = np->ob_item; + for (i = 0; i < len; i++) { + PyObject *v = src[i]; + Py_INCREF(v); + dest[i] = v; + } + return (PyObject *)np; +} + +PyObject * +_PyList_Copy(PyObject *a) +{ + return list_copy_impl((PyListObject *)a); } /*[clinic input] @@ -2482,26 +2539,32 @@ list_subscript(PyListObject* self, PyObject* item) return list_item(self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, cur, i; + Py_ssize_t start, stop, step, slicelength, llen, cur, i; PyObject* result; PyObject* it; PyObject **src, **dest; + again: if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, + llen = Py_SIZE(self); + slicelength = PySlice_AdjustIndices(llen, &start, &stop, step); if (slicelength <= 0) { return PyList_New(0); } - else if (step == 1) { - return list_slice(self, start, stop); - } else { result = PyList_New(slicelength); if (!result) return NULL; + if (llen != Py_SIZE(self)) { + /* Durnit. The allocations caused the list to resize. + * Just start over, this shouldn't normally happen. + */ + Py_DECREF(result); + goto again; + } src = self->ob_item; dest = ((PyListObject *)result)->ob_item; @@ -2622,8 +2685,7 @@ list_ass_subscript(PyListObject* self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - seq = list_slice((PyListObject*)value, 0, - PyList_GET_SIZE(value)); + seq = list_copy_impl((PyListObject*)value); } else { seq = PySequence_Fast(value,