diff --git a/doc/release/1.14.0-notes.rst b/doc/release/1.14.0-notes.rst index 5db0df5da2f6..6919585e4ebb 100644 --- a/doc/release/1.14.0-notes.rst +++ b/doc/release/1.14.0-notes.rst @@ -60,6 +60,21 @@ let us know. C API changes ============= +Change internal UPDATEIFCOPY semantics +-------------------------------------- +When ndarrays are created with the ``UPDATEIFCOPY`` flag, a temporary copy of +the data is created. The writeback to the original data occurrs at ndarray +deallocation. Refactor the writeback into a new PyArray_ResolveUpdateIfCopy +function and use it internally wherever such an array is created. The only +visible change in behavior is that if compiled with ``-DDEPRECATE_UPDATEIFCOPY`` +or if run on PyPy, and writeback resolution has not occurred before calling +``array_dealloc``, NumPy will now raise a DeprecationWarning. + +Additionally, deprecrate the NPY_ARRAY_UPDATEIFCOPY flag, replace it with +NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT. This was done to encourage downstream +consumers of the C-API to properly use the PyArray_ResolveUpdateIfCopy +function. Calling ndarray creation functions such as PyArray_FromArray where +flags uses NPY_ARRAY_UPDATEIFCOPY will raise a DeprecationWarning. New Features ============ diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 6e6547129b2f..aec313f86e6d 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -36,5 +36,5 @@ 0x0000000a = 9b8bce614655d3eb02acddcb508203cb # Version 11 (NumPy 1.13) Added PyArray_MapIterArrayCopyIfOverlap -# Version 11 (NumPy 1.14) No Change +# Version 11 (NumPy 1.14) Added PyArray_ResolveUpdateIfCopy 0x0000000b = edb1ba83730c650fd9bc5772a919cda7 diff --git a/numpy/core/code_generators/numpy_api.py b/numpy/core/code_generators/numpy_api.py index d1406e3b288a..8b2095b9f54e 100644 --- a/numpy/core/code_generators/numpy_api.py +++ b/numpy/core/code_generators/numpy_api.py @@ -346,6 +346,8 @@ # End 1.10 API 'PyArray_MapIterArrayCopyIfOverlap': (301,), # End 1.13 API + 'PyArray_ResolveUpdateIfCopy': (302,), + # End 1.14 API } ufunc_types_api = { diff --git a/numpy/core/include/numpy/ndarrayobject.h b/numpy/core/include/numpy/ndarrayobject.h index f26d64efbd41..5b96d0a70922 100644 --- a/numpy/core/include/numpy/ndarrayobject.h +++ b/numpy/core/include/numpy/ndarrayobject.h @@ -174,10 +174,10 @@ static NPY_INLINE void PyArray_XDECREF_ERR(PyArrayObject *arr) { if (arr != NULL) { - if (PyArray_FLAGS(arr) & NPY_ARRAY_UPDATEIFCOPY) { + if (PyArray_FLAGS(arr) & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) { PyArrayObject *base = (PyArrayObject *)PyArray_BASE(arr); PyArray_ENABLEFLAGS(base, NPY_ARRAY_WRITEABLE); - PyArray_CLEARFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); } Py_DECREF(arr); } diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index e0df189f9c6a..a3f4e105a8c4 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -870,7 +870,8 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); * This flag may be requested in constructor functions. * This flag may be tested for in PyArray_FLAGS(arr). */ -#define NPY_ARRAY_UPDATEIFCOPY 0x1000 +#define NPY_ARRAY_UPDATEIFCOPY 0x1000 /* Deprecated in 1.14 */ +#define NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT 0x2000 /* * NOTE: there are also internal flags defined in multiarray/arrayobject.h, @@ -894,11 +895,11 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); #define NPY_ARRAY_IN_ARRAY (NPY_ARRAY_CARRAY_RO) #define NPY_ARRAY_OUT_ARRAY (NPY_ARRAY_CARRAY) #define NPY_ARRAY_INOUT_ARRAY (NPY_ARRAY_CARRAY | \ - NPY_ARRAY_UPDATEIFCOPY) + NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) #define NPY_ARRAY_IN_FARRAY (NPY_ARRAY_FARRAY_RO) #define NPY_ARRAY_OUT_FARRAY (NPY_ARRAY_FARRAY) #define NPY_ARRAY_INOUT_FARRAY (NPY_ARRAY_FARRAY | \ - NPY_ARRAY_UPDATEIFCOPY) + NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) #define NPY_ARRAY_UPDATE_ALL (NPY_ARRAY_C_CONTIGUOUS | \ NPY_ARRAY_F_CONTIGUOUS | \ diff --git a/numpy/core/include/numpy/npy_1_7_deprecated_api.h b/numpy/core/include/numpy/npy_1_7_deprecated_api.h index 4c318bc4784c..dd6f6fa88d33 100644 --- a/numpy/core/include/numpy/npy_1_7_deprecated_api.h +++ b/numpy/core/include/numpy/npy_1_7_deprecated_api.h @@ -46,7 +46,7 @@ #define NPY_ALIGNED NPY_ARRAY_ALIGNED #define NPY_NOTSWAPPED NPY_ARRAY_NOTSWAPPED #define NPY_WRITEABLE NPY_ARRAY_WRITEABLE -#define NPY_UPDATEIFCOPY NPY_ARRAY_UPDATEIFCOPY +#define NPY_UPDATEIFCOPY NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT #define NPY_BEHAVED NPY_ARRAY_BEHAVED #define NPY_BEHAVED_NS NPY_ARRAY_BEHAVED_NS #define NPY_CARRAY NPY_ARRAY_CARRAY diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index 36d48af9fde8..54a535bbd96b 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -112,7 +112,7 @@ PyArray_SetUpdateIfCopyBase(PyArrayObject *arr, PyArrayObject *base) * references. */ ((PyArrayObject_fields *)arr)->base = (PyObject *)base; - PyArray_ENABLEFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY); + PyArray_ENABLEFLAGS(arr, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); PyArray_CLEARFLAGS(base, NPY_ARRAY_WRITEABLE); return 0; @@ -372,10 +372,63 @@ PyArray_TypeNumFromName(char *str) return NPY_NOTYPE; } +/*NUMPY_API + * + * If UPDATEIFCOPY and self has data, reset the base WRITEABLE flag, + * copy the local data to base, release the local data, and set flags + * appropriately. Return 0 if not relevant, 1 if success, < 0 on failure + */ +NPY_NO_EXPORT int +PyArray_ResolveUpdateIfCopy(PyArrayObject * self) +{ + PyArrayObject_fields *fa = (PyArrayObject_fields *)self; + if (fa && fa->base) { + if (fa->flags & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) { + /* + * UPDATEIFCOPY means that fa->base's data + * should be updated with the contents + * of self. + * fa->base->flags is not WRITEABLE to protect the relationship + * unlock it. + */ + int retval = 0; + PyArray_ENABLEFLAGS(((PyArrayObject *)fa->base), + NPY_ARRAY_WRITEABLE); + PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); + retval = PyArray_CopyAnyInto((PyArrayObject *)fa->base, self); + if (retval < 0) { + /* this should never happen, how did the two copies of data + * get out of sync? + */ + return retval; + } + if ((fa->flags & NPY_ARRAY_OWNDATA) && fa->data) { + /* Free internal references if an Object array */ + if (PyDataType_FLAGCHK(fa->descr, NPY_ITEM_REFCOUNT)) { + Py_INCREF(self); /*hold on to self */ + PyArray_XDECREF(self); + Py_DECREF(self); + } + npy_free_cache(fa->data, PyArray_NBYTES(self)); + fa->data = NULL; + PyArray_CLEARFLAGS(self, NPY_ARRAY_OWNDATA); + } + return 1; + } + } + return 0; + } + /*********************** end C-API functions **********************/ /* array object functions */ +#ifdef PYPY_VERSION + #ifndef DEPRECATE_UPDATEIFCOPY + #define DEPRECATE_UPDATEIFCOPY + #endif +#endif + static void array_dealloc(PyArrayObject *self) { @@ -387,27 +440,24 @@ array_dealloc(PyArrayObject *self) PyObject_ClearWeakRefs((PyObject *)self); } if (fa->base) { - /* - * UPDATEIFCOPY means that base points to an - * array that should be updated with the contents - * of this array upon destruction. - * fa->base->flags must have been WRITEABLE - * (checked previously) and it was locked here - * thus, unlock it. - */ - if (fa->flags & NPY_ARRAY_UPDATEIFCOPY) { - PyArray_ENABLEFLAGS(((PyArrayObject *)fa->base), - NPY_ARRAY_WRITEABLE); - Py_INCREF(self); /* hold on to self in next call */ - if (PyArray_CopyAnyInto((PyArrayObject *)fa->base, self) < 0) { - PyErr_Print(); - PyErr_Clear(); - } - /* - * Don't need to DECREF -- because we are deleting - *self already... + int retval; + Py_INCREF(self); /* hold on to self in next call */ + retval = PyArray_ResolveUpdateIfCopy(self); + if (retval < 0) + { + PyErr_Print(); + PyErr_Clear(); + } +#ifdef DEPRECATE_UPDATEIFCOPY + if (retval > 0) { + DEPRECATE("UPDATEIFCOPY resolution in array_dealloc is " + "incompatible with PyPy and will be removed in " + "the future"); + /* dealloc must not raise an error, even if warning filters are set */ + PyErr_Clear(); } +#endif /* * In any case base is pointing to something that we need * to DECREF -- either a view or a buffer object @@ -482,7 +532,7 @@ PyArray_DebugPrint(PyArrayObject *obj) printf(" NPY_ALIGNED"); if (fobj->flags & NPY_ARRAY_WRITEABLE) printf(" NPY_WRITEABLE"); - if (fobj->flags & NPY_ARRAY_UPDATEIFCOPY) + if (fobj->flags & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) printf(" NPY_UPDATEIFCOPY"); printf("\n"); @@ -507,8 +557,6 @@ PyArray_SetDatetimeParseFunction(PyObject *op) { } - - /*NUMPY_API */ NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/calculation.c b/numpy/core/src/multiarray/calculation.c index 379e5c3d26c6..da3700705a74 100644 --- a/numpy/core/src/multiarray/calculation.c +++ b/numpy/core/src/multiarray/calculation.c @@ -118,7 +118,7 @@ PyArray_ArgMax(PyArrayObject *op, int axis, PyArrayObject *out) } rp = (PyArrayObject *)PyArray_FromArray(out, PyArray_DescrFromType(NPY_INTP), - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); if (rp == NULL) { goto fail; } @@ -136,6 +136,7 @@ PyArray_ArgMax(PyArrayObject *op, int axis, PyArrayObject *out) Py_DECREF(ap); /* Trigger the UPDATEIFCOPY if necessary */ if (out != NULL && out != rp) { + PyArray_ResolveUpdateIfCopy(rp); Py_DECREF(rp); rp = out; Py_INCREF(rp); @@ -233,7 +234,7 @@ PyArray_ArgMin(PyArrayObject *op, int axis, PyArrayObject *out) } rp = (PyArrayObject *)PyArray_FromArray(out, PyArray_DescrFromType(NPY_INTP), - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); if (rp == NULL) { goto fail; } @@ -251,6 +252,7 @@ PyArray_ArgMin(PyArrayObject *op, int axis, PyArrayObject *out) Py_DECREF(ap); /* Trigger the UPDATEIFCOPY if necessary */ if (out != NULL && out != rp) { + PyArray_ResolveUpdateIfCopy(rp); Py_DECREF(rp); rp = out; Py_INCREF(rp); @@ -1117,7 +1119,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o oflags = NPY_ARRAY_FARRAY; else oflags = NPY_ARRAY_CARRAY; - oflags |= NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_FORCECAST; + oflags |= NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT | NPY_ARRAY_FORCECAST; Py_INCREF(indescr); newout = (PyArrayObject*)PyArray_FromArray(out, indescr, oflags); if (newout == NULL) { @@ -1153,6 +1155,7 @@ PyArray_Clip(PyArrayObject *self, PyObject *min, PyObject *max, PyArrayObject *o Py_XDECREF(maxa); Py_DECREF(newin); /* Copy back into out if out was not already a nice array. */ + PyArray_ResolveUpdateIfCopy(newout); Py_DECREF(newout); return (PyObject *)out; diff --git a/numpy/core/src/multiarray/cblasfuncs.c b/numpy/core/src/multiarray/cblasfuncs.c index 3b0b2f4f6d75..271e7159ae3a 100644 --- a/numpy/core/src/multiarray/cblasfuncs.c +++ b/numpy/core/src/multiarray/cblasfuncs.c @@ -770,6 +770,7 @@ cblas_matrixproduct(int typenum, PyArrayObject *ap1, PyArrayObject *ap2, Py_DECREF(ap2); /* Trigger possible copyback into `result` */ + PyArray_ResolveUpdateIfCopy(out_buf); Py_DECREF(out_buf); return PyArray_Return(result); diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index fa4a10c0e13b..60182f080421 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -327,7 +327,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) } array = (PyArrayObject *)PyArray_FromArray((PyArrayObject *)array0, NULL, - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); if (array == NULL) { goto fail; } @@ -402,6 +402,7 @@ arr_insert(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict) Py_XDECREF(values); Py_XDECREF(mask); + PyArray_ResolveUpdateIfCopy(array); Py_DECREF(array); Py_RETURN_NONE; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index de164823003b..534487c964d5 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1009,7 +1009,7 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd, } } else { - fa->flags = (flags & ~NPY_ARRAY_UPDATEIFCOPY); + fa->flags = (flags & ~NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); } fa->descr = descr; fa->base = (PyObject *)NULL; @@ -1703,7 +1703,8 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, /* If we got dimensions and dtype instead of an array */ if (arr == NULL) { - if (flags & NPY_ARRAY_UPDATEIFCOPY) { + if ((flags & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) || + (flags & NPY_ARRAY_UPDATEIFCOPY)) { Py_XDECREF(newtype); PyErr_SetString(PyExc_TypeError, "UPDATEIFCOPY used for non-array input."); @@ -1811,6 +1812,7 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * NPY_ARRAY_NOTSWAPPED, * NPY_ARRAY_ENSURECOPY, * NPY_ARRAY_UPDATEIFCOPY, + * NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT, * NPY_ARRAY_FORCECAST, * NPY_ARRAY_ENSUREARRAY, * NPY_ARRAY_ELEMENTSTRIDES @@ -1835,10 +1837,13 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, * Fortran arrays are always behaved (aligned, * notswapped, and writeable) and not (C) CONTIGUOUS (if > 1d). * - * NPY_ARRAY_UPDATEIFCOPY flag sets this flag in the returned array if a copy - * is made and the base argument points to the (possibly) misbehaved array. - * When the new array is deallocated, the original array held in base - * is updated with the contents of the new array. + * NPY_ARRAY_UPDATEIFCOPY is deprecated in favor of + * NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT in 1.14 + + * NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT flag sets this flag in the returned + * array if a copy is made and the base argument points to the (possibly) + * misbehaved array. Before returning to python, PyArray_ResolveUpdateIfCopy + * must be called to update the contents of the orignal array from the copy. * * NPY_ARRAY_FORCECAST will cause a cast to occur regardless of whether or not * it is safe. @@ -2005,7 +2010,16 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags) return NULL; } - if (flags & NPY_ARRAY_UPDATEIFCOPY) { + if ((flags & NPY_ARRAY_UPDATEIFCOPY) || + (flags & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT)) { + if (flags & NPY_ARRAY_UPDATEIFCOPY) { + if (DEPRECATE("Use NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT and" + " be sure to call PyArray_ResolveUpdateIfCopy") < 0) { + /* 2017-07-24, 1.14 */ + Py_DECREF(ret); + return NULL; + } + } Py_INCREF(arr); if (PyArray_SetUpdateIfCopyBase(ret, arr) < 0) { Py_DECREF(ret); diff --git a/numpy/core/src/multiarray/flagsobject.c b/numpy/core/src/multiarray/flagsobject.c index 7f56ddb038aa..837cb72baf03 100644 --- a/numpy/core/src/multiarray/flagsobject.c +++ b/numpy/core/src/multiarray/flagsobject.c @@ -208,7 +208,7 @@ arrayflags_dealloc(PyArrayFlagsObject *self) _define_get(NPY_ARRAY_C_CONTIGUOUS, contiguous) _define_get(NPY_ARRAY_F_CONTIGUOUS, fortran) -_define_get(NPY_ARRAY_UPDATEIFCOPY, updateifcopy) +_define_get(NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT, updateifcopy) _define_get(NPY_ARRAY_OWNDATA, owndata) _define_get(NPY_ARRAY_ALIGNED, aligned) _define_get(NPY_ARRAY_WRITEABLE, writeable) @@ -595,7 +595,7 @@ arrayflags_print(PyArrayFlagsObject *self) "OWNDATA", _torf_(fl, NPY_ARRAY_OWNDATA), "WRITEABLE", _torf_(fl, NPY_ARRAY_WRITEABLE), "ALIGNED", _torf_(fl, NPY_ARRAY_ALIGNED), - "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY)); + "UPDATEIFCOPY", _torf_(fl, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT)); } diff --git a/numpy/core/src/multiarray/getset.c b/numpy/core/src/multiarray/getset.c index 538256369890..8cd1b49b6e77 100644 --- a/numpy/core/src/multiarray/getset.c +++ b/numpy/core/src/multiarray/getset.c @@ -365,10 +365,10 @@ array_data_set(PyArrayObject *self, PyObject *op) PyDataMem_FREE(PyArray_DATA(self)); } if (PyArray_BASE(self)) { - if (PyArray_FLAGS(self) & NPY_ARRAY_UPDATEIFCOPY) { + if (PyArray_FLAGS(self) & NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) { PyArray_ENABLEFLAGS((PyArrayObject *)PyArray_BASE(self), NPY_ARRAY_WRITEABLE); - PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); } Py_DECREF(PyArray_BASE(self)); ((PyArrayObject_fields *)self)->base = NULL; @@ -604,7 +604,7 @@ array_struct_get(PyArrayObject *self) inter->itemsize = PyArray_DESCR(self)->elsize; inter->flags = PyArray_FLAGS(self); /* reset unused flags */ - inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_OWNDATA); + inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT | NPY_ARRAY_OWNDATA); if (PyArray_ISNOTSWAPPED(self)) inter->flags |= NPY_ARRAY_NOTSWAPPED; /* * Copy shape and strides over since these can be reset diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 21bcd6cadd90..68e8c210c5e5 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -88,7 +88,7 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, } else { int flags = NPY_ARRAY_CARRAY | - NPY_ARRAY_UPDATEIFCOPY; + NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT; if ((PyArray_NDIM(out) != nd) || !PyArray_CompareLists(PyArray_DIMS(out), shape, nd)) { @@ -235,6 +235,7 @@ PyArray_TakeFrom(PyArrayObject *self0, PyObject *indices0, int axis, Py_XDECREF(self); if (out != NULL && out != obj) { Py_INCREF(out); + PyArray_ResolveUpdateIfCopy(obj); Py_DECREF(obj); obj = out; } @@ -273,7 +274,7 @@ PyArray_PutTo(PyArrayObject *self, PyObject* values0, PyObject *indices0, if (!PyArray_ISCONTIGUOUS(self)) { PyArrayObject *obj; - int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY; + int flags = NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT; if (clipmode == NPY_RAISE) { flags |= NPY_ARRAY_ENSURECOPY; @@ -407,6 +408,7 @@ PyArray_PutTo(PyArrayObject *self, PyObject* values0, PyObject *indices0, Py_XDECREF(values); Py_XDECREF(indices); if (copied) { + PyArray_ResolveUpdateIfCopy(self); Py_DECREF(self); } Py_RETURN_NONE; @@ -448,7 +450,7 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) dtype = PyArray_DESCR(self); Py_INCREF(dtype); obj = (PyArrayObject *)PyArray_FromArray(self, dtype, - NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY); + NPY_ARRAY_CARRAY | NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); if (obj != self) { copied = 1; } @@ -524,6 +526,7 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) Py_XDECREF(values); Py_XDECREF(mask); if (copied) { + PyArray_ResolveUpdateIfCopy(self); Py_DECREF(self); } Py_RETURN_NONE; @@ -694,7 +697,7 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out, } else { int flags = NPY_ARRAY_CARRAY | - NPY_ARRAY_UPDATEIFCOPY | + NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT | NPY_ARRAY_FORCECAST; if ((PyArray_NDIM(out) != multi->nd) @@ -769,6 +772,7 @@ PyArray_Choose(PyArrayObject *ip, PyObject *op, PyArrayObject *out, npy_free_cache(mps, n * sizeof(mps[0])); if (out != NULL && out != obj) { Py_INCREF(out); + PyArray_ResolveUpdateIfCopy(obj); Py_DECREF(obj); obj = out; } diff --git a/numpy/core/src/multiarray/iterators.c b/numpy/core/src/multiarray/iterators.c index 01910a657261..74ee75f01448 100644 --- a/numpy/core/src/multiarray/iterators.c +++ b/numpy/core/src/multiarray/iterators.c @@ -1151,6 +1151,7 @@ iter_richcompare(PyArrayIterObject *self, PyObject *other, int cmp_op) return NULL; } ret = array_richcompare(new, other, cmp_op); + PyArray_ResolveUpdateIfCopy(new); Py_DECREF(new); return ret; } diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index ffccf4e055c8..2286e7324e41 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -1778,7 +1778,7 @@ array_setstate(PyArrayObject *self, PyObject *args) Py_XDECREF(PyArray_BASE(self)); fa->base = NULL; - PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); if (PyArray_DIMS(self) != NULL) { npy_free_cache_dim_array(self); @@ -2324,7 +2324,7 @@ array_setflags(PyArrayObject *self, PyObject *args, PyObject *kwds) return NULL; } else { - PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY); + PyArray_CLEARFLAGS(self, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT); Py_XDECREF(fa->base); fa->base = NULL; } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index ed5fa07b3cfa..8a5f8c93eed6 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -1062,6 +1062,7 @@ PyArray_MatrixProduct2(PyObject *op1, PyObject *op2, PyArrayObject* out) Py_DECREF(ap2); /* Trigger possible copy-back into `result` */ + PyArray_ResolveUpdateIfCopy(out_buf); Py_DECREF(out_buf); return (PyObject *)result; @@ -4464,7 +4465,7 @@ set_flaginfo(PyObject *d) _addnew(FORTRAN, NPY_ARRAY_F_CONTIGUOUS, F); _addnew(CONTIGUOUS, NPY_ARRAY_C_CONTIGUOUS, C); _addnew(ALIGNED, NPY_ARRAY_ALIGNED, A); - _addnew(UPDATEIFCOPY, NPY_ARRAY_UPDATEIFCOPY, U); + _addnew(UPDATEIFCOPY, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT, U); _addnew(WRITEABLE, NPY_ARRAY_WRITEABLE, W); _addone(C_CONTIGUOUS, NPY_ARRAY_C_CONTIGUOUS); _addone(F_CONTIGUOUS, NPY_ARRAY_F_CONTIGUOUS); diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 7a7d674a402c..f75afc8b4ef4 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -693,7 +693,7 @@ npyiter_convert_ops(PyObject *op_in, PyObject *op_flags_in, int fromanyflags = 0; if (op_flags[iop]&(NPY_ITER_READWRITE|NPY_ITER_WRITEONLY)) { - fromanyflags |= NPY_ARRAY_UPDATEIFCOPY; + fromanyflags |= NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT; } ao = (PyArrayObject *)PyArray_FROM_OF((PyObject *)op[iop], fromanyflags); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index cc867fe04380..9975ff27104d 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -1209,7 +1209,7 @@ gentype_struct_get(PyObject *self) inter->two = 2; inter->nd = 0; inter->flags = PyArray_FLAGS(arr); - inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY | NPY_ARRAY_OWNDATA); + inter->flags &= ~(NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT | NPY_ARRAY_OWNDATA); inter->flags |= NPY_ARRAY_NOTSWAPPED; inter->typekind = PyArray_DESCR(arr)->kind; inter->itemsize = PyArray_DESCR(arr)->elsize; diff --git a/numpy/core/src/multiarray/temp_elide.c b/numpy/core/src/multiarray/temp_elide.c index abca0ecd630e..df19afe99ed9 100644 --- a/numpy/core/src/multiarray/temp_elide.c +++ b/numpy/core/src/multiarray/temp_elide.c @@ -286,7 +286,7 @@ can_elide_temp(PyArrayObject * alhs, PyObject * orhs, int * cannot) !PyArray_ISNUMBER(alhs) || !(PyArray_FLAGS(alhs) & NPY_ARRAY_OWNDATA) || !PyArray_ISWRITEABLE(alhs) || - PyArray_CHKFLAGS(alhs, NPY_ARRAY_UPDATEIFCOPY) || + PyArray_CHKFLAGS(alhs, NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT) || PyArray_NBYTES(alhs) < NPY_MIN_ELIDE_BYTES) { return 0; } diff --git a/numpy/core/src/umath/reduction.c b/numpy/core/src/umath/reduction.c index e870620140b3..69cbecbf6af5 100644 --- a/numpy/core/src/umath/reduction.c +++ b/numpy/core/src/umath/reduction.c @@ -610,6 +610,7 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, } } else { + PyArray_ResolveUpdateIfCopy(result); /* prevent spurious warnings */ Py_DECREF(result); result = out; Py_INCREF(result); @@ -618,6 +619,7 @@ PyUFunc_ReduceWrapper(PyArrayObject *operand, PyArrayObject *out, return result; fail: + PyArray_ResolveUpdateIfCopy(result); /* prevent spurious warnings */ Py_XDECREF(result); Py_XDECREF(op_view); if (iter != NULL) { diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 2d59895541e0..a90b8ec6f23b 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1501,7 +1501,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, PyObject **arr_prep, PyObject *arr_prep_args) { - int i, nin = ufunc->nin, nout = ufunc->nout; + int retval, i, nin = ufunc->nin, nout = ufunc->nout; int nop = nin + nout; npy_uint32 op_flags[NPY_MAXARGS]; NpyIter *iter; @@ -1687,8 +1687,16 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, NPY_AUXDATA_FREE(innerloopdata); } + retval = 0; + nop = NpyIter_GetNOp(iter); + for(i=0; i< nop; ++i) { + if (PyArray_ResolveUpdateIfCopy(NpyIter_GetOperandArray(iter)[i]) < 0) { + retval = -1; + } + } + NpyIter_Deallocate(iter); - return 0; + return retval; } static PyObject * @@ -2285,6 +2293,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, goto fail; } + /* Write back any temporary data from PyArray_SetUpdateIfCopyBase */ + for(i=nin; i< nop; ++i) + if (PyArray_ResolveUpdateIfCopy(NpyIter_GetOperandArray(iter)[i]) < 0) + goto fail; + PyArray_free(inner_strides); NpyIter_Deallocate(iter); /* The caller takes ownership of all the references in op */ @@ -3217,6 +3230,9 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, } finish: + /* Write back any temporary data from PyArray_SetUpdateIfCopyBase */ + if (PyArray_ResolveUpdateIfCopy(op[0]) < 0) + goto fail; Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); NpyIter_Deallocate(iter_inner); @@ -3599,6 +3615,9 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, } finish: + if (op[0] && PyArray_ResolveUpdateIfCopy(op[0]) < 0) { + goto fail; + } Py_XDECREF(op_dtypes[0]); NpyIter_Deallocate(iter); diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index c182a4d59cbb..daafd616cfc3 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -6901,6 +6901,72 @@ def test_ctypes_is_not_available(self): _internal.ctypes = ctypes +class TestUpdateIfCopy(TestCase): + # all these tests use the UPDATEIFCOPY mechanism + def test_argmax_with_out(self): + mat = np.eye(5) + out = np.empty(5, dtype='i2') + res = np.argmax(mat, 0, out=out) + assert_equal(res, range(5)) + + def test_argmin_with_out(self): + mat = -np.eye(5) + out = np.empty(5, dtype='i2') + res = np.argmin(mat, 0, out=out) + assert_equal(res, range(5)) + + def test_clip_with_out(self): + mat = np.eye(5) + out = np.eye(5, dtype='i2') + res = np.clip(mat, a_min=-10, a_max=0, out=out) + assert_equal(np.sum(out), 0) + + def test_insert_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + # uses arr_insert + np.place(a, a>2, [44, 55]) + assert_equal(a, np.array([[0, 44], [1, 55], [2, 44]])) + + def test_put_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + np.put(a, [0, 2], [44, 55]) + assert_equal(a, np.array([[44, 3], [55, 4], [2, 5]])) + + def test_putmask_noncontiguous(self): + a = np.arange(6).reshape(2,3).T # force non-c-contiguous + # uses arr_putmask + np.putmask(a, a>2, a**2) + assert_equal(a, np.array([[0, 9], [1, 16], [2, 25]])) + + def test_take_mode_raise(self): + a = np.arange(6, dtype='int') + out = np.empty(2, dtype='int') + np.take(a, [0, 2], out=out, mode='raise') + assert_equal(out, np.array([0, 2])) + + def test_choose_mod_raise(self): + a = np.array([[1, 0, 1], [0, 1, 0], [1, 0, 1]]) + out = np.empty((3,3), dtype='int') + choices = [-10, 10] + np.choose(a, choices, out=out, mode='raise') + assert_equal(out, np.array([[ 10, -10, 10], + [-10, 10, -10], + [ 10, -10, 10]])) + + def test_flatiter__array__(self): + a = np.arange(9).reshape(3,3) + b = a.T.flat + c = b.__array__() + # triggers the UPDATEIFCOPY resolution, assuming refcount semantics + del c + + def test_dot_out(self): + # if HAVE_CBLAS, will use UPDATEIFCOPY + a = np.arange(9, dtype=float).reshape(3,3) + b = np.dot(a, a, out=a) + assert_equal(b, np.array([[15, 18, 21], [42, 54, 66], [69, 90, 111]])) + + def test_orderconverter_with_nonASCII_unicode_ordering(): # gh-7475 a = np.arange(5) diff --git a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c index 2da6a2c5de10..e62e54a71d79 100644 --- a/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c +++ b/numpy/f2py/tests/src/array_from_pyobj/wrapmodule.c @@ -198,7 +198,7 @@ PyMODINIT_FUNC inittest_array_from_pyobj_ext(void) { PyDict_SetItemString(d, "ENSUREARRAY", PyInt_FromLong(NPY_ARRAY_ENSUREARRAY)); PyDict_SetItemString(d, "ALIGNED", PyInt_FromLong(NPY_ARRAY_ALIGNED)); PyDict_SetItemString(d, "WRITEABLE", PyInt_FromLong(NPY_ARRAY_WRITEABLE)); - PyDict_SetItemString(d, "UPDATEIFCOPY", PyInt_FromLong(NPY_ARRAY_UPDATEIFCOPY)); + PyDict_SetItemString(d, "UPDATEIFCOPY", PyInt_FromLong(NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT)); PyDict_SetItemString(d, "BEHAVED", PyInt_FromLong(NPY_ARRAY_BEHAVED)); PyDict_SetItemString(d, "BEHAVED_NS", PyInt_FromLong(NPY_ARRAY_BEHAVED_NS)); diff --git a/numpy/random/mtrand/numpy.pxd b/numpy/random/mtrand/numpy.pxd index d5b0d74caf68..07aab5e500cf 100644 --- a/numpy/random/mtrand/numpy.pxd +++ b/numpy/random/mtrand/numpy.pxd @@ -40,7 +40,7 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_ALIGNED NPY_ARRAY_NOTSWAPPED NPY_ARRAY_WRITEABLE - NPY_ARRAY_UPDATEIFCOPY + NPY_ARRAY_UPDATEIFCOPY_CLEAR_B4_EXIT NPY_ARR_HAS_DESCR NPY_ARRAY_BEHAVED